Server Metrics Reference

The na-user service exports a variety of metrics suitable for monitoring performance. The metrics are suitable for importing into metrics monitoring systems such as Prometheus or Datadog.

Metrics are available on the https://na-user.example.com/metrics API endpoint. Below is a sample of output from a call to that API endpoint with a description of the available metrics.

An example call to, e.g.

curl -k -s 'https://na-user.example.com/metrics'

Will result in output similar to this:

# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0
go_gc_duration_seconds{quantile="0.25"} 0
go_gc_duration_seconds{quantile="0.5"} 0
go_gc_duration_seconds{quantile="0.75"} 0
go_gc_duration_seconds{quantile="1"} 0
go_gc_duration_seconds_sum 0
go_gc_duration_seconds_count 0
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 11
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.21.1"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 2.034488e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 2.034488e+06
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 4483
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 1089
# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
# TYPE go_memstats_gc_cpu_fraction gauge
go_memstats_gc_cpu_fraction 0
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 3.186936e+06
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 2.034488e+06
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 4.431872e+06
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 3.334144e+06
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 14468
# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes gauge
go_memstats_heap_released_bytes 4.431872e+06
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 7.766016e+06
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 0
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 0
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 15557
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 4800
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 15600
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 78456
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 81480
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 4.194304e+06
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 1.273437e+06
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 622592
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 622592
# HELP go_memstats_sys_bytes Number of bytes obtained from system.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 1.2950544e+07
# HELP go_threads Number of OS threads created.
# TYPE go_threads gauge
go_threads 9
# HELP na_user_http_200_ok_total Number of HTTP Status 200 OK responses
# TYPE na_user_http_200_ok_total counter
na_user_http_200_ok_total 2
# HELP na_user_http_400_bad_request_total Number of HTTP Status 400 BadRequest responses
# TYPE na_user_http_400_bad_request_total counter
na_user_http_400_bad_request_total 0
# HELP na_user_http_401_unauthorized_total Number of HTTP Status 401 Unauthorized responses
# TYPE na_user_http_401_unauthorized_total counter
na_user_http_401_unauthorized_total 0
# HELP na_user_http_403_forbidden_total Number of HTTP Status 403 Forbidden responses
# TYPE na_user_http_403_forbidden_total counter
na_user_http_403_forbidden_total 0
# HELP na_user_http_404_not_found_total Number of HTTP Status 404 NotFound responses
# TYPE na_user_http_404_not_found_total counter
na_user_http_404_not_found_total 0
# HELP na_user_http_405_method_not_allowed_total Number of HTTP Status 405 MethodNotAllowed responses
# TYPE na_user_http_405_method_not_allowed_total counter
na_user_http_405_method_not_allowed_total 0
# HELP na_user_http_409_conflict_total Number of HTTP Status 409 Conflict responses
# TYPE na_user_http_409_conflict_total counter
na_user_http_409_conflict_total 0
# HELP na_user_http_500_internal_server_error_total Number of HTTP Status 500 InternalServerError responses
# TYPE na_user_http_500_internal_server_error_total counter
na_user_http_500_internal_server_error_total 0
# HELP na_user_http_503_service_unavailable_total Number of HTTP Status 503 ServiceUnavailable responses
# TYPE na_user_http_503_service_unavailable_total counter
na_user_http_503_service_unavailable_total 0
# HELP na_user_http_errors_total Number of HTTP Error (non 200) responses
# TYPE na_user_http_errors_total counter
na_user_http_errors_total 0
# HELP na_user_requests_auth_error_total Number of errors during authentication
# TYPE na_user_requests_auth_error_total counter
na_user_requests_auth_error_total 0
# HELP na_user_requests_auth_login_total Number of requests to /auth/login
# TYPE na_user_requests_auth_login_total counter
na_user_requests_auth_login_total 0
# HELP na_user_requests_auth_logout_total Number of requests to /auth/logout
# TYPE na_user_requests_auth_logout_total counter
na_user_requests_auth_logout_total 0
# HELP na_user_requests_auth_password_set_total Number of requests to /auth/password/set
# TYPE na_user_requests_auth_password_set_total counter
na_user_requests_auth_password_set_total 0
# HELP na_user_requests_groups_adduser_total Number of requests to /groups/adduser
# TYPE na_user_requests_groups_adduser_total counter
na_user_requests_groups_adduser_total 0
# HELP na_user_requests_groups_create_total Number of requests to /groups/create
# TYPE na_user_requests_groups_create_total counter
na_user_requests_groups_create_total 0
# HELP na_user_requests_groups_delete_total Number of requests to /groups/delete
# TYPE na_user_requests_groups_delete_total counter
na_user_requests_groups_delete_total 0
# HELP na_user_requests_groups_deluser_total Number of requests to /groups/deluser
# TYPE na_user_requests_groups_deluser_total counter
na_user_requests_groups_deluser_total 0
# HELP na_user_requests_groups_exists_total Number of requests to /groups/exists
# TYPE na_user_requests_groups_exists_total counter
na_user_requests_groups_exists_total 0
# HELP na_user_requests_groups_get_total Number of requests to /groups/get
# TYPE na_user_requests_groups_get_total counter
na_user_requests_groups_get_total 0
# HELP na_user_requests_groups_list_total Number of requests to /groups/list
# TYPE na_user_requests_groups_list_total counter
na_user_requests_groups_list_total 0
# HELP na_user_requests_groups_members_gid_total Number of requests to /groups/members/gid
# TYPE na_user_requests_groups_members_gid_total counter
na_user_requests_groups_members_gid_total 0
# HELP na_user_requests_groups_members_groupname_total Number of requests to /groups/members/groupname
# TYPE na_user_requests_groups_members_groupname_total counter
na_user_requests_groups_members_groupname_total 0
# HELP na_user_requests_groups_policies_add_total Number of PUT requests to /groups/:gid/policies/:pid
# TYPE na_user_requests_groups_policies_add_total counter
na_user_requests_groups_policies_add_total 0
# HELP na_user_requests_groups_policies_del_total Number of DELETE requests to /groups/:gid/policies/:pid
# TYPE na_user_requests_groups_policies_del_total counter
na_user_requests_groups_policies_del_total 0
# HELP na_user_requests_groups_policies_list_total Number of GET requests to /groups/:gid/policies
# TYPE na_user_requests_groups_policies_list_total counter
na_user_requests_groups_policies_list_total 0
# HELP na_user_requests_groups_search_total Number of requests to /groups/search
# TYPE na_user_requests_groups_search_total counter
na_user_requests_groups_search_total 0
# HELP na_user_requests_groups_update_total Number of requests to /groups/update
# TYPE na_user_requests_groups_update_total counter
na_user_requests_groups_update_total 0
# HELP na_user_requests_health_total Number of requests to /health
# TYPE na_user_requests_health_total counter
na_user_requests_health_total 0
# HELP na_user_requests_index_total Number of requests to /
# TYPE na_user_requests_index_total counter
na_user_requests_index_total 0
# HELP na_user_requests_keys_create_total Number of requests to /keys/create
# TYPE na_user_requests_keys_create_total counter
na_user_requests_keys_create_total 0
# HELP na_user_requests_keys_delete_total Number of requests to /keys/delete
# TYPE na_user_requests_keys_delete_total counter
na_user_requests_keys_delete_total 0
# HELP na_user_requests_keys_exists_total Number of requests to /keys/exists
# TYPE na_user_requests_keys_exists_total counter
na_user_requests_keys_exists_total 0
# HELP na_user_requests_keys_get_total Number of requests to /keys/get
# TYPE na_user_requests_keys_get_total counter
na_user_requests_keys_get_total 0
# HELP na_user_requests_keys_list_total Number of requests to /keys/list
# TYPE na_user_requests_keys_list_total counter
na_user_requests_keys_list_total 0
# HELP na_user_requests_keys_search_total Number of requests to /keys/search
# TYPE na_user_requests_keys_search_total counter
na_user_requests_keys_search_total 0
# HELP na_user_requests_keys_update_total Number of requests to /keys/update
# TYPE na_user_requests_keys_update_total counter
na_user_requests_keys_update_total 0
# HELP na_user_requests_metrics_total Number of requests to /metrics
# TYPE na_user_requests_metrics_total counter
na_user_requests_metrics_total 2
# HELP na_user_requests_permissions_create_total Number of POST requests to /permissions/create
# TYPE na_user_requests_permissions_create_total counter
na_user_requests_permissions_create_total 0
# HELP na_user_requests_permissions_delete_total Number of DELETE requests to /permissions/delete
# TYPE na_user_requests_permissions_delete_total counter
na_user_requests_permissions_delete_total 0
# HELP na_user_requests_permissions_exists_total Number of GET requests to /permissions/exists
# TYPE na_user_requests_permissions_exists_total counter
na_user_requests_permissions_exists_total 0
# HELP na_user_requests_permissions_get_total Number of GET requests to /permissions/get
# TYPE na_user_requests_permissions_get_total counter
na_user_requests_permissions_get_total 0
# HELP na_user_requests_permissions_list_total Number of GET requests to /permissions/list
# TYPE na_user_requests_permissions_list_total counter
na_user_requests_permissions_list_total 0
# HELP na_user_requests_permissions_search_total Number of POST requests to /permissions/search
# TYPE na_user_requests_permissions_search_total counter
na_user_requests_permissions_search_total 0
# HELP na_user_requests_permissions_update_total Number of POST requests to /permissions/update
# TYPE na_user_requests_permissions_update_total counter
na_user_requests_permissions_update_total 0
# HELP na_user_requests_policies_create_total Number of POST requests to /policies/create
# TYPE na_user_requests_policies_create_total counter
na_user_requests_policies_create_total 0
# HELP na_user_requests_policies_delete_total Number of DELETE requests to /policies/delete
# TYPE na_user_requests_policies_delete_total counter
na_user_requests_policies_delete_total 0
# HELP na_user_requests_policies_exists_total Number of GET requests to /policies/exists
# TYPE na_user_requests_policies_exists_total counter
na_user_requests_policies_exists_total 0
# HELP na_user_requests_policies_get_total Number of GET requests to /policies/get
# TYPE na_user_requests_policies_get_total counter
na_user_requests_policies_get_total 0
# HELP na_user_requests_policies_list_total Number of GET requests to /policies/list
# TYPE na_user_requests_policies_list_total counter
na_user_requests_policies_list_total 0
# HELP na_user_requests_policies_permissions_add_total Number of PUT requests to /policies/:pid/permissions/:permid
# TYPE na_user_requests_policies_permissions_add_total counter
na_user_requests_policies_permissions_add_total 0
# HELP na_user_requests_policies_permissions_del_total Number of DELETE requests to /policies/:pid/permissions/:permid
# TYPE na_user_requests_policies_permissions_del_total counter
na_user_requests_policies_permissions_del_total 0
# HELP na_user_requests_policies_permissions_list_total Number of GET requests to /policies/:pid/permissions
# TYPE na_user_requests_policies_permissions_list_total counter
na_user_requests_policies_permissions_list_total 0
# HELP na_user_requests_policies_resources_add_total Number of PUT requests to /policies/:pid/resources/:rid
# TYPE na_user_requests_policies_resources_add_total counter
na_user_requests_policies_resources_add_total 0
# HELP na_user_requests_policies_resources_del_total Number of DELETE requests to /policies/:pid/resources/:rid
# TYPE na_user_requests_policies_resources_del_total counter
na_user_requests_policies_resources_del_total 0
# HELP na_user_requests_policies_resources_list_total Number of GET requests to /policies/:pid/resources
# TYPE na_user_requests_policies_resources_list_total counter
na_user_requests_policies_resources_list_total 0
# HELP na_user_requests_policies_search_total Number of POST requests to /policies/search
# TYPE na_user_requests_policies_search_total counter
na_user_requests_policies_search_total 0
# HELP na_user_requests_policies_update_total Number of POST requests to /policies/update
# TYPE na_user_requests_policies_update_total counter
na_user_requests_policies_update_total 0
# HELP na_user_requests_resources_create_total Number of POST requests to /resources/create
# TYPE na_user_requests_resources_create_total counter
na_user_requests_resources_create_total 0
# HELP na_user_requests_resources_delete_total Number of DELETE requests to /resources/delete
# TYPE na_user_requests_resources_delete_total counter
na_user_requests_resources_delete_total 0
# HELP na_user_requests_resources_exists_total Number of GET requests to /resources/exists
# TYPE na_user_requests_resources_exists_total counter
na_user_requests_resources_exists_total 0
# HELP na_user_requests_resources_get_total Number of GET requests to /resources/get
# TYPE na_user_requests_resources_get_total counter
na_user_requests_resources_get_total 0
# HELP na_user_requests_resources_list_total Number of GET requests to /resources/list
# TYPE na_user_requests_resources_list_total counter
na_user_requests_resources_list_total 0
# HELP na_user_requests_resources_search_total Number of POST requests to /resources/search
# TYPE na_user_requests_resources_search_total counter
na_user_requests_resources_search_total 0
# HELP na_user_requests_resources_update_total Number of POST requests to /resources/update
# TYPE na_user_requests_resources_update_total counter
na_user_requests_resources_update_total 0
# HELP na_user_requests_total Total number of API requests
# TYPE na_user_requests_total counter
na_user_requests_total 3
# HELP na_user_requests_users_create_total Number of requests to /users/create
# TYPE na_user_requests_users_create_total counter
na_user_requests_users_create_total 0
# HELP na_user_requests_users_delete_total Number of requests to /users/delete
# TYPE na_user_requests_users_delete_total counter
na_user_requests_users_delete_total 0
# HELP na_user_requests_users_exists_total Number of requests to /users/exists
# TYPE na_user_requests_users_exists_total counter
na_user_requests_users_exists_total 0
# HELP na_user_requests_users_get_total Number of requests to /users/get
# TYPE na_user_requests_users_get_total counter
na_user_requests_users_get_total 0
# HELP na_user_requests_users_list_total Number of requests to /users/list
# TYPE na_user_requests_users_list_total counter
na_user_requests_users_list_total 0
# HELP na_user_requests_users_policies_add_total Number of PUT requests to /users/:uid/policies/:pid
# TYPE na_user_requests_users_policies_add_total counter
na_user_requests_users_policies_add_total 0
# HELP na_user_requests_users_policies_del_total Number of DELETE requests to /users/:uid/policies/:pid
# TYPE na_user_requests_users_policies_del_total counter
na_user_requests_users_policies_del_total 0
# HELP na_user_requests_users_policies_list_total Number of GET requests to /users/:uid/policies
# TYPE na_user_requests_users_policies_list_total counter
na_user_requests_users_policies_list_total 0
# HELP na_user_requests_users_search_total Number of requests to /users/search
# TYPE na_user_requests_users_search_total counter
na_user_requests_users_search_total 0
# HELP na_user_requests_users_update_total Number of requests to /users/update
# TYPE na_user_requests_users_update_total counter
na_user_requests_users_update_total 0
# HELP na_user_requests_version_total Number of requests to /version
# TYPE na_user_requests_version_total counter
na_user_requests_version_total 0
# HELP na_user_response_time_seconds Request response times
# TYPE na_user_response_time_seconds summary
na_user_response_time_seconds{quantile="0.5"} 8.208e-05
na_user_response_time_seconds{quantile="0.9"} 0.001068557
na_user_response_time_seconds{quantile="0.99"} 0.001068557
na_user_response_time_seconds_sum 0.001150637
na_user_response_time_seconds_count 2
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 0
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 10
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 1.2111872e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.69620457091e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 1.269219328e+09
# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes -1
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 1
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0