Support legacy Prometheus metric names for source average (#5531)
* Support older prometheus metric unit naming and allow removal of units from metrics * Update swagger * Added bats tests, improved efficiency of checking units
Chris Akritidis committed
Mar 4, 2019 at 18:36 UTC
56ac9f514566d0cbfa6125a28c96b759d4de1ff9
10 files changed
+400
-5
backends/prometheus/README.md
+8
@@ -373,6 +373,14 @@ netdata sends all metrics prefixed with `netdata_`. You can change this in `netd
373
374
It can also be changed from the URL, by appending `&prefix=netdata`.
375
376
+### Metric Units
377
+
378
+The default source `average` adds the unit of measurement to the name of each metric (e.g. `_KiB_persec`).
379
+To hide the units and get the same metric names as with the other sources, append to the URL `&hideunits=yes`.
380
+
381
+The units were standardized in v1.12, with the effect of changing the metric names.
382
+To get the metric names as they were before v1.12, append to the URL `&oldunits=yes`
383
+
384
### Accuracy of `average` and `sum` data sources
385
386
When the data source is set to `average` or `sum`, netdata remembers the last access of each client accessing prometheus metrics and uses this last access time to respond with the `average` or `sum` of all the entries in the database since that. This means that prometheus servers are not losing data when they access netdata with data source = `average` or `sum`.
backends/prometheus/backend_prometheus.c
+42
-3
@@ -78,11 +78,50 @@ static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable
78
return n;
79
}
80
81
-static inline char *prometheus_units_copy(char *d, const char *s, size_t usable) {
81
+static inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits) {
82
const char *sorig = s;
83
char *ret = d;
84
size_t n;
85
86
+ // Fix for issue 5227
87
+ if (unlikely(showoldunits)) {
88
+ static struct {
89
+ const char *newunit;
90
+ uint32_t hash;
91
+ const char *oldunit;
92
+ } units[] = {
93
+ {"KiB/s", 0, "kilobytes/s"}
94
+ , {"MiB/s", 0, "MB/s"}
95
+ , {"GiB/s", 0, "GB/s"}
96
+ , {"KiB" , 0, "KB"}
97
+ , {"MiB" , 0, "MB"}
98
+ , {"GiB" , 0, "GB"}
99
+ , {"inodes" , 0, "Inodes"}
100
+ , {"percentage" , 0, "percent"}
101
+ , {"faults/s" , 0, "page faults/s"}
102
+ , {"KiB/operation", 0, "kilobytes per operation"}
103
+ , {"milliseconds/operation", 0, "ms per operation"}
104
+ , {NULL, 0, NULL}
105
+ };
106
+ static int initialized = 0;
107
+ int i;
108
+
109
+ if(unlikely(!initialized)) {
110
+ for (i = 0; units[i].newunit; i++)
111
+ units[i].hash = simple_hash(units[i].newunit);
112
+ initialized = 1;
113
+ }
114
+
115
+ uint32_t hash = simple_hash(s);
116
+ for(i = 0; units[i].newunit ; i++) {
117
+ if(unlikely(hash == units[i].hash && !strcmp(s, units[i].newunit))) {
118
+ // info("matched extension for filename '%s': '%s'", filename, last_dot);
119
+ s=units[i].oldunit;
120
+ sorig = s;
121
+ break;
122
+ }
123
+ }
124
+ }
125
*d++ = '_';
126
for(n = 1; *s && n < usable ; d++, s++, n++) {
127
register char c = *s;
@@ -275,8 +314,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER
314
homogeneus = 0;
315
}
316
else {
278
- if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)
279
- prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX);
317
+ if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & PROMETHEUS_OUTPUT_HIDEUNITS))
318
+ prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS);
319
}
320
321
if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
backends/prometheus/backend_prometheus.h
+3
-1
@@ -11,7 +11,9 @@ typedef enum prometheus_output_flags {
11
PROMETHEUS_OUTPUT_TYPES = (1 << 1),
12
PROMETHEUS_OUTPUT_NAMES = (1 << 2),
13
PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3),
14
- PROMETHEUS_OUTPUT_VARIABLES = (1 << 4)
14
+ PROMETHEUS_OUTPUT_VARIABLES = (1 << 4),
15
+ PROMETHEUS_OUTPUT_OLDUNITS = (1 << 5),
16
+ PROMETHEUS_OUTPUT_HIDEUNITS = (1 << 6)
17
} PROMETHEUS_OUTPUT_OPTIONS;
18
19
extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
tests/backends/prometheus-avg-oldunits.txt
new
+92
@@ -0,0 +1,92 @@
1
+nd_cpu_core_throttling_events_persec_average
2
+nd_cpu_cpu_percent_average
3
+nd_cpu_interrupts_interrupts_persec_average
4
+nd_cpu_softirqs_softirqs_persec_average
5
+nd_cpu_softnet_stat_events_persec_average
6
+nd_disk_avgsz_kilobytes_per_operation_average
7
+nd_disk_await_ms_per_operation_average
8
+nd_disk_backlog_milliseconds_average
9
+nd_disk_inodes_Inodes_average
10
+nd_disk_io_kilobytes_persec_average
11
+nd_disk_iotime_milliseconds_persec_average
12
+nd_disk_mops_merged_operations_persec_average
13
+nd_disk_ops_operations_persec_average
14
+nd_disk_space_GB_average
15
+nd_disk_svctm_ms_per_operation_average
16
+nd_disk_util___of_time_working_average
17
+nd_ip_bcast_kilobits_persec_average
18
+nd_ip_bcastpkts_packets_persec_average
19
+nd_ip_ecnpkts_packets_persec_average
20
+nd_ip_inerrors_packets_persec_average
21
+nd_ip_mcast_kilobits_persec_average
22
+nd_ip_mcastpkts_packets_persec_average
23
+nd_ip_tcpconnaborts_connections_persec_average
24
+nd_ip_tcpofo_packets_persec_average
25
+nd_ip_tcpreorders_packets_persec_average
26
+nd_ipv4_errors_packets_persec_average
27
+nd_ipv4_icmp_errors_packets_persec_average
28
+nd_ipv4_icmpmsg_packets_persec_average
29
+nd_ipv4_icmp_packets_persec_average
30
+nd_ipv4_packets_packets_persec_average
31
+nd_ipv4_sockstat_sockets_sockets_average
32
+nd_ipv4_sockstat_tcp_mem_KB_average
33
+nd_ipv4_sockstat_tcp_sockets_sockets_average
34
+nd_ipv4_sockstat_udp_mem_KB_average
35
+nd_ipv4_sockstat_udp_sockets_sockets_average
36
+nd_ipv4_tcperrors_packets_persec_average
37
+nd_ipv4_tcphandshake_events_persec_average
38
+nd_ipv4_tcpopens_connections_persec_average
39
+nd_ipv4_tcppackets_packets_persec_average
40
+nd_ipv4_tcpsock_active_connections_average
41
+nd_ipv4_udperrors_events_persec_average
42
+nd_ipv4_udppackets_packets_persec_average
43
+nd_ipv6_ect_packets_persec_average
44
+nd_ipv6_errors_packets_persec_average
45
+nd_ipv6_icmpechos_messages_persec_average
46
+nd_ipv6_icmperrors_errors_persec_average
47
+nd_ipv6_icmp_messages_persec_average
48
+nd_ipv6_icmpmldv2_reports_persec_average
49
+nd_ipv6_icmpneighbor_messages_persec_average
50
+nd_ipv6_icmprouter_messages_persec_average
51
+nd_ipv6_icmptypes_messages_persec_average
52
+nd_ipv6_mcast_kilobits_persec_average
53
+nd_ipv6_mcastpkts_packets_persec_average
54
+nd_ipv6_packets_packets_persec_average
55
+nd_ipv6_sockstat6_raw_sockets_sockets_average
56
+nd_ipv6_sockstat6_tcp_sockets_sockets_average
57
+nd_ipv6_sockstat6_udp_sockets_sockets_average
58
+nd_ipv6_udperrors_events_persec_average
59
+nd_ipv6_udppackets_packets_persec_average
60
+nd_mem_available_MB_average
61
+nd_mem_committed_MB_average
62
+nd_mem_kernel_MB_average
63
+nd_mem_pgfaults_page_faults_persec_average
64
+nd_mem_slab_MB_average
65
+nd_mem_transparent_hugepages_MB_average
66
+nd_mem_writeback_MB_average
67
+nd_net_drops_drops_persec_average
68
+nd_net_net_kilobits_persec_average
69
+nd_net_packets_packets_persec_average
70
+nd_system_active_processes_processes_average
71
+nd_system_cpu_percent_average
72
+nd_system_ctxt_context_switches_persec_average
73
+nd_system_entropy_entropy_average
74
+nd_system_forks_processes_persec_average
75
+nd_system_idlejitter_microseconds_lost_persec_average
76
+nd_system_interrupts_interrupts_persec_average
77
+nd_system_intr_interrupts_persec_average
78
+nd_system_io_kilobytes_persec_average
79
+nd_system_ipc_semaphore_arrays_arrays_average
80
+nd_system_ipc_semaphores_semaphores_average
81
+nd_system_ip_kilobits_persec_average
82
+nd_system_ipv6_kilobits_persec_average
83
+nd_system_load_load_average
84
+nd_system_net_kilobits_persec_average
85
+nd_system_pgpgio_kilobytes_persec_average
86
+nd_system_processes_processes_average
87
+nd_system_ram_MB_average
88
+nd_system_softirqs_softirqs_persec_average
89
+nd_system_softnet_stat_events_persec_average
90
+nd_system_swapio_kilobytes_persec_average
91
+nd_system_swap_MB_average
92
+nd_system_uptime_seconds_average
tests/backends/prometheus-avg.txt
new
+92
@@ -0,0 +1,92 @@
1
+nd_cpu_core_throttling_events_persec_average
2
+nd_cpu_cpu_percentage_average
3
+nd_cpu_interrupts_interrupts_persec_average
4
+nd_cpu_softirqs_softirqs_persec_average
5
+nd_cpu_softnet_stat_events_persec_average
6
+nd_disk_avgsz_KiB_operation_average
7
+nd_disk_await_milliseconds_operation_average
8
+nd_disk_backlog_milliseconds_average
9
+nd_disk_inodes_inodes_average
10
+nd_disk_io_KiB_persec_average
11
+nd_disk_iotime_milliseconds_persec_average
12
+nd_disk_mops_merged_operations_persec_average
13
+nd_disk_ops_operations_persec_average
14
+nd_disk_space_GiB_average
15
+nd_disk_svctm_milliseconds_operation_average
16
+nd_disk_util___of_time_working_average
17
+nd_ip_bcast_kilobits_persec_average
18
+nd_ip_bcastpkts_packets_persec_average
19
+nd_ip_ecnpkts_packets_persec_average
20
+nd_ip_inerrors_packets_persec_average
21
+nd_ip_mcast_kilobits_persec_average
22
+nd_ip_mcastpkts_packets_persec_average
23
+nd_ip_tcpconnaborts_connections_persec_average
24
+nd_ip_tcpofo_packets_persec_average
25
+nd_ip_tcpreorders_packets_persec_average
26
+nd_ipv4_errors_packets_persec_average
27
+nd_ipv4_icmp_errors_packets_persec_average
28
+nd_ipv4_icmpmsg_packets_persec_average
29
+nd_ipv4_icmp_packets_persec_average
30
+nd_ipv4_packets_packets_persec_average
31
+nd_ipv4_sockstat_sockets_sockets_average
32
+nd_ipv4_sockstat_tcp_mem_KiB_average
33
+nd_ipv4_sockstat_tcp_sockets_sockets_average
34
+nd_ipv4_sockstat_udp_mem_KiB_average
35
+nd_ipv4_sockstat_udp_sockets_sockets_average
36
+nd_ipv4_tcperrors_packets_persec_average
37
+nd_ipv4_tcphandshake_events_persec_average
38
+nd_ipv4_tcpopens_connections_persec_average
39
+nd_ipv4_tcppackets_packets_persec_average
40
+nd_ipv4_tcpsock_active_connections_average
41
+nd_ipv4_udperrors_events_persec_average
42
+nd_ipv4_udppackets_packets_persec_average
43
+nd_ipv6_ect_packets_persec_average
44
+nd_ipv6_errors_packets_persec_average
45
+nd_ipv6_icmpechos_messages_persec_average
46
+nd_ipv6_icmperrors_errors_persec_average
47
+nd_ipv6_icmp_messages_persec_average
48
+nd_ipv6_icmpmldv2_reports_persec_average
49
+nd_ipv6_icmpneighbor_messages_persec_average
50
+nd_ipv6_icmprouter_messages_persec_average
51
+nd_ipv6_icmptypes_messages_persec_average
52
+nd_ipv6_mcast_kilobits_persec_average
53
+nd_ipv6_mcastpkts_packets_persec_average
54
+nd_ipv6_packets_packets_persec_average
55
+nd_ipv6_sockstat6_raw_sockets_sockets_average
56
+nd_ipv6_sockstat6_tcp_sockets_sockets_average
57
+nd_ipv6_sockstat6_udp_sockets_sockets_average
58
+nd_ipv6_udperrors_events_persec_average
59
+nd_ipv6_udppackets_packets_persec_average
60
+nd_mem_available_MiB_average
61
+nd_mem_committed_MiB_average
62
+nd_mem_kernel_MiB_average
63
+nd_mem_pgfaults_faults_persec_average
64
+nd_mem_slab_MiB_average
65
+nd_mem_transparent_hugepages_MiB_average
66
+nd_mem_writeback_MiB_average
67
+nd_net_drops_drops_persec_average
68
+nd_net_net_kilobits_persec_average
69
+nd_net_packets_packets_persec_average
70
+nd_system_active_processes_processes_average
71
+nd_system_cpu_percentage_average
72
+nd_system_ctxt_context_switches_persec_average
73
+nd_system_entropy_entropy_average
74
+nd_system_forks_processes_persec_average
75
+nd_system_idlejitter_microseconds_lost_persec_average
76
+nd_system_interrupts_interrupts_persec_average
77
+nd_system_intr_interrupts_persec_average
78
+nd_system_io_KiB_persec_average
79
+nd_system_ipc_semaphore_arrays_arrays_average
80
+nd_system_ipc_semaphores_semaphores_average
81
+nd_system_ip_kilobits_persec_average
82
+nd_system_ipv6_kilobits_persec_average
83
+nd_system_load_load_average
84
+nd_system_net_kilobits_persec_average
85
+nd_system_pgpgio_KiB_persec_average
86
+nd_system_processes_processes_average
87
+nd_system_ram_MiB_average
88
+nd_system_softirqs_softirqs_persec_average
89
+nd_system_softnet_stat_events_persec_average
90
+nd_system_swapio_KiB_persec_average
91
+nd_system_swap_MiB_average
92
+nd_system_uptime_seconds_average
tests/backends/prometheus-raw.txt
new
+92
@@ -0,0 +1,92 @@
1
+nd_cpu_core_throttling_total
2
+nd_cpu_cpu_total
3
+nd_cpu_interrupts_total
4
+nd_cpu_softirqs_total
5
+nd_cpu_softnet_stat_total
6
+nd_disk_avgsz
7
+nd_disk_await
8
+nd_disk_backlog_total
9
+nd_disk_inodes
10
+nd_disk_iotime_total
11
+nd_disk_io_total
12
+nd_disk_mops_total
13
+nd_disk_ops_total
14
+nd_disk_space
15
+nd_disk_svctm
16
+nd_disk_util_total
17
+nd_ip_bcastpkts_total
18
+nd_ip_bcast_total
19
+nd_ip_ecnpkts_total
20
+nd_ip_inerrors_total
21
+nd_ip_mcastpkts_total
22
+nd_ip_mcast_total
23
+nd_ip_tcpconnaborts_total
24
+nd_ip_tcpofo_total
25
+nd_ip_tcpreorders_total
26
+nd_ipv4_errors_total
27
+nd_ipv4_icmp_errors_total
28
+nd_ipv4_icmpmsg_total
29
+nd_ipv4_icmp_total
30
+nd_ipv4_packets_total
31
+nd_ipv4_sockstat_sockets
32
+nd_ipv4_sockstat_tcp_mem
33
+nd_ipv4_sockstat_tcp_sockets
34
+nd_ipv4_sockstat_udp_mem
35
+nd_ipv4_sockstat_udp_sockets
36
+nd_ipv4_tcperrors_total
37
+nd_ipv4_tcphandshake_total
38
+nd_ipv4_tcpopens_total
39
+nd_ipv4_tcppackets_total
40
+nd_ipv4_tcpsock
41
+nd_ipv4_udperrors_total
42
+nd_ipv4_udppackets_total
43
+nd_ipv6_ect_total
44
+nd_ipv6_errors_total
45
+nd_ipv6_icmpechos_total
46
+nd_ipv6_icmperrors_total
47
+nd_ipv6_icmpmldv2_total
48
+nd_ipv6_icmpneighbor_total
49
+nd_ipv6_icmprouter_total
50
+nd_ipv6_icmp_total
51
+nd_ipv6_icmptypes_total
52
+nd_ipv6_mcastpkts_total
53
+nd_ipv6_mcast_total
54
+nd_ipv6_packets_total
55
+nd_ipv6_sockstat6_raw_sockets
56
+nd_ipv6_sockstat6_tcp_sockets
57
+nd_ipv6_sockstat6_udp_sockets
58
+nd_ipv6_udperrors_total
59
+nd_ipv6_udppackets_total
60
+nd_mem_available
61
+nd_mem_committed
62
+nd_mem_kernel
63
+nd_mem_pgfaults_total
64
+nd_mem_slab
65
+nd_mem_transparent_hugepages
66
+nd_mem_writeback
67
+nd_net_drops_total
68
+nd_net_net_total
69
+nd_net_packets_total
70
+nd_system_active_processes
71
+nd_system_cpu_total
72
+nd_system_ctxt_total
73
+nd_system_entropy
74
+nd_system_forks_total
75
+nd_system_idlejitter
76
+nd_system_interrupts_total
77
+nd_system_intr_total
78
+nd_system_io_total
79
+nd_system_ipc_semaphore_arrays
80
+nd_system_ipc_semaphores
81
+nd_system_ip_total
82
+nd_system_ipv6_total
83
+nd_system_load
84
+nd_system_net_total
85
+nd_system_pgpgio_total
86
+nd_system_processes
87
+nd_system_ram
88
+nd_system_softirqs_total
89
+nd_system_softnet_stat_total
90
+nd_system_swap
91
+nd_system_swapio_total
92
+nd_system_uptime
tests/backends/prometheus.bats
new
+31
@@ -0,0 +1,31 @@
1
+#!/usr/bin/env bats
2
+
3
+validate_metrics() {
4
+ fname="${1}"
5
+ params="${2}"
6
+
7
+ curl -sS "http://localhost:19999/api/v1/allmetrics?format=prometheus&prefix=nd×tamps=no${params}" |
8
+ grep -E 'nd_system_|nd_cpu_|nd_system_|nd_net_|nd_disk_|nd_ip_|nd_ipv4_|nd_ipv6_|nd_mem_' |
9
+ sed -ne 's/{.*//p' | sort | uniq > tests/backends/new-${fname}
10
+ diff tests/backends/${fname} tests/backends/new-${fname}
11
+ rm tests/backends/new-${fname}
12
+}
13
+
14
+
15
+if [ ! -f .gitignore ]; then
16
+ echo "Need to run as ./tests/backends/$(basename "$0") from top level directory of git repository" >&2
17
+ exit 1
18
+fi
19
+
20
+
21
+@test "prometheus raw" {
22
+ validate_metrics prometheus-raw.txt "&data=raw"
23
+}
24
+
25
+@test "prometheus avg" {
26
+ validate_metrics prometheus-avg.txt ""
27
+}
28
+
29
+@test "prometheus avg oldunits" {
30
+ validate_metrics prometheus-avg-oldunits.txt "&oldunits=yes"
31
+}
web/api/exporters/allmetrics.c
+2
-1
@@ -11,7 +11,8 @@ struct prometheus_output_options {
11
{ "names", PROMETHEUS_OUTPUT_NAMES },
12
{ "timestamps", PROMETHEUS_OUTPUT_TIMESTAMPS },
13
{ "variables", PROMETHEUS_OUTPUT_VARIABLES },
14
-
14
+ { "oldunits", PROMETHEUS_OUTPUT_OLDUNITS },
15
+ { "hideunits", PROMETHEUS_OUTPUT_HIDEUNITS },
16
// terminator
17
{ NULL, PROMETHEUS_OUTPUT_NONE },
18
};
web/api/netdata-swagger.json
+24
@@ -506,6 +506,30 @@
506
],
507
"default": "yes"
508
},
509
+ {
510
+ "name": "oldunits",
511
+ "in": "query",
512
+ "description": "When enabled, netdata will show metric names for the default source=average as they appeared before 1.12, by using the legacy unit naming conventions",
513
+ "required": false,
514
+ "type": "string",
515
+ "enum": [
516
+ "yes",
517
+ "no"
518
+ ],
519
+ "default": "yes"
520
+ },
521
+ {
522
+ "name": "hideunits",
523
+ "in": "query",
524
+ "description": "When enabled, netdata will not include the units in the metric names, for the default source=average.",
525
+ "required": false,
526
+ "type": "string",
527
+ "enum": [
528
+ "yes",
529
+ "no"
530
+ ],
531
+ "default": "yes"
532
+ },
533
{
534
"name": "server",
535
"in": "query",
web/api/netdata-swagger.yaml
+14
@@ -333,6 +333,20 @@ paths:
333
type: string
334
enum: [ 'yes', 'no' ]
335
default: 'yes'
336
+ - name: oldunits
337
+ in: query
338
+ description: 'When enabled, netdata will show metric names for the default source=average as they appeared before 1.12, by using the legacy unit naming conventions'
339
+ required: false
340
+ type: string
341
+ enum: [ 'yes', 'no' ]
342
+ default: 'yes'
343
+ - name: hideunits
344
+ in: query
345
+ description: 'When enabled, netdata will not include the units in the metric names, for the default source=average.'
346
+ required: false
347
+ type: string
348
+ enum: [ 'yes', 'no' ]
349
+ default: 'yes'
350
- name: server
351
in: query
352
description: 'Set a distinct name of the client querying prometheus metrics. Netdata will use the client IP if this is not set.'