Add chart labels to Prometheus. (#15099)
thiagoftsm committed
May 24, 2023 at 19:19 UTC
e39e809ac63eb73376c5fde1c769fb8c3915a8e7
3 files changed
+83
-21
exporting/README.md
+1
-1
@@ -284,7 +284,7 @@ Configure individual connectors and override any global settings with the follow
284
and names are human friendly labels (also unique). Most charts and metrics have the same ID and name, but in several
285
cases they are different: disks with device-mapper, interrupts, QoS classes, statsd synthetic charts, etc.
286
287
-- `send configured labels = yes | no` controls if labels defined in the `[host labels]` section in `netdata.conf`
287
+- `send configured labels = yes | no` controls if host labels defined in the `[host labels]` section in `netdata.conf`
288
should be sent to the external database
289
290
- `send automatic labels = yes | no` controls if automatically created labels, like `_os_name` or `_architecture`
exporting/WALKTHROUGH.md
+9
-8
@@ -74,10 +74,10 @@ this is your first time using Netdata I suggest you take a look around. The amou
74
Next I want to draw your attention to a particular endpoint. Navigate to
75
<http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes> In your browser. This is the endpoint which
76
publishes all the metrics in a format which Prometheus understands. Let's take a look at one of these metrics.
77
-`netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000` This
78
-metric is representing several things which I will go in more details in the section on Prometheus. For now understand
79
-that this metric: `netdata_system_cpu_percentage_average` has several labels: (`chart`, `family`, `dimension`). This
80
-corresponds with the first cpu chart you see on the Netdata dashboard.
77
+`netdata_disk_space_GiB_average{chart="disk_space._run",dimension="avail",family="/run",mount_point="/run",filesystem="tmpfs",mount_root="/"} 0.0298195 1684951093000`
78
+This metric is representing several things which I will go in more details in the section on Prometheus. For now understand
79
+that this metric: `netdata_disk_space_GiB_average` has several labels: (`chart`, `family`, `dimension`, `mountt_point`, `filesystem`, `mount_root`).
80
+This corresponds with disk space you see on the Netdata dashboard.
81
82
83
@@ -138,12 +138,13 @@ As explained we have two key elements in Prometheus metrics. We have the _metric
138
granularity between metrics. Let's use our previous example to further explain.
139
140
```conf
141
-netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000
141
+netdata_disk_space_GiB_average{chart="disk_space._run",dimension="avail",family="/run",mount_point="/run",filesystem="tmpfs",mount_root="/"} 0.0298195 1684951093000
142
```
143
144
-Here our metric is `netdata_system_cpu_percentage_average` and our labels are `chart`, `family`, and `dimension`. The
145
-last two values constitute the actual metric value for the metric type (gauge, counter, etc…). We can begin graphing
146
-system metrics with this information, but first we need to hook up Prometheus to poll Netdata stats.
144
+Here our metric is `netdata_disk_space_GiB_average` and our common labels are `chart`, `family`, and `dimension`. The
145
+last two values constitute the actual metric value for the metric type (gauge, counter, etc…). We also have specific
146
+label for this chart named `mount_point`,`filesystem`, and `mount_root`. We can begin graphing system metrics with this information,
147
+but first we need to hook up Prometheus to poll Netdata stats.
148
149
Let's move our attention to Prometheus's configuration. Prometheus gets it config from the file located (in our example)
150
at `/opt/prometheus/prometheus.yml`. I won't spend an extensive amount of time going over the configuration values
exporting/prometheus/prometheus.c
+73
-12
@@ -326,6 +326,53 @@ void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
326
rrdlabels_walkthrough_read(host->rrdlabels, format_prometheus_label_callback, &tmp);
327
}
328
329
+/**
330
+ * Format host labels for the Prometheus exporter
331
+ * We are using a structure instead a direct buffer to expand options quickly.
332
+ *
333
+ * @param labels_buffer is the buffer used to add labels.
334
+ */
335
+
336
+struct format_prometheus_chart_label_callback {
337
+ BUFFER *labels_buffer;
338
+};
339
+
340
+static int format_prometheus_chart_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
341
+ struct format_prometheus_chart_label_callback *d = (struct format_prometheus_chart_label_callback *)data;
342
+
343
+ (void)ls;
344
+
345
+ if (name[0] == '_' )
346
+ return 1;
347
+
348
+ char k[PROMETHEUS_ELEMENT_MAX + 1];
349
+ char v[PROMETHEUS_ELEMENT_MAX + 1];
350
+
351
+ prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
352
+ prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
353
+
354
+ if (*k && *v) {
355
+ buffer_sprintf(d->labels_buffer, ",%s=\"%s\"", k, v);
356
+ }
357
+ return 1;
358
+}
359
+
360
+void format_chart_labels_prometheus(struct format_prometheus_chart_label_callback *plabel,
361
+ const char *chart,
362
+ const char *family,
363
+ const char *dim,
364
+ RRDSET *st)
365
+{
366
+ if (likely(plabel->labels_buffer))
367
+ buffer_reset(plabel->labels_buffer);
368
+ else {
369
+ plabel->labels_buffer = buffer_create(1024, NULL);
370
+ }
371
+ buffer_sprintf(plabel->labels_buffer, "chart=\"%s\",dimension=\"%s\",family=\"%s\"", chart, dim, family);
372
+
373
+ rrdlabels_walkthrough_read(st->rrdlabels, format_prometheus_chart_label_callback, plabel);
374
+}
375
+
376
struct host_variables_callback_options {
377
RRDHOST *host;
378
BUFFER *wb;
@@ -462,9 +509,17 @@ static void generate_as_collected_prom_help(BUFFER *wb, struct gen_parameters *p
509
* @param p parameters for generating the metric string.
510
* @param homogeneous a flag for homogeneous charts.
511
* @param prometheus_collector a flag for metrics from prometheus collector.
512
+ * @param chart_labels the dictionary with chart labels
513
*/
466
-static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters *p, int homogeneous, int prometheus_collector)
514
+static void generate_as_collected_prom_metric(BUFFER *wb,
515
+ struct gen_parameters *p,
516
+ int homogeneous,
517
+ int prometheus_collector,
518
+ DICTIONARY *chart_labels)
519
{
520
+ struct format_prometheus_chart_label_callback local_label;
521
+ local_label.labels_buffer = wb;
522
+
523
buffer_sprintf(wb, "%s_%s", p->prefix, p->context);
524
525
if (!homogeneous)
@@ -475,7 +530,11 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters
530
if (homogeneous)
531
buffer_sprintf(wb, ",dimension=\"%s\"", p->dimension);
532
478
- buffer_sprintf(wb, ",family=\"%s\"%s} ", p->family, p->labels);
533
+ buffer_sprintf(wb, ",family=\"%s\"", p->family);
534
+
535
+ rrdlabels_walkthrough_read(chart_labels, format_prometheus_chart_label_callback, &local_label);
536
+
537
+ buffer_sprintf(wb, "%s} ", p->labels);
538
539
if (prometheus_collector)
540
buffer_sprintf(
@@ -564,6 +623,10 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
623
624
// for each chart
625
RRDSET *st;
626
+
627
+ static struct format_prometheus_chart_label_callback plabels = {
628
+ .labels_buffer = NULL,
629
+ };
630
rrdset_foreach_read(st, host) {
631
632
if (likely(can_send_rrdset(instance, st, filter))) {
@@ -655,7 +718,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
718
if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
719
buffer_sprintf(wb, "# TYPE %s_%s%s %s\n", prefix, context, suffix, p.type);
720
658
- generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector);
721
+ generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector, st->rrdlabels);
722
}
723
else {
724
// the dimensions of the chart, do not have the same algorithm, multiplier or divisor
@@ -673,7 +736,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
736
buffer_sprintf(
737
wb, "# TYPE %s_%s_%s%s %s\n", prefix, context, dimension, suffix, p.type);
738
676
- generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector);
739
+ generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector, st->rrdlabels);
740
}
741
}
742
else {
@@ -694,6 +757,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
757
(output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd),
758
PROMETHEUS_ELEMENT_MAX);
759
760
+ format_chart_labels_prometheus(&plabels, chart, family, dimension, st);
761
+
762
if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))
763
buffer_sprintf(
764
wb,
@@ -713,30 +778,26 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
778
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
779
buffer_sprintf(
780
wb,
716
- "%s_%s%s%s{chart=\"%s\",dimension=\"%s\",family=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
781
+ "%s_%s%s%s{%s%s} " NETDATA_DOUBLE_FORMAT
782
" %llu\n",
783
prefix,
784
context,
785
units,
786
suffix,
722
- chart,
723
- dimension,
724
- family,
787
+ buffer_tostring(plabels.labels_buffer),
788
labels,
789
value,
790
last_time * MSEC_PER_SEC);
791
else
792
buffer_sprintf(
793
wb,
731
- "%s_%s%s%s{chart=\"%s\",dimension=\"%s\",family=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
794
+ "%s_%s%s%s{%s%s} " NETDATA_DOUBLE_FORMAT
795
"\n",
796
prefix,
797
context,
798
units,
799
suffix,
737
- chart,
738
- dimension,
739
- family,
800
+ buffer_tostring(plabels.labels_buffer),
801
labels,
802
value);
803
}