Remove stale labels after host label reload (#22571)
* Remove stale labels after host label reload reload_host_labels() unmarked all labels and re-added what the config / k8s / auto sources provided, but never removed unmarked entries. Labels dropped from netdata.conf or no longer returned by get-kubernetes-labels.sh persisted indefinitely. Add the matching rrdlabels_remove_all_unmarked() call; RRDLABEL_FLAG_DONT_DELETE entries are still preserved. * Preserve k8s labels when get-kubernetes-labels.sh fails If the kubernetes labels script is missing, fails to spawn, or exits non-zero, the prune step in reload_host_labels() would delete every RRDLABEL_SRC_K8S label as if it were stale. Make rrdhost_load_kubernetes_labels() report its outcome, and on failure re-mark all RRDLABEL_SRC_K8S entries with RRDLABEL_FLAG_OLD via a new rrdlabels_mark_source_as_old() helper so they survive the next remove_all_unmarked() pass. * Add unit tests for preserving K8S labels during prune operations
Stelios Fragkakis committed
May 27, 2026 at 22:35 UTC
c43da6b7b04ae6de9807f3c8b29211633929ddba
3 files changed
+129
-5
src/database/rrdhost-labels.c
+26
-5
@@ -111,7 +111,11 @@ static void rrdhost_load_config_labels(void) {
111
inicfg_foreach_value_in_section(&netdata_config, CONFIG_SECTION_HOST_LABEL, config_label_cb, NULL);
112
}
113
114
-static void rrdhost_load_kubernetes_labels(void) {
114
+// Returns true if the kubernetes labels script ran to a clean exit. A false
115
+// return tells the caller the refresh was best-effort: callers must preserve
116
+// the previously-loaded k8s labels (see reload_host_labels()) so a transient
117
+// script failure does not silently delete them.
118
+static bool rrdhost_load_kubernetes_labels(void) {
119
char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
120
sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
121
@@ -120,11 +124,11 @@ static void rrdhost_load_kubernetes_labels(void) {
124
"Kubernetes pod label fetching script %s not found.",
125
label_script);
126
123
- return;
127
+ return false;
128
}
129
130
POPEN_INSTANCE *instance = spawn_popen_run(label_script);
127
- if(!instance) return;
131
+ if(!instance) return false;
132
133
char buffer[1000 + 1];
134
while (fgets(buffer, 1000, spawn_popen_stdout(instance)) != NULL)
@@ -133,10 +137,14 @@ static void rrdhost_load_kubernetes_labels(void) {
137
// Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
138
// Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
139
int rc = spawn_popen_wait(instance);
136
- if(rc)
140
+ if(rc) {
141
nd_log(NDLS_DAEMON, NDLP_ERR,
142
"%s exited abnormally. Failed to get kubernetes labels.",
143
label_script);
144
+ return false;
145
+ }
146
+
147
+ return true;
148
}
149
150
static void rrdhost_load_auto_labels(void) {
@@ -178,9 +186,22 @@ void reload_host_labels(void) {
186
187
// priority is important here
188
rrdhost_load_config_labels();
181
- rrdhost_load_kubernetes_labels();
189
+ bool k8s_loaded = rrdhost_load_kubernetes_labels();
190
rrdhost_load_auto_labels();
191
192
+ // If the kubernetes loader did not run cleanly (script missing, spawn
193
+ // failure, or non-zero exit), the previously-loaded k8s labels were not
194
+ // refreshed and must survive the prune below -- otherwise a transient
195
+ // script failure would silently delete them. Re-mark them so the next
196
+ // step keeps them.
197
+ if (!k8s_loaded)
198
+ rrdlabels_mark_source_as_old(localhost->rrdlabels, RRDLABEL_SRC_K8S);
199
+
200
+ // drop entries that the loaders did not re-add (e.g. a label removed from
201
+ // netdata.conf, or no longer returned by get-kubernetes-labels.sh).
202
+ // RRDLABEL_FLAG_DONT_DELETE entries are preserved.
203
+ rrdlabels_remove_all_unmarked(localhost->rrdlabels);
204
+
205
rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
206
207
stream_send_host_labels(localhost);
src/database/rrdlabels.c
+97
@@ -581,6 +581,24 @@ void rrdlabels_remove_all_unmarked(RRDLABELS *labels)
581
spinlock_unlock(&labels->spinlock);
582
}
583
584
+void rrdlabels_mark_source_as_old(RRDLABELS *labels, RRDLABEL_SRC src_match)
585
+{
586
+ if (!labels) return;
587
+
588
+ Pvoid_t *PValue;
589
+ Word_t Index = 0;
590
+ bool first_then_next = true;
591
+
592
+ spinlock_lock(&labels->spinlock);
593
+
594
+ while ((PValue = JudyLFirstThenNext(labels->JudyL, &Index, &first_then_next))) {
595
+ if ((*((RRDLABEL_SRC *)PValue)) & src_match)
596
+ *((RRDLABEL_SRC *)PValue) |= RRDLABEL_FLAG_OLD;
597
+ }
598
+
599
+ spinlock_unlock(&labels->spinlock);
600
+}
601
+
602
// ----------------------------------------------------------------------------
603
// rrdlabels_walkthrough_read()
604
@@ -1437,6 +1455,84 @@ static int rrdlabels_unittest_migrate_check()
1455
return rc;
1456
}
1457
1458
+// Exercises the unmark + re-add + remove_all_unmarked cycle that
1459
+// reload_host_labels() relies on, plus the rrdlabels_mark_source_as_old()
1460
+// preservation path used when the kubernetes loader fails.
1461
+static int rrdlabels_unittest_mark_source_as_old(void) {
1462
+ fprintf(stderr, "\n%s() tests\n", __FUNCTION__);
1463
+ int errors = 0;
1464
+
1465
+ // Subtest A: the prune drops entries that no loader re-added.
1466
+ {
1467
+ RRDLABELS *l = rrdlabels_create();
1468
+ rrdlabels_add(l, "cfg_key", "cv", RRDLABEL_SRC_CONFIG);
1469
+ rrdlabels_add(l, "k8s_key", "kv", RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S);
1470
+ rrdlabels_add(l, "aclk_key", "av", RRDLABEL_SRC_AUTO | RRDLABEL_SRC_ACLK);
1471
+
1472
+ rrdlabels_unmark_all(l);
1473
+ // simulate a successful config + aclk loader; the k8s loader added nothing
1474
+ rrdlabels_add(l, "cfg_key", "cv", RRDLABEL_SRC_CONFIG);
1475
+ rrdlabels_add(l, "aclk_key", "av", RRDLABEL_SRC_AUTO | RRDLABEL_SRC_ACLK);
1476
+ rrdlabels_remove_all_unmarked(l);
1477
+
1478
+ if (rrdlabels_exist(l, "k8s_key")) {
1479
+ fprintf(stderr, " FAIL (A): k8s_key should have been pruned (not re-added, not preserved)\n");
1480
+ errors++;
1481
+ }
1482
+ if (!rrdlabels_exist(l, "cfg_key") || !rrdlabels_exist(l, "aclk_key")) {
1483
+ fprintf(stderr, " FAIL (A): re-added labels should have survived the prune\n");
1484
+ errors++;
1485
+ }
1486
+ rrdlabels_destroy(l);
1487
+ }
1488
+
1489
+ // Subtest B: mark_source_as_old(K8S) preserves k8s entries through the prune
1490
+ // (the k8s-loader-failure preservation path in reload_host_labels()).
1491
+ {
1492
+ RRDLABELS *l = rrdlabels_create();
1493
+ rrdlabels_add(l, "cfg_key", "cv", RRDLABEL_SRC_CONFIG);
1494
+ rrdlabels_add(l, "k8s_key", "kv", RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S);
1495
+ rrdlabels_add(l, "k8s_key2", "kv2", RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S);
1496
+
1497
+ rrdlabels_unmark_all(l);
1498
+ // simulate successful config loader; k8s loader failed
1499
+ rrdlabels_add(l, "cfg_key", "cv", RRDLABEL_SRC_CONFIG);
1500
+ rrdlabels_mark_source_as_old(l, RRDLABEL_SRC_K8S);
1501
+ rrdlabels_remove_all_unmarked(l);
1502
+
1503
+ if (!rrdlabels_exist(l, "k8s_key") || !rrdlabels_exist(l, "k8s_key2")) {
1504
+ fprintf(stderr, " FAIL (B): K8S labels should be preserved via mark_source_as_old\n");
1505
+ errors++;
1506
+ }
1507
+ if (!rrdlabels_exist(l, "cfg_key")) {
1508
+ fprintf(stderr, " FAIL (B): cfg_key should still be present after the prune\n");
1509
+ errors++;
1510
+ }
1511
+ rrdlabels_destroy(l);
1512
+ }
1513
+
1514
+ // Subtest C: mark_source_as_old(K8S) only preserves labels whose source
1515
+ // mask intersects K8S; entries with other-only sources are still pruned.
1516
+ {
1517
+ RRDLABELS *l = rrdlabels_create();
1518
+ rrdlabels_add(l, "aclk_key", "av", RRDLABEL_SRC_AUTO | RRDLABEL_SRC_ACLK);
1519
+ rrdlabels_add(l, "cfg_key", "cv", RRDLABEL_SRC_CONFIG);
1520
+
1521
+ rrdlabels_unmark_all(l);
1522
+ rrdlabels_mark_source_as_old(l, RRDLABEL_SRC_K8S);
1523
+ rrdlabels_remove_all_unmarked(l);
1524
+
1525
+ if (rrdlabels_exist(l, "aclk_key") || rrdlabels_exist(l, "cfg_key")) {
1526
+ fprintf(stderr, " FAIL (C): non-K8S labels must not be preserved by mark_source_as_old(K8S)\n");
1527
+ errors++;
1528
+ }
1529
+ rrdlabels_destroy(l);
1530
+ }
1531
+
1532
+ fprintf(stderr, "%s: %d errors\n", __FUNCTION__, errors);
1533
+ return errors;
1534
+}
1535
+
1536
struct pattern_array *trim_and_add_key_to_values(struct pattern_array *pa, const char *key, STRING *input);
1537
static int rrdlabels_unittest_check_pattern_list(RRDLABELS *labels, const char *pattern, bool expected) {
1538
fprintf(stderr, "rrdlabels_match_simple_pattern(labels, \"%s\") ... ", pattern);
@@ -1619,6 +1715,7 @@ int rrdlabels_unittest(void) {
1715
errors += rrdlabels_unittest_host_chart_labels();
1716
errors += rrdlabels_unittest_double_check();
1717
errors += rrdlabels_unittest_migrate_check();
1718
+ errors += rrdlabels_unittest_mark_source_as_old();
1719
errors += rrdlabels_unittest_pattern_check();
1720
1721
fprintf(stderr, "%d errors found\n", errors);
src/database/rrdlabels.h
+6
@@ -46,6 +46,12 @@ void rrdlabels_get_value_strcpyz(RRDLABELS *labels, char *dst, size_t dst_len, c
46
void rrdlabels_unmark_all(RRDLABELS *labels);
47
void rrdlabels_remove_all_unmarked(RRDLABELS *labels);
48
49
+// OR RRDLABEL_FLAG_OLD onto every entry whose source bits intersect src_match.
50
+// Used after a best-effort loader (e.g. the kubernetes labels script) fails,
51
+// so labels from that source survive the next rrdlabels_remove_all_unmarked()
52
+// prune even though the loader did not re-add them this round.
53
+void rrdlabels_mark_source_as_old(RRDLABELS *labels, RRDLABEL_SRC src_match);
54
+
55
int rrdlabels_walkthrough_read(RRDLABELS *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
56
int rrdlabels_walkthrough_read_string(RRDLABELS *labels, int (*callback)(STRING *name, STRING *value, RRDLABEL_SRC ls, void *data), void *data);
57
void rrdlabels_log_to_buffer(RRDLABELS *labels, BUFFER *wb);