Fix registry save integer overflow and add failure backoff (#20437)
The registry save functions were accumulating byte counts from fprintf() calls, which could exceed INT_MAX on large registries, causing the dictionary_walkthrough_read() to return negative values and fail. This fix: - Changes save functions to return item counts (1 per saved item) instead of accumulated byte counts - Adds exponential backoff (60s, 120s, 240s... up to 1 hour) to prevent repeated save attempts after failures - Tracks consecutive failures to implement the backoff mechanism - Returns 0 on success instead of -1 from registry_db_save()
Costa Tsaousis committed
Jun 7, 2025 at 09:02 UTC
5b73350b4752463da61e4b4f75918235c74247a2
2 files changed
+44
-19
src/registry/registry_db.c
+40
-19
@@ -45,15 +45,14 @@ static int registry_machine_save(const DICTIONARY_ITEM *item __maybe_unused, voi
45
for(REGISTRY_MACHINE_URL *mu = m->machine_urls; mu ; mu = mu->next) {
46
int rc = registry_machine_save_url(mu, fp);
47
if(rc < 0)
48
- return rc;
49
-
50
- ret += rc;
48
+ return -1; // Error saving URL
49
}
50
+ return 1; // Successfully saved 1 machine
51
}
52
53
// error handling is done at registry_db_save()
54
56
- return ret;
55
+ return -1; // Error writing machine record
56
}
57
58
static inline int registry_person_save_url(REGISTRY_PERSON_URL *pu, FILE *fp) {
@@ -91,15 +90,14 @@ static inline int registry_person_save(const DICTIONARY_ITEM *item __maybe_unuse
90
for(REGISTRY_PERSON_URL *pu = p->person_urls; pu ;pu = pu->next) {
91
int rc = registry_person_save_url(pu, fp);
92
if(rc < 0)
94
- return rc;
95
- else
96
- ret += rc;
93
+ return -1; // Error saving URL
94
}
95
+ return 1; // Successfully saved 1 person
96
}
97
98
// error handling is done at registry_db_save()
99
102
- return ret;
100
+ return -1; // Error writing person record
101
}
102
103
// ----------------------------------------------------------------------------
@@ -112,6 +110,19 @@ int registry_db_save(void) {
110
if(unlikely(!registry_db_should_be_saved()))
111
return -2;
112
113
+ // Implement exponential backoff for save failures
114
+ if(registry.consecutive_save_failures > 0) {
115
+ time_t now = now_realtime_sec();
116
+ time_t backoff_seconds = 60 * (1 << (registry.consecutive_save_failures - 1)); // 60s, 120s, 240s, etc.
117
+ if(backoff_seconds > 3600) backoff_seconds = 3600; // Cap at 1 hour
118
+
119
+ if((now - registry.last_save_failure) < backoff_seconds) {
120
+ netdata_log_debug(D_REGISTRY, "REGISTRY: skipping save due to backoff (failed %d times, waiting %ld seconds)",
121
+ registry.consecutive_save_failures, backoff_seconds);
122
+ return -3;
123
+ }
124
+ }
125
+
126
nd_log_limits_unlimited();
127
128
char tmp_filename[FILENAME_MAX + 1];
@@ -125,30 +136,36 @@ int registry_db_save(void) {
136
if(!fp) {
137
netdata_log_error("REGISTRY: Cannot create file: %s", tmp_filename);
138
nd_log_limits_reset();
139
+ registry.consecutive_save_failures++;
140
+ registry.last_save_failure = now_realtime_sec();
141
return -1;
142
}
143
144
// dictionary_walkthrough_read() has its own locking, so this is safe to do
145
146
netdata_log_debug(D_REGISTRY, "REGISTRY: saving all machines");
134
- int bytes1 = dictionary_walkthrough_read(registry.machines, registry_machine_save, fp);
135
- if(bytes1 < 0) {
136
- netdata_log_error("REGISTRY: Cannot save registry machines - return value %d", bytes1);
147
+ int machines_saved = dictionary_walkthrough_read(registry.machines, registry_machine_save, fp);
148
+ if(machines_saved < 0) {
149
+ netdata_log_error("REGISTRY: Cannot save registry machines - return value %d", machines_saved);
150
fclose(fp);
151
nd_log_limits_reset();
139
- return bytes1;
152
+ registry.consecutive_save_failures++;
153
+ registry.last_save_failure = now_realtime_sec();
154
+ return machines_saved;
155
}
141
- netdata_log_debug(D_REGISTRY, "REGISTRY: saving machines took %d bytes", bytes1);
156
+ netdata_log_debug(D_REGISTRY, "REGISTRY: saved %d machines", machines_saved);
157
158
netdata_log_debug(D_REGISTRY, "Saving all persons");
144
- int bytes2 = dictionary_walkthrough_read(registry.persons, registry_person_save, fp);
145
- if(bytes2 < 0) {
146
- netdata_log_error("REGISTRY: Cannot save registry persons - return value %d", bytes2);
159
+ int persons_saved = dictionary_walkthrough_read(registry.persons, registry_person_save, fp);
160
+ if(persons_saved < 0) {
161
+ netdata_log_error("REGISTRY: Cannot save registry persons - return value %d", persons_saved);
162
fclose(fp);
163
nd_log_limits_reset();
149
- return bytes2;
164
+ registry.consecutive_save_failures++;
165
+ registry.last_save_failure = now_realtime_sec();
166
+ return persons_saved;
167
}
151
- netdata_log_debug(D_REGISTRY, "REGISTRY: saving persons took %d bytes", bytes2);
168
+ netdata_log_debug(D_REGISTRY, "REGISTRY: saved %d persons", persons_saved);
169
170
// save the totals
171
fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",
@@ -200,13 +217,17 @@ int registry_db_save(void) {
217
// discard the current registry log
218
registry_log_recreate();
219
registry.log_count = 0;
220
+
221
+ // Reset failure tracking on success
222
+ registry.consecutive_save_failures = 0;
223
+ registry.last_save_failure = 0;
224
}
225
}
226
227
// continue operations
228
nd_log_limits_reset();
229
209
- return -1;
230
+ return 0; // Success
231
}
232
233
// ----------------------------------------------------------------------------
src/registry/registry_internals.h
+4
@@ -49,6 +49,10 @@ struct registry {
49
// open files
50
FILE *log_fp;
51
52
+ // save failure tracking
53
+ time_t last_save_failure;
54
+ int consecutive_save_failures;
55
+
56
// the database
57
DICTIONARY *persons; // dictionary of REGISTRY_PERSON *, with key the REGISTRY_PERSON.guid
58
DICTIONARY *machines; // dictionary of REGISTRY_MACHINE *, with key the REGISTRY_MACHINE.guid