hydra-queue-runner-reexporter: Treat notification metrics as optional
This should unbreak https://status.nixos.org/grafana which currently doesn't show any details about the hydra queues and runners.
Andreas Rammhold committed
Sep 4, 2019 at 13:43 UTC
c5a96beaf24cab58cfd22d5a3a45df14b1ff8fb8
1 file changed
+20
-6
delft/prometheus/hydra-queue-runner-reexporter.py
+20
-6
@@ -24,6 +24,13 @@ class EvaporatingDict:
24
else:
25
return val
26
27
+ def preserving_read_default(self, key, default):
28
+ try:
29
+ val = self.preserving_read(key)
30
+ return val
31
+ except KeyError:
32
+ return default
33
+
34
def destructive_read(self, key):
35
val = self.preserving_read(key)
36
del self._state[key]
@@ -208,32 +215,33 @@ class HydraScrapeImporter:
215
yield self.trivial_counter(
216
"notifications_total",
217
"Total number of notifications sent",
211
- self.preserving_read("nrNotificationsDone") + self.preserving_read("nrNotificationsFailed")
218
+ self.preserving_read_default("nrNotificationsDone", 0) +
219
+ self.preserving_read_default("nrNotificationsFailed", 0)
220
)
221
yield self.trivial_counter(
222
"notifications_done",
223
"Number of notifications completed",
216
- self.destructive_read("nrNotificationsDone")
224
+ self.destructive_read_default("nrNotificationsDone", 0)
225
)
226
yield self.trivial_counter(
227
"notifications_failed",
228
"Number of notifications failed",
221
- self.destructive_read("nrNotificationsFailed")
229
+ self.destructive_read_default("nrNotificationsFailed", 0)
230
)
231
yield self.trivial_counter(
232
"notifications_in_progress",
233
"Number of notifications in_progress",
226
- self.destructive_read("nrNotificationsInProgress")
234
+ self.destructive_read_default("nrNotificationsInProgress", 0)
235
)
236
yield self.trivial_counter(
237
"notifications_pending",
238
"Number of notifications pending",
231
- self.destructive_read("nrNotificationsPending")
239
+ self.destructive_read_default("nrNotificationsPending", 0)
240
)
241
yield self.trivial_counter(
242
"notifications_seconds",
243
"Time spent delivering notifications",
236
- self.destructive_read("nrNotificationTimeMs") / 1000
244
+ self.destructive_read_default("nrNotificationTimeMs", 0) / 1000
245
)
246
self.unused_metric("nrNotificationTimeAvgMs")
247
@@ -394,9 +402,15 @@ class HydraScrapeImporter:
402
def preserving_read(self, key):
403
return self._status.preserving_read(key)
404
405
+ def preserving_read_default(self, key, default):
406
+ return self._status.preserving_read_default(key, default)
407
+
408
def destructive_read(self, key):
409
return self._status.destructive_read(key)
410
411
+ def destructive_read_default(self, key, default):
412
+ return self._status.destructive_read_default(key, default)
413
+
414
def uncollected_status(self):
415
return self._status.remaining_state()
416