channels: monitor a bit
Graham Christensen committed
Dec 26, 2019 at 03:34 UTC
3f896e9c482a8fee07cdcd22c4d2907a34d14552
3 files changed
+69
delft/eris.nix
+13
@@ -10,6 +10,7 @@ in {
10
./eris/packet-spot-market-prices.nix
11
./eris/github-project-monitor.nix
12
./eris/alertmanager-irc-forwarder.nix
13
+ ./eris/channel-monitor.nix
14
];
15
deployment.targetEnv = "hetzner";
16
deployment.hetzner.mainIPv4 = "138.201.32.77";
@@ -360,6 +361,18 @@ in {
361
];
362
}
363
364
+ {
365
+ job_name = "channel-updates";
366
+ metrics_path = "/";
367
+ static_configs = [
368
+ {
369
+ targets = [
370
+ "127.0.0.1:9402"
371
+ ];
372
+ }
373
+ ];
374
+ }
375
+
376
];
377
};
378
delft/eris/channel-exporter.py
new
+42
@@ -0,0 +1,42 @@
1
+#!/usr/bin/env python3
2
+
3
+import requests
4
+from dateutil.parser import parse
5
+from prometheus_client import Counter, Histogram, Gauge, start_http_server
6
+import time
7
+import sys
8
+from pprint import pprint
9
+
10
+CHANNEL_REQUEST_TIME = Histogram(
11
+ "channel_request_time", "Time spent requesting channel data"
12
+)
13
+CHANNEL_UPDATE_TIME = Gauge(
14
+ "channel_update_time",
15
+ "Total number of failures to fetch spot market prices",
16
+ ["channel"],
17
+)
18
+CHANNEL_REQUEST_FAILURES = Counter(
19
+ "channel_request_failures_total",
20
+ "Number of channel status requests which have failed",
21
+)
22
+
23
+
24
+@CHANNEL_REQUEST_TIME.time()
25
+def measure_channel(name):
26
+ try:
27
+ with CHANNEL_REQUEST_FAILURES.count_exceptions():
28
+ result = requests.get(f"https://nixos.org/channels/{name}")
29
+ return parse(result.headers["last-modified"]).timestamp()
30
+ except Exception as e:
31
+ pprint(e)
32
+
33
+
34
+if __name__ == "__main__":
35
+ start_http_server(9402)
36
+
37
+ while True:
38
+ for channel in sys.argv[1:]:
39
+ measurement = measure_channel(channel)
40
+ if measurement is not None:
41
+ CHANNEL_UPDATE_TIME.labels(channel=channel).set(measurement)
42
+ time.sleep(60)
delft/eris/channel-monitor.nix
new
+14
@@ -0,0 +1,14 @@
1
+{ lib, pkgs, ... }:
2
+let
3
+ channels = builtins.attrNames (import ../../channels.nix).channels;
4
+in {
5
+ systemd.services.channel-update-exporter = {
6
+ description = "Check all active channels' last-update times";
7
+ path = [ (pkgs.python3.withPackages (pypkgs: with pypkgs; [ requests dateutil prometheus_client ])) ];
8
+ wantedBy = [ "multi-user.target" ];
9
+ serviceConfig = {
10
+ DynamicUser = true;
11
+ ExecStart = "${./channel-exporter.py} ${lib.concatStringsSep " " channels}";
12
+ };
13
+ };
14
+}
\ No newline at end of file