prometheus: monitor nixos versions
Graham Christensen committed
Sep 30, 2019 at 22:35 UTC
d71a0eddd6dc9a26ae69b8835afc7e717861cb3a
4 files changed
+104
-3
delft/common.nix
+1
-1
@@ -12,7 +12,7 @@ with lib;
12
rev = "531a2338101f0b6db2d9d512c3f98145f7b75397";
13
}) + "/module.nix")
14
../modules/common.nix
15
- ./prometheus/node.nix
15
+ ../modules/prometheus
16
];
17
18
system.stateVersion = "14.12";
delft/eris.nix
+17
@@ -4,6 +4,9 @@ let
4
5
macs = filterAttrs (_: v: (v.macosGuest or {}).enable or false) resources.machines;
6
in {
7
+ imports = [
8
+ ../modules/prometheus
9
+ ];
10
deployment.targetEnv = "hetzner";
11
deployment.hetzner.mainIPv4 = "138.201.32.77";
12
@@ -78,6 +81,20 @@ in {
81
}
82
];
83
}
84
+ {
85
+ job_name = "nixos";
86
+ static_configs = [
87
+ {
88
+ targets = flip mapAttrsToList resources.machines (machine: v: "${v.networking.hostName}:9300");
89
+ }
90
+ {
91
+ targets = [
92
+ "nixos.org:9300"
93
+ ];
94
+ labels.role = "webserver";
95
+ }
96
+ ];
97
+ }
98
{
99
job_name = "packet_nodes";
100
file_sd_configs = [
modules/prometheus/default.nix
+23
-2
@@ -1,9 +1,12 @@
1
{ pkgs, ... }:
2
{
3
- networking.firewall.allowedTCPPorts = [ 9100 ];
3
+ networking.firewall.allowedTCPPorts = [
4
+ 9100 # node-exporter
5
+ 9300 # prometheus-nixos-exporter
6
+ ];
7
services.prometheus.exporters.node = {
8
enable = true;
6
-
9
+ enabledCollectors = [ "systemd" ];
10
extraFlags = [
11
"--collector.textfile.directory=/var/lib/prometheus-node-exporter-text-files"
12
];
@@ -15,4 +18,22 @@
18
cd /var/lib/prometheus-node-exporter-text-files
19
${./system-version-exporter.sh} | ${pkgs.moreutils}/bin/sponge system-version.prom
20
'';
21
+
22
+
23
+ systemd.services.prometheus-nixos-exporter = {
24
+ wantedBy = [ "multi-user.target" ];
25
+ after = [ "network.target" ];
26
+ path= [ pkgs.nix pkgs.bash ];
27
+ serviceConfig = {
28
+ Restart = "always";
29
+ RestartSec = "60s";
30
+ ExecStart = let
31
+ python = pkgs.python3.withPackages (p: [
32
+ p.prometheus_client
33
+ ]);
34
+ in ''
35
+ ${python}/bin/python ${./nixos-exporter.py}
36
+ '';
37
+ };
38
+ };
39
}
modules/prometheus/nixos-exporter.py
new
+63
@@ -0,0 +1,63 @@
1
+#!/usr/bin/env nix-shell
2
+#!nix-shell -i python3 -p python3 -p python3Packages.prometheus_client
3
+
4
+
5
+import subprocess
6
+import json
7
+from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
8
+from prometheus_client import CollectorRegistry, generate_latest, start_http_server
9
+from pprint import pprint
10
+import time
11
+
12
+class NixosSystemCollector:
13
+ def collect(self):
14
+ # note: Gauges because of rollbacks.
15
+ current_system = GaugeMetricFamily(
16
+ "nixos_current_system_time_seconds",
17
+ "The time the system's current generation was registered in the Nix database.",
18
+ labels=["version_id"]
19
+ )
20
+ current_system.add_metric([self.get_version_id("/run/current-system")],
21
+ self.get_time("/run/current-system"))
22
+ yield current_system
23
+
24
+ booted_system = GaugeMetricFamily(
25
+ "nixos_booted_system_time_seconds",
26
+ "The time the system's booted generation was registered in the Nix database.",
27
+ labels=["version_id"]
28
+ )
29
+ booted_system.add_metric([self.get_version_id("/run/booted-system")], self.get_time("/run/booted-system"))
30
+ yield booted_system
31
+
32
+ def get_version_id(self, path):
33
+ result = subprocess.run(
34
+ [ "bash", "-c", f"source {path}/etc/os-release; echo $VERSION_ID" ],
35
+ stdout=subprocess.PIPE
36
+ )
37
+ if result.returncode == 0:
38
+ return result.stdout.decode("utf-8").strip()
39
+
40
+ return None
41
+
42
+ def get_time(self, path):
43
+ # nix path-info --json /run/booted-system | jq .[0].registrationTime
44
+ result = subprocess.run(
45
+ [ "nix", "path-info", "--json", path ],
46
+ stdout=subprocess.PIPE
47
+ )
48
+ if result.returncode == 0:
49
+ parsed = json.loads(result.stdout)
50
+ return parsed[0]['registrationTime']
51
+
52
+ return 0
53
+
54
+registry = CollectorRegistry()
55
+registry.register(NixosSystemCollector())
56
+
57
+if __name__ == '__main__':
58
+ # Start up the server to expose the metrics.
59
+ start_http_server(9300, registry=registry)
60
+
61
+ while True:
62
+ time.sleep(100000)
63
+