| 1 | { config, ... }: |
| 2 | |
| 3 | { |
| 4 | services.prometheus = { |
| 5 | alertmanagers = [ |
| 6 | { |
| 7 | scheme = "http"; |
| 8 | static_configs = [ |
| 9 | { targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ]; } |
| 10 | ]; |
| 11 | } |
| 12 | ]; |
| 13 | |
| 14 | alertmanager = { |
| 15 | enable = true; |
| 16 | |
| 17 | # Allow alertmanager to start even if it doesn't find an RFC1918 IP on |
| 18 | # the machine's network interfaces. |
| 19 | extraFlags = [ "--cluster.listen-address=''" ]; |
| 20 | |
| 21 | webExternalUrl = "http://alerts.nixos.org"; |
| 22 | configuration = { |
| 23 | global = { }; |
| 24 | route = { |
| 25 | receiver = "ignore"; |
| 26 | group_wait = "30s"; |
| 27 | group_interval = "5m"; |
| 28 | repeat_interval = "24h"; |
| 29 | group_by = [ "alertname" ]; |
| 30 | |
| 31 | routes = [ |
| 32 | { |
| 33 | receiver = "go-neb"; |
| 34 | group_wait = "30s"; |
| 35 | match.severity = "warning"; |
| 36 | } |
| 37 | ]; |
| 38 | }; |
| 39 | receivers = [ |
| 40 | { |
| 41 | # with no *_config, this will drop all alerts directed to it |
| 42 | name = "ignore"; |
| 43 | } |
| 44 | { |
| 45 | name = "go-neb"; |
| 46 | webhook_configs = [ |
| 47 | { |
| 48 | url = "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U"; |
| 49 | send_resolved = true; |
| 50 | } |
| 51 | ]; |
| 52 | } |
| 53 | ]; |
| 54 | }; |
| 55 | }; |
| 56 | }; |
| 57 | |
| 58 | services.nginx.virtualHosts."alerts.nixos.org" = { |
| 59 | enableACME = true; |
| 60 | forceSSL = true; |
| 61 | |
| 62 | locations."/" = { |
| 63 | proxyPass = "http://localhost:9093"; |
| 64 | }; |
| 65 | }; |
| 66 | |
| 67 | age.secrets."alertmanager-oauth2-proxy-env".file = ../../secrets/alertmanager-oauth2-proxy-env.age; |
| 68 | |
| 69 | services.oauth2-proxy = { |
| 70 | enable = true; |
| 71 | |
| 72 | # oidc provider |
| 73 | provider = "github"; |
| 74 | clientID = "Ov23liDt1q76okEJpVVE"; |
| 75 | keyFile = config.age.secrets."alertmanager-oauth2-proxy-env".path; |
| 76 | |
| 77 | # filter criteria |
| 78 | email.domains = [ "*" ]; |
| 79 | github = { |
| 80 | org = "NixOS"; |
| 81 | team = "infra"; |
| 82 | }; |
| 83 | |
| 84 | # protected domains |
| 85 | nginx = { |
| 86 | domain = "alerts.nixos.org"; |
| 87 | virtualHosts."alerts.nixos.org" = { }; |
| 88 | }; |
| 89 | }; |
| 90 | |
| 91 | age.secrets.alertmanager-matrix-forwarder = { |
| 92 | file = ../../secrets/alertmanager-matrix-forwarder.age; |
| 93 | owner = config.systemd.services.go-neb.serviceConfig.User; |
| 94 | }; |
| 95 | |
| 96 | # Create user so that we can set the ownership of the key to |
| 97 | # it. DynamicUser will not take full effect as a result of this. |
| 98 | users.users.go-neb = { |
| 99 | isSystemUser = true; |
| 100 | group = "go-neb"; |
| 101 | }; |
| 102 | users.groups.go-neb = { }; |
| 103 | |
| 104 | systemd.services.go-neb.serviceConfig.SupplementaryGroups = [ "keys" ]; |
| 105 | |
| 106 | nixpkgs.config.permittedInsecurePackages = [ "olm-3.2.16" ]; |
| 107 | |
| 108 | services.go-neb = { |
| 109 | enable = true; |
| 110 | bindAddress = "localhost:4050"; |
| 111 | baseUrl = "http://localhost"; |
| 112 | secretFile = config.age.secrets.alertmanager-matrix-forwarder.path; |
| 113 | config = { |
| 114 | clients = [ |
| 115 | { |
| 116 | UserId = "@bot:nixos.org"; |
| 117 | AccessToken = "$CHANGEME"; |
| 118 | HomeServerUrl = "https://matrix.nixos.org"; |
| 119 | Sync = true; |
| 120 | AutoJoinRooms = true; |
| 121 | DisplayName = "Bot"; |
| 122 | } |
| 123 | ]; |
| 124 | services = [ |
| 125 | { |
| 126 | ID = "alertmanager_service"; |
| 127 | Type = "alertmanager"; |
| 128 | UserId = "@bot:nixos.org"; |
| 129 | Config = { |
| 130 | webhook_url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U"; |
| 131 | rooms = { |
| 132 | # infra-alerts:nixos.org |
| 133 | "!QLQqibtFaVtDgurUAE:nixos.org" = { |
| 134 | text_template = '' |
| 135 | {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}} |
| 136 | ''; |
| 137 | |
| 138 | # $$severity otherwise envsubst replaces $severity with an empty string |
| 139 | html_template = '' |
| 140 | {{range .Alerts -}} |
| 141 | {{ $$severity := index .Labels "severity" }} |
| 142 | {{ if eq .Status "firing" }} |
| 143 | {{ if eq $$severity "critical"}} |
| 144 | <font color='red'><b>[FIRING - CRITICAL]</b></font> |
| 145 | {{ else if eq $$severity "warning"}} |
| 146 | <font color='orange'><b>[FIRING - WARNING]</b></font> |
| 147 | {{ else }} |
| 148 | <b>[FIRING - {{ $$severity }}]</b> |
| 149 | {{ end }} |
| 150 | {{ else }} |
| 151 | <font color='green'><b>[RESOLVED]</b></font> |
| 152 | {{ end }} |
| 153 | {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}} |
| 154 | ( |
| 155 | {{ if .Annotations.grafana }} |
| 156 | <a href="{{ index .Annotations "grafana" }}">📈 Grafana</a>, |
| 157 | {{ end }} |
| 158 | <a href="{{ .GeneratorURL }}">🔥 Prometheus</a>, |
| 159 | <a href="{{ .SilenceURL }}">🔕 Silence</a> |
| 160 | )<br/> |
| 161 | {{end -}}''; |
| 162 | msg_type = "m.text"; # Must be either `m.text` or `m.notice` |
| 163 | }; |
| 164 | }; |
| 165 | }; |
| 166 | } |
| 167 | ]; |
| 168 | }; |
| 169 | }; |
| 170 | } |