@cryptotaxi247 / infra / commits / a3f90834

removing status.nixos.org

also: - status.nixos.org/prometheus -> prometheus.nixos.org - status.nixos.org/grafana -> grafana.nixos.org

Rok Garbas committed Dec 3, 2020 at 14:55 UTC a3f90834ed066ed6706520474c0df2455e544550
7 files changed +42 -410
delft/apache-errors/503.html
+22 -22
@@ -2,28 +2,28 @@
2
3 <html>
4
5 - <head>
6 - <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
7 - <title>Hydra is down</title>
8 - <style type="text/css" media="screen">
9 - body {
10 - font-family: Helvetica, Arial, sans-serif;
11 - color: rgba(0, 0, 0, 0.7);
12 - }
13 - </style>
14 - </head>
5 +<head>
6 + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
7 + <title>Hydra is down</title>
8 + <style type="text/css" media="screen">
9 + body {
10 + font-family: Helvetica, Arial, sans-serif;
11 + color: rgba(0, 0, 0, 0.7);
12 + }
13 + </style>
14 +</head>
15
16 - <body>
17 - <center>
18 - <img src="/apache-errors/warning.png" alt="Warning" />
19 - <p>Looks like Hydra is having some problems. Sorry about that!</p>
20 - <p style="font-size: 90%;">
21 - <a href="https://nixos.org/">NixOS Homepage</a> |
22 - <a href="https://status.nixos.org/prometheus/alerts">System Alerts</a> |
23 - <a href="https://status.nixos.org/grafana/">Dashboards</a> |
24 - <a href="https://github.com/NixOS/nixpkgs/labels/infrastructure">Related Issues</a>
25 - </p>
26 - </center>
27 - </body>
16 +<body>
17 + <center>
18 + <img src="/apache-errors/warning.png" alt="Warning" />
19 + <p>Looks like Hydra is having some problems. Sorry about that!</p>
20 + <p style="font-size: 90%;">
21 + <a href="https://nixos.org/">NixOS Homepage</a> |
22 + <a href="https://prometheus.nixos.org/alerts">System Alerts</a> |
23 + <a href="https://grafana.nixos.org/">Dashboards</a> |
24 + <a href="https://github.com/NixOS/nixpkgs/labels/infrastructure">Related Issues</a>
25 + </p>
26 + </center>
27 +</body>
28
29 </html>
delft/eris.nix
+16 -13
@@ -40,12 +40,15 @@ in {
40
41 services.nginx = {
42 enable = true;
43 - virtualHosts."status.nixos.org" = {
43 + virtualHosts."grafana.nixos.org" = {
44 enableACME = true;
45 forceSSL = true;
46 - root = ./eris/status-page;
47 - locations."/grafana/".proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}/";
48 - locations."/prometheus".proxyPass = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
46 + locations."/".proxyPass = "http://${config.services.grafana.addr}:${toString config.services.grafana.port}/";
47 + };
48 + virtualHosts."prometheus.nixos.org" = {
49 + enableACME = true;
50 + forceSSL = true;
51 + locations."/".proxyPass = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
52 };
53 };
54
@@ -53,7 +56,7 @@ in {
56 enable = true;
57 extraFlags = [
58 "--storage.tsdb.retention=${toString (150 * 24)}h"
56 - "--web.external-url=https://status.nixos.org/prometheus/"
59 + "--web.external-url=https://prometheus.nixos.org/"
60 ];
61
62 alertmanagers = [
@@ -114,7 +117,7 @@ in {
117 expr = ''hydra_machine_build_duration_bucket{le="259200"} - ignoring(le) hydra_machine_build_duration_bucket{le="172800"} > 0'';
118 for = "30m";
119 labels.severity = "page";
117 - annotations.summary = "https://status.nixos.org/grafana/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
120 + annotations.summary = "https://grafana.nixos.org/d/j0hJAY1Wk/in-progress-build-duration-heatmap";
121 }
122 ];
123 }
@@ -127,7 +130,7 @@ in {
130 expr = ''node_filesystem_files_free{mountpoint="/"} <= 10000'';
131 for = "30m";
132 labels.severity = "page";
130 - annotations.summary = "https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
133 + annotations.summary = "https://grafana.nixos.org/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
134 }
135
136 {
@@ -135,7 +138,7 @@ in {
138 expr = ''node_filesystem_avail_bytes{mountpoint="/"} <= 10000000000'';
139 for = "30m";
140 labels.severity = "page";
138 - annotations.summary = "https://status.nixos.org/grafana/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
141 + annotations.summary = "https://grafana.nixos.org/d/5LANB9pZk/per-instance-metrics?orgId=1&refresh=30s&var-instance={{ $labels.instance }}";
142 }
143 ];
144 }
@@ -148,13 +151,13 @@ in {
151 expr = ''node_systemd_unit_state{name=~"^rfc39-sync.service$", state="failed"} == 1'';
152 for = "30m";
153 labels.severity = "page";
151 - annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
154 + annotations.summary = "https://grafana.nixos.org/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
155 }
156 {
157 alert = "ChannelUpdateStuck";
158 expr = ''max_over_time(node_systemd_unit_state{name=~"^update-nix.*.service$",state=~"failed"}[5m]) == 1'';
159 for = "30m";
157 - annotations.summary = "https://status.nixos.org/grafana/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
160 + annotations.summary = "https://grafana.nixos.org/d/fBW4tL1Wz/scheduled-task-state-channels-website?orgId=1&refresh=10s";
161 }
162 ];
163 }
@@ -311,7 +314,7 @@ in {
314 static_configs = [
315 {
316 targets = [
314 - "status.nixos.org:9200"
317 + "prometheus.nixos.org:9200"
318 ];
319 }
320 ];
@@ -409,8 +412,8 @@ in {
412 auth.anonymous.enable = true;
413 users.allowSignUp = true;
414 addr = "0.0.0.0";
412 - domain = "status.nixos.org";
413 - rootUrl = "https://status.nixos.org/grafana/";
415 + domain = "grafana.nixos.org";
416 + rootUrl = "https://grafana.nixos.org/";
417 };
418
419 systemd.services.prometheus-hydra-exporter = {
delft/eris/alertmanager-irc-forwarder.nix
+2 -2
@@ -21,7 +21,7 @@
21
22 environment = {
23 FLASK_APP = ./prometheus-alertmanager-irc-notifier.py;
24 - EXTERNAL_URL = "https://status.nixos.org/prometheus/alerts";
24 + EXTERNAL_URL = "https://prometheus.nixos.org/alerts";
25 };
26
27 serviceConfig = {
@@ -30,4 +30,4 @@
30 EnvironmentFile = "/run/keys/alertmanager-irc-forwarder.env";
31 };
32 };
33 -}
\ No newline at end of file
33 +}
delft/eris/status-page/index.html deleted
-87
@@ -1,87 +0,0 @@
1 -<!DOCTYPE html>
2 -
3 -<html lang="en">
4 -
5 - <head>
6 - <title>NixOS Infra Status</title>
7 -
8 - <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
9 - <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
10 -
11 - <script type="text/javascript" src="https://nixos.org/js/jquery.min.js"></script>
12 - <meta name="viewport" content="width=device-width, initial-scale=1.0" />
13 - <script type="text/javascript" src="https://nixos.org/bootstrap/js/bootstrap.min.js"></script>
14 - <link rel="stylesheet" href="https://nixos.org/bootstrap/css/bootstrap.min.css" />
15 - <link rel="stylesheet" href="https://nixos.org/bootstrap/css/bootstrap-responsive.min.css" />
16 - <link rel="stylesheet" href="https://nixos.org/css/nixos-site.css" type="text/css" />
17 - <link rel="stylesheet" href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" />
18 - <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.24.0/moment-with-locales.min.js" integrity="sha256-AdQN98MVZs44Eq2yTwtoKufhnU+uZ7v2kXnD5vqzZVo=" crossorigin="anonymous"></script>
19 - <link rel="shortcut icon" type="image/png" href="https://nixos.org/favicon.png" />
20 - <script type="text/javascript" src="/status.js"></script>
21 - <link rel="stylesheet" href="/status.css" />
22 -
23 - </head>
24 -
25 - <body>
26 -
27 - <div class="navbar navbar-fixed-top">
28 - <div class="navbar-inner">
29 - <div class="container">
30 - <button type="button" class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
31 - <span class="icon-bar"></span>
32 - <span class="icon-bar"></span>
33 - <span class="icon-bar"></span>
34 - </button>
35 - <li class="dropdown brand">
36 - <a class="dropdown-toggle" href="#" data-toggle="dropdown">
37 - <img src="https://nixos.org/logo/nix-wiki.png" alt="NixOS" class="logo" /> NixOS Status <b class="caret"></b>
38 - </a>
39 - <ul class="dropdown-menu">
40 - <li><a href="https://nixos.org">NixOS.org</a></li>
41 - </ul>
42 - </li>
43 -
44 - <div class="nav-collapse collapse">
45 - <ul class="nav pull-left">
46 - <li><a href="/grafana">Grafana</a></li>
47 - <li><a href="/prometheus">Prometheus</a></li>
48 - </ul>
49 - </div>
50 - </div>
51 - </div>
52 - </div>
53 -
54 - <div class="container main">
55 - <div class="nixos-org-index">
56 - <h1>NixOS</h1>
57 - </div>
58 -
59 - <div id="alerts"></div>
60 -
61 - <div class="hero12">
62 - <h2>Nix Channel Status</h2>
63 - <p>See <a href="http://howoldis.herokuapp.com/">HowOldIs</a> and the <a href="https://nixos.wiki/wiki/Nix_channels">NixOS Wiki</a> for information on how channel updates progress.</p>
64 - </div>
65 -
66 - <div class="hero12">
67 - <noscript>
68 - <p class="jsfallback">This page requires Javascript.</p>
69 - </noscript>
70 -
71 - <table class="table table-striped">
72 - <thead>
73 - <tr>
74 - <th>Channel</th>
75 - <th>Last updated</th>
76 - <th>Commit</th>
77 - <th>Hydra job for tests</th>
78 - <th>Channel Status</th>
79 - </tr>
80 - </thead>
81 - <tbody id="channel-status"></tbody>
82 - </table>
83 - </div>
84 - </div>
85 - </body>
86 -
87 -</html>
delft/eris/status-page/status.css deleted
-45
@@ -1,45 +0,0 @@
1 -
2 -.stale {
3 - opacity: 60%;
4 -}
5 -
6 -.channel {
7 - font-family: monospace;
8 - max-width: 20em;
9 -}
10 -
11 -.agetd {
12 - max-width: 15em;
13 -}
14 -
15 -#alerts {
16 - margin-top: 10px;
17 -}
18 -
19 -.issue-age {
20 - display: inline-block;
21 - min-width: 6em;
22 -}
23 -
24 -.github {
25 - max-width: 10em;
26 - overflow: hidden;
27 - text-overflow: ellipsis;
28 - font-family: monospace;
29 -}
30 -
31 -.hydra {
32 - max-width: 20em;
33 -}
34 -
35 -#channel-status {
36 - max-width: 100%;
37 -}
38 -
39 -.jsfallback {
40 - font-weight: bold;
41 - font-style: italics;
42 - text-align: center;
43 - padding-top: 3em;
44 - padding-bottom: 3em;
45 -}
delft/eris/status-page/status.js deleted
-239
@@ -1,239 +0,0 @@
1 -function init() {
2 - return new Promise(resolve => {
3 - window.onload = () => {
4 - var tbody = document.getElementById("channel-status");
5 - tbody.innerHTML = "<tr><td class='jsfallback' colspan='5'>Loading data from Prometheus...</td></tr>";
6 - resolve();
7 - };
8 - });
9 -}
10 -
11 -async function fetchIssues(label) {
12 - const response = await fetch(`https://api.github.com/repos/NixOS/nixpkgs/issues?labels=${label}`);
13 - return await response.json();
14 -}
15 -
16 -fetchIssues("1.severity%3A%20channel%20blocker")
17 - .then(data => data.map(issue => {
18 - var el = document.createElement('div');
19 - el.classList = "alert alert-warning";
20 - el.innerHTML = '<span class="issue-age"></span> <a class="issue-link"></a>';
21 - const since = moment(issue['created_at']).fromNow();
22 - el.getElementsByClassName('issue-age')[0].innerText = since;
23 - el.getElementsByClassName('issue-link')[0].href = issue['html_url'];
24 - el.getElementsByClassName('issue-link')[0].innerText = issue['title'];
25 - if (issue['labels'].find(label => label['name'] == 'infrastructure')) {
26 - el.innerHTML += ' <span class="label label-important">Infrastructure</span>'
27 - }
28 - return el;
29 - }))
30 - .then(elems => {
31 - var alerts = document.getElementById('alerts');
32 - elems.forEach(el => alerts.appendChild(el));
33 - });
34 -
35 -function aggregateByChannel(result) {
36 - return result.reduce((acc, {
37 - channel,
38 - value
39 - }) => ({
40 - ...acc,
41 - [channel]: value
42 - }), {});
43 -}
44 -
45 -async function fetchMetrics(queryType, queryArgs = {}) {
46 - const params = new URLSearchParams();
47 - for (const [key, value] of Object.entries(queryArgs)) {
48 - params.set(key, String(value));
49 - }
50 -
51 - const response = await fetch(`https://status.nixos.org/prometheus/api/v1/${queryType}?${params}`);
52 - const {
53 - data
54 - } = await response.json();
55 -
56 - return data.result;
57 -}
58 -
59 -const revisionData = fetchMetrics('query', {
60 - query: 'channel_revision'
61 - })
62 - .then(records => (
63 - records.map(({
64 - metric
65 - }) => ({
66 - channel: metric.channel,
67 - value: {
68 - revision: metric.revision,
69 - short_revision: metric.revision.substring(0, 12),
70 - github_url: `https://github.com/NixOS/nixpkgs/commit/${metric.revision}`,
71 - },
72 - }))
73 - ))
74 - .then(aggregateByChannel);
75 -
76 -
77 -const updateTimeData = fetchMetrics('query', {
78 - query: 'channel_update_time'
79 - })
80 - .then(records => (
81 - records.map(({
82 - metric,
83 - value
84 - }) => ({
85 - channel: metric.channel,
86 - value: {
87 - update_time: value[1],
88 - },
89 - }))
90 - ))
91 - .then(aggregateByChannel);
92 -
93 -var earliestStart = moment.utc("2019-12-30T01:00:00Z");
94 -var idealStart = moment.utc().subtract(30, "days");
95 -var start;
96 -if (idealStart > earliestStart) {
97 - start = idealStart;
98 -} else {
99 - start = earliestStart;
100 -}
101 -var end = moment.utc().format();
102 -const jobsetData = fetchMetrics('query_range', {
103 - query: 'hydra_job_failed',
104 - start: start.format(),
105 - end,
106 - step: '1h'
107 - })
108 - .then(records => (
109 - records.map(({
110 - metric,
111 - values
112 - }) => {
113 - const project = metric.project;
114 - const jobset = metric.jobset;
115 - const job = metric.exported_job;
116 -
117 - return {
118 - channel: metric.channel,
119 - value: {
120 - current: metric.current == 1,
121 - project,
122 - jobset,
123 - job,
124 - job_history: values.map(state => state[1] == 0),
125 - oldest_status: values[0][0],
126 - hydra_url: `https://hydra.nixos.org/job/${project}/${jobset}/${job}`,
127 - },
128 - };
129 - })
130 - ))
131 - .then(aggregateByChannel);
132 -
133 -function split_channel(channel) {
134 - var parts = channel.split("-");
135 - return {
136 - "time": parts[1],
137 - "collection": parts[0],
138 - "qualifier": parts[2] || "",
139 - }
140 -}
141 -
142 -function normalize_channel(channel) {
143 - parts = split_channel(channel);
144 - return [parts['time'], parts['collection'], parts['qualifier']].join("-");
145 -}
146 -
147 -function cmp_channels(left, right) {
148 - return normalize_channel(left) < normalize_channel(right)
149 -}
150 -
151 -init()
152 - .then(() => Promise.all([revisionData, updateTimeData, jobsetData]))
153 - .then(([revisions, update_times, jobsets]) => {
154 - var combined = [];
155 -
156 - for (let [channel, jobset] of Object.entries(jobsets)) {
157 - jobset['oldest_status_relative'] = moment.unix(jobset['oldest_status']).fromNow()
158 - // Ensure each jobset here is in each other dataset, guaranteeing we have
159 - // complete data.
160 - jobset['channel'] = channel;
161 - if (revisions[channel] != undefined) {
162 - jobset['revision'] = revisions[channel]['revision'];
163 - jobset['short_revision'] = revisions[channel]['short_revision'];
164 - jobset['github_url'] = revisions[channel]['github_url'];
165 - } else {
166 - continue
167 - }
168 - if (update_times[channel] != undefined) {
169 - var m = moment.unix(update_times[channel]['update_time']);
170 - jobset['update_time_relative'] = m.fromNow()
171 - jobset['update_time_local'] = m.format()
172 - // do not use color indications on outdated channels
173 - if (jobset['current']) {
174 - if (m > moment().subtract(3, 'days')) {
175 - jobset['update_age'] = "success";
176 - } else if (m > moment().subtract(10, 'days')) {
177 - jobset['update_age'] = "warning";
178 - } else {
179 - jobset['update_age'] = "important";
180 - }
181 - }
182 - } else {
183 - continue
184 - }
185 -
186 - combined.push(jobset);
187 - }
188 -
189 - combined.sort((left, right) => cmp_channels(left['channel'], right['channel']));
190 -
191 - return combined;
192 - })
193 - .then(data => {
194 - return data.map(record => {
195 - var row = document.createElement('tr');
196 - row.innerHTML = '<td class="channel" /><td><span class="age label"></span></td><td class="github"><a class="revision" /></td><td class="hydra"><a class="hydra-link" /></td><td class="status"></td>';
197 - var status = row.getElementsByClassName("status")[0];
198 -
199 - if (record['current']) {
200 - row.classList.add("current")
201 - } else {
202 - row.classList.add("stale")
203 - status.innerHTML += '<span class="label label-important">End of life</span>';
204 - }
205 - row.getElementsByClassName("channel")[0].innerText = record['channel'];
206 -
207 - var date = row.getElementsByClassName("age")[0];
208 - date.innerText = record['update_time_relative'];
209 - date.title = record['update_time_local'];
210 - date.classList.add("label-" + record['update_age']);
211 -
212 - var revisions = row.getElementsByClassName("revision")[0];
213 - revisions.innerText = record['short_revision'];
214 - revisions.href = record['github_url'];
215 -
216 - var hydraLink = row.getElementsByClassName("hydra-link")[0];
217 - hydraLink.href = record['hydra_url'];
218 - hydraLink.innerText = [record['project'], record['jobset'], record['job']].join('/');
219 -
220 - var hydra = row.getElementsByClassName("hydra")[0];
221 - hydra.style.backgroundImage = "linear-gradient(to right, " +
222 - (record['job_history'].map(val => val ? "#b5ffb5" : "#ff9e9e")).join(", ") +
223 - ")";
224 - hydra.title = `The Hydra job's state over time, since ${record['oldest_status_relative']}`;
225 -
226 - if (record['job_history'][record['job_history'].length - 1] == 0) {
227 - status.innerHTML += '<span class="label label-important">Build problem</span>';
228 -
229 - }
230 - return row;
231 - })
232 - })
233 - .then(rows => {
234 - var tbody = document.getElementById("channel-status");
235 - tbody.innerHTML = "";
236 - rows.forEach(row => {
237 - tbody.appendChild(row);
238 - });
239 - });
ngi0/hydra/hydra.nix
+2 -2
@@ -29,7 +29,7 @@ in
29 # patchelf:master:3
30 xxx-jobset-repeats = nixos:reproducibility:1
31
32 - # https://status.nixos.org/prometheus/graph?g0.range_input=2w&g0.expr=hydra_memory_tokens_in_use&g0.tab=0
32 + # https://prometheus.nixos.org/graph?g0.range_input=2w&g0.expr=hydra_memory_tokens_in_use&g0.tab=0
33 nar_buffer_size = ${let gb = 8; in toString (gb * 1024 * 1024 * 1024)}
34
35 #upload_logs_to_binary_cache = true
@@ -44,4 +44,4 @@ in
44
45 max_concurrent_evals = 2
46 '';
47 -}
\ No newline at end of file
47 +}