The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import * as Alchemy from "alchemy"; |
| 2 | import * as Axiom from "@alchemy.run/axiom"; |
| 3 | import * as Output from "alchemy/Output"; |
| 4 | import * as Config from "effect/Config"; |
| 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as Option from "effect/Option"; |
| 7 | |
| 8 | import { relayResourceNameForStage } from "./deploymentConfig.ts"; |
| 9 | |
| 10 | const RUNBOOK_URL = |
| 11 | "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#alerts"; |
| 12 | |
| 13 | const LINK_ROUTE = `"/v1/client/environment-links"`; |
| 14 | const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`; |
| 15 | // The Cloudflare SDK client spans record the host but not the full URL. |
| 16 | const CLOUDFLARE_API = `['attributes.server.address'] == "api.cloudflare.com"`; |
| 17 | const STATUS = `toint(['attributes.http.response.status_code'])`; |
| 18 | // The tunnel reaper records its counters on this span every five minutes, |
| 19 | // including while cleanup is off. |
| 20 | const SWEEP = `name == "relay.managed_endpoint_reaper.sweep"`; |
| 21 | const sweepCounter = (field: string) => |
| 22 | `toint(['attributes.custom']['relay.managed_endpoint_reaper.${field}'])`; |
| 23 | |
| 24 | // Status codes, not span errors: a request cut off by the relay deadline ends |
| 25 | // as an interrupted span with no error flag, but still answers 504. |
| 26 | // Below `minimum` requests the rate reads 0, so a quiet window cannot page. |
| 27 | const serverFailurePercent = (dataset: string, routes: string, minimum: number) => |
| 28 | [ |
| 29 | `['${dataset}']`, |
| 30 | `| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`, |
| 31 | `| summarize total = count(), failed = countif(${STATUS} >= 500)`, |
| 32 | `| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`, |
| 33 | ].join("\n"); |
| 34 | |
| 35 | type Severity = "page" | "warn"; |
| 36 | |
| 37 | interface RelayMonitor { |
| 38 | readonly id: string; |
| 39 | readonly severity: Severity; |
| 40 | readonly name: string; |
| 41 | /** Why it fires and the healthy baseline. The runbook link is appended. */ |
| 42 | readonly description: string; |
| 43 | readonly query: (dataset: string) => string; |
| 44 | readonly operator: "Above" | "AboveOrEqual" | "Below"; |
| 45 | readonly threshold: number; |
| 46 | readonly rangeMinutes: number; |
| 47 | readonly intervalMinutes?: number; |
| 48 | readonly triggerFromNRuns?: number; |
| 49 | readonly alertOnNoData?: boolean; |
| 50 | } |
| 51 | |
| 52 | // Thresholds come from 2026-10-16..19 prod data: the worst healthy window, |
| 53 | // and the outage (Cloudflare API 429s stalling every provision). |
| 54 | // Paging monitors need two bad runs in a row; warnings catch the slow |
| 55 | // degradations an outage threshold has to ignore. |
| 56 | const relayMonitors: ReadonlyArray<RelayMonitor> = [ |
| 57 | { |
| 58 | id: "RelayLinkFailureRateMonitor", |
| 59 | severity: "page", |
| 60 | name: "T3 Connect relay: links failing", |
| 61 | description: |
| 62 | "Share of POST /v1/client/environment-links (turning on T3 Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.", |
| 63 | query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20), |
| 64 | operator: "Above", |
| 65 | threshold: 80, |
| 66 | rangeMinutes: 15, |
| 67 | triggerFromNRuns: 2, |
| 68 | }, |
| 69 | { |
| 70 | id: "RelayTunnelFailureRateMonitor", |
| 71 | severity: "page", |
| 72 | name: "T3 Connect relay: tunnel recovery failing", |
| 73 | description: |
| 74 | "Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.", |
| 75 | query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200), |
| 76 | operator: "Above", |
| 77 | threshold: 60, |
| 78 | rangeMinutes: 15, |
| 79 | triggerFromNRuns: 2, |
| 80 | }, |
| 81 | { |
| 82 | id: "RelayConnectReachabilityMonitor", |
| 83 | severity: "page", |
| 84 | name: "T3 Connect relay: environments unreachable", |
| 85 | description: |
| 86 | "Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.", |
| 87 | query: (dataset) => |
| 88 | [ |
| 89 | `['${dataset}']`, |
| 90 | `| where name == "relay.environment_connector.connect"`, |
| 91 | `| extend environment = tostring(['attributes.custom']['relay.environment_id'])`, |
| 92 | `| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`, |
| 93 | `| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`, |
| 94 | ].join("\n"), |
| 95 | operator: "Below", |
| 96 | threshold: 55, |
| 97 | rangeMinutes: 30, |
| 98 | triggerFromNRuns: 2, |
| 99 | }, |
| 100 | { |
| 101 | id: "RelayPushFailureRateMonitor", |
| 102 | severity: "page", |
| 103 | name: "T3 Connect relay: push delivery failing", |
| 104 | description: |
| 105 | "Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.", |
| 106 | query: (dataset) => |
| 107 | [ |
| 108 | `['${dataset}']`, |
| 109 | `| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`, |
| 110 | `| summarize total = count(), failed = countif(error == true)`, |
| 111 | `| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`, |
| 112 | ].join("\n"), |
| 113 | operator: "Above", |
| 114 | threshold: 10, |
| 115 | rangeMinutes: 15, |
| 116 | triggerFromNRuns: 2, |
| 117 | }, |
| 118 | { |
| 119 | id: "RelayCloudflareRateLimitedMonitor", |
| 120 | severity: "page", |
| 121 | name: "T3 Connect relay: Cloudflare API rate limited", |
| 122 | description: |
| 123 | "Cloudflare API 429s, plus calls the relay refused while paused after one, per 10 min. Sustained at this level, links and recoveries stall. Peak 429s before #16810: 1,286; during it: ~11k.", |
| 124 | query: (dataset) => |
| 125 | `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`, |
| 126 | operator: "AboveOrEqual", |
| 127 | threshold: 500, |
| 128 | rangeMinutes: 10, |
| 129 | triggerFromNRuns: 2, |
| 130 | }, |
| 131 | { |
| 132 | id: "RelayTrafficDroppedMonitor", |
| 133 | severity: "page", |
| 134 | name: "T3 Connect relay: traffic dropped", |
| 135 | description: |
| 136 | "Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.", |
| 137 | query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`, |
| 138 | operator: "Below", |
| 139 | threshold: 10_000, |
| 140 | rangeMinutes: 10, |
| 141 | triggerFromNRuns: 2, |
| 142 | alertOnNoData: true, |
| 143 | }, |
| 144 | { |
| 145 | id: "RelayCloudflareRateLimitWarningMonitor", |
| 146 | severity: "warn", |
| 147 | name: "T3 Connect relay: Cloudflare API near its limit", |
| 148 | description: |
| 149 | "Any sustained Cloudflare API 429s or paused calls. 429 bursts ran for days before #16810; raise the account limit or cut call volume before they grow.", |
| 150 | query: (dataset) => |
| 151 | `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`, |
| 152 | operator: "AboveOrEqual", |
| 153 | threshold: 20, |
| 154 | rangeMinutes: 10, |
| 155 | triggerFromNRuns: 2, |
| 156 | }, |
| 157 | { |
| 158 | id: "RelayLinkChronicFailureMonitor", |
| 159 | severity: "warn", |
| 160 | name: "T3 Connect relay: links degraded", |
| 161 | description: |
| 162 | "Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.", |
| 163 | query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100), |
| 164 | operator: "Above", |
| 165 | threshold: 20, |
| 166 | rangeMinutes: 360, |
| 167 | intervalMinutes: 30, |
| 168 | }, |
| 169 | { |
| 170 | id: "RelayDatabaseLatencyMonitor", |
| 171 | severity: "warn", |
| 172 | name: "T3 Connect relay: database slow", |
| 173 | description: |
| 174 | "p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.", |
| 175 | query: (dataset) => |
| 176 | `['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`, |
| 177 | operator: "Above", |
| 178 | threshold: 500, |
| 179 | rangeMinutes: 15, |
| 180 | triggerFromNRuns: 2, |
| 181 | }, |
| 182 | { |
| 183 | id: "RelayTunnelCleanupStoppedMonitor", |
| 184 | // Pages because nothing else surfaces it: unnoticed, it ran 2026-11-15..18 while 27k+ idle tunnels piled up. |
| 185 | severity: "page", |
| 186 | name: "T3 Connect relay: tunnel cleanup stopped", |
| 187 | description: |
| 188 | "Tunnel cleanup sweeps that finished in 2 h. Healthy: 24. The cron is not running, or every sweep fails, as during the 2026-10-19 Cloudflare 429s and from 2026-11-15 when listing tunnels timed out. Idle tunnels pile up and are billed meanwhile.", |
| 189 | query: (dataset) => |
| 190 | `['${dataset}']\n| where ${SWEEP} and error != true\n| summarize sweeps = count()`, |
| 191 | operator: "Below", |
| 192 | threshold: 1, |
| 193 | rangeMinutes: 120, |
| 194 | intervalMinutes: 15, |
| 195 | triggerFromNRuns: 2, |
| 196 | alertOnNoData: true, |
| 197 | }, |
| 198 | { |
| 199 | id: "RelayTunnelCleanupFailingMonitor", |
| 200 | severity: "warn", |
| 201 | name: "T3 Connect relay: tunnel deletions failing", |
| 202 | description: |
| 203 | "Cloudflare tunnel deletions that failed in 30 min. Healthy: 0; the worst 30 min so far had 3. The relay logs a warning with the Cloudflare error for each.", |
| 204 | query: (dataset) => |
| 205 | `['${dataset}']\n| where ${SWEEP}\n| summarize failed = sum(${sweepCounter("failed")})`, |
| 206 | operator: "Above", |
| 207 | threshold: 10, |
| 208 | rangeMinutes: 30, |
| 209 | triggerFromNRuns: 2, |
| 210 | }, |
| 211 | ]; |
| 212 | |
| 213 | interface QueryChart { |
| 214 | readonly id: string; |
| 215 | readonly name: string; |
| 216 | readonly type: "TimeSeries" | "Table"; |
| 217 | readonly query: { readonly apl: string }; |
| 218 | } |
| 219 | |
| 220 | const timeSeries = (id: string, name: string, apl: string): QueryChart => ({ |
| 221 | id, |
| 222 | name, |
| 223 | type: "TimeSeries", |
| 224 | query: { apl }, |
| 225 | }); |
| 226 | |
| 227 | const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => { |
| 228 | const source = `['${dataset}']`; |
| 229 | const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`; |
| 230 | const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`; |
| 231 | const clientConnections = `${source}\n| where name == "relay.connection.attempt" and error != true\n| extend surface = tostring(['resource.custom']['t3.client.surface'])\n| where isnotempty(surface)`; |
| 232 | return [ |
| 233 | timeSeries( |
| 234 | "traffic", |
| 235 | "Requests and agent activity publishes", |
| 236 | `${source}\n| where kind == "server" or name == "relay.agent_activity_publisher.publish"\n| summarize requests = countif(kind == "server"), publishes = countif(name == "relay.agent_activity_publisher.publish") by bin_auto(_time)`, |
| 237 | ), |
| 238 | timeSeries( |
| 239 | "links", |
| 240 | "Links by outcome", |
| 241 | `${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`, |
| 242 | ), |
| 243 | timeSeries( |
| 244 | "link-latency", |
| 245 | "Link latency (9 s deadline)", |
| 246 | `${linkRequests}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`, |
| 247 | ), |
| 248 | timeSeries( |
| 249 | "reachability", |
| 250 | "Environments reachable per 30 min (%)", |
| 251 | `${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`, |
| 252 | ), |
| 253 | timeSeries( |
| 254 | "tunnel", |
| 255 | "Tunnel registration and recovery by outcome", |
| 256 | `${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`, |
| 257 | ), |
| 258 | timeSeries( |
| 259 | "cloudflare-api", |
| 260 | "Cloudflare API calls by status", |
| 261 | // Calls refused during a rate-limit pause never reach Cloudflare, so |
| 262 | // they are their own series or a storm would look like a quiet period. |
| 263 | `${source}\n| where ${CLOUDFLARE_API} or name == "relay.cloudflare_api_pause.rejected"\n| summarize count() by bin_auto(_time), status = case(name == "relay.cloudflare_api_pause.rejected", "paused", isnull(${STATUS}), "no response", tostring(${STATUS}))`, |
| 264 | ), |
| 265 | timeSeries( |
| 266 | "provision-paths", |
| 267 | "Tunnel provisions by path", |
| 268 | `${source}\n| where name == "relay.managed_endpoint_provider.provision"\n| summarize reused_ready_tunnel = countif(error != true and tobool(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel']) == true), full_provision = countif(error != true and isnull(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel'])), failed = countif(error == true) by bin_auto(_time)`, |
| 269 | ), |
| 270 | timeSeries( |
| 271 | "postgres", |
| 272 | "Postgres query time", |
| 273 | `${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`, |
| 274 | ), |
| 275 | timeSeries( |
| 276 | "push", |
| 277 | "Push delivery jobs", |
| 278 | `${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`, |
| 279 | ), |
| 280 | timeSeries( |
| 281 | "tunnel-cleanup", |
| 282 | "Tunnel cleanup per sweep", |
| 283 | `${source}\n| where ${SWEEP}\n| summarize deleted = sum(${sweepCounter("deleted")}), deleted_legacy = sum(${sweepCounter("deletedLegacy")}), would_delete = sum(${sweepCounter("wouldDelete")}), would_delete_legacy = sum(${sweepCounter("wouldDeleteLegacy")}), failed = sum(${sweepCounter("failed")}), sweeps_failed = countif(error == true) by bin_auto(_time)`, |
| 284 | ), |
| 285 | // Cloudflare's own counts for this stage, including tunnels the reaper |
| 286 | // skips, so this is the idle tunnels still billed. |
| 287 | timeSeries( |
| 288 | "tunnel-backlog", |
| 289 | "Idle tunnels in Cloudflare", |
| 290 | `${source}\n| where ${SWEEP}\n| summarize down = max(${sweepCounter("totalDown")}), never_connected = max(${sweepCounter("totalInactive")}) by bin_auto(_time)`, |
| 291 | ), |
| 292 | timeSeries( |
| 293 | "server-errors", |
| 294 | "5xx by route", |
| 295 | `${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`, |
| 296 | ), |
| 297 | { |
| 298 | id: "link-failures", |
| 299 | name: "Link failures by reason", |
| 300 | type: "Table", |
| 301 | query: { |
| 302 | apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`, |
| 303 | }, |
| 304 | }, |
| 305 | // Connecting, as users see it: web, desktop and mobile export their |
| 306 | // connection spans into this dataset, in the same traces as the relay's |
| 307 | // and the host's spans. An attempt lasts until initial sync finishes. |
| 308 | timeSeries( |
| 309 | "connect-time", |
| 310 | "Connecting: time to connected by surface", |
| 311 | `${clientConnections}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time), surface`, |
| 312 | ), |
| 313 | timeSeries( |
| 314 | "connect-outcomes", |
| 315 | "Connecting: attempts by outcome", |
| 316 | `${source}\n| where name == "relay.connection.attempt" and isnotempty(['resource.custom']['t3.client.surface'])\n| summarize connected = countif(error != true), failed = countif(error == true) by bin_auto(_time)`, |
| 317 | ), |
| 318 | { |
| 319 | id: "connect-phases", |
| 320 | name: "Connecting: where the time goes", |
| 321 | type: "Table", |
| 322 | query: { |
| 323 | // Failed direct route checks stay in: waiting out an unreachable LAN |
| 324 | // address before falling back to the relay is the slowest step. |
| 325 | apl: [ |
| 326 | source, |
| 327 | `| where isnotempty(['resource.custom']['t3.client.surface'])`, |
| 328 | `| extend url = tostring(['attributes.url.full'])`, |
| 329 | `| extend phase = case(`, |
| 330 | ` name == "ConnectionDriver.checkRoute", "0. Direct route check, before any relay fallback",`, |
| 331 | ` name == "relay.connection.cloudSessionToken.resolve", "1. Clerk session token",`, |
| 332 | ` name == "clientRuntime.managedRelay.exchangeAccessToken", "2. Relay token exchange",`, |
| 333 | ` kind == "client" and url startswith "https://relay.t3.codes/v1/environments/" and url endswith "/connect", "3. Relay connect, minting a host credential",`, |
| 334 | ` name == "environment.authorization.descriptor", "4. Host descriptor",`, |
| 335 | ` name == "environment.authorization.accessToken.exchange", "5. Host token exchange",`, |
| 336 | ` name == "clientRuntime.connection.remote.createDpopSocketUrl", "6. Socket ticket",`, |
| 337 | ` name == "environment.initialSync", "7. Initial sync",`, |
| 338 | ` name == "relay.connection.attempt", "Total: time to connected",`, |
| 339 | ` "")`, |
| 340 | `| where phase != "" and (error != true or phase startswith "0.")`, |
| 341 | `| summarize calls = count(), p50 = percentile(duration, 50), p90 = percentile(duration, 90) by phase`, |
| 342 | `| order by phase asc`, |
| 343 | ].join("\n"), |
| 344 | }, |
| 345 | }, |
| 346 | timeSeries( |
| 347 | "relay-latency-by-continent", |
| 348 | "Relay token and connect latency by user continent (p50)", |
| 349 | `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and (['attributes.http.route'] == "/v1/client/dpop-token" or ['attributes.http.route'] == "/v1/environments/:environmentId/connect")\n| extend continent = tostring(['attributes.custom']['relay.edge.continent'])\n| where isnotempty(continent)\n| summarize p50 = percentile(duration, 50) by bin_auto(_time), continent`, |
| 350 | ), |
| 351 | ]; |
| 352 | }; |
| 353 | |
| 354 | export const notifierConfig = (name: string) => |
| 355 | Config.String(name).pipe( |
| 356 | Config.option, |
| 357 | Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))), |
| 358 | ); |
| 359 | |
| 360 | /** |
| 361 | * A deployment opts in by naming the notifier that pages (incident.io for |
| 362 | * prod). Personal stages and forks leave it unset and get no monitors or |
| 363 | * dashboard. Warnings go to their own notifier when one is set, and are only |
| 364 | * visible in Axiom otherwise. |
| 365 | */ |
| 366 | export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) { |
| 367 | const { stage } = yield* Alchemy.Stack; |
| 368 | const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID"); |
| 369 | if (Option.isNone(page)) return; |
| 370 | const notifiers: Record<Severity, Option.Option<string>> = { |
| 371 | page, |
| 372 | warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"), |
| 373 | }; |
| 374 | |
| 375 | for (const monitor of relayMonitors) { |
| 376 | yield* Axiom.Monitor(monitor.id, { |
| 377 | name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`, |
| 378 | description: `${monitor.description} Runbook: ${RUNBOOK_URL}`, |
| 379 | type: "Threshold", |
| 380 | aplQuery: Output.map(traces.name, monitor.query), |
| 381 | operator: monitor.operator, |
| 382 | threshold: monitor.threshold, |
| 383 | intervalMinutes: monitor.intervalMinutes ?? 5, |
| 384 | rangeMinutes: monitor.rangeMinutes, |
| 385 | triggerFromNRuns: monitor.triggerFromNRuns ?? 1, |
| 386 | alertOnNoData: monitor.alertOnNoData ?? false, |
| 387 | notifierIds: Option.toArray(notifiers[monitor.severity]), |
| 388 | }); |
| 389 | } |
| 390 | |
| 391 | yield* Axiom.Dashboard("RelayHealthDashboard", { |
| 392 | dashboard: Output.map(traces.name, (dataset) => { |
| 393 | const charts = relayDashboardCharts(dataset); |
| 394 | return { |
| 395 | name: relayResourceNameForStage("t3-code-relay-health", stage), |
| 396 | // Shared with the whole org. An empty owner only means that for API |
| 397 | // tokens; deploys with a personal token would make it private. |
| 398 | owner: "X-AXIOM-EVERYONE", |
| 399 | description: `T3 Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`, |
| 400 | refreshTime: 60, |
| 401 | schemaVersion: 2, |
| 402 | timeWindowStart: "qr-now-24h", |
| 403 | timeWindowEnd: "qr-now", |
| 404 | charts: [...charts], |
| 405 | // Two charts per row on Axiom's 12-column grid. |
| 406 | layout: charts.map((chart, index) => ({ |
| 407 | i: chart.id, |
| 408 | x: (index % 2) * 6, |
| 409 | y: Math.floor(index / 2) * 6, |
| 410 | w: 6, |
| 411 | h: 6, |
| 412 | })), |
| 413 | }; |
| 414 | }), |
| 415 | }); |
| 416 | }); |