@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 416 lines 19.6 KB
Raw
1 import * as Alchemy from "alchemy";
2 import * as Axiom from "@alchemy.run/axiom";
3 import * as Output from "alchemy/Output";
4 import * as Config from "effect/Config";
5 import * as Effect from "effect/Effect";
6 import * as Option from "effect/Option";
7
8 import { relayResourceNameForStage } from "./deploymentConfig.ts";
9
10 const RUNBOOK_URL =
11 "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#alerts";
12
13 const LINK_ROUTE = `"/v1/client/environment-links"`;
14 const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`;
15 // The Cloudflare SDK client spans record the host but not the full URL.
16 const CLOUDFLARE_API = `['attributes.server.address'] == "api.cloudflare.com"`;
17 const STATUS = `toint(['attributes.http.response.status_code'])`;
18 // The tunnel reaper records its counters on this span every five minutes,
19 // including while cleanup is off.
20 const SWEEP = `name == "relay.managed_endpoint_reaper.sweep"`;
21 const sweepCounter = (field: string) =>
22 `toint(['attributes.custom']['relay.managed_endpoint_reaper.${field}'])`;
23
24 // Status codes, not span errors: a request cut off by the relay deadline ends
25 // as an interrupted span with no error flag, but still answers 504.
26 // Below `minimum` requests the rate reads 0, so a quiet window cannot page.
27 const serverFailurePercent = (dataset: string, routes: string, minimum: number) =>
28 [
29 `['${dataset}']`,
30 `| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`,
31 `| summarize total = count(), failed = countif(${STATUS} >= 500)`,
32 `| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`,
33 ].join("\n");
34
35 type Severity = "page" | "warn";
36
37 interface RelayMonitor {
38 readonly id: string;
39 readonly severity: Severity;
40 readonly name: string;
41 /** Why it fires and the healthy baseline. The runbook link is appended. */
42 readonly description: string;
43 readonly query: (dataset: string) => string;
44 readonly operator: "Above" | "AboveOrEqual" | "Below";
45 readonly threshold: number;
46 readonly rangeMinutes: number;
47 readonly intervalMinutes?: number;
48 readonly triggerFromNRuns?: number;
49 readonly alertOnNoData?: boolean;
50 }
51
52 // Thresholds come from 2026-10-16..19 prod data: the worst healthy window,
53 // and the outage (Cloudflare API 429s stalling every provision).
54 // Paging monitors need two bad runs in a row; warnings catch the slow
55 // degradations an outage threshold has to ignore.
56 const relayMonitors: ReadonlyArray<RelayMonitor> = [
57 {
58 id: "RelayLinkFailureRateMonitor",
59 severity: "page",
60 name: "T3 Connect relay: links failing",
61 description:
62 "Share of POST /v1/client/environment-links (turning on T3 Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.",
63 query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20),
64 operator: "Above",
65 threshold: 80,
66 rangeMinutes: 15,
67 triggerFromNRuns: 2,
68 },
69 {
70 id: "RelayTunnelFailureRateMonitor",
71 severity: "page",
72 name: "T3 Connect relay: tunnel recovery failing",
73 description:
74 "Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.",
75 query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200),
76 operator: "Above",
77 threshold: 60,
78 rangeMinutes: 15,
79 triggerFromNRuns: 2,
80 },
81 {
82 id: "RelayConnectReachabilityMonitor",
83 severity: "page",
84 name: "T3 Connect relay: environments unreachable",
85 description:
86 "Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.",
87 query: (dataset) =>
88 [
89 `['${dataset}']`,
90 `| where name == "relay.environment_connector.connect"`,
91 `| extend environment = tostring(['attributes.custom']['relay.environment_id'])`,
92 `| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`,
93 `| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`,
94 ].join("\n"),
95 operator: "Below",
96 threshold: 55,
97 rangeMinutes: 30,
98 triggerFromNRuns: 2,
99 },
100 {
101 id: "RelayPushFailureRateMonitor",
102 severity: "page",
103 name: "T3 Connect relay: push delivery failing",
104 description:
105 "Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.",
106 query: (dataset) =>
107 [
108 `['${dataset}']`,
109 `| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`,
110 `| summarize total = count(), failed = countif(error == true)`,
111 `| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`,
112 ].join("\n"),
113 operator: "Above",
114 threshold: 10,
115 rangeMinutes: 15,
116 triggerFromNRuns: 2,
117 },
118 {
119 id: "RelayCloudflareRateLimitedMonitor",
120 severity: "page",
121 name: "T3 Connect relay: Cloudflare API rate limited",
122 description:
123 "Cloudflare API 429s, plus calls the relay refused while paused after one, per 10 min. Sustained at this level, links and recoveries stall. Peak 429s before #16810: 1,286; during it: ~11k.",
124 query: (dataset) =>
125 `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
126 operator: "AboveOrEqual",
127 threshold: 500,
128 rangeMinutes: 10,
129 triggerFromNRuns: 2,
130 },
131 {
132 id: "RelayTrafficDroppedMonitor",
133 severity: "page",
134 name: "T3 Connect relay: traffic dropped",
135 description:
136 "Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.",
137 query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`,
138 operator: "Below",
139 threshold: 10_000,
140 rangeMinutes: 10,
141 triggerFromNRuns: 2,
142 alertOnNoData: true,
143 },
144 {
145 id: "RelayCloudflareRateLimitWarningMonitor",
146 severity: "warn",
147 name: "T3 Connect relay: Cloudflare API near its limit",
148 description:
149 "Any sustained Cloudflare API 429s or paused calls. 429 bursts ran for days before #16810; raise the account limit or cut call volume before they grow.",
150 query: (dataset) =>
151 `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
152 operator: "AboveOrEqual",
153 threshold: 20,
154 rangeMinutes: 10,
155 triggerFromNRuns: 2,
156 },
157 {
158 id: "RelayLinkChronicFailureMonitor",
159 severity: "warn",
160 name: "T3 Connect relay: links degraded",
161 description:
162 "Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.",
163 query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100),
164 operator: "Above",
165 threshold: 20,
166 rangeMinutes: 360,
167 intervalMinutes: 30,
168 },
169 {
170 id: "RelayDatabaseLatencyMonitor",
171 severity: "warn",
172 name: "T3 Connect relay: database slow",
173 description:
174 "p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.",
175 query: (dataset) =>
176 `['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`,
177 operator: "Above",
178 threshold: 500,
179 rangeMinutes: 15,
180 triggerFromNRuns: 2,
181 },
182 {
183 id: "RelayTunnelCleanupStoppedMonitor",
184 // Pages because nothing else surfaces it: unnoticed, it ran 2026-11-15..18 while 27k+ idle tunnels piled up.
185 severity: "page",
186 name: "T3 Connect relay: tunnel cleanup stopped",
187 description:
188 "Tunnel cleanup sweeps that finished in 2 h. Healthy: 24. The cron is not running, or every sweep fails, as during the 2026-10-19 Cloudflare 429s and from 2026-11-15 when listing tunnels timed out. Idle tunnels pile up and are billed meanwhile.",
189 query: (dataset) =>
190 `['${dataset}']\n| where ${SWEEP} and error != true\n| summarize sweeps = count()`,
191 operator: "Below",
192 threshold: 1,
193 rangeMinutes: 120,
194 intervalMinutes: 15,
195 triggerFromNRuns: 2,
196 alertOnNoData: true,
197 },
198 {
199 id: "RelayTunnelCleanupFailingMonitor",
200 severity: "warn",
201 name: "T3 Connect relay: tunnel deletions failing",
202 description:
203 "Cloudflare tunnel deletions that failed in 30 min. Healthy: 0; the worst 30 min so far had 3. The relay logs a warning with the Cloudflare error for each.",
204 query: (dataset) =>
205 `['${dataset}']\n| where ${SWEEP}\n| summarize failed = sum(${sweepCounter("failed")})`,
206 operator: "Above",
207 threshold: 10,
208 rangeMinutes: 30,
209 triggerFromNRuns: 2,
210 },
211 ];
212
213 interface QueryChart {
214 readonly id: string;
215 readonly name: string;
216 readonly type: "TimeSeries" | "Table";
217 readonly query: { readonly apl: string };
218 }
219
220 const timeSeries = (id: string, name: string, apl: string): QueryChart => ({
221 id,
222 name,
223 type: "TimeSeries",
224 query: { apl },
225 });
226
227 const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
228 const source = `['${dataset}']`;
229 const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`;
230 const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`;
231 const clientConnections = `${source}\n| where name == "relay.connection.attempt" and error != true\n| extend surface = tostring(['resource.custom']['t3.client.surface'])\n| where isnotempty(surface)`;
232 return [
233 timeSeries(
234 "traffic",
235 "Requests and agent activity publishes",
236 `${source}\n| where kind == "server" or name == "relay.agent_activity_publisher.publish"\n| summarize requests = countif(kind == "server"), publishes = countif(name == "relay.agent_activity_publisher.publish") by bin_auto(_time)`,
237 ),
238 timeSeries(
239 "links",
240 "Links by outcome",
241 `${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
242 ),
243 timeSeries(
244 "link-latency",
245 "Link latency (9 s deadline)",
246 `${linkRequests}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
247 ),
248 timeSeries(
249 "reachability",
250 "Environments reachable per 30 min (%)",
251 `${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`,
252 ),
253 timeSeries(
254 "tunnel",
255 "Tunnel registration and recovery by outcome",
256 `${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
257 ),
258 timeSeries(
259 "cloudflare-api",
260 "Cloudflare API calls by status",
261 // Calls refused during a rate-limit pause never reach Cloudflare, so
262 // they are their own series or a storm would look like a quiet period.
263 `${source}\n| where ${CLOUDFLARE_API} or name == "relay.cloudflare_api_pause.rejected"\n| summarize count() by bin_auto(_time), status = case(name == "relay.cloudflare_api_pause.rejected", "paused", isnull(${STATUS}), "no response", tostring(${STATUS}))`,
264 ),
265 timeSeries(
266 "provision-paths",
267 "Tunnel provisions by path",
268 `${source}\n| where name == "relay.managed_endpoint_provider.provision"\n| summarize reused_ready_tunnel = countif(error != true and tobool(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel']) == true), full_provision = countif(error != true and isnull(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel'])), failed = countif(error == true) by bin_auto(_time)`,
269 ),
270 timeSeries(
271 "postgres",
272 "Postgres query time",
273 `${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
274 ),
275 timeSeries(
276 "push",
277 "Push delivery jobs",
278 `${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`,
279 ),
280 timeSeries(
281 "tunnel-cleanup",
282 "Tunnel cleanup per sweep",
283 `${source}\n| where ${SWEEP}\n| summarize deleted = sum(${sweepCounter("deleted")}), deleted_legacy = sum(${sweepCounter("deletedLegacy")}), would_delete = sum(${sweepCounter("wouldDelete")}), would_delete_legacy = sum(${sweepCounter("wouldDeleteLegacy")}), failed = sum(${sweepCounter("failed")}), sweeps_failed = countif(error == true) by bin_auto(_time)`,
284 ),
285 // Cloudflare's own counts for this stage, including tunnels the reaper
286 // skips, so this is the idle tunnels still billed.
287 timeSeries(
288 "tunnel-backlog",
289 "Idle tunnels in Cloudflare",
290 `${source}\n| where ${SWEEP}\n| summarize down = max(${sweepCounter("totalDown")}), never_connected = max(${sweepCounter("totalInactive")}) by bin_auto(_time)`,
291 ),
292 timeSeries(
293 "server-errors",
294 "5xx by route",
295 `${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`,
296 ),
297 {
298 id: "link-failures",
299 name: "Link failures by reason",
300 type: "Table",
301 query: {
302 apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`,
303 },
304 },
305 // Connecting, as users see it: web, desktop and mobile export their
306 // connection spans into this dataset, in the same traces as the relay's
307 // and the host's spans. An attempt lasts until initial sync finishes.
308 timeSeries(
309 "connect-time",
310 "Connecting: time to connected by surface",
311 `${clientConnections}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time), surface`,
312 ),
313 timeSeries(
314 "connect-outcomes",
315 "Connecting: attempts by outcome",
316 `${source}\n| where name == "relay.connection.attempt" and isnotempty(['resource.custom']['t3.client.surface'])\n| summarize connected = countif(error != true), failed = countif(error == true) by bin_auto(_time)`,
317 ),
318 {
319 id: "connect-phases",
320 name: "Connecting: where the time goes",
321 type: "Table",
322 query: {
323 // Failed direct route checks stay in: waiting out an unreachable LAN
324 // address before falling back to the relay is the slowest step.
325 apl: [
326 source,
327 `| where isnotempty(['resource.custom']['t3.client.surface'])`,
328 `| extend url = tostring(['attributes.url.full'])`,
329 `| extend phase = case(`,
330 ` name == "ConnectionDriver.checkRoute", "0. Direct route check, before any relay fallback",`,
331 ` name == "relay.connection.cloudSessionToken.resolve", "1. Clerk session token",`,
332 ` name == "clientRuntime.managedRelay.exchangeAccessToken", "2. Relay token exchange",`,
333 ` kind == "client" and url startswith "https://relay.t3.codes/v1/environments/" and url endswith "/connect", "3. Relay connect, minting a host credential",`,
334 ` name == "environment.authorization.descriptor", "4. Host descriptor",`,
335 ` name == "environment.authorization.accessToken.exchange", "5. Host token exchange",`,
336 ` name == "clientRuntime.connection.remote.createDpopSocketUrl", "6. Socket ticket",`,
337 ` name == "environment.initialSync", "7. Initial sync",`,
338 ` name == "relay.connection.attempt", "Total: time to connected",`,
339 ` "")`,
340 `| where phase != "" and (error != true or phase startswith "0.")`,
341 `| summarize calls = count(), p50 = percentile(duration, 50), p90 = percentile(duration, 90) by phase`,
342 `| order by phase asc`,
343 ].join("\n"),
344 },
345 },
346 timeSeries(
347 "relay-latency-by-continent",
348 "Relay token and connect latency by user continent (p50)",
349 `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and (['attributes.http.route'] == "/v1/client/dpop-token" or ['attributes.http.route'] == "/v1/environments/:environmentId/connect")\n| extend continent = tostring(['attributes.custom']['relay.edge.continent'])\n| where isnotempty(continent)\n| summarize p50 = percentile(duration, 50) by bin_auto(_time), continent`,
350 ),
351 ];
352 };
353
354 export const notifierConfig = (name: string) =>
355 Config.String(name).pipe(
356 Config.option,
357 Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
358 );
359
360 /**
361 * A deployment opts in by naming the notifier that pages (incident.io for
362 * prod). Personal stages and forks leave it unset and get no monitors or
363 * dashboard. Warnings go to their own notifier when one is set, and are only
364 * visible in Axiom otherwise.
365 */
366 export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) {
367 const { stage } = yield* Alchemy.Stack;
368 const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
369 if (Option.isNone(page)) return;
370 const notifiers: Record<Severity, Option.Option<string>> = {
371 page,
372 warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
373 };
374
375 for (const monitor of relayMonitors) {
376 yield* Axiom.Monitor(monitor.id, {
377 name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
378 description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
379 type: "Threshold",
380 aplQuery: Output.map(traces.name, monitor.query),
381 operator: monitor.operator,
382 threshold: monitor.threshold,
383 intervalMinutes: monitor.intervalMinutes ?? 5,
384 rangeMinutes: monitor.rangeMinutes,
385 triggerFromNRuns: monitor.triggerFromNRuns ?? 1,
386 alertOnNoData: monitor.alertOnNoData ?? false,
387 notifierIds: Option.toArray(notifiers[monitor.severity]),
388 });
389 }
390
391 yield* Axiom.Dashboard("RelayHealthDashboard", {
392 dashboard: Output.map(traces.name, (dataset) => {
393 const charts = relayDashboardCharts(dataset);
394 return {
395 name: relayResourceNameForStage("t3-code-relay-health", stage),
396 // Shared with the whole org. An empty owner only means that for API
397 // tokens; deploys with a personal token would make it private.
398 owner: "X-AXIOM-EVERYONE",
399 description: `T3 Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`,
400 refreshTime: 60,
401 schemaVersion: 2,
402 timeWindowStart: "qr-now-24h",
403 timeWindowEnd: "qr-now",
404 charts: [...charts],
405 // Two charts per row on Axiom's 12-column grid.
406 layout: charts.map((chart, index) => ({
407 i: chart.id,
408 x: (index % 2) * 6,
409 y: Math.floor(index / 2) * 6,
410 w: 6,
411 h: 6,
412 })),
413 };
414 }),
415 });
416 });