@devries-cameron20 / t3code Mirror · read-only / commits / 570d4b03ee
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 14:30 UTC

feat(mobile): report crashes to Axiom with symbolicated stacks (#19093)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Gabriel De Andrade committed Oct 27, 2026 at 16:20 UTC 570d4b03eef879bd60be3d74444046858c94dacd
28 files changed +2418 -16
.github/workflows/mobile-eas-production.yml
+19
@@ -275,6 +275,7 @@ jobs:
275 --environment production \
276 --platform ${{ inputs.platform }} \
277 --message "$OTA_MESSAGE" \
278 + --emit-metadata \
279 --non-interactive
280
281 # Reconcile each version separately because the major is in the fingerprint.
@@ -313,8 +314,11 @@ jobs:
314 --environment production \
315 --platform "$platform" \
316 --message "$message" \
317 + --emit-metadata \
318 --non-interactive; then
319 echo ":white_check_mark: $platform/$profile: OTA published to production (fingerprint \`$hash\`)" >> "$GITHUB_STEP_SUMMARY"
320 + # The next target's export replaces dist; keep this one for the map upload.
321 + rm -rf "dist-$platform-$profile" && mv dist "dist-$platform-$profile"
322 else
323 failed=1
324 echo ":x: $platform/$profile: OTA publish failed (fingerprint \`$hash\`)" >> "$GITHUB_STEP_SUMMARY"
@@ -354,3 +358,18 @@ jobs:
358 fi
359 done
360 exit "$failed"
361 +
362 + # Lets the relay symbolicate crash stacks from the updates published
363 + # above. A step of its own so the R2 credentials reach only this script,
364 + # not `eas update` and the dependencies it runs. Never fails the job.
365 + - name: Upload OTA source maps
366 + if: ${{ !cancelled() && steps.expo-token.outputs.present == 'true' }}
367 + working-directory: apps/mobile
368 + env:
369 + MOBILE_SOURCEMAPS_R2: ${{ secrets.MOBILE_SOURCEMAPS_R2 }}
370 + run: |
371 + for dir in dist dist-*; do
372 + if [ -f "$dir/eas-update-metadata.json" ]; then
373 + node scripts/upload-sourcemaps.mts ota "$dir"
374 + fi
375 + done
apps/mobile/app.config.ts
+7
@@ -17,6 +17,12 @@ const runtimeVersionPolicy =
17 const personalTeamBundleIdentifier = repoEnv.T3CODE_IOS_PERSONAL_TEAM_BUNDLE_ID?.trim();
18 const IOS_BUNDLE_IDENTIFIER_PATTERN = /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/;
19
20 +// Compiled into the native project. Without a relay, Observe's native side could dispatch to
21 +// Expo's own ingest before observe.ts turns it off; `.invalid` never resolves (RFC 6761).
22 +const observeEndpointUrl = repoEnv.T3CODE_RELAY_URL?.trim()
23 + ? new URL("/v1/mobile/observe", repoEnv.T3CODE_RELAY_URL.trim()).toString()
24 + : "https://observe.invalid";
25 +
26 const fromRepoRoot = (relativePath: string) => `../../${relativePath}`;
27 const androidAdaptiveForeground = "./assets/android-icon-foreground.png";
28
@@ -450,6 +456,7 @@ const config: ExpoConfig = {
456 },
457 eas: {
458 projectId: "d763fcb8-d37c-41ea-a773-b54a0ab4a454",
459 + observe: { endpointUrl: observeEndpointUrl },
460 },
461 },
462 owner: "pingdotgg",
apps/mobile/eas.json
+2 -1
@@ -66,7 +66,8 @@
66 "corepack": true,
67 "env": {
68 "APP_VARIANT": "production",
69 - "NODE_OPTIONS": "--max-old-space-size=4096"
69 + "NODE_OPTIONS": "--max-old-space-size=4096",
70 + "SOURCEMAP_FILE": "/tmp/t3code-ios-main.jsbundle.map"
71 },
72 "channel": "production",
73 "environment": "production",
apps/mobile/index.ts
+2
@@ -1,5 +1,7 @@
1 // Must stay first: the render counter wraps the DevTools hook before any React commit.
2 import "./src/features/development/render-counter-entry";
3 +// Next, so errors thrown while the rest of the app loads are recorded too.
4 +import "./src/features/observability/observe";
5 import { registerRootComponent } from "expo";
6 import "react-native-gesture-handler";
7 import { LogBox } from "react-native";
apps/mobile/package.json
+3
@@ -46,6 +46,7 @@
46 "measure-renders": "node scripts/measure-renders.mts",
47 "generate": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/generate-uniwind-themes.mts",
48 "generate:highlights": "node scripts/generate-highlights.mts",
49 + "eas-build-on-success": "node scripts/upload-sourcemaps.mts build",
50 "test": "vp test run --project unit --project react",
51 "typecheck": "tsc --noEmit"
52 },
@@ -103,6 +104,7 @@
104 "expo-linking": "~58.0.13",
105 "expo-network": "~58.0.3",
106 "expo-notifications": "~58.1.5",
107 + "expo-observe": "~58.0.15",
108 "expo-paste-input": "^0.1.15",
109 "expo-quick-actions": "^6.0.2",
110 "expo-secure-store": "~58.0.2",
@@ -145,6 +147,7 @@
147 "@types/mdast": "^4.0.4",
148 "@types/react": "catalog:",
149 "@types/react-dom": "catalog:",
150 + "aws4fetch": "1.0.20",
151 "babel-preset-expo": "~58.0.17",
152 "jsdom": "^30.0.1",
153 "tailwindcss": "^4.0.0",
apps/mobile/scripts/sourcemap-upload.mts new
+165
@@ -0,0 +1,165 @@
1 +import * as NodeFS from "node:fs";
2 +import * as NodePath from "node:path";
3 +import { mobileSourceMapKey, type MobilePlatform } from "@t3tools/shared/mobileSourceMaps";
4 +import { AwsClient } from "aws4fetch";
5 +
6 +export interface SourceMapUpload {
7 + readonly key: string;
8 + readonly path: string;
9 +}
10 +
11 +/** The `MOBILE_SOURCEMAPS_R2` secret: the relay stack's `mobileSourceMapsUploadCredentials`. */
12 +interface UploadCredentials {
13 + readonly endpoint: string;
14 + readonly bucketName: string;
15 + readonly accessKeyId: string;
16 + readonly secretAccessKey: string;
17 +}
18 +
19 +const isPlatform = (value: unknown): value is MobilePlatform =>
20 + value === "ios" || value === "android";
21 +
22 +const readJson = (path: string): unknown => JSON.parse(NodeFS.readFileSync(path, "utf8"));
23 +
24 +const isRecord = (value: unknown): value is Record<string, unknown> =>
25 + typeof value === "object" && value !== null && !Array.isArray(value);
26 +
27 +export function parseCredentials(value: string | undefined): UploadCredentials | undefined {
28 + if (!value?.trim()) return undefined;
29 + let parsed: unknown;
30 + try {
31 + parsed = JSON.parse(value);
32 + } catch {
33 + // JSON.parse's message quotes the input, which here is the secret.
34 + throw new Error("MOBILE_SOURCEMAPS_R2 is not valid JSON.");
35 + }
36 + if (!isRecord(parsed)) throw new Error("MOBILE_SOURCEMAPS_R2 is not a JSON object.");
37 + const { endpoint, bucketName, accessKeyId, secretAccessKey } = parsed;
38 + if (
39 + typeof endpoint !== "string" ||
40 + typeof bucketName !== "string" ||
41 + typeof accessKeyId !== "string" ||
42 + typeof secretAccessKey !== "string"
43 + ) {
44 + throw new Error(
45 + "MOBILE_SOURCEMAPS_R2 needs endpoint, bucketName, accessKeyId and secretAccessKey.",
46 + );
47 + }
48 + return { endpoint, bucketName, accessKeyId, secretAccessKey };
49 +}
50 +
51 +/** One map per update `eas update --emit-metadata` published, next to the bundle `metadata.json` names. */
52 +export function otaSourceMapUploads(distDir: string): ReadonlyArray<SourceMapUpload> {
53 + const metadata = readJson(NodePath.join(distDir, "metadata.json"));
54 + const easMetadata = readJson(NodePath.join(distDir, "eas-update-metadata.json"));
55 + const fileMetadata = isRecord(metadata) ? metadata.fileMetadata : undefined;
56 + const updates = isRecord(easMetadata) ? easMetadata.updates : undefined;
57 + if (!isRecord(fileMetadata) || !Array.isArray(updates)) {
58 + throw new Error(`${distDir} is missing Expo export or EAS update metadata.`);
59 + }
60 + return updates.flatMap((update): Array<SourceMapUpload> => {
61 + if (!isRecord(update) || !isPlatform(update.platform) || typeof update.id !== "string") {
62 + return [];
63 + }
64 + const platformFiles = fileMetadata[update.platform];
65 + const bundle = isRecord(platformFiles) ? platformFiles.bundle : undefined;
66 + if (typeof bundle !== "string") return [];
67 + return [
68 + {
69 + key: mobileSourceMapKey(update.platform, "ota", update.id),
70 + path: NodePath.join(distDir, `${bundle}.map`),
71 + },
72 + ];
73 + });
74 +}
75 +
76 +/** Android's Gradle plugin writes the map under the release variant; iOS to `SOURCEMAP_FILE` (eas.json). */
77 +export function buildSourceMapUpload(
78 + env: Readonly<Record<string, string | undefined>>,
79 + projectDir: string,
80 +): SourceMapUpload {
81 + const platform = env.EAS_BUILD_PLATFORM;
82 + const buildId = env.EAS_BUILD_ID;
83 + if (!isPlatform(platform) || !buildId) {
84 + throw new Error("EAS_BUILD_PLATFORM and EAS_BUILD_ID are only set on EAS Build.");
85 + }
86 + if (platform === "ios") {
87 + if (!env.SOURCEMAP_FILE) throw new Error("SOURCEMAP_FILE is not set for this iOS build.");
88 + return { key: mobileSourceMapKey("ios", "build", buildId), path: env.SOURCEMAP_FILE };
89 + }
90 + const variantsDir = NodePath.join(projectDir, "android/app/build/generated/sourcemaps/react");
91 + const variant = NodeFS.existsSync(variantsDir)
92 + ? NodeFS.readdirSync(variantsDir).find((name) => /release$/iu.test(name))
93 + : undefined;
94 + if (!variant) throw new Error(`No release source map under ${variantsDir}.`);
95 + return {
96 + key: mobileSourceMapKey("android", "build", buildId),
97 + path: NodePath.join(variantsDir, variant, "index.android.bundle.map"),
98 + };
99 +}
100 +
101 +/**
102 + * EAS writes the app's debug symbols next to the `.ipa`, so native frames in iOS crash
103 + * reports can be read with `atos`. Undefined when the build left none.
104 + */
105 +export function buildDsymUpload(
106 + env: Readonly<Record<string, string | undefined>>,
107 + projectDir: string,
108 +): SourceMapUpload | undefined {
109 + if (env.EAS_BUILD_PLATFORM !== "ios" || !env.EAS_BUILD_ID) return undefined;
110 + const buildDir = NodePath.join(projectDir, "ios/build");
111 + const dsym = NodeFS.existsSync(buildDir)
112 + ? NodeFS.readdirSync(buildDir).find((name) => name.endsWith(".app.dSYM.zip"))
113 + : undefined;
114 + if (!dsym) return undefined;
115 + return { key: `ios/dsym/${env.EAS_BUILD_ID}.zip`, path: NodePath.join(buildDir, dsym) };
116 +}
117 +
118 +/** Only what the relay reads: source text stays out of the bucket, and the rest is dead weight. */
119 +function stripToMappings(text: string): string {
120 + const map: unknown = JSON.parse(text);
121 + if (!isRecord(map)) throw new Error("Source map is not a JSON object.");
122 + const { version, sources, sourceRoot, mappings } = map;
123 + return JSON.stringify({ version, sources, sourceRoot, mappings });
124 +}
125 +
126 +export function uploadSourceMap(
127 + credentials: UploadCredentials,
128 + upload: SourceMapUpload,
129 +): Promise<number> {
130 + return putObject(
131 + credentials,
132 + upload.key,
133 + stripToMappings(NodeFS.readFileSync(upload.path, "utf8")),
134 + "application/json",
135 + );
136 +}
137 +
138 +export function uploadDsym(credentials: UploadCredentials, upload: SourceMapUpload) {
139 + const body = new Uint8Array(NodeFS.readFileSync(upload.path));
140 + return putObject(credentials, upload.key, body, "application/zip");
141 +}
142 +
143 +async function putObject(
144 + credentials: UploadCredentials,
145 + key: string,
146 + body: string | Uint8Array<ArrayBuffer>,
147 + contentType: string,
148 +): Promise<number> {
149 + const client = new AwsClient({
150 + accessKeyId: credentials.accessKeyId,
151 + secretAccessKey: credentials.secretAccessKey,
152 + service: "s3",
153 + region: "auto",
154 + });
155 + const url = `${credentials.endpoint.replace(/\/$/u, "")}/${credentials.bucketName}/${key}`;
156 + const response = await client.fetch(url, {
157 + method: "PUT",
158 + body,
159 + headers: { "content-type": contentType },
160 + });
161 + if (!response.ok) {
162 + throw new Error(`R2 answered ${response.status}: ${(await response.text()).slice(0, 500)}`);
163 + }
164 + return body.length;
165 +}
apps/mobile/scripts/sourcemap-upload.test.ts new
+136
@@ -0,0 +1,136 @@
1 +import * as NodeFS from "node:fs";
2 +import * as NodeHttp from "node:http";
3 +import type * as NodeNet from "node:net";
4 +import * as NodeOS from "node:os";
5 +import * as NodePath from "node:path";
6 +import { afterEach, describe, expect, it } from "vite-plus/test";
7 +
8 +import {
9 + buildDsymUpload,
10 + buildSourceMapUpload,
11 + otaSourceMapUploads,
12 + uploadSourceMap,
13 +} from "./sourcemap-upload.mts";
14 +
15 +const directories: Array<string> = [];
16 +const tempDir = () => {
17 + const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "sourcemap-upload-"));
18 + directories.push(dir);
19 + return dir;
20 +};
21 +const writeFile = (path: string, contents: string) => {
22 + NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true });
23 + NodeFS.writeFileSync(path, contents);
24 +};
25 +
26 +afterEach(() => {
27 + for (const dir of directories.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true });
28 +});
29 +
30 +describe("uploadSourceMap", () => {
31 + it("PUTs each published update's map, only what the relay reads, under its update id", async () => {
32 + const dist = tempDir();
33 + writeFile(
34 + NodePath.join(dist, "metadata.json"),
35 + JSON.stringify({
36 + fileMetadata: {
37 + android: { bundle: "_expo/static/js/android/index-abc.hbc" },
38 + ios: { bundle: "_expo/static/js/ios/index-def.hbc" },
39 + },
40 + }),
41 + );
42 + writeFile(
43 + NodePath.join(dist, "eas-update-metadata.json"),
44 + JSON.stringify({
45 + updates: [
46 + { id: "update-android", platform: "android" },
47 + { id: "update-ios", platform: "ios" },
48 + { id: "update-web", platform: "web" },
49 + ],
50 + }),
51 + );
52 + for (const platform of ["android", "ios"]) {
53 + writeFile(
54 + NodePath.join(
55 + dist,
56 + `_expo/static/js/${platform}/index-${platform === "ios" ? "def" : "abc"}.hbc.map`,
57 + ),
58 + JSON.stringify({
59 + version: 3,
60 + sources: [`${platform}.ts`],
61 + sourcesContent: ["secret"],
62 + names: ["render"],
63 + x_facebook_sources: [[null]],
64 + mappings: "",
65 + }),
66 + );
67 + }
68 +
69 + const requests: Array<{ method?: string; url?: string; auth?: string; body: string }> = [];
70 + const server = NodeHttp.createServer((request, response) => {
71 + let body = "";
72 + request.on("data", (chunk) => (body += chunk));
73 + request.on("end", () => {
74 + requests.push({
75 + method: request.method,
76 + url: request.url,
77 + auth: request.headers.authorization,
78 + body,
79 + });
80 + response.end();
81 + });
82 + });
83 + await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
84 + try {
85 + const address = server.address() as NodeNet.AddressInfo;
86 + const credentials = {
87 + endpoint: `http://127.0.0.1:${address.port}`,
88 + bucketName: "maps",
89 + accessKeyId: "key-id",
90 + secretAccessKey: "secret-key",
91 + };
92 + for (const upload of otaSourceMapUploads(dist)) {
93 + await uploadSourceMap(credentials, upload);
94 + }
95 + expect(requests).toEqual(
96 + ["android", "ios"].map((platform) => ({
97 + method: "PUT",
98 + url: `/maps/${platform}/ota/update-${platform}.map`,
99 + auth: expect.stringMatching(/^AWS4-HMAC-SHA256 Credential=key-id\/\d{8}\/auto\/s3\//),
100 + body: JSON.stringify({ version: 3, sources: [`${platform}.ts`], mappings: "" }),
101 + })),
102 + );
103 + } finally {
104 + server.close();
105 + }
106 + });
107 +});
108 +
109 +describe("buildSourceMapUpload", () => {
110 + it("names the EAS build's embedded bundle map under its build id", () => {
111 + const project = tempDir();
112 + const androidMap = NodePath.join(
113 + project,
114 + "android/app/build/generated/sourcemaps/react/release/index.android.bundle.map",
115 + );
116 + writeFile(androidMap, "{}");
117 + expect(
118 + buildSourceMapUpload({ EAS_BUILD_PLATFORM: "android", EAS_BUILD_ID: "build-1" }, project),
119 + ).toEqual({ key: "android/build/build-1.map", path: androidMap });
120 + expect(
121 + buildSourceMapUpload(
122 + { EAS_BUILD_PLATFORM: "ios", EAS_BUILD_ID: "build-2", SOURCEMAP_FILE: "/tmp/main.map" },
123 + project,
124 + ),
125 + ).toEqual({ key: "ios/build/build-2.map", path: "/tmp/main.map" });
126 + });
127 +
128 + it("names an iOS build's dSYMs under its build id", () => {
129 + const project = tempDir();
130 + const dsym = NodePath.join(project, "ios/build/T3Code.app.dSYM.zip");
131 + writeFile(dsym, "zip");
132 + expect(
133 + buildDsymUpload({ EAS_BUILD_PLATFORM: "ios", EAS_BUILD_ID: "build-3" }, project),
134 + ).toEqual({ key: "ios/dsym/build-3.zip", path: dsym });
135 + });
136 +});
apps/mobile/scripts/upload-sourcemaps.mts new
+61
@@ -0,0 +1,61 @@
1 +#!/usr/bin/env node
2 +// Usage: `ota <dist>` after `eas update --emit-metadata`, or `build` from eas-build-on-success.
3 +// Problems are warnings, never failures: an unreadable stack is not worth failing a release over.
4 +import {
5 + buildDsymUpload,
6 + buildSourceMapUpload,
7 + otaSourceMapUploads,
8 + parseCredentials,
9 + uploadDsym,
10 + uploadSourceMap,
11 + type SourceMapUpload,
12 +} from "./sourcemap-upload.mts";
13 +
14 +const warn = (message: string) => {
15 + // GitHub Actions shows this as an annotation; elsewhere it is a log line.
16 + console.warn(process.env.GITHUB_ACTIONS ? `::warning::${message}` : `warning: ${message}`);
17 +};
18 +
19 +const errorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error));
20 +
21 +async function main() {
22 + const [mode, distDir] = process.argv.slice(2);
23 + const credentials = parseCredentials(process.env.MOBILE_SOURCEMAPS_R2);
24 + if (!credentials) {
25 + warn("MOBILE_SOURCEMAPS_R2 is not set; crash stacks from this bundle stay minified.");
26 + return;
27 + }
28 + let uploads: ReadonlyArray<SourceMapUpload>;
29 + if (mode === "ota" && distDir) {
30 + uploads = otaSourceMapUploads(distDir);
31 + } else if (mode === "build") {
32 + uploads = [buildSourceMapUpload(process.env, process.cwd())];
33 + } else {
34 + throw new Error("Usage: upload-sourcemaps.mts ota <dist-dir> | build");
35 + }
36 + if (uploads.length === 0) warn("Found no source maps to upload.");
37 + for (const upload of uploads) {
38 + try {
39 + const bytes = await uploadSourceMap(credentials, upload);
40 + console.log(`Uploaded ${upload.key} (${(bytes / 1e6).toFixed(1)} MB)`);
41 + } catch (error) {
42 + warn(`Could not upload ${upload.key}: ${errorMessage(error)}`);
43 + }
44 + }
45 + if (mode !== "build" || process.env.EAS_BUILD_PLATFORM !== "ios") return;
46 + const dsym = buildDsymUpload(process.env, process.cwd());
47 + if (!dsym) {
48 + warn("Found no dSYM under ios/build; native iOS frames from this build stay unreadable.");
49 + return;
50 + }
51 + try {
52 + const bytes = await uploadDsym(credentials, dsym);
53 + console.log(`Uploaded ${dsym.key} (${(bytes / 1e6).toFixed(1)} MB)`);
54 + } catch (error) {
55 + warn(`Could not upload ${dsym.key}: ${errorMessage(error)}`);
56 + }
57 +}
58 +
59 +await main().catch((error: unknown) => {
60 + warn(`Source map upload skipped: ${errorMessage(error)}`);
61 +});
apps/mobile/src/components/RenderErrorBoundary.tsx
+4 -1
@@ -1,3 +1,4 @@
1 +import { Observe } from "expo-observe";
2 import { Component, type ReactNode } from "react";
3 import { ScrollView, View } from "react-native";
4
@@ -60,7 +61,9 @@ export class RenderErrorBoundary extends Component<
61 return { failed: true, error };
62 }
63
63 - override componentDidCatch(_error: unknown, info: { componentStack?: string }) {
64 + override componentDidCatch(error: unknown, info: { componentStack?: string }) {
65 + // A caught render error never reaches Observe's global handler.
66 + Observe.reportError(error);
67 this.setState({ componentStack: info.componentStack });
68 }
69
apps/mobile/src/features/diagnostics/SettingsDiagnosticsRouteScreen.tsx
+55 -1
@@ -3,8 +3,9 @@ import { ScreenScrollView as ScrollView } from "../../components/ui/ScreenScroll
3 import * as Application from "expo-application";
4 import Constants from "expo-constants";
5 import * as Updates from "expo-updates";
6 -import { useAtomValue } from "@effect/atom-react";
6 +import { useAtomSet, useAtomValue } from "@effect/atom-react";
7 import * as Effect from "effect/Effect";
8 +import * as Exit from "effect/Exit";
9 import { AsyncResult, Atom } from "effect/reactivity";
10 import { useState } from "react";
11 import { View } from "react-native";
@@ -13,9 +14,12 @@ import { useSafeAreaInsets } from "react-native-safe-area-context";
14 import { SymbolView } from "../../components/ui/AppSymbol";
15 import { AppText as Text } from "../../components/ui/AppText";
16 import { tryCopyTextWithHaptic } from "../../lib/copyTextWithHaptic";
17 +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
18 +import { crashReportingAvailable, setCrashReportsEnabled } from "../observability/observe";
19 import { SettingsActionRow } from "../settings/components/SettingsActionRow";
20 import { SettingsScreen } from "../settings/components/SettingsScreen";
21 import { SettingsSection } from "../settings/components/SettingsSection";
22 +import { SettingsSwitchRow } from "../settings/components/SettingsSwitchRow";
23 import {
24 formatStartupCrashReport,
25 parseStartupCrashRecords,
@@ -86,6 +90,8 @@ export function SettingsDiagnosticsRouteScreen() {
90 className="flex-1"
91 contentContainerClassName="gap-6 px-5 pt-4 pb-[18px]"
92 >
93 + {crashReportingAvailable ? <CrashReportsSetting /> : null}
94 +
95 <SettingsSection title="Startup crashes">
96 {state.status === "loading" ? (
97 <View className="items-center gap-3 px-6 py-8">
@@ -131,6 +137,54 @@ export function SettingsDiagnosticsRouteScreen() {
137 );
138 }
139
140 +function CrashReportsSetting() {
141 + const preferences = useAtomValue(mobilePreferencesAtom);
142 + const savePreferences = useAtomSet(updateMobilePreferencesAtom, { mode: "promiseExit" });
143 + // One change at a time, so a slow save never turns sending back on after a later opt-out.
144 + const [saving, setSaving] = useState(false);
145 + const ready = AsyncResult.isSuccess(preferences) && !preferences.waiting;
146 + const enabled = AsyncResult.isSuccess(preferences)
147 + ? preferences.value.crashReportsEnabled !== false
148 + : true;
149 +
150 + const change = (value: boolean) => {
151 + setSaving(true);
152 + // Sending stops before the save and starts only after it.
153 + if (!value) void setCrashReportsEnabled(false);
154 + savePreferences({ crashReportsEnabled: value })
155 + .then((exit) => {
156 + if (Exit.isFailure(exit)) {
157 + console.warn("[diagnostics] could not save the crash report setting", exit.cause);
158 + return value ? undefined : setCrashReportsEnabled(true);
159 + }
160 + return value ? setCrashReportsEnabled(true) : undefined;
161 + })
162 + .catch((error: unknown) => {
163 + console.warn("[diagnostics] could not apply the crash report setting", error);
164 + })
165 + .finally(() => setSaving(false));
166 + };
167 +
168 + return (
169 + <View className="gap-3">
170 + <SettingsSection title="Crash reports">
171 + <SettingsSwitchRow
172 + icon="tray.and.arrow.up"
173 + label="Send crash reports"
174 + disabled={!ready || saving}
175 + value={enabled}
176 + onValueChange={change}
177 + />
178 + </SettingsSection>
179 + <Text className="px-2 text-sm leading-normal text-foreground-muted">
180 + When the app crashes or hits an error, it sends the error message, the stack trace, the app
181 + version, and the device model and OS version to the T3 Code team. Each launch also sends
182 + startup timings. Reports carry a random ID for this install.
183 + </Text>
184 + </View>
185 + );
186 +}
187 +
188 function EmptyState(props: {
189 readonly icon: "exclamationmark.triangle" | "checkmark.circle";
190 readonly title: string;
apps/mobile/src/features/observability/observe.ts new
+63
@@ -0,0 +1,63 @@
1 +import Constants from "expo-constants";
2 +import { Observe } from "expo-observe";
3 +
4 +// Importing expo-observe installs its global JS error handler; reports leave only through the relay.
5 +const relayUrl: unknown = Constants.expoConfig?.extra?.relay?.url;
6 +const appVariant: unknown = Constants.expoConfig?.extra?.appVariant;
7 +
8 +export const crashReportingAvailable = typeof relayUrl === "string" && relayUrl.trim() !== "";
9 +
10 +interface PromiseRejectionTracker {
11 + readonly enablePromiseRejectionTracker?: (options: {
12 + readonly allRejections: boolean;
13 + readonly onUnhandled: (id: number, rejection: unknown) => void;
14 + }) => void;
15 +}
16 +
17 +// React Native tracks unhandled rejections only in development, for LogBox; Observe has no hook.
18 +if (!__DEV__) {
19 + const hermes: PromiseRejectionTracker | undefined = Reflect.get(globalThis, "HermesInternal");
20 + hermes?.enablePromiseRejectionTracker?.({
21 + allRejections: true,
22 + onUnhandled: (_id, rejection) => Observe.reportError(rejection),
23 + });
24 +}
25 +
26 +let latestConfiguration = 0;
27 +
28 +// While sending is off, a dispatch pass drops what it finds, so one runs before turning it back on.
29 +// Best effort: Observe skips the pass while it is backing off from a failed send.
30 +export async function setCrashReportsEnabled(enabled: boolean) {
31 + const configuration = ++latestConfiguration;
32 + const dispatchingEnabled = crashReportingAvailable && enabled;
33 + if (dispatchingEnabled) {
34 + await Observe.dispatchEvents();
35 + if (configuration !== latestConfiguration) return;
36 + }
37 + Observe.configure({
38 + dispatchingEnabled,
39 + ...(typeof appVariant === "string" ? { environment: appVariant } : {}),
40 + });
41 +}
42 +
43 +if (!crashReportingAvailable) {
44 + void setCrashReportsEnabled(false);
45 +} else {
46 + // Observe keeps its last configuration across launches; configuring it off while the setting
47 + // loads would drop the last session's native crash. Imported lazily so the persistence graph
48 + // never runs ahead of the imports `index.ts` orders first.
49 + const changedSinceLoad = () => latestConfiguration !== 0;
50 + void import("../../persistence/imperative")
51 + .then(({ loadPreferences }) => loadPreferences())
52 + .then(
53 + (preferences) =>
54 + changedSinceLoad()
55 + ? undefined
56 + : setCrashReportsEnabled(preferences.crashReportsEnabled !== false),
57 + // Observe keeps the last choice it was given, which stands in for the unreadable one.
58 + (error: unknown) => console.warn("[observe] could not read the crash report setting", error),
59 + )
60 + .catch((error: unknown) => {
61 + console.warn("[observe] could not apply the crash report setting", error);
62 + });
63 +}
apps/mobile/src/persistence/mobile-preferences.ts
+6
@@ -103,6 +103,8 @@ export interface Preferences {
103 readonly codeFontSize?: number | null;
104 readonly codeWordBreak?: boolean;
105 readonly mathRenderingEnabled?: boolean;
106 + /** Unset means on: crash reports are sent unless the user turns them off. */
107 + readonly crashReportsEnabled?: boolean;
108 readonly connectOnboardingOptOutAccounts?: ReadonlyArray<string>;
109 readonly collapsedProjectGroups?: readonly string[];
110 readonly dismissedDatabaseStorageNotices?: readonly string[];
@@ -183,6 +185,7 @@ function sanitizePreferences(parsed: Preferences): Preferences {
185 codeFontSize?: number | null;
186 codeWordBreak?: boolean;
187 mathRenderingEnabled?: boolean;
188 + crashReportsEnabled?: boolean;
189 connectOnboardingOptOutAccounts?: ReadonlyArray<string>;
190 collapsedProjectGroups?: readonly string[];
191 dismissedDatabaseStorageNotices?: readonly string[];
@@ -272,6 +275,9 @@ function sanitizePreferences(parsed: Preferences): Preferences {
275 if (typeof parsed.codeWordBreak === "boolean") preferences.codeWordBreak = parsed.codeWordBreak;
276 if (typeof parsed.mathRenderingEnabled === "boolean")
277 preferences.mathRenderingEnabled = parsed.mathRenderingEnabled;
278 + if (typeof parsed.crashReportsEnabled === "boolean") {
279 + preferences.crashReportsEnabled = parsed.crashReportsEnabled;
280 + }
281 if (Array.isArray(parsed.connectOnboardingOptOutAccounts)) {
282 preferences.connectOnboardingOptOutAccounts = parsed.connectOnboardingOptOutAccounts.filter(
283 (account): account is string => typeof account === "string",
docs/operations/relay-observability.md
+93
@@ -10,6 +10,11 @@ The relay Alchemy stack owns a shared Axiom trace setup:
10 - `t3-code-mobile-otel-ingest-prod`, the dataset-scoped mobile ingest token
11 - `t3-code-relay-client-otel-ingest-prod`, the dataset-scoped first-party relay-client ingest token
12 - `t3-code-relay-recent-spans-prod`, a view of recent request and endpoint spans
13 +- `t3-code-mobile-logs-prod` and `t3-code-mobile-metrics-prod`, the mobile app's crash, error, and
14 + startup data from `expo-observe` (see [Mobile crashes and errors](#mobile-crashes-and-errors))
15 +- `t3-code-mobile-telemetry-ingest-prod`, the ingest token the Worker forwards that data with
16 +- `t3-code-mobile-sourcemaps-prod`, the R2 bucket of mobile JavaScript source maps, and
17 + `t3-code-mobile-sourcemaps-upload-prod`, a Cloudflare token that can only write objects to it
18
19 Alchemy stages append their sanitized stage name to isolate resources, for example
20 `t3-code-relay-traces-dev-julius` for a personal stage.
@@ -159,3 +164,91 @@ see [Managed tunnel cleanup rollout](release.md#managed-tunnel-cleanup-rollout).
164 **Links degraded / database slow.** Warnings for slow degradation. Link failures above 20% over six
165 hours have so far meant provisioning running into the 9 s deadline, with Postgres latency as the
166 usual cause.
167 +
168 +## Mobile crashes and errors
169 +
170 +The mobile app reports through [`expo-observe`](https://docs.expo.dev/eas/observe/). Its
171 +`endpointUrl` points at the relay (`/v1/mobile/observe`, set in `apps/mobile/app.config.ts` from
172 +`T3CODE_RELAY_URL`), because Observe cannot send auth headers and prefixes every path with the EAS
173 +project id. The relay forwards each request to Axiom with its own token: logs (JavaScript errors,
174 +native crashes, log events) to `t3-code-mobile-logs-*`, metrics to `t3-code-mobile-metrics-*`, and
175 +spans to the shared traces dataset. Expo's Observe dashboard receives nothing. The endpoint URL is
176 +compiled into the native app, so pointing it elsewhere takes a store build.
177 +
178 +Hermes stacks from release builds are minified bytecode offsets. On the way through, the relay
179 +rewrites each JavaScript exception's `exception.stacktrace` with the bundle's source map from R2
180 +and records the outcome on the log record:
181 +
182 +- `relay.symbolication.status`: `symbolicated`, or why not. `no_bundle_id` is a development or
183 + local build. `source_map_missing` means no map was uploaded for the bundle;
184 + `source_map_too_large`, `source_map_invalid` and `source_map_unavailable` (R2 unreadable, retried
185 + shortly) mean one was. `source_map_limit` is a request naming more bundles than the relay decodes
186 + at once, and `no_bundle_frames` a stack with no frames from the bundle.
187 +- `relay.symbolication.source_map`: the R2 key used
188 +- `relay.symbolication.raw_stacktrace`: the original stack, when it was rewritten
189 +
190 +iOS frames lose their install path (`/private/var/containers/Bundle/Application/<uuid>/...`), which
191 +differs per install, so `exception.stacktrace` groups across users.
192 +
193 +A fatal JavaScript error ends in a native abort, which Observe reports as a second crash with a
194 +generic React Native stack. The relay marks that one `relay.crash.js_fatal` when both arrive in one
195 +request, as they do on the next launch. Leave it out to count real native crashes:
196 +
197 +```apl
198 +['t3-code-mobile-logs-prod']
199 +| where ['attributes.expo.error.source'] == 'nativeCrash'
200 +| where isnull(['attributes.relay.crash.js_fatal'])
201 +| summarize count() by ['attributes.exception.type'], ['attributes.exception.message']
202 +```
203 +
204 +Native crash stacks (`expo.error.source == 'nativeCrash'`) are not symbolicated. On iOS, app frames
205 +read like `main (T3Code + 19476)`: the offset into the app binary. The build hook uploads that
206 +build's dSYMs to `ios/dsym/<easBuildId>.zip`; download it with an account R2 token, then:
207 +
208 +```sh
209 +unzip <easBuildId>.zip
210 +atos -arch arm64 -o T3Code.app.dSYM/Contents/Resources/DWARF/T3Code -l 0x100000000 \
211 + $(printf '0x%x' $((0x100000000 + 19476)))
212 +```
213 +
214 +Maps are keyed by the bundle that crashed: `<platform>/ota/<updateId>.map` for OTA updates and
215 +`<platform>/build/<easBuildId>.map` for the bundle embedded in a store build. The production
216 +mobile workflow uploads OTA maps after `eas update`, and the `eas-build-on-success` hook in
217 +`apps/mobile/package.json` uploads a build's map, both with `apps/mobile/scripts/upload-sourcemaps.mts`.
218 +They need the `MOBILE_SOURCEMAPS_R2` secret, a JSON value the stack outputs as
219 +`mobileSourceMapsUploadCredentials` from a token that can only write to the bucket. Creating that
220 +token needs **Account API Tokens: Edit** on the deploy token. Deploy output shows it redacted; read
221 +it from state, where it is under `__redacted__`:
222 +
223 +```sh
224 +cd infra/relay && vp exec alchemy state read T3CodeRelay/prod/output
225 +```
226 +
227 +Set it as a GitHub Actions secret, for OTA updates, and as an EAS environment variable with
228 +`secret` visibility in the `production` environment only, for builds. Keep it out of `preview`:
229 +PR preview builds run the PR's code in that environment. In CI only the upload step gets it, not
230 +`eas update` and the dependencies it runs. If the deploy replaces the token, update both.
231 +
232 +The `t3-code-mobile-health` dashboard is the place to look every few days: crash and handled
233 +error counts, top crashes with the installs and app versions they hit, crashes by version, whether
234 +stacks are symbolicated, and whether reports reach the relay at all. A crash counts once: the
235 +native abort after a JS crash (`relay.crash.js_fatal`) is left out. Startup times are metrics, so
236 +they are not on it; query `t3-code-mobile-metrics-*` with MPL. Prod and PR previews get the
237 +dashboard, and deployments with `RELAY_ALERT_NOTIFIER_ID` get these monitors
238 +([`infra/relay/src/mobileTelemetry/monitoring.ts`](../../infra/relay/src/mobileTelemetry/monitoring.ts)):
239 +
240 +- **crash spike** (pages): crashes per 15 min well above the same time over the last 7 days. Until
241 + a week of data exists it has no baseline. A spike right after a release usually means a bad OTA
242 + update; roll it back in EAS.
243 +- **one crash hitting many installs** (pages): a single crash on 25 installs within an hour.
244 +- **reports stopped** (warning): no startup metrics forwarded for an hour, so crashes are not
245 + arriving either.
246 +- **crash stacks not symbolicated** (warning): over half of release JS stacks stayed minified for
247 + 6 h. `relay.symbolication.status` names the cause.
248 +
249 +Thresholds are starting points, not measured baselines; revisit them after two weeks of
250 +production data.
251 +
252 +Without it, uploads are skipped with a warning and stacks stay minified. Each ingest request is one
253 +`relay.mobile_telemetry.ingest` span; `relay.mobile_telemetry.outcome` is `forwarded`,
254 +`rate_limited`, `body_too_large`, `upstream_unavailable`, or `upstream_timeout`.
docs/user/telemetry.md
+10
@@ -21,6 +21,16 @@ environment before starting it. Relay tracing also stops when `T3CODE_OTEL_SDK_D
21 The desktop app reads `T3CODE_TELEMETRY_ENABLED` from your shell profile (for example `~/.zshrc`) on macOS and
22 Linux, so export it there and restart the app. On Windows, set it as a user environment variable.
23
24 +## Mobile app crash reports
25 +
26 +When the iOS or Android app crashes or hits an error, it sends a report to T3 Code: the error
27 +message, the stack trace, the app version, and the device model and OS version. Each launch also
28 +sends startup timings, and every report carries a random ID for the install. Reports go through
29 +the T3 Connect relay to Axiom. They do not include your threads, prompts, or connection details,
30 +but an error message can quote a value the app was handling.
31 +
32 +To stop sending them, turn off **Send crash reports** in **Settings > About T3 Code > Diagnostics**.
33 +
34 ## Export diagnostics to your own receiver
35
36 To send traces, metrics, or logs to an OpenTelemetry receiver, open
infra/relay/alchemy.run.ts
+7 -1
@@ -10,7 +10,7 @@ import * as Planetscale from "@alchemy.run/planetscale";
10
11 import { PublishClientConfig, tokenDigest } from "./src/clientConfig.ts";
12 import * as RelayDb from "./src/db.ts";
13 -import { RelayObservability } from "./src/observability.ts";
13 +import { MobileSourceMapsUploadCredentials, RelayObservability } from "./src/observability.ts";
14 import { isPreviewStage } from "./src/previewStages.ts";
15 import { PreviewTunnelSweep, PreviewTunnelSweepProvider } from "./src/previewTunnelSweep.ts";
16 import { ManagedEndpointZone, RelayApiZone, RelayDeploymentConfig } from "./src/zone.ts";
@@ -35,6 +35,7 @@ export default Alchemy.Stack(
35 const managedEndpointZone = yield* ManagedEndpointZone.pipe(Effect.orDie);
36 const relayApiZone = yield* RelayApiZone.pipe(Effect.orDie);
37 const observability = yield* RelayObservability;
38 + const mobileSourceMapsUploadCredentials = yield* MobileSourceMapsUploadCredentials;
39 const api = yield* RelayWorker.Api;
40 if (isPreviewStage(stage)) {
41 const { managedEndpointZoneName } = yield* RelayDeploymentConfig;
@@ -72,6 +73,11 @@ export default Alchemy.Stack(
73 clientTracingUrl: observability.traces.otelTracesEndpoint,
74 clientTracingDataset: observability.traces.name,
75 clientTracingToken: observability.clientIngestToken.token,
76 + mobileLogsDataset: observability.mobileLogs.name,
77 + mobileMetricsDataset: observability.mobileMetrics.name,
78 + mobileSourceMapsBucket: observability.mobileSourceMaps.bucketName,
79 + // The MOBILE_SOURCEMAPS_R2 secret for GitHub Actions and EAS Build.
80 + mobileSourceMapsUploadCredentials,
81 };
82 }).pipe(Effect.provide(RelayWorker.layer)),
83 );
infra/relay/src/mobileTelemetry/MobileTelemetry.test.ts new
+382
@@ -0,0 +1,382 @@
1 +import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
2 +import * as NodeServices from "@effect/platform-node/NodeServices";
3 +import { describe, expect, it } from "@effect/vitest";
4 +import { RelayApi } from "@t3tools/contracts/relay";
5 +import * as Deferred from "effect/Deferred";
6 +import * as Effect from "effect/Effect";
7 +import * as Fiber from "effect/Fiber";
8 +import * as Layer from "effect/Layer";
9 +import * as Redacted from "effect/Redacted";
10 +import * as Schema from "effect/Schema";
11 +import * as Etag from "effect/http/Etag";
12 +import * as HttpClient from "effect/http/HttpClient";
13 +import type * as HttpClientError from "effect/http/HttpClientError";
14 +import type * as HttpClientRequest from "effect/http/HttpClientRequest";
15 +import * as HttpClientResponse from "effect/http/HttpClientResponse";
16 +import * as HttpEffect from "effect/http/HttpEffect";
17 +import * as HttpRouter from "effect/http/HttpRouter";
18 +import * as HttpServerRequest from "effect/http/HttpServerRequest";
19 +import * as HttpServerResponse from "effect/http/HttpServerResponse";
20 +import * as HttpApi from "effect/http-api/HttpApi";
21 +import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
22 +
23 +import { RELAY_HTTP_ROUTER_CONFIG } from "../http/Api.ts";
24 +import * as RelayHttpApi from "../http/Api.ts";
25 +import * as MobileTelemetry from "./MobileTelemetry.ts";
26 +
27 +const PROJECT_ID = "d763fcb8-d37c-41ea-a773-b54a0ab4a454";
28 +const UPDATE_ID = "0198e8a4-1111-7000-8000-000000000001";
29 +const BUILD_ID = "0198e8a4-2222-7000-8000-000000000002";
30 +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
31 +const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
32 +
33 +const destinations = {
34 + traces: { url: "https://axiom.test/v1/traces", dataset: "mobile-traces" },
35 + logs: { url: "https://axiom.test/v1/logs", dataset: "mobile-logs" },
36 + metrics: { url: "https://axiom.test/v1/metrics", dataset: "mobile-metrics" },
37 +} as const;
38 +
39 +/** Maps generated column 12 of the one-line bundle to `src/App.tsx` 1:2. */
40 +const SOURCE_MAP = encodeJson({ version: 3, sources: ["src/App.tsx"], mappings: "AAAA,UAAE" });
41 +const STACK = "Error: x\n at render (address at index.android.bundle:1:12)";
42 +
43 +interface Harness {
44 + readonly execute?: (
45 + request: HttpClientRequest.HttpClientRequest,
46 + ) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>;
47 + /** The source maps bucket, by key. */
48 + readonly objects?: Readonly<Record<string, string>>;
49 +}
50 +
51 +const makeHarness = (options: Harness = {}) => {
52 + const sent: Array<HttpClientRequest.HttpClientRequest> = [];
53 + const execute =
54 + options.execute ??
55 + ((request: HttpClientRequest.HttpClientRequest) =>
56 + Effect.succeed(
57 + HttpClientResponse.fromWeb(
58 + request,
59 + new Response("{}", { status: 200, headers: { "content-type": "application/json" } }),
60 + ),
61 + ));
62 + const layerTelemetry = MobileTelemetry.layer.pipe(
63 + Layer.provide(
64 + Layer.mergeAll(
65 + Layer.succeed(MobileTelemetry.MobileTelemetrySettings, {
66 + axiomToken: Redacted.make("axiom-ingest-token"),
67 + destinations,
68 + }),
69 + Layer.succeed(MobileTelemetry.MobileTelemetryRateLimiter, {
70 + allow: () => Effect.succeed(true),
71 + }),
72 + MobileTelemetry.layerSourceMaps.pipe(
73 + Layer.provide(
74 + Layer.succeed(MobileTelemetry.SourceMapObjects, {
75 + get: (key) =>
76 + Effect.sync(() => {
77 + const object = options.objects?.[key];
78 + return object === undefined
79 + ? undefined
80 + : { size: object.length, text: Effect.succeed(object) };
81 + }),
82 + }),
83 + ),
84 + ),
85 + Layer.succeed(
86 + HttpClient.HttpClient,
87 + HttpClient.make((request) => {
88 + sent.push(request);
89 + return execute(request);
90 + }),
91 + ),
92 + ),
93 + ),
94 + );
95 + const httpEffect = HttpRouter.toHttpEffect(
96 + Layer.mergeAll(
97 + HttpApiBuilder.layer(HttpApi.make("RelayApi").add(RelayApi.groups.mobileTelemetry)).pipe(
98 + Layer.provide(MobileTelemetry.layerApi.pipe(Layer.provide(layerTelemetry))),
99 + Layer.provide([NodeServices.layer, NodeHttpPlatform.layer, Etag.layerWeak]),
100 + ),
101 + RelayHttpApi.layerNotFoundRoute,
102 + ),
103 + ).pipe(Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG));
104 + const send = (request: Request) =>
105 + Effect.gen(function* () {
106 + const handler = yield* httpEffect;
107 + const response = yield* Deferred.make<HttpServerResponse.HttpServerResponse>();
108 + yield* HttpEffect.toHandled(handler, (_request, value) =>
109 + Deferred.succeed(response, value),
110 + ).pipe(
111 + Effect.provideService(
112 + HttpServerRequest.HttpServerRequest,
113 + HttpServerRequest.fromWeb(request),
114 + ),
115 + );
116 + return yield* Deferred.await(response);
117 + });
118 + return { sent, send };
119 +};
120 +
121 +const ingest = (signal: string, body: string) =>
122 + new Request(`https://relay.test/v1/mobile/observe/${PROJECT_ID}/v1/${signal}`, {
123 + method: "POST",
124 + headers: { "content-type": "application/json", "cf-connecting-ip": "203.0.113.7" },
125 + body,
126 + });
127 +
128 +const sentText = (request: HttpClientRequest.HttpClientRequest) =>
129 + request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "";
130 +
131 +const str = (key: string, value: string) => ({ key, value: { stringValue: value } });
132 +
133 +/** One resource's JS exception, as Observe reports it from the app's bundle. */
134 +const exceptionResource = (resource: ReadonlyArray<ReturnType<typeof str>>) => ({
135 + resource: { attributes: resource },
136 + scopeLogs: [
137 + {
138 + logRecords: [
139 + {
140 + attributes: [str("expo.error.source", "global"), str("exception.stacktrace", STACK)],
141 + },
142 + ],
143 + },
144 + ],
145 +});
146 +
147 +/** Each forwarded exception record's string attributes. */
148 +const forwardedExceptions = (request: HttpClientRequest.HttpClientRequest) => {
149 + const payload = decodeJson(sentText(request)) as {
150 + resourceLogs: Array<{
151 + scopeLogs: Array<{
152 + logRecords: Array<{ attributes: Array<{ key: string; value: { stringValue: string } }> }>;
153 + }>;
154 + }>;
155 + };
156 + return payload.resourceLogs.map((resourceLogs) =>
157 + Object.fromEntries(
158 + resourceLogs.scopeLogs[0]!.logRecords[0]!.attributes.map(({ key, value }) => [
159 + key,
160 + value.stringValue,
161 + ]),
162 + ),
163 + );
164 +};
165 +
166 +describe("MobileTelemetry", () => {
167 + it.effect("forwards each signal to its Axiom dataset with the relay's token", () =>
168 + Effect.gen(function* () {
169 + const harness = makeHarness();
170 + const body = encodeJson({ resourceMetrics: [] });
171 + for (const signal of ["metrics", "traces", "logs"] as const) {
172 + const response = yield* harness.send(ingest(signal, body));
173 + expect(response.status).toBe(200);
174 + }
175 + expect(
176 + harness.sent.map((request) => ({
177 + url: request.url,
178 + authorization: request.headers.authorization,
179 + dataset: request.headers["x-axiom-dataset"],
180 + body: sentText(request),
181 + })),
182 + ).toEqual(
183 + (["metrics", "traces", "logs"] as const).map((signal) => ({
184 + url: destinations[signal].url,
185 + authorization: "Bearer axiom-ingest-token",
186 + dataset: destinations[signal].dataset,
187 + body,
188 + })),
189 + );
190 + }),
191 + );
192 +
193 + it.effect("passes Axiom's status and Retry-After back for Observe's retries", () =>
194 + Effect.gen(function* () {
195 + const harness = makeHarness({
196 + execute: (request) =>
197 + Effect.succeed(
198 + HttpClientResponse.fromWeb(
199 + request,
200 + new Response('{"message":"slow down"}', {
201 + status: 429,
202 + headers: { "content-type": "application/json", "retry-after": "30" },
203 + }),
204 + ),
205 + ),
206 + });
207 + const response = yield* harness.send(ingest("metrics", "{}"));
208 + expect(response.status).toBe(429);
209 + expect(response.headers["retry-after"]).toBe("30");
210 + }),
211 + );
212 +
213 + it.effect("symbolicates JS exceptions with the OTA update's map, else the build's", () =>
214 + Effect.gen(function* () {
215 + const harness = makeHarness({
216 + objects: {
217 + [`android/ota/${UPDATE_ID}.map`]: SOURCE_MAP,
218 + [`ios/build/${BUILD_ID}.map`]: SOURCE_MAP,
219 + },
220 + });
221 + const body = encodeJson({
222 + resourceLogs: [
223 + exceptionResource([
224 + str("os.type", "linux"),
225 + str("expo.app.update_id", UPDATE_ID),
226 + str("expo.eas_build.id", BUILD_ID),
227 + ]),
228 + exceptionResource([str("os.type", "darwin"), str("expo.eas_build.id", BUILD_ID)]),
229 + ],
230 + });
231 + expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
232 + const symbolicated = (key: string) => ({
233 + "expo.error.source": "global",
234 + "exception.stacktrace": "Error: x\n at render (src/App.tsx:1:3)",
235 + "relay.symbolication.source_map": key,
236 + "relay.symbolication.status": "symbolicated",
237 + "relay.symbolication.raw_stacktrace": STACK,
238 + });
239 + expect(forwardedExceptions(harness.sent[0]!)).toEqual([
240 + symbolicated(`android/ota/${UPDATE_ID}.map`),
241 + symbolicated(`ios/build/${BUILD_ID}.map`),
242 + ]);
243 + }),
244 + );
245 +
246 + it.effect("forwards the stack untouched when the OTA update's map is missing", () =>
247 + Effect.gen(function* () {
248 + // The build's map describes the embedded bundle, not the update that threw.
249 + const harness = makeHarness({
250 + objects: { [`android/build/${BUILD_ID}.map`]: SOURCE_MAP },
251 + });
252 + const body = encodeJson({
253 + resourceLogs: [
254 + exceptionResource([
255 + str("os.type", "linux"),
256 + str("expo.app.update_id", UPDATE_ID),
257 + str("expo.eas_build.id", BUILD_ID),
258 + ]),
259 + ],
260 + });
261 + expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
262 + expect(forwardedExceptions(harness.sent[0]!)).toEqual([
263 + {
264 + "expo.error.source": "global",
265 + "exception.stacktrace": STACK,
266 + "relay.symbolication.status": "source_map_missing",
267 + },
268 + ]);
269 + }),
270 + );
271 +
272 + it.effect("says why a stack stayed minified, without the iOS install path", () =>
273 + Effect.gen(function* () {
274 + const harness = makeHarness({ objects: { [`ios/build/${BUILD_ID}.map`]: "{}" } });
275 + const iosStack =
276 + "Error: x\n at render (address at /private/var/containers/Bundle/Application/" +
277 + "0C1D/T3Code.app/main.jsbundle:1:12)";
278 + const record = (stack: string) => ({
279 + scopeLogs: [{ logRecords: [{ attributes: [str("exception.stacktrace", stack)] }] }],
280 + });
281 + const body = encodeJson({
282 + resourceLogs: [
283 + { resource: { attributes: [str("os.type", "linux")] }, ...record(STACK) },
284 + {
285 + resource: {
286 + attributes: [str("os.type", "darwin"), str("expo.eas_build.id", BUILD_ID)],
287 + },
288 + ...record(iosStack),
289 + },
290 + ],
291 + });
292 + expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
293 + expect(forwardedExceptions(harness.sent[0]!)).toEqual([
294 + { "exception.stacktrace": STACK, "relay.symbolication.status": "no_bundle_id" },
295 + {
296 + "exception.stacktrace": "Error: x\n at render (address at main.jsbundle:1:12)",
297 + "relay.symbolication.status": "source_map_invalid",
298 + },
299 + ]);
300 + }),
301 + );
302 +
303 + it.effect("marks the native abort that follows a fatal JS error in the same session", () =>
304 + Effect.gen(function* () {
305 + const harness = makeHarness();
306 + const fatal = { key: "expo.error.is_fatal", value: { boolValue: true } };
307 + const crash = (session: string, source: string) => ({
308 + attributes: [str("expo.error.source", source), str("session.id", session), fatal],
309 + });
310 + const body = encodeJson({
311 + resourceLogs: [
312 + {
313 + resource: { attributes: [str("os.type", "darwin")] },
314 + scopeLogs: [
315 + {
316 + logRecords: [
317 + crash("a", "global"),
318 + crash("a", "nativeCrash"),
319 + crash("b", "nativeCrash"),
320 + ],
321 + },
322 + ],
323 + },
324 + ],
325 + });
326 + expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
327 + const records = (
328 + decodeJson(sentText(harness.sent[0]!)) as {
329 + resourceLogs: Array<{
330 + scopeLogs: Array<{ logRecords: Array<{ attributes: Array<{ key: string }> }> }>;
331 + }>;
332 + }
333 + ).resourceLogs[0]!.scopeLogs[0]!.logRecords;
334 + expect(
335 + records.map((record) =>
336 + record.attributes.some((attribute) => attribute.key === "relay.crash.js_fatal"),
337 + ),
338 + ).toEqual([false, true, false]);
339 + }),
340 + );
341 +
342 + it.effect("answers 413 for a body over the cap without forwarding it", () =>
343 + Effect.gen(function* () {
344 + const harness = makeHarness();
345 + const tooLarge = "x".repeat(MobileTelemetry.MOBILE_TELEMETRY_MAX_BODY_BYTES + 1);
346 + expect((yield* harness.send(ingest("logs", tooLarge))).status).toBe(413);
347 + expect(harness.sent).toEqual([]);
348 + }),
349 + );
350 +
351 + it.effect("decodes one source map at a time", () =>
352 + Effect.gen(function* () {
353 + const releaseFirst = yield* Deferred.make<void>();
354 + const firstStarted = yield* Deferred.make<void>();
355 + let firstDone = false;
356 + let secondSawFirstDone: boolean | undefined;
357 + const sourceMaps = yield* MobileTelemetry.SourceMaps.pipe(
358 + Effect.provide(MobileTelemetry.layerSourceMaps),
359 + Effect.provideService(MobileTelemetry.SourceMapObjects, {
360 + get: (key) =>
361 + key === "first"
362 + ? Deferred.succeed(firstStarted, undefined).pipe(
363 + Effect.andThen(Deferred.await(releaseFirst)),
364 + Effect.as({ size: SOURCE_MAP.length, text: Effect.succeed(SOURCE_MAP) }),
365 + Effect.ensuring(Effect.sync(() => (firstDone = true))),
366 + )
367 + : Effect.sync(() => {
368 + secondSawFirstDone = firstDone;
369 + return { size: SOURCE_MAP.length, text: Effect.succeed(SOURCE_MAP) };
370 + }),
371 + }),
372 + );
373 + const first = yield* Effect.forkChild(sourceMaps.find("first"));
374 + yield* Deferred.await(firstStarted);
375 + const second = yield* Effect.forkChild(sourceMaps.find("second"), { startImmediately: true });
376 + yield* Deferred.succeed(releaseFirst, undefined);
377 + yield* Fiber.join(first);
378 + expect(yield* Fiber.join(second)).toMatchObject({ key: "second" });
379 + expect(secondSawFirstDone).toBe(true);
380 + }),
381 + );
382 +});
infra/relay/src/mobileTelemetry/MobileTelemetry.ts new
+339
@@ -0,0 +1,339 @@
1 +import * as Cache from "effect/Cache";
2 +import * as Context from "effect/Context";
3 +import * as Duration from "effect/Duration";
4 +import * as Effect from "effect/Effect";
5 +import * as Exit from "effect/Exit";
6 +import * as Layer from "effect/Layer";
7 +import * as Option from "effect/Option";
8 +import * as Redacted from "effect/Redacted";
9 +import * as Result from "effect/Result";
10 +import * as Schema from "effect/Schema";
11 +import * as Semaphore from "effect/Semaphore";
12 +import * as Stream from "effect/Stream";
13 +import * as HttpClient from "effect/http/HttpClient";
14 +import * as HttpClientRequest from "effect/http/HttpClientRequest";
15 +import type * as HttpServerRequest from "effect/http/HttpServerRequest";
16 +import * as HttpServerResponse from "effect/http/HttpServerResponse";
17 +import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
18 +import { RelayApi, type RelayMobileTelemetrySignal } from "@t3tools/contracts/relay";
19 +
20 +import {
21 + applySymbolication,
22 + bundleSourceMapKey,
23 + collectJsExceptions,
24 + markJsFatalCrashes,
25 + type SourceMapMiss,
26 +} from "./otlpLogs.ts";
27 +import { decodeSourceMap, type CompactSourceMap } from "./sourceMap.ts";
28 +
29 +/** Observe splits batches the endpoint answers 413 for, so this only bounds one request. */
30 +export const MOBILE_TELEMETRY_MAX_BODY_BYTES = 2 * 1024 * 1024;
31 +/** Per client IP. An app flushes a few batches per launch and background. */
32 +export const MOBILE_TELEMETRY_RATE_LIMIT = { limit: 120, periodSeconds: 60 } as const;
33 +/** Inside the relay's 9 s request deadline, so a slow Axiom answers as `upstream_timeout`. */
34 +const UPSTREAM_TIMEOUT = Duration.seconds(7);
35 +const MAX_UPSTREAM_RESPONSE_BYTES = 64 * 1024;
36 +/** A decoded production map is about 20 MB; two fit an isolate and cover iOS and Android. */
37 +const SOURCE_MAP_CACHE_CAPACITY = 2;
38 +/** Our iOS map is about 12 MB once uploads drop what the relay never reads; larger is skipped unread. */
39 +const MAX_SOURCE_MAP_BYTES = 16 * 1024 * 1024;
40 +/** A request can carry thousands of frames, each rewritten to a longer source path. */
41 +const MAX_SYMBOLICATION_GROWTH = 1024 * 1024;
42 +/** One app names one bundle, or two across an OTA update; more would evict cached maps. */
43 +const MAX_SOURCE_MAPS_PER_REQUEST = 2;
44 +/** A map can land in R2 a moment after its update ships, so a miss is retried. */
45 +const MISSING_SOURCE_MAP_TTL = Duration.minutes(5);
46 +const FAILED_SOURCE_MAP_TTL = Duration.seconds(30);
47 +
48 +const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown);
49 +const decodeJson = Schema.decodeUnknownOption(UnknownFromJsonString);
50 +const encodeJson = Schema.encodeSync(UnknownFromJsonString);
51 +const parseJson = (text: string): unknown => Option.getOrUndefined(decodeJson(text));
52 +
53 +interface AxiomDestination {
54 + /** Axiom's OTLP/HTTP endpoint for the signal, such as `.../v1/logs`. */
55 + readonly url: string;
56 + readonly dataset: string;
57 +}
58 +
59 +export class MobileTelemetrySettings extends Context.Service<
60 + MobileTelemetrySettings,
61 + {
62 + readonly axiomToken: Redacted.Redacted<string>;
63 + readonly destinations: Readonly<Record<RelayMobileTelemetrySignal, AxiomDestination>>;
64 + }
65 +>()("t3code-relay/mobileTelemetry/MobileTelemetry/MobileTelemetrySettings") {}
66 +
67 +export class MobileTelemetryRateLimiter extends Context.Service<
68 + MobileTelemetryRateLimiter,
69 + { readonly allow: (key: string) => Effect.Effect<boolean> }
70 +>()("t3code-relay/mobileTelemetry/MobileTelemetry/MobileTelemetryRateLimiter") {}
71 +
72 +export class SourceMapObjectError extends Schema.TaggedError<SourceMapObjectError>()(
73 + "SourceMapObjectError",
74 + { key: Schema.String, cause: Schema.Defect() },
75 +) {}
76 +
77 +/** An object in the source maps bucket, read only once its size is known to fit. */
78 +interface StoredSourceMap {
79 + readonly size: number;
80 + readonly text: Effect.Effect<string, SourceMapObjectError>;
81 +}
82 +
83 +interface FoundSourceMap {
84 + readonly key: string;
85 + readonly map: CompactSourceMap;
86 +}
87 +
88 +export class SourceMaps extends Context.Service<
89 + SourceMaps,
90 + {
91 + readonly find: (key: string) => Effect.Effect<FoundSourceMap | SourceMapMiss>;
92 + }
93 +>()("t3code-relay/mobileTelemetry/MobileTelemetry/SourceMaps") {}
94 +
95 +export class SourceMapObjects extends Context.Service<
96 + SourceMapObjects,
97 + {
98 + readonly get: (key: string) => Effect.Effect<StoredSourceMap | undefined, SourceMapObjectError>;
99 + }
100 +>()("t3code-relay/mobileTelemetry/MobileTelemetry/SourceMapObjects") {}
101 +
102 +const makeSourceMaps = Effect.gen(function* () {
103 + const objects = yield* SourceMapObjects;
104 + // The cache only joins lookups of one key; a decode holds ~40 MB, so different keys take turns.
105 + const decoding = yield* Semaphore.make(1);
106 + const cache = yield* Cache.makeWith(
107 + (key: string) =>
108 + Effect.gen(function* () {
109 + const stored = yield* objects.get(key);
110 + if (stored === undefined) return "source_map_missing" as const;
111 + if (stored.size > MAX_SOURCE_MAP_BYTES) {
112 + yield* Effect.logWarning("Stored mobile source map is too large", {
113 + key,
114 + size: stored.size,
115 + });
116 + return "source_map_too_large" as const;
117 + }
118 + const map = decodeSourceMap(parseJson(yield* stored.text));
119 + if (typeof map !== "string") return { key, map } satisfies FoundSourceMap;
120 + yield* Effect.logWarning("Stored mobile source map could not be decoded", { key, map });
121 + return map === "too_large"
122 + ? ("source_map_too_large" as const)
123 + : ("source_map_invalid" as const);
124 + }).pipe(Semaphore.withPermit(decoding)),
125 + {
126 + capacity: SOURCE_MAP_CACHE_CAPACITY,
127 + timeToLive: (exit) =>
128 + Exit.isFailure(exit)
129 + ? FAILED_SOURCE_MAP_TTL
130 + : typeof exit.value === "string"
131 + ? MISSING_SOURCE_MAP_TTL
132 + : Duration.infinity,
133 + },
134 + );
135 + return SourceMaps.of({
136 + find: Effect.fn("MobileTelemetry.findSourceMap")(function* (key: string) {
137 + return yield* Cache.get(cache, key).pipe(
138 + Effect.catch((error) =>
139 + Effect.logWarning("Mobile source map lookup failed", { key: error.key }).pipe(
140 + Effect.as("source_map_unavailable" as const),
141 + ),
142 + ),
143 + );
144 + }),
145 + });
146 +});
147 +
148 +/** Build once per Worker isolate, not per request: decoding a production map costs ~150 ms. */
149 +export const layerSourceMaps = Layer.effect(SourceMaps, makeSourceMaps);
150 +
151 +export class MobileTelemetry extends Context.Service<
152 + MobileTelemetry,
153 + {
154 + readonly handle: (
155 + request: HttpServerRequest.HttpServerRequest,
156 + params: { readonly projectId: string; readonly signal: RelayMobileTelemetrySignal },
157 + ) => Effect.Effect<HttpServerResponse.HttpServerResponse>;
158 + }
159 +>()("t3code-relay/mobileTelemetry/MobileTelemetry") {}
160 +
161 +class BodyTooLarge extends Schema.TaggedError<BodyTooLarge>()("BodyTooLarge", {}) {}
162 +
163 +const errorResponse = (status: number, error: string, headers?: Record<string, string>) =>
164 + HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) });
165 +
166 +const readCapped = <E>(stream: Stream.Stream<Uint8Array, E>, maxBytes: number) =>
167 + Effect.suspend(() => {
168 + const chunks: Array<Uint8Array> = [];
169 + let total = 0;
170 + return stream.pipe(
171 + Stream.runForEach((chunk) => {
172 + total += chunk.length;
173 + if (total > maxBytes) return Effect.fail(new BodyTooLarge());
174 + chunks.push(chunk);
175 + return Effect.void;
176 + }),
177 + Effect.map(() => {
178 + const body = new Uint8Array(total);
179 + let offset = 0;
180 + for (const chunk of chunks) {
181 + body.set(chunk, offset);
182 + offset += chunk.length;
183 + }
184 + return body;
185 + }),
186 + );
187 + });
188 +
189 +const hasNoBody = (request: HttpServerRequest.HttpServerRequest) =>
190 + request.source instanceof Request && request.source.body === null;
191 +
192 +const make = Effect.gen(function* () {
193 + const settings = yield* MobileTelemetrySettings;
194 + const rateLimiter = yield* MobileTelemetryRateLimiter;
195 + const sourceMaps = yield* SourceMaps;
196 + const httpClient = yield* HttpClient.HttpClient;
197 +
198 + const symbolicateLogs = Effect.fn("relay.mobile_telemetry.symbolicate")(function* (
199 + body: Uint8Array,
200 + ) {
201 + const payload = parseJson(new TextDecoder().decode(body));
202 + const exceptions = collectJsExceptions(payload);
203 + const jsFatalCrashes = markJsFatalCrashes(payload);
204 + if (exceptions.length === 0 && jsFatalCrashes === 0) return undefined;
205 + let symbolicated = 0;
206 + let growthLeft = MAX_SYMBOLICATION_GROWTH;
207 + const keys = new Set<string>();
208 + for (const exception of exceptions) {
209 + const key = bundleSourceMapKey(exception.identity);
210 + if (key !== undefined && keys.size < MAX_SOURCE_MAPS_PER_REQUEST) keys.add(key);
211 + const outcome = applySymbolication(
212 + exception,
213 + key === undefined
214 + ? "no_bundle_id"
215 + : keys.has(key)
216 + ? yield* sourceMaps.find(key)
217 + : "source_map_limit",
218 + growthLeft,
219 + );
220 + growthLeft -= outcome.growth;
221 + if (outcome.symbolicated) symbolicated += 1;
222 + }
223 + yield* Effect.annotateCurrentSpan({
224 + "relay.mobile_telemetry.exceptions": exceptions.length,
225 + "relay.mobile_telemetry.symbolicated": symbolicated,
226 + "relay.mobile_telemetry.js_fatal_crashes": jsFatalCrashes,
227 + });
228 + return new TextEncoder().encode(encodeJson(payload));
229 + });
230 +
231 + const forward = Effect.fn("relay.mobile_telemetry.forward")(function* (
232 + destination: AxiomDestination,
233 + body: Uint8Array,
234 + ) {
235 + const response = yield* httpClient.execute(
236 + HttpClientRequest.post(destination.url, {
237 + headers: {
238 + authorization: `Bearer ${Redacted.value(settings.axiomToken)}`,
239 + "x-axiom-dataset": destination.dataset,
240 + },
241 + }).pipe(HttpClientRequest.bodyUint8Array(body, "application/json")),
242 + );
243 + const responseBody = yield* readCapped(response.stream, MAX_UPSTREAM_RESPONSE_BYTES).pipe(
244 + Effect.catchIf(
245 + (error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",
246 + () => Effect.succeed(new Uint8Array(0)),
247 + ),
248 + );
249 + return { response, body: responseBody };
250 + }, Effect.timeoutOption(UPSTREAM_TIMEOUT));
251 +
252 + const handle = Effect.fn("relay.mobile_telemetry.ingest")(function* (
253 + request: HttpServerRequest.HttpServerRequest,
254 + {
255 + projectId,
256 + signal,
257 + }: { readonly projectId: string; readonly signal: RelayMobileTelemetrySignal },
258 + ) {
259 + const outcome = (value: string) =>
260 + Effect.annotateCurrentSpan({ "relay.mobile_telemetry.outcome": value });
261 + yield* Effect.annotateCurrentSpan({
262 + "relay.mobile_telemetry.project_id": projectId,
263 + "relay.mobile_telemetry.signal": signal,
264 + });
265 +
266 + const clientIp = request.headers["cf-connecting-ip"] ?? "unknown";
267 + if (!(yield* rateLimiter.allow(clientIp))) {
268 + yield* outcome("rate_limited");
269 + return errorResponse(429, "rate_limited", {
270 + "retry-after": String(MOBILE_TELEMETRY_RATE_LIMIT.periodSeconds),
271 + });
272 + }
273 +
274 + const read = yield* (
275 + hasNoBody(request)
276 + ? Effect.succeed(new Uint8Array(0))
277 + : readCapped(request.stream, MOBILE_TELEMETRY_MAX_BODY_BYTES)
278 + ).pipe(Effect.result);
279 + if (Result.isFailure(read)) {
280 + const tooLarge = read.failure._tag === "BodyTooLarge";
281 + yield* outcome(tooLarge ? "body_too_large" : "body_unreadable");
282 + return errorResponse(tooLarge ? 413 : 400, tooLarge ? "body_too_large" : "invalid_body");
283 + }
284 + yield* Effect.annotateCurrentSpan({ "relay.mobile_telemetry.bytes": read.success.length });
285 +
286 + // Forwarding matters more than symbolication: on any defect the payload goes through as sent.
287 + const body =
288 + signal === "logs"
289 + ? ((yield* symbolicateLogs(read.success).pipe(
290 + Effect.catchCause((cause) =>
291 + Effect.logWarning("Mobile stack symbolication failed", { cause }).pipe(
292 + Effect.as(undefined),
293 + ),
294 + ),
295 + )) ?? read.success)
296 + : read.success;
297 +
298 + const upstream = yield* forward(settings.destinations[signal], body).pipe(Effect.result);
299 + if (Result.isFailure(upstream)) {
300 + yield* outcome("upstream_unavailable");
301 + return errorResponse(502, "upstream_unavailable");
302 + }
303 + if (Option.isNone(upstream.success)) {
304 + yield* outcome("upstream_timeout");
305 + return errorResponse(504, "upstream_timeout");
306 + }
307 + const { response, body: responseBody } = upstream.success.value;
308 + yield* outcome("forwarded");
309 + yield* Effect.annotateCurrentSpan({
310 + "relay.mobile_telemetry.upstream_status": response.status,
311 + });
312 + // Observe's retry and partial-success handling reads only these from Axiom's answer.
313 + const contentType = response.headers["content-type"];
314 + const retryAfter = response.headers["retry-after"];
315 + const headers = retryAfter === undefined ? undefined : { "retry-after": retryAfter };
316 + if (responseBody.length === 0) {
317 + return HttpServerResponse.empty({ status: response.status, ...(headers ? { headers } : {}) });
318 + }
319 + return HttpServerResponse.uint8Array(responseBody, {
320 + status: response.status,
321 + ...(headers ? { headers } : {}),
322 + ...(contentType ? { contentType } : {}),
323 + });
324 + });
325 +
326 + return MobileTelemetry.of({ handle });
327 +});
328 +
329 +export const layer = Layer.effect(MobileTelemetry, make);
330 +
331 +/** Raw so the body is read once, capped. */
332 +export const layerApi = HttpApiBuilder.group(
333 + RelayApi,
334 + "mobileTelemetry",
335 + Effect.fnUntraced(function* (handlers) {
336 + const telemetry = yield* MobileTelemetry;
337 + return handlers.handleRaw("ingest", ({ request, params }) => telemetry.handle(request, params));
338 + }),
339 +);
infra/relay/src/mobileTelemetry/monitoring.ts new
+297
@@ -0,0 +1,297 @@
1 +import * as Alchemy from "alchemy";
2 +import * as Axiom from "@alchemy.run/axiom";
3 +import * as Output from "alchemy/Output";
4 +import * as Effect from "effect/Effect";
5 +import * as Option from "effect/Option";
6 +
7 +import { relayResourceNameForStage } from "../deploymentConfig.ts";
8 +import { notifierConfig } from "../monitoring.ts";
9 +import { isPreviewStage } from "../previewStages.ts";
10 +
11 +const RUNBOOK_URL =
12 + "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#mobile-crashes-and-errors";
13 +
14 +// A new dataset has none of these fields until a report carries them, and an
15 +// unknown field fails the whole query, so every one has a default.
16 +const field = (name: string, fallback: string) => `column_ifexists('${name}', ${fallback})`;
17 +const reports = (dataset: string) =>
18 + [
19 + `['${dataset}']`,
20 + `| extend source = tostring(${field("attributes.expo.error.source", "''")}),`,
21 + ` fatal = tobool(${field("attributes.expo.error.is_fatal", "false")}),`,
22 + ` js_fatal = tobool(${field("attributes.relay.crash.js_fatal", "false")}),`,
23 + ` platform = tostring(${field("resource.os.name", "''")}),`,
24 + ` install = tostring(${field("resource.expo.eas_client.id", "''")}),`,
25 + ` error_type = tostring(${field("attributes.exception.type", "''")}),`,
26 + ` message = substring(tostring(${field("attributes.exception.message", "''")}), 0, 140),`,
27 + ` version = strcat(tostring(${field("service.version", "''")}), ' (', tostring(${field("resource.expo.app.build_number", "''")}), ')')`,
28 + `| where isnotempty(source) and isnotempty(platform)`,
29 + // A fatal JS error also ends in a native abort; counting both would double every JS crash.
30 + `| extend kind = case(source == 'nativeCrash' and js_fatal, 'abort after JS crash', source == 'nativeCrash', 'native crash', fatal, 'JS crash', 'handled error')`,
31 + ].join("\n");
32 +const crashes = (dataset: string) =>
33 + `${reports(dataset)}\n| where kind in ('JS crash', 'native crash')`;
34 +const handledErrors = (dataset: string) => `${reports(dataset)}\n| where kind == 'handled error'`;
35 +const ingest = (traces: string) =>
36 + `['${traces}']\n| where name == 'relay.mobile_telemetry.ingest'\n| extend signal = tostring(['attributes.custom']['relay.mobile_telemetry.signal']), outcome = tostring(['attributes.custom']['relay.mobile_telemetry.outcome'])`;
37 +
38 +type Severity = "page" | "warn";
39 +
40 +interface Datasets {
41 + readonly logs: string;
42 + readonly traces: string;
43 +}
44 +
45 +interface MobileMonitor {
46 + readonly id: string;
47 + readonly severity: Severity;
48 + readonly name: string;
49 + /** Why it fires. The runbook link is appended. */
50 + readonly description: string;
51 + readonly query: (datasets: Datasets) => string;
52 + readonly props: Pick<
53 + Axiom.MonitorProps,
54 + | "type"
55 + | "operator"
56 + | "threshold"
57 + | "rangeMinutes"
58 + | "intervalMinutes"
59 + | "compareDays"
60 + | "tolerance"
61 + | "alertOnNoData"
62 + | "triggerFromNRuns"
63 + >;
64 +}
65 +
66 +// No production baseline exists yet: the spike monitor learns its own, and the
67 +// thresholds below are starting points to tune after two weeks of data.
68 +const mobileMonitors: ReadonlyArray<MobileMonitor> = [
69 + {
70 + id: "MobileCrashSpikeMonitor",
71 + severity: "page",
72 + name: "T3 Code mobile: crash spike",
73 + description:
74 + "Crashes (JS and native, each counted once) per 15 min well above the same time over the last 7 days. Usually a bad OTA update or store build: compare app versions on the dashboard.",
75 + query: ({ logs }) => `${crashes(logs)}\n| summarize crashes = count() by bin_auto(_time)`,
76 + props: {
77 + type: "AnomalyDetection",
78 + operator: "Above",
79 + compareDays: 7,
80 + tolerance: 50,
81 + rangeMinutes: 15,
82 + intervalMinutes: 15,
83 + },
84 + },
85 + {
86 + id: "MobileCrashSpreadMonitor",
87 + severity: "page",
88 + name: "T3 Code mobile: one crash hitting many installs",
89 + description:
90 + "The most widespread crash reached this many installs in an hour. Its message and versions are at the top of the dashboard's Top crashes table.",
91 + query: ({ logs }) =>
92 + `${crashes(logs)}\n| summarize installs = dcount(install) by platform, error_type, message\n| summarize worst = max(installs)`,
93 + props: {
94 + type: "Threshold",
95 + operator: "AboveOrEqual",
96 + threshold: 25,
97 + rangeMinutes: 60,
98 + intervalMinutes: 15,
99 + },
100 + },
101 + {
102 + id: "MobileTelemetryStoppedMonitor",
103 + severity: "warn",
104 + name: "T3 Code mobile: reports stopped",
105 + description:
106 + "No startup metrics forwarded in an hour. Every launch sends some, so the relay route, the Axiom token, or the app's endpoint is broken, and crashes go unseen too.",
107 + query: ({ traces }) =>
108 + `${ingest(traces)}\n| where signal == 'metrics' and outcome == 'forwarded'\n| summarize forwarded = count()`,
109 + props: {
110 + type: "Threshold",
111 + operator: "Below",
112 + threshold: 1,
113 + rangeMinutes: 60,
114 + intervalMinutes: 15,
115 + alertOnNoData: true,
116 + triggerFromNRuns: 2,
117 + },
118 + },
119 + {
120 + id: "MobileSymbolicationFailingMonitor",
121 + severity: "warn",
122 + name: "T3 Code mobile: crash stacks not symbolicated",
123 + description:
124 + "Share of release JS exceptions over 6 h whose stack stayed minified. Map uploads are failing or the relay cannot read them; relay.symbolication.status says which.",
125 + query: ({ logs }) =>
126 + [
127 + `['${logs}']`,
128 + `| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})`,
129 + `| where isnotempty(status) and status !in ('no_bundle_id', 'no_bundle_frames')`,
130 + `| summarize total = count(), minified = countif(status != 'symbolicated')`,
131 + `| project minified_pct = iff(total >= 20, 100.0 * minified / total, 0.0)`,
132 + ].join("\n"),
133 + props: {
134 + type: "Threshold",
135 + operator: "Above",
136 + threshold: 50,
137 + rangeMinutes: 360,
138 + intervalMinutes: 60,
139 + },
140 + },
141 +];
142 +
143 +interface QueryChart {
144 + readonly id: string;
145 + readonly name: string;
146 + readonly type: "TimeSeries" | "Table" | "Statistic";
147 + readonly query: { readonly apl: string };
148 + /** Grid columns out of 12. */
149 + readonly width: 4 | 6 | 12;
150 +}
151 +
152 +const chart = (
153 + type: QueryChart["type"],
154 + width: QueryChart["width"],
155 + id: string,
156 + name: string,
157 + apl: string,
158 +): QueryChart => ({ id, name, type, width, query: { apl } });
159 +
160 +const mobileDashboardCharts = ({ logs, traces }: Datasets): ReadonlyArray<QueryChart> => [
161 + chart("Statistic", 4, "crashes", "Crashes", `${crashes(logs)}\n| summarize count()`),
162 + chart(
163 + "Statistic",
164 + 4,
165 + "crashed-installs",
166 + "Installs that crashed",
167 + `${crashes(logs)}\n| summarize dcount(install)`,
168 + ),
169 + chart(
170 + "Statistic",
171 + 4,
172 + "handled-errors",
173 + "Handled errors",
174 + `${handledErrors(logs)}\n| summarize count()`,
175 + ),
176 + chart(
177 + "TimeSeries",
178 + 6,
179 + "crashes-over-time",
180 + "Crashes by platform",
181 + `${crashes(logs)}\n| summarize count() by bin_auto(_time), series = strcat(platform, ' ', kind)`,
182 + ),
183 + chart(
184 + "TimeSeries",
185 + 6,
186 + "handled-over-time",
187 + "Handled errors by platform",
188 + `${handledErrors(logs)}\n| summarize count() by bin_auto(_time), platform`,
189 + ),
190 + chart(
191 + "Table",
192 + 12,
193 + "top-crashes",
194 + "Top crashes",
195 + `${crashes(logs)}\n| summarize installs = dcount(install), crashes = count(), first_seen = min(_time), last_seen = max(_time), versions = make_set(version, 3) by platform, kind, error_type, message\n| order by installs desc, crashes desc\n| take 15`,
196 + ),
197 + chart(
198 + "Table",
199 + 12,
200 + "top-handled",
201 + "Top handled errors",
202 + `${handledErrors(logs)}\n| summarize installs = dcount(install), errors = count(), last_seen = max(_time), versions = make_set(version, 3) by platform, error_type, message\n| order by installs desc, errors desc\n| take 15`,
203 + ),
204 + chart(
205 + "Table",
206 + 6,
207 + "by-version",
208 + "Crashes by app version",
209 + `${crashes(logs)}\n| summarize crashes = count(), installs = dcount(install) by platform, version\n| order by crashes desc\n| take 10`,
210 + ),
211 + chart(
212 + "TimeSeries",
213 + 6,
214 + "symbolication",
215 + "JS stacks by symbolication outcome",
216 + `['${logs}']\n| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})\n| where isnotempty(status)\n| summarize count() by bin_auto(_time), status`,
217 + ),
218 + chart(
219 + "TimeSeries",
220 + 12,
221 + "ingest",
222 + "Reports reaching the relay",
223 + `${ingest(traces)}\n| summarize count() by bin_auto(_time), series = strcat(signal, ' ', outcome)`,
224 + ),
225 +];
226 +
227 +/** Lays charts left to right, wrapping at Axiom's 12-column grid. */
228 +function layout(charts: ReadonlyArray<QueryChart>) {
229 + let x = 0;
230 + let y = 0;
231 + let rowHeight = 0;
232 + return charts.map((chart) => {
233 + const h = chart.type === "Statistic" ? 3 : 6;
234 + if (x + chart.width > 12) {
235 + x = 0;
236 + y += rowHeight;
237 + rowHeight = 0;
238 + }
239 + const cell = { i: chart.id, x, y, w: chart.width, h };
240 + x += chart.width;
241 + rowHeight = Math.max(rowHeight, h);
242 + return cell;
243 + });
244 +}
245 +
246 +/**
247 + * Monitors follow the relay's: a deployment opts in by naming its paging notifier. The
248 + * dashboard also comes up on PR previews, so a change to mobile telemetry can be checked
249 + * there; the preview's teardown removes it.
250 + */
251 +export const MobileMonitoring = Effect.fn("MobileMonitoring")(function* (
252 + logs: Axiom.Dataset,
253 + traces: Axiom.Dataset,
254 +) {
255 + const { stage } = yield* Alchemy.Stack;
256 + const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
257 + if (Option.isNone(page) && !isPreviewStage(stage)) return;
258 + const datasets = Output.map(Output.all(logs.name, traces.name), ([logs, traces]) => ({
259 + logs,
260 + traces,
261 + }));
262 +
263 + if (Option.isSome(page)) {
264 + const notifiers: Record<Severity, Option.Option<string>> = {
265 + page,
266 + warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
267 + };
268 + for (const monitor of mobileMonitors) {
269 + yield* Axiom.Monitor(monitor.id, {
270 + ...monitor.props,
271 + name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
272 + description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
273 + aplQuery: Output.map(datasets, monitor.query),
274 + triggerFromNRuns: monitor.props.triggerFromNRuns ?? 1,
275 + alertOnNoData: monitor.props.alertOnNoData ?? false,
276 + notifierIds: Option.toArray(notifiers[monitor.severity]),
277 + });
278 + }
279 + }
280 +
281 + yield* Axiom.Dashboard("MobileHealthDashboard", {
282 + dashboard: Output.map(datasets, (names) => {
283 + const charts = mobileDashboardCharts(names);
284 + return {
285 + name: relayResourceNameForStage("t3-code-mobile-health", stage),
286 + owner: "X-AXIOM-EVERYONE",
287 + description: `T3 Code mobile app crashes and errors. Crashes count JS and native once each. Runbook: ${RUNBOOK_URL}`,
288 + refreshTime: 300,
289 + schemaVersion: 2,
290 + timeWindowStart: "qr-now-7d",
291 + timeWindowEnd: "qr-now",
292 + charts: charts.map(({ width: _width, ...chart }) => chart),
293 + layout: layout(charts),
294 + };
295 + }),
296 + });
297 +});
infra/relay/src/mobileTelemetry/otlpLogs.ts new
+178
@@ -0,0 +1,178 @@
1 +import { mobileSourceMapKey, type MobilePlatform } from "@t3tools/shared/mobileSourceMaps";
2 +
3 +import { stripBundlePaths, symbolicateStack, type CompactSourceMap } from "./sourceMap.ts";
4 +
5 +/** The JS bundle a record came from, read from its OTLP resource. */
6 +interface BundleIdentity {
7 + readonly platform: MobilePlatform;
8 + /** Observe reports one only for an OTA launch. */
9 + readonly updateId: string | undefined;
10 + readonly easBuildId: string | undefined;
11 +}
12 +
13 +const BUNDLE_ID_PATTERN = /^[A-Za-z0-9-]{1,64}$/u;
14 +
15 +/**
16 + * The map for the bundle that threw. A build map never stands in for a
17 + * missing OTA map: its offsets belong to a different bundle.
18 + */
19 +export function bundleSourceMapKey(identity: BundleIdentity): string | undefined {
20 + if (identity.updateId !== undefined) {
21 + return BUNDLE_ID_PATTERN.test(identity.updateId)
22 + ? mobileSourceMapKey(identity.platform, "ota", identity.updateId)
23 + : undefined;
24 + }
25 + if (identity.easBuildId !== undefined && BUNDLE_ID_PATTERN.test(identity.easBuildId)) {
26 + return mobileSourceMapKey(identity.platform, "build", identity.easBuildId);
27 + }
28 + return undefined;
29 +}
30 +
31 +interface JsException {
32 + readonly identity: BundleIdentity;
33 + readonly stack: string;
34 + /** The record's attribute list, updated in place by `applySymbolication`. */
35 + readonly attributes: Array<unknown>;
36 +}
37 +
38 +const isRecord = (value: unknown): value is Record<string, unknown> =>
39 + typeof value === "object" && value !== null && !Array.isArray(value);
40 +
41 +const arrayAt = (value: unknown, key: string): Array<unknown> | undefined => {
42 + const child = isRecord(value) ? value[key] : undefined;
43 + return Array.isArray(child) ? child : undefined;
44 +};
45 +
46 +const stringAttribute = (attributes: ReadonlyArray<unknown>, key: string): string | undefined => {
47 + for (const attribute of attributes) {
48 + if (!isRecord(attribute) || attribute.key !== key) continue;
49 + const value = isRecord(attribute.value) ? attribute.value.stringValue : undefined;
50 + return typeof value === "string" ? value : undefined;
51 + }
52 + return undefined;
53 +};
54 +
55 +const platformOf = (attributes: ReadonlyArray<unknown>): MobilePlatform | undefined => {
56 + const osType = stringAttribute(attributes, "os.type");
57 + if (osType === "darwin") return "ios";
58 + if (osType === "linux") return "android";
59 + return undefined;
60 +};
61 +
62 +/** JavaScript exception records with a stack; native crash frames are not in the bundle. */
63 +export function collectJsExceptions(payload: unknown): ReadonlyArray<JsException> {
64 + const exceptions: Array<JsException> = [];
65 + for (const resourceLogs of arrayAt(payload, "resourceLogs") ?? []) {
66 + const resourceAttributes =
67 + arrayAt(isRecord(resourceLogs) ? resourceLogs.resource : undefined, "attributes") ?? [];
68 + const platform = platformOf(resourceAttributes);
69 + if (platform === undefined) continue;
70 + const identity: BundleIdentity = {
71 + platform,
72 + updateId:
73 + stringAttribute(resourceAttributes, "expo.app.updates.id") ??
74 + stringAttribute(resourceAttributes, "expo.app.update_id"),
75 + easBuildId: stringAttribute(resourceAttributes, "expo.eas_build.id"),
76 + };
77 + for (const scopeLogs of arrayAt(resourceLogs, "scopeLogs") ?? []) {
78 + for (const logRecord of arrayAt(scopeLogs, "logRecords") ?? []) {
79 + const attributes = arrayAt(logRecord, "attributes");
80 + if (!attributes) continue;
81 + const stack = stringAttribute(attributes, "exception.stacktrace");
82 + if (stack === undefined) continue;
83 + if (stringAttribute(attributes, "expo.error.source") === "nativeCrash") continue;
84 + exceptions.push({ identity, stack, attributes });
85 + }
86 + }
87 + }
88 + return exceptions;
89 +}
90 +
91 +const stringValue = (key: string, value: string) => ({ key, value: { stringValue: value } });
92 +
93 +const setAttribute = (attributes: Array<unknown>, key: string, value: string) => {
94 + const index = attributes.findIndex((attribute) => isRecord(attribute) && attribute.key === key);
95 + if (index === -1) attributes.push(stringValue(key, value));
96 + else attributes[index] = stringValue(key, value);
97 +};
98 +
99 +/** Why a stack stayed minified, recorded as `relay.symbolication.status`. */
100 +export type SourceMapMiss =
101 + /** A development or local build: no update or EAS build to name a map. */
102 + | "no_bundle_id"
103 + | "source_map_missing"
104 + | "source_map_too_large"
105 + | "source_map_invalid"
106 + /** The bucket could not be read; retried shortly. */
107 + | "source_map_unavailable"
108 + /** The request named more bundles than the relay decodes for one request. */
109 + | "source_map_limit";
110 +
111 +/**
112 + * Rewrites one exception's stack, growing it by at most `maxGrowth`, and
113 + * records the outcome on the record so a stack still minified in Axiom
114 + * explains itself.
115 + */
116 +export function applySymbolication(
117 + exception: JsException,
118 + found: { readonly key: string; readonly map: CompactSourceMap } | SourceMapMiss,
119 + maxGrowth: number,
120 +): { readonly symbolicated: boolean; readonly growth: number } {
121 + const stack = stripBundlePaths(exception.stack);
122 + if (stack !== exception.stack) setAttribute(exception.attributes, "exception.stacktrace", stack);
123 + if (typeof found === "string") {
124 + setAttribute(exception.attributes, "relay.symbolication.status", found);
125 + return { symbolicated: false, growth: 0 };
126 + }
127 + const result = symbolicateStack(stack, found.map, maxGrowth);
128 + setAttribute(exception.attributes, "relay.symbolication.source_map", found.key);
129 + if (result.mappedFrames === 0) {
130 + setAttribute(exception.attributes, "relay.symbolication.status", "no_bundle_frames");
131 + return { symbolicated: false, growth: 0 };
132 + }
133 + setAttribute(exception.attributes, "relay.symbolication.status", "symbolicated");
134 + setAttribute(exception.attributes, "relay.symbolication.raw_stacktrace", stack);
135 + setAttribute(exception.attributes, "exception.stacktrace", result.stack);
136 + return { symbolicated: true, growth: result.growth };
137 +}
138 +
139 +const isFatal = (attributes: ReadonlyArray<unknown>) =>
140 + attributes.some(
141 + (attribute) =>
142 + isRecord(attribute) &&
143 + attribute.key === "expo.error.is_fatal" &&
144 + isRecord(attribute.value) &&
145 + attribute.value.boolValue === true,
146 + );
147 +
148 +/**
149 + * A fatal JS error ends in a native abort, which Observe reports as a second crash with the
150 + * same generic stack in the same session. Marks those `relay.crash.js_fatal` so they don't
151 + * bury real native crashes. Both arrive together on the next launch.
152 + */
153 +export function markJsFatalCrashes(payload: unknown): number {
154 + const records: Array<Array<unknown>> = [];
155 + const fatalSessions = new Set<string>();
156 + for (const resourceLogs of arrayAt(payload, "resourceLogs") ?? []) {
157 + for (const scopeLogs of arrayAt(resourceLogs, "scopeLogs") ?? []) {
158 + for (const logRecord of arrayAt(scopeLogs, "logRecords") ?? []) {
159 + const attributes = arrayAt(logRecord, "attributes");
160 + if (!attributes || !isFatal(attributes)) continue;
161 + const session = stringAttribute(attributes, "session.id");
162 + if (session === undefined) continue;
163 + if (stringAttribute(attributes, "expo.error.source") === "nativeCrash") {
164 + records.push(attributes);
165 + } else {
166 + fatalSessions.add(session);
167 + }
168 + }
169 + }
170 + }
171 + let marked = 0;
172 + for (const attributes of records) {
173 + if (!fatalSessions.has(stringAttribute(attributes, "session.id")!)) continue;
174 + attributes.push({ key: "relay.crash.js_fatal", value: { boolValue: true } });
175 + marked += 1;
176 + }
177 + return marked;
178 +}
infra/relay/src/mobileTelemetry/sourceMap.test.ts new
+60
@@ -0,0 +1,60 @@
1 +import { describe, expect, it } from "@effect/vitest";
2 +
3 +import { decodeSourceMap, symbolicateStack, type CompactSourceMap } from "./sourceMap.ts";
4 +
5 +// Real `hermesc -O -output-source-map` output and the stack its VM printed: a frame's column is the
6 +// map's 0-based generated column. In `t.js`, alpha throws at 2:18, beta calls it at 5:19, gamma calls
7 +// beta at 8:13, and the top level calls gamma at 10:6.
8 +const HERMES_MAP = {
9 + version: 3,
10 + sources: ["t.js"],
11 + mappings:
12 + "AAAA,iBAAA,OAAA,MAAA,KAAA,MAAA,KAAA,MASA,OAAK,IAAE,EATP,EACY,MAAK,KAAA,WAAA,IAAA,IAAf,EAEF,EACa,OAAK,OAAT,IAAP,EAEF,AACE,EAAM,OAAI,IAAI,EAAE,IAAY,MAAO,OAAF,KAAW,EAC9C;",
13 + x_hermes_function_offsets: { "0": [0, 65, 99, 121] },
14 +};
15 +const HERMES_STACK = [
16 + "Error: boom",
17 + " at alpha (address at t.hbc:1:89)",
18 + " at beta (address at t.hbc:1:108)",
19 + " at gamma (address at t.hbc:1:130)",
20 + " at global (address at t.hbc:1:59)",
21 +].join("\n");
22 +
23 +describe("symbolicateStack", () => {
24 + it("rewrites Hermes bundle frames to their original locations and keeps the rest", () => {
25 + const map = decodeSourceMap(HERMES_MAP) as CompactSourceMap;
26 + const result = symbolicateStack(
27 + `${HERMES_STACK}\n at apply (native)\n at tryCallOne (address at InternalBytecode.js:1:89)`,
28 + map,
29 + Number.POSITIVE_INFINITY,
30 + );
31 + expect(result).toEqual({
32 + stack: [
33 + "Error: boom",
34 + " at alpha (t.js:2:18)",
35 + " at beta (t.js:5:19)",
36 + " at gamma (t.js:8:13)",
37 + " at global (t.js:10:6)",
38 + " at apply (native)",
39 + " at tryCallOne (address at InternalBytecode.js:1:89)",
40 + ].join("\n"),
41 + bundleFrames: 4,
42 + mappedFrames: 4,
43 + growth: -50,
44 + });
45 + });
46 +});
47 +
48 +describe("decodeSourceMap", () => {
49 + it("fits a map as large as our iOS release map, read from an OTA bundle's path", () => {
50 + // 2.1M segments, every other one repeating its predecessor's position, like Hermes output.
51 + const mappings = `AAAA,${"CAAC,CAAA,".repeat(1_050_000)}CAAC`;
52 + const map = decodeSourceMap({ version: 3, sources: ["src/App.tsx"], mappings });
53 + const frame =
54 + " at render (address at /var/mobile/Containers/Data/Application/A1/Library/" +
55 + "Application Support/.expo-internal/bundle:1:4)";
56 + expect(symbolicateStack(frame, map as CompactSourceMap, Number.POSITIVE_INFINITY).stack).toBe(
57 + " at render (src/App.tsx:1:3)",
58 + );
59 + });
60 +});
infra/relay/src/mobileTelemetry/sourceMap.ts new
+287
@@ -0,0 +1,287 @@
1 +// A production Hermes map has over a million segments. General-purpose decoders hold
2 +// each as its own array, past a Worker isolate's memory budget; typed arrays fit.
3 +
4 +const BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
5 +const BASE64_VALUES = new Int8Array(128).fill(-1);
6 +for (let index = 0; index < BASE64.length; index += 1) {
7 + BASE64_VALUES[BASE64.charCodeAt(index)] = index;
8 +}
9 +const COMMA = 44;
10 +const SEMICOLON = 59;
11 +/** The relay caches two decoded maps in a 128 MB isolate. */
12 +const MAX_DECODED_BYTES = 32 * 1024 * 1024;
13 +
14 +export interface CompactSourceMap {
15 + readonly sources: ReadonlyArray<string>;
16 + /** Segment index where each generated line starts, plus one past the last line. */
17 + readonly lineStarts: Int32Array;
18 + readonly generatedColumns: Int32Array;
19 + /** -1 for segments that map to no source. */
20 + readonly sourceIndexes: Int32Array;
21 + readonly originalLines: Int32Array;
22 + readonly originalColumns: Int32Array;
23 +}
24 +
25 +interface OriginalPosition {
26 + readonly source: string;
27 + /** 1-based. */
28 + readonly line: number;
29 + /** 0-based, as source maps store it. */
30 + readonly column: number;
31 +}
32 +
33 +const isStringArray = (value: unknown): value is ReadonlyArray<string> =>
34 + Array.isArray(value) && value.every((item) => typeof item === "string");
35 +
36 +/**
37 + * Calls `visit` per segment in order; `sourceIndex` is -1 for a segment that maps to no source.
38 + * False if a segment is malformed.
39 + */
40 +function walkSegments(
41 + mappings: string,
42 + visit: (
43 + line: number,
44 + generatedColumn: number,
45 + sourceIndex: number,
46 + originalLine: number,
47 + originalColumn: number,
48 + ) => void,
49 +): boolean {
50 + // Names keep the running offsets right but are not stored: they name an identifier, not the function.
51 + const fields = [0, 0, 0, 0, 0];
52 + let line = 0;
53 + let generatedColumn = 0;
54 + let sourceIndex = 0;
55 + let originalLine = 0;
56 + let originalColumn = 0;
57 + let index = 0;
58 +
59 + while (index < mappings.length) {
60 + const code = mappings.charCodeAt(index);
61 + if (code === SEMICOLON) {
62 + line += 1;
63 + generatedColumn = 0;
64 + index += 1;
65 + continue;
66 + }
67 + if (code === COMMA) {
68 + index += 1;
69 + continue;
70 + }
71 +
72 + let fieldCount = 0;
73 + while (index < mappings.length) {
74 + const next = mappings.charCodeAt(index);
75 + if (next === COMMA || next === SEMICOLON) break;
76 + let value = 0;
77 + let shift = 0;
78 + let digit: number;
79 + do {
80 + const char = mappings.charCodeAt(index);
81 + digit = char < 128 ? BASE64_VALUES[char]! : -1;
82 + if (digit === -1) return false;
83 + value += (digit & 31) * 2 ** shift;
84 + shift += 5;
85 + index += 1;
86 + } while (digit & 32);
87 + const negative = value % 2 === 1;
88 + value = Math.floor(value / 2);
89 + if (fieldCount < fields.length) fields[fieldCount] = negative ? -value : value;
90 + fieldCount += 1;
91 + }
92 + if (fieldCount !== 1 && fieldCount !== 4 && fieldCount !== 5) return false;
93 +
94 + generatedColumn += fields[0]!;
95 + if (fieldCount === 1) {
96 + visit(line, generatedColumn, -1, 0, 0);
97 + } else {
98 + sourceIndex += fields[1]!;
99 + originalLine += fields[2]!;
100 + originalColumn += fields[3]!;
101 + visit(line, generatedColumn, sourceIndex, originalLine, originalColumn);
102 + }
103 + }
104 + return true;
105 +}
106 +
107 +/**
108 + * `too_large` past the decode budget; `invalid` for anything but a version 3 map, including
109 + * index maps, which React Native never emits.
110 + */
111 +export function decodeSourceMap(raw: unknown): CompactSourceMap | "too_large" | "invalid" {
112 + if (typeof raw !== "object" || raw === null) return "invalid";
113 + const version = Reflect.get(raw, "version");
114 + const mappings = Reflect.get(raw, "mappings");
115 + const sources = Reflect.get(raw, "sources");
116 + const sourceRoot = Reflect.get(raw, "sourceRoot");
117 + if (version !== 3 || typeof mappings !== "string" || !isStringArray(sources)) return "invalid";
118 +
119 + // Hermes maps most bytecode instructions to the same position as the one before; a lookup
120 + // lands on the earlier segment instead with the same answer. Dropping them shrinks our iOS
121 + // map from 2.1M segments to 1.2M, so the first pass counts what the second keeps.
122 + let lineCount = 1;
123 + for (let index = 0; index < mappings.length; index += 1) {
124 + if (mappings.charCodeAt(index) === SEMICOLON) lineCount += 1;
125 + }
126 + let kept = 0;
127 + let lastLine = -1;
128 + let lastSource = -2;
129 + let lastOriginalLine = 0;
130 + let lastOriginalColumn = 0;
131 + const repeats = (line: number, source: number, originalLine: number, originalColumn: number) => {
132 + const same =
133 + line === lastLine &&
134 + source === lastSource &&
135 + (source === -1 ||
136 + (originalLine === lastOriginalLine && originalColumn === lastOriginalColumn));
137 + lastLine = line;
138 + lastSource = source;
139 + lastOriginalLine = originalLine;
140 + lastOriginalColumn = originalColumn;
141 + return same;
142 + };
143 + const counted = walkSegments(mappings, (line, _column, source, originalLine, originalColumn) => {
144 + if (!repeats(line, source, originalLine, originalColumn)) kept += 1;
145 + });
146 + if (!counted) return "invalid";
147 +
148 + const prefixLength = typeof sourceRoot === "string" ? sourceRoot.length + 1 : 0;
149 + let sourceChars = 0;
150 + for (const source of sources) sourceChars += source.length + prefixLength;
151 + // A path takes up to two bytes a character, plus its string header and array slot.
152 + const decodedBytes = kept * 16 + (lineCount + 1) * 4 + sourceChars * 2 + sources.length * 32;
153 + if (decodedBytes > MAX_DECODED_BYTES) return "too_large";
154 +
155 + const lineStarts = new Int32Array(lineCount + 1);
156 + const generatedColumns = new Int32Array(kept);
157 + const sourceIndexes = new Int32Array(kept);
158 + const originalLines = new Int32Array(kept);
159 + const originalColumns = new Int32Array(kept);
160 + let segment = 0;
161 + let currentLine = 0;
162 + lastLine = -1;
163 + lastSource = -2;
164 + walkSegments(mappings, (line, generatedColumn, source, originalLine, originalColumn) => {
165 + if (repeats(line, source, originalLine, originalColumn)) return;
166 + while (currentLine < line) {
167 + currentLine += 1;
168 + lineStarts[currentLine] = segment;
169 + }
170 + generatedColumns[segment] = generatedColumn;
171 + sourceIndexes[segment] = source;
172 + originalLines[segment] = originalLine;
173 + originalColumns[segment] = originalColumn;
174 + segment += 1;
175 + });
176 + while (currentLine < lineCount) {
177 + currentLine += 1;
178 + lineStarts[currentLine] = segment;
179 + }
180 +
181 + const prefix = typeof sourceRoot === "string" && sourceRoot !== "" ? sourceRoot : undefined;
182 + return {
183 + sources: prefix ? sources.map((source) => `${prefix.replace(/\/?$/u, "/")}${source}`) : sources,
184 + lineStarts,
185 + generatedColumns,
186 + sourceIndexes,
187 + originalLines,
188 + originalColumns,
189 + };
190 +}
191 +
192 +/** `line` is 1-based and `column` 0-based, as in `@jridgewell/trace-mapping`. */
193 +function originalPositionFor(
194 + map: CompactSourceMap,
195 + line: number,
196 + column: number,
197 +): OriginalPosition | undefined {
198 + const lineIndex = line - 1;
199 + if (lineIndex < 0 || lineIndex >= map.lineStarts.length - 1) return undefined;
200 + const lineStart = map.lineStarts[lineIndex]!;
201 + // An exact hit takes the first segment at `column`: Hermes maps a function's first
202 + // instruction twice (declaration, then first statement), and other libraries take the first.
203 + let low = lineStart;
204 + let high = map.lineStarts[lineIndex + 1]!;
205 + while (low < high) {
206 + const middle = (low + high) >>> 1;
207 + if (map.generatedColumns[middle]! < column) low = middle + 1;
208 + else high = middle;
209 + }
210 + const exact = low < map.lineStarts[lineIndex + 1]! && map.generatedColumns[low] === column;
211 + const found = exact ? low : low - 1;
212 + if (found < lineStart) return undefined;
213 + const sourceIndex = map.sourceIndexes[found]!;
214 + const source = sourceIndex === -1 ? undefined : map.sources[sourceIndex];
215 + if (source === undefined) return undefined;
216 + return {
217 + source,
218 + line: map.originalLines[found]! + 1,
219 + column: map.originalColumns[found]!,
220 + };
221 +}
222 +
223 +/**
224 + * `at render (address at index.android.bundle:1:48213)`: Hermes prints the bytecode offset,
225 + * the composed map's generated column. On iOS the path is absolute, and an OTA bundle's
226 + * contains a space (`Application Support`).
227 + */
228 +const BUNDLE_FRAME = /address at (.+?):(\d+):(\d+)(?=\)?$)/u;
229 +/** Hermes's own runtime, not in the map. */
230 +const HERMES_RUNTIME = "InternalBytecode.js";
231 +
232 +const baseName = (path: string) => path.slice(path.lastIndexOf("/") + 1);
233 +
234 +/** iOS bundle paths carry per-install container UUIDs, which split one crash into many groups. */
235 +export function stripBundlePaths(stack: string): string {
236 + return stack
237 + .split("\n")
238 + .map((line) => {
239 + const match = BUNDLE_FRAME.exec(line);
240 + if (!match || !match[1]!.includes("/")) return line;
241 + const start = match.index + "address at ".length;
242 + return line.slice(0, start) + baseName(match[1]!) + line.slice(start + match[1]!.length);
243 + })
244 + .join("\n");
245 +}
246 +
247 +interface SymbolicatedStack {
248 + readonly stack: string;
249 + /** Frames in the app bundle. */
250 + readonly bundleFrames: number;
251 + /** Bundle frames the map resolved. */
252 + readonly mappedFrames: number;
253 + /** How much longer the stack got. */
254 + readonly growth: number;
255 +}
256 +
257 +/** Once the stack would grow by more than `maxGrowth`, the remaining frames keep their offsets. */
258 +export function symbolicateStack(
259 + stack: string,
260 + map: CompactSourceMap,
261 + maxGrowth: number,
262 +): SymbolicatedStack {
263 + let bundleFrames = 0;
264 + let mappedFrames = 0;
265 + let growth = 0;
266 + let exhausted = false;
267 + const lines = stack.split("\n").map((line) => {
268 + const match = BUNDLE_FRAME.exec(line);
269 + if (!match || baseName(match[1]!) === HERMES_RUNTIME) return line;
270 + bundleFrames += 1;
271 + if (exhausted) return line;
272 + const position = originalPositionFor(map, Number(match[2]), Number(match[3]));
273 + if (!position) return line;
274 + const lineAndColumn = `:${position.line}:${position.column + 1}`;
275 + const added = position.source.length + lineAndColumn.length - match[0].length;
276 + if (growth + added > maxGrowth) {
277 + exhausted = true;
278 + return line;
279 + }
280 + growth += added;
281 + mappedFrames += 1;
282 + // Spliced, not `replace`d: a replacement string would expand `$&` and `$'` in the path.
283 + const end = match.index + match[0].length;
284 + return line.slice(0, match.index) + position.source + lineAndColumn + line.slice(end);
285 + });
286 + return { stack: lines.join("\n"), bundleFrames, mappedFrames, growth };
287 +}
infra/relay/src/monitoring.ts
+1 -1
@@ -343,7 +343,7 @@ const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
343 ];
344 };
345
346 -const notifierConfig = (name: string) =>
346 +export const notifierConfig = (name: string) =>
347 Config.String(name).pipe(
348 Config.option,
349 Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
infra/relay/src/observability.ts
+83 -1
@@ -1,14 +1,20 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - one sha256 inside a synchronous Output.map.
2 +import * as NodeCrypto from "node:crypto";
3 +
4 import * as Alchemy from "alchemy";
5 import * as Axiom from "@alchemy.run/axiom";
6 +import * as Cloudflare from "@alchemy.run/cloudflare";
7 import * as Output from "alchemy/Output";
8 import * as Cause from "effect/Cause";
9 import * as Effect from "effect/Effect";
10 import * as Exit from "effect/Exit";
11 import * as Option from "effect/Option";
12 +import * as Redacted from "effect/Redacted";
13 import * as Schema from "effect/Schema";
14 import * as Tracer from "effect/Tracer";
15
16 import { relayResourceNameForStage } from "./deploymentConfig.ts";
17 +import { MobileMonitoring } from "./mobileTelemetry/monitoring.ts";
18 import { RelayMonitoring } from "./monitoring.ts";
19
20 const relayRecentSpansQuery = (dataset: string) =>
@@ -56,6 +62,37 @@ export const RelayObservability = Effect.gen(function* () {
62 })),
63 });
64
65 + // expo-observe data, forwarded by src/mobileTelemetry. Mobile spans share the traces dataset.
66 + const mobileLogs = yield* Axiom.Dataset("MobileLogsDataset", {
67 + name: relayResourceNameForStage("t3-code-mobile-logs", stage),
68 + kind: "otel:logs:v1",
69 + description: "T3 Code mobile app errors, crashes, and log events from expo-observe.",
70 + retentionDays: 30,
71 + useRetentionPeriod: true,
72 + });
73 + const mobileMetrics = yield* Axiom.Dataset("MobileMetricsDataset", {
74 + name: relayResourceNameForStage("t3-code-mobile-metrics", stage),
75 + kind: "otel:metrics:v1",
76 + description: "T3 Code mobile app startup and update metrics from expo-observe.",
77 + retentionDays: 30,
78 + useRetentionPeriod: true,
79 + });
80 + const mobileTelemetryIngestToken = yield* Axiom.ApiToken("MobileTelemetryAxiomIngestToken", {
81 + name: relayResourceNameForStage("t3-code-mobile-telemetry-ingest", stage),
82 + description: "Owned by Alchemy. Scoped ingest token the relay forwards mobile telemetry with.",
83 + datasetCapabilities: Output.map(
84 + Output.all(traces.name, mobileLogs.name, mobileMetrics.name),
85 + (datasets) =>
86 + Object.fromEntries(datasets.map((dataset) => [dataset, { ingest: ["create" as const] }])),
87 + ),
88 + });
89 +
90 + const mobileSourceMaps = yield* Cloudflare.R2.Bucket("MobileSourceMapsBucket", {
91 + name: mobileSourceMapsBucketName(stage),
92 + // R2 refuses to delete a non-empty bucket, which would half tear down every non-prod stage.
93 + forceDestroy: stage !== "prod",
94 + });
95 +
96 yield* Axiom.View("RelayRecentSpansView", {
97 name: relayResourceNameForStage("t3-code-relay-recent-spans", stage),
98 description: "Recent relay HTTP request spans.",
@@ -64,8 +101,53 @@ export const RelayObservability = Effect.gen(function* () {
101 });
102
103 yield* RelayMonitoring(traces);
104 + yield* MobileMonitoring(mobileLogs, traces);
105 +
106 + return {
107 + traces,
108 + workerIngestToken,
109 + mobileIngestToken,
110 + clientIngestToken,
111 + mobileLogs,
112 + mobileMetrics,
113 + mobileTelemetryIngestToken,
114 + mobileSourceMaps,
115 + } as const;
116 +});
117
68 - return { traces, workerIngestToken, mobileIngestToken, clientIngestToken } as const;
118 +const mobileSourceMapsBucketName = (stage: string) =>
119 + relayResourceNameForStage("t3-code-mobile-sourcemaps", stage);
120 +
121 +/**
122 + * Write-only S3 credentials for the source maps bucket: the `MOBILE_SOURCEMAPS_R2` secret.
123 + * R2's S3 access key id is the API token's id and the secret the SHA-256 of its value.
124 + */
125 +export const MobileSourceMapsUploadCredentials = Effect.gen(function* () {
126 + const { stage } = yield* Alchemy.Stack;
127 + const { accountId } = yield* yield* Cloudflare.CloudflareEnvironment;
128 + const bucketName = mobileSourceMapsBucketName(stage);
129 + const token = yield* Cloudflare.ApiToken.AccountApiToken("MobileSourceMapsUploadToken", {
130 + name: relayResourceNameForStage("t3-code-mobile-sourcemaps-upload", stage),
131 + policies: [
132 + {
133 + effect: "allow",
134 + permissionGroups: ["Workers R2 Storage Bucket Item Write"],
135 + resources: { [`com.cloudflare.edge.r2.bucket.${accountId}_default_${bucketName}`]: "*" },
136 + },
137 + ],
138 + });
139 + return Output.map(Output.all(token.tokenId, token.value), ([tokenId, value]) =>
140 + Redacted.make(
141 + JSON.stringify({
142 + endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
143 + bucketName,
144 + accessKeyId: tokenId,
145 + secretAccessKey: NodeCrypto.createHash("sha256")
146 + .update(Redacted.value(value))
147 + .digest("hex"),
148 + }),
149 + ),
150 + );
151 });
152
153 export const withSpanAttributes =
infra/relay/src/worker.ts
+78 -8
@@ -69,6 +69,7 @@ import * as HookForwarder from "./hooks/HookForwarder.ts";
69 import * as HeldHooks from "./hooks/HeldHooks.ts";
70 import * as HookInbox from "./hooks/HookInbox.ts";
71 import * as HookInboxObject from "./hooks/HookInboxObject.ts";
72 +import * as MobileTelemetry from "./mobileTelemetry/MobileTelemetry.ts";
73
74 const layerWebcrypto = Layer.succeed(
75 Crypto.Crypto,
@@ -140,7 +141,7 @@ export const layer = Api.make(
141 const relayApiZone = yield* RelayApiZone;
142 const managedEndpointZone = yield* ManagedEndpointZone;
143 const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;
143 - yield* RelayObservability;
144 + const observability = yield* RelayObservability;
145
146 const apnsEnabled = yield* Config.Boolean("APNS_ENABLED").pipe(Config.withDefault(true));
147 const apnsCredentials = apnsEnabled
@@ -201,6 +202,21 @@ export const layer = Api.make(
202 },
203 });
204 const hookInboxes = yield* HookInboxObject.HookInboxObject;
205 + const mobileTelemetryRateLimit = yield* Cloudflare.RateLimit("MOBILE_TELEMETRY_RATE_LIMIT", {
206 + namespaceId: 1003,
207 + simple: {
208 + limit: MobileTelemetry.MOBILE_TELEMETRY_RATE_LIMIT.limit,
209 + period: MobileTelemetry.MOBILE_TELEMETRY_RATE_LIMIT.periodSeconds,
210 + },
211 + });
212 + const mobileSourceMaps = yield* Cloudflare.R2.ReadBucket(observability.mobileSourceMaps);
213 + const mobileTelemetryToken = yield* observability.mobileTelemetryIngestToken.token;
214 + const mobileTracesUrl = yield* observability.traces.otelTracesEndpoint;
215 + const mobileTracesDataset = yield* observability.traces.name;
216 + const mobileLogsUrl = yield* observability.mobileLogs.otelLogsEndpoint;
217 + const mobileLogsDataset = yield* observability.mobileLogs.name;
218 + const mobileMetricsUrl = yield* observability.mobileMetrics.otelMetricsEndpoint;
219 + const mobileMetricsDataset = yield* observability.mobileMetrics.name;
220
221 const alchemyRuntimeContext: Alchemy.BaseRuntimeContext = yield* Cloudflare.Worker;
222
@@ -325,18 +341,19 @@ export const layer = Api.make(
341 Layer.provideMerge(layerWebcrypto),
342 );
343
328 - // Fails open: a limiter outage must not drop webhooks the environment would accept.
344 + // Fails open: a limiter outage must not drop webhooks or app telemetry.
345 const allowWith =
330 - (limiter: typeof hookRateLimit) =>
346 + (
347 + limiter: typeof hookRateLimit,
348 + failedOpenAttribute = "relay.hook.rate_limiter_failed_open",
349 + ) =>
350 (key: string): Effect.Effect<boolean> =>
351 limiter.limit({ key }).pipe(
352 Effect.map((result) => result.success),
353 Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
354 Effect.catch((error) =>
336 - Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe(
337 - Effect.andThen(
338 - Effect.annotateCurrentSpan({ "relay.hook.rate_limiter_failed_open": true }),
339 - ),
355 + Effect.logWarning("Rate limiter unavailable", { error: error.message }).pipe(
356 + Effect.andThen(Effect.annotateCurrentSpan({ [failedOpenAttribute]: true })),
357 Effect.as(true),
358 ),
359 ),
@@ -346,12 +363,64 @@ export const layer = Api.make(
363 allowEndpoint: allowWith(hookEndpointRateLimit),
364 });
365
349 - const layerApp = Layer.merge(
366 + // `fetch` rebuilds its layers per request; the source map cache must outlive one.
367 + const mobileSourceMapLookups = yield* MobileTelemetry.SourceMaps.pipe(
368 + Effect.provide(MobileTelemetry.layerSourceMaps),
369 + Effect.provideService(MobileTelemetry.SourceMapObjects, {
370 + get: (key) =>
371 + mobileSourceMaps.get(key).pipe(
372 + Effect.map((object) =>
373 + object === null
374 + ? undefined
375 + : {
376 + size: object.size,
377 + text: object.text().pipe(
378 + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
379 + Effect.mapError(
380 + (cause) => new MobileTelemetry.SourceMapObjectError({ key, cause }),
381 + ),
382 + ),
383 + },
384 + ),
385 + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
386 + Effect.mapError((cause) => new MobileTelemetry.SourceMapObjectError({ key, cause })),
387 + ),
388 + }),
389 + );
390 + const layerMobileTelemetry = MobileTelemetry.layer.pipe(
391 + Layer.provide(
392 + Layer.mergeAll(
393 + Layer.effect(
394 + MobileTelemetry.MobileTelemetrySettings,
395 + Effect.gen(function* () {
396 + return MobileTelemetry.MobileTelemetrySettings.of({
397 + axiomToken: yield* mobileTelemetryToken,
398 + destinations: {
399 + traces: { url: yield* mobileTracesUrl, dataset: yield* mobileTracesDataset },
400 + logs: { url: yield* mobileLogsUrl, dataset: yield* mobileLogsDataset },
401 + metrics: { url: yield* mobileMetricsUrl, dataset: yield* mobileMetricsDataset },
402 + },
403 + });
404 + }),
405 + ),
406 + Layer.succeed(MobileTelemetry.MobileTelemetryRateLimiter, {
407 + allow: allowWith(
408 + mobileTelemetryRateLimit,
409 + "relay.mobile_telemetry.rate_limiter_failed_open",
410 + ),
411 + }),
412 + Layer.succeed(MobileTelemetry.SourceMaps, mobileSourceMapLookups),
413 + ),
414 + ),
415 + );
416 +
417 + const layerApp = Layer.mergeAll(
418 layerRelayApi,
419 HookForwarder.layerApi.pipe(
420 Layer.provide(HookForwarder.layer),
421 Layer.provide(layerHookRateLimiter),
422 ),
423 + MobileTelemetry.layerApi.pipe(Layer.provide(layerMobileTelemetry)),
424 ).pipe(
425 Layer.provideMerge(RelayHttpApi.layerClientAuth),
426 Layer.provideMerge(RelayHttpApi.layerDpopClientAuth),
@@ -481,6 +550,7 @@ export const layer = Api.make(
550 Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding),
551 Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
552 Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
553 + Layer.provideMerge(Cloudflare.R2.ReadBucketBinding),
554 Layer.provideMerge(HookInboxObject.layer),
555 // Disable the duplicate runtime span; ours applies route/header/token redaction and rejects sender traceparent.
556 Layer.provideMerge(
packages/contracts/src/relay.ts
+17
@@ -1250,6 +1250,22 @@ const RelayHooksGroup = HttpApiGroup.make("hooks")
1250 )
1251 .annotate(OpenApi.Description, "Forward webhook requests to an environment.");
1252
1253 +export const RelayMobileTelemetrySignal = Schema.Literals(["logs", "metrics", "traces"]);
1254 +export type RelayMobileTelemetrySignal = typeof RelayMobileTelemetrySignal.Type;
1255 +
1256 +// expo-observe appends `<easProjectId>/v1/<signal>` to its endpoint URL and cannot send credentials.
1257 +const RelayMobileTelemetryParams = Schema.Struct({
1258 + projectId: Schema.String.check(Schema.isGUID()),
1259 + signal: RelayMobileTelemetrySignal,
1260 +});
1261 +const RelayMobileTelemetryGroup = HttpApiGroup.make("mobileTelemetry")
1262 + .add(
1263 + HttpApiEndpoint.post("ingest", "/v1/mobile/observe/:projectId/v1/:signal", {
1264 + params: RelayMobileTelemetryParams,
1265 + }),
1266 + )
1267 + .annotate(OpenApi.Description, "Receive mobile app telemetry from expo-observe.");
1268 +
1269 export const RelayApi = HttpApi.make("RelayApi")
1270 .add(
1271 RelayHealthGroup,
@@ -1260,6 +1276,7 @@ export const RelayApi = HttpApi.make("RelayApi")
1276 RelayDpopClientGroup,
1277 RelayServerGroup,
1278 RelayHooksGroup,
1279 + RelayMobileTelemetryGroup,
1280 )
1281 .annotate(OpenApi.Title, "T3 Code Relay API")
1282 .annotate(OpenApi.Version, "1.0.0")
packages/shared/package.json
+4
@@ -35,6 +35,10 @@
35 "types": "./src/projectFavicon.ts",
36 "import": "./src/projectFavicon.ts"
37 },
38 + "./mobileSourceMaps": {
39 + "types": "./src/mobileSourceMaps.ts",
40 + "import": "./src/mobileSourceMaps.ts"
41 + },
42 "./model": {
43 "types": "./src/model.ts",
44 "import": "./src/model.ts"
packages/shared/src/mobileSourceMaps.ts new
+10
@@ -0,0 +1,10 @@
1 +export type MobilePlatform = "ios" | "android";
2 +
3 +export type MobileSourceMapKind = "ota" | "build";
4 +
5 +/** The relay's R2 key for a bundle's source map: an OTA update's, or an EAS build's embedded one. */
6 +export const mobileSourceMapKey = (
7 + platform: MobilePlatform,
8 + kind: MobileSourceMapKind,
9 + id: string,
10 +) => `${platform}/${kind}/${id}.map`;
pnpm-lock.yaml
+49 -2
@@ -680,6 +680,9 @@ importers:
680 expo-notifications:
681 specifier: ~58.1.5
682 version: 58.1.5(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)
683 + expo-observe:
684 + specifier: ~58.0.15
685 + version: 58.0.16(677910f8cba89c74801637ca226a8913)
686 expo-paste-input:
687 specifier: ^0.1.15
688 version: 0.1.15(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
@@ -801,6 +804,9 @@ importers:
804 '@types/react-dom':
805 specifier: 'catalog:'
806 version: 19.3.0(@types/react@19.3.0)
807 + aws4fetch:
808 + specifier: 1.0.20
809 + version: 1.0.20
810 babel-preset-expo:
811 specifier: ~58.0.17
812 version: 58.0.17(@babel/core@7.29.7(supports-color@8.1.1))(@babel/runtime@7.29.7)(expo-widgets@58.0.20)(expo@58.0.14)(react-refresh@0.14.2)(supports-color@8.1.1)
@@ -8568,6 +8574,13 @@ packages:
8574 resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
8575 engines: {node: '>=12.0.0'}
8576
8577 + expo-app-metrics@58.0.14:
8578 + resolution: {integrity: sha512-Bj/Bx3s5woOKThUcqucmDOdOAxmyHr232KbnOBD8SoA/iUo7E4RBWvRT8DwZOmLJ7H1wFK2ckFTiNoDFB+Ag+Q==}
8579 + peerDependencies:
8580 + expo: '*'
8581 + react: '*'
8582 + react-native: '*'
8583 +
8584 expo-application@58.1.2:
8585 resolution: {integrity: sha512-WYFdNHGaxe/j69oYREhK0cKsKYszuIuz399WXYCP+M1grfO+FCWB/RbuMgaXJsLWlzqhNjA/zLgfJ+33FwrQvw==}
8586 peerDependencies:
@@ -8779,6 +8792,23 @@ packages:
8792 react: '*'
8793 react-native: '*'
8794
8795 + expo-observe@58.0.16:
8796 + resolution: {integrity: sha512-STG1qp5VKA3f0jFQuf+waoO8QNIG6fpog8Hqi2UnklA08ec8a+w+hJm0wL4DgyO8dy5oDdiTVUnpqaloeUC2lQ==}
8797 + peerDependencies:
8798 + '@react-navigation/native': ^7.0.0
8799 + expo: '*'
8800 + expo-router: 58.0.20
8801 + react: '*'
8802 + react-native: '*'
8803 + react-native-reanimated: 4.7.1
8804 + peerDependenciesMeta:
8805 + '@react-navigation/native':
8806 + optional: true
8807 + expo-router:
8808 + optional: true
8809 + react-native-reanimated:
8810 + optional: true
8811 +
8812 expo-paste-input@0.1.15:
8813 resolution: {integrity: sha512-wV0z1s0/Q4MB+lNiSwbMfck+1Im6sDIcgyCi5Y0aF35n5kcM4FHSOjmRfzn3UbkMlvxmYfDiHeHvIqn22qatqA==}
8814 peerDependencies:
@@ -17192,8 +17222,7 @@ snapshots:
17222
17223 aws4@1.13.2: {}
17224
17195 - aws4fetch@1.0.20:
17196 - optional: true
17225 + aws4fetch@1.0.20: {}
17226
17227 axobject-query@4.1.0: {}
17228
@@ -18201,6 +18230,13 @@ snapshots:
18230
18231 expect-type@1.4.0: {}
18232
18233 + expo-app-metrics@58.0.14(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18234 + dependencies:
18235 + expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
18236 + expo-updates-interface: 58.0.1(expo@58.0.14)
18237 + react: 19.3.0
18238 + react-native: 0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6)
18239 +
18240 expo-application@58.1.2(expo@58.0.14):
18241 dependencies:
18242 expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
@@ -18436,6 +18472,17 @@ snapshots:
18472 - supports-color
18473 - typescript
18474
18475 + expo-observe@58.0.16(677910f8cba89c74801637ca226a8913):
18476 + dependencies:
18477 + expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
18478 + expo-app-metrics: 58.0.14(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
18479 + expo-eas-client: 58.0.2
18480 + react: 19.3.0
18481 + react-native: 0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6)
18482 + optionalDependencies:
18483 + '@react-navigation/native': 7.5.1(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
18484 + react-native-reanimated: 4.7.1(patch_hash=95ee26e5fcab5f50bc4b4c47b3f7e6278a0b20850205efb9f6ee7e70a22d2b59)(react-native-worklets@0.13.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1))(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
18485 +
18486 expo-paste-input@0.1.15(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18487 dependencies:
18488 expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)