The best way to control Claude Code, Codex and more from any computer or phone
feat(mobile): report crashes to Axiom with symbolicated stacks (#19093)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Gabriel De Andrade committed
Oct 27, 2026 at 16:20 UTC
570d4b03eef879bd60be3d74444046858c94dacd
28 files changed
+2418
-16
.github/workflows/mobile-eas-production.yml
+19
@@ -275,6 +275,7 @@ jobs:
275
--environment production \
276
--platform ${{ inputs.platform }} \
277
--message "$OTA_MESSAGE" \
278
+ --emit-metadata \
279
--non-interactive
280
281
# Reconcile each version separately because the major is in the fingerprint.
@@ -313,8 +314,11 @@ jobs:
314
--environment production \
315
--platform "$platform" \
316
--message "$message" \
317
+ --emit-metadata \
318
--non-interactive; then
319
echo ":white_check_mark: $platform/$profile: OTA published to production (fingerprint \`$hash\`)" >> "$GITHUB_STEP_SUMMARY"
320
+ # The next target's export replaces dist; keep this one for the map upload.
321
+ rm -rf "dist-$platform-$profile" && mv dist "dist-$platform-$profile"
322
else
323
failed=1
324
echo ":x: $platform/$profile: OTA publish failed (fingerprint \`$hash\`)" >> "$GITHUB_STEP_SUMMARY"
@@ -354,3 +358,18 @@ jobs:
358
fi
359
done
360
exit "$failed"
361
+
362
+ # Lets the relay symbolicate crash stacks from the updates published
363
+ # above. A step of its own so the R2 credentials reach only this script,
364
+ # not `eas update` and the dependencies it runs. Never fails the job.
365
+ - name: Upload OTA source maps
366
+ if: ${{ !cancelled() && steps.expo-token.outputs.present == 'true' }}
367
+ working-directory: apps/mobile
368
+ env:
369
+ MOBILE_SOURCEMAPS_R2: ${{ secrets.MOBILE_SOURCEMAPS_R2 }}
370
+ run: |
371
+ for dir in dist dist-*; do
372
+ if [ -f "$dir/eas-update-metadata.json" ]; then
373
+ node scripts/upload-sourcemaps.mts ota "$dir"
374
+ fi
375
+ done
apps/mobile/app.config.ts
+7
@@ -17,6 +17,12 @@ const runtimeVersionPolicy =
17
const personalTeamBundleIdentifier = repoEnv.T3CODE_IOS_PERSONAL_TEAM_BUNDLE_ID?.trim();
18
const IOS_BUNDLE_IDENTIFIER_PATTERN = /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/;
19
20
+// Compiled into the native project. Without a relay, Observe's native side could dispatch to
21
+// Expo's own ingest before observe.ts turns it off; `.invalid` never resolves (RFC 6761).
22
+const observeEndpointUrl = repoEnv.T3CODE_RELAY_URL?.trim()
23
+ ? new URL("/v1/mobile/observe", repoEnv.T3CODE_RELAY_URL.trim()).toString()
24
+ : "https://observe.invalid";
25
+
26
const fromRepoRoot = (relativePath: string) => `../../${relativePath}`;
27
const androidAdaptiveForeground = "./assets/android-icon-foreground.png";
28
@@ -450,6 +456,7 @@ const config: ExpoConfig = {
456
},
457
eas: {
458
projectId: "d763fcb8-d37c-41ea-a773-b54a0ab4a454",
459
+ observe: { endpointUrl: observeEndpointUrl },
460
},
461
},
462
owner: "pingdotgg",
apps/mobile/eas.json
+2
-1
@@ -66,7 +66,8 @@
66
"corepack": true,
67
"env": {
68
"APP_VARIANT": "production",
69
- "NODE_OPTIONS": "--max-old-space-size=4096"
69
+ "NODE_OPTIONS": "--max-old-space-size=4096",
70
+ "SOURCEMAP_FILE": "/tmp/t3code-ios-main.jsbundle.map"
71
},
72
"channel": "production",
73
"environment": "production",
apps/mobile/index.ts
+2
@@ -1,5 +1,7 @@
1
// Must stay first: the render counter wraps the DevTools hook before any React commit.
2
import "./src/features/development/render-counter-entry";
3
+// Next, so errors thrown while the rest of the app loads are recorded too.
4
+import "./src/features/observability/observe";
5
import { registerRootComponent } from "expo";
6
import "react-native-gesture-handler";
7
import { LogBox } from "react-native";
apps/mobile/package.json
+3
@@ -46,6 +46,7 @@
46
"measure-renders": "node scripts/measure-renders.mts",
47
"generate": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/generate-uniwind-themes.mts",
48
"generate:highlights": "node scripts/generate-highlights.mts",
49
+ "eas-build-on-success": "node scripts/upload-sourcemaps.mts build",
50
"test": "vp test run --project unit --project react",
51
"typecheck": "tsc --noEmit"
52
},
@@ -103,6 +104,7 @@
104
"expo-linking": "~58.0.13",
105
"expo-network": "~58.0.3",
106
"expo-notifications": "~58.1.5",
107
+ "expo-observe": "~58.0.15",
108
"expo-paste-input": "^0.1.15",
109
"expo-quick-actions": "^6.0.2",
110
"expo-secure-store": "~58.0.2",
@@ -145,6 +147,7 @@
147
"@types/mdast": "^4.0.4",
148
"@types/react": "catalog:",
149
"@types/react-dom": "catalog:",
150
+ "aws4fetch": "1.0.20",
151
"babel-preset-expo": "~58.0.17",
152
"jsdom": "^30.0.1",
153
"tailwindcss": "^4.0.0",
apps/mobile/scripts/sourcemap-upload.mts
new
+165
@@ -0,0 +1,165 @@
1
+import * as NodeFS from "node:fs";
2
+import * as NodePath from "node:path";
3
+import { mobileSourceMapKey, type MobilePlatform } from "@t3tools/shared/mobileSourceMaps";
4
+import { AwsClient } from "aws4fetch";
5
+
6
+export interface SourceMapUpload {
7
+ readonly key: string;
8
+ readonly path: string;
9
+}
10
+
11
+/** The `MOBILE_SOURCEMAPS_R2` secret: the relay stack's `mobileSourceMapsUploadCredentials`. */
12
+interface UploadCredentials {
13
+ readonly endpoint: string;
14
+ readonly bucketName: string;
15
+ readonly accessKeyId: string;
16
+ readonly secretAccessKey: string;
17
+}
18
+
19
+const isPlatform = (value: unknown): value is MobilePlatform =>
20
+ value === "ios" || value === "android";
21
+
22
+const readJson = (path: string): unknown => JSON.parse(NodeFS.readFileSync(path, "utf8"));
23
+
24
+const isRecord = (value: unknown): value is Record<string, unknown> =>
25
+ typeof value === "object" && value !== null && !Array.isArray(value);
26
+
27
+export function parseCredentials(value: string | undefined): UploadCredentials | undefined {
28
+ if (!value?.trim()) return undefined;
29
+ let parsed: unknown;
30
+ try {
31
+ parsed = JSON.parse(value);
32
+ } catch {
33
+ // JSON.parse's message quotes the input, which here is the secret.
34
+ throw new Error("MOBILE_SOURCEMAPS_R2 is not valid JSON.");
35
+ }
36
+ if (!isRecord(parsed)) throw new Error("MOBILE_SOURCEMAPS_R2 is not a JSON object.");
37
+ const { endpoint, bucketName, accessKeyId, secretAccessKey } = parsed;
38
+ if (
39
+ typeof endpoint !== "string" ||
40
+ typeof bucketName !== "string" ||
41
+ typeof accessKeyId !== "string" ||
42
+ typeof secretAccessKey !== "string"
43
+ ) {
44
+ throw new Error(
45
+ "MOBILE_SOURCEMAPS_R2 needs endpoint, bucketName, accessKeyId and secretAccessKey.",
46
+ );
47
+ }
48
+ return { endpoint, bucketName, accessKeyId, secretAccessKey };
49
+}
50
+
51
+/** One map per update `eas update --emit-metadata` published, next to the bundle `metadata.json` names. */
52
+export function otaSourceMapUploads(distDir: string): ReadonlyArray<SourceMapUpload> {
53
+ const metadata = readJson(NodePath.join(distDir, "metadata.json"));
54
+ const easMetadata = readJson(NodePath.join(distDir, "eas-update-metadata.json"));
55
+ const fileMetadata = isRecord(metadata) ? metadata.fileMetadata : undefined;
56
+ const updates = isRecord(easMetadata) ? easMetadata.updates : undefined;
57
+ if (!isRecord(fileMetadata) || !Array.isArray(updates)) {
58
+ throw new Error(`${distDir} is missing Expo export or EAS update metadata.`);
59
+ }
60
+ return updates.flatMap((update): Array<SourceMapUpload> => {
61
+ if (!isRecord(update) || !isPlatform(update.platform) || typeof update.id !== "string") {
62
+ return [];
63
+ }
64
+ const platformFiles = fileMetadata[update.platform];
65
+ const bundle = isRecord(platformFiles) ? platformFiles.bundle : undefined;
66
+ if (typeof bundle !== "string") return [];
67
+ return [
68
+ {
69
+ key: mobileSourceMapKey(update.platform, "ota", update.id),
70
+ path: NodePath.join(distDir, `${bundle}.map`),
71
+ },
72
+ ];
73
+ });
74
+}
75
+
76
+/** Android's Gradle plugin writes the map under the release variant; iOS to `SOURCEMAP_FILE` (eas.json). */
77
+export function buildSourceMapUpload(
78
+ env: Readonly<Record<string, string | undefined>>,
79
+ projectDir: string,
80
+): SourceMapUpload {
81
+ const platform = env.EAS_BUILD_PLATFORM;
82
+ const buildId = env.EAS_BUILD_ID;
83
+ if (!isPlatform(platform) || !buildId) {
84
+ throw new Error("EAS_BUILD_PLATFORM and EAS_BUILD_ID are only set on EAS Build.");
85
+ }
86
+ if (platform === "ios") {
87
+ if (!env.SOURCEMAP_FILE) throw new Error("SOURCEMAP_FILE is not set for this iOS build.");
88
+ return { key: mobileSourceMapKey("ios", "build", buildId), path: env.SOURCEMAP_FILE };
89
+ }
90
+ const variantsDir = NodePath.join(projectDir, "android/app/build/generated/sourcemaps/react");
91
+ const variant = NodeFS.existsSync(variantsDir)
92
+ ? NodeFS.readdirSync(variantsDir).find((name) => /release$/iu.test(name))
93
+ : undefined;
94
+ if (!variant) throw new Error(`No release source map under ${variantsDir}.`);
95
+ return {
96
+ key: mobileSourceMapKey("android", "build", buildId),
97
+ path: NodePath.join(variantsDir, variant, "index.android.bundle.map"),
98
+ };
99
+}
100
+
101
+/**
102
+ * EAS writes the app's debug symbols next to the `.ipa`, so native frames in iOS crash
103
+ * reports can be read with `atos`. Undefined when the build left none.
104
+ */
105
+export function buildDsymUpload(
106
+ env: Readonly<Record<string, string | undefined>>,
107
+ projectDir: string,
108
+): SourceMapUpload | undefined {
109
+ if (env.EAS_BUILD_PLATFORM !== "ios" || !env.EAS_BUILD_ID) return undefined;
110
+ const buildDir = NodePath.join(projectDir, "ios/build");
111
+ const dsym = NodeFS.existsSync(buildDir)
112
+ ? NodeFS.readdirSync(buildDir).find((name) => name.endsWith(".app.dSYM.zip"))
113
+ : undefined;
114
+ if (!dsym) return undefined;
115
+ return { key: `ios/dsym/${env.EAS_BUILD_ID}.zip`, path: NodePath.join(buildDir, dsym) };
116
+}
117
+
118
+/** Only what the relay reads: source text stays out of the bucket, and the rest is dead weight. */
119
+function stripToMappings(text: string): string {
120
+ const map: unknown = JSON.parse(text);
121
+ if (!isRecord(map)) throw new Error("Source map is not a JSON object.");
122
+ const { version, sources, sourceRoot, mappings } = map;
123
+ return JSON.stringify({ version, sources, sourceRoot, mappings });
124
+}
125
+
126
+export function uploadSourceMap(
127
+ credentials: UploadCredentials,
128
+ upload: SourceMapUpload,
129
+): Promise<number> {
130
+ return putObject(
131
+ credentials,
132
+ upload.key,
133
+ stripToMappings(NodeFS.readFileSync(upload.path, "utf8")),
134
+ "application/json",
135
+ );
136
+}
137
+
138
+export function uploadDsym(credentials: UploadCredentials, upload: SourceMapUpload) {
139
+ const body = new Uint8Array(NodeFS.readFileSync(upload.path));
140
+ return putObject(credentials, upload.key, body, "application/zip");
141
+}
142
+
143
+async function putObject(
144
+ credentials: UploadCredentials,
145
+ key: string,
146
+ body: string | Uint8Array<ArrayBuffer>,
147
+ contentType: string,
148
+): Promise<number> {
149
+ const client = new AwsClient({
150
+ accessKeyId: credentials.accessKeyId,
151
+ secretAccessKey: credentials.secretAccessKey,
152
+ service: "s3",
153
+ region: "auto",
154
+ });
155
+ const url = `${credentials.endpoint.replace(/\/$/u, "")}/${credentials.bucketName}/${key}`;
156
+ const response = await client.fetch(url, {
157
+ method: "PUT",
158
+ body,
159
+ headers: { "content-type": contentType },
160
+ });
161
+ if (!response.ok) {
162
+ throw new Error(`R2 answered ${response.status}: ${(await response.text()).slice(0, 500)}`);
163
+ }
164
+ return body.length;
165
+}
apps/mobile/scripts/sourcemap-upload.test.ts
new
+136
@@ -0,0 +1,136 @@
1
+import * as NodeFS from "node:fs";
2
+import * as NodeHttp from "node:http";
3
+import type * as NodeNet from "node:net";
4
+import * as NodeOS from "node:os";
5
+import * as NodePath from "node:path";
6
+import { afterEach, describe, expect, it } from "vite-plus/test";
7
+
8
+import {
9
+ buildDsymUpload,
10
+ buildSourceMapUpload,
11
+ otaSourceMapUploads,
12
+ uploadSourceMap,
13
+} from "./sourcemap-upload.mts";
14
+
15
+const directories: Array<string> = [];
16
+const tempDir = () => {
17
+ const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "sourcemap-upload-"));
18
+ directories.push(dir);
19
+ return dir;
20
+};
21
+const writeFile = (path: string, contents: string) => {
22
+ NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true });
23
+ NodeFS.writeFileSync(path, contents);
24
+};
25
+
26
+afterEach(() => {
27
+ for (const dir of directories.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true });
28
+});
29
+
30
+describe("uploadSourceMap", () => {
31
+ it("PUTs each published update's map, only what the relay reads, under its update id", async () => {
32
+ const dist = tempDir();
33
+ writeFile(
34
+ NodePath.join(dist, "metadata.json"),
35
+ JSON.stringify({
36
+ fileMetadata: {
37
+ android: { bundle: "_expo/static/js/android/index-abc.hbc" },
38
+ ios: { bundle: "_expo/static/js/ios/index-def.hbc" },
39
+ },
40
+ }),
41
+ );
42
+ writeFile(
43
+ NodePath.join(dist, "eas-update-metadata.json"),
44
+ JSON.stringify({
45
+ updates: [
46
+ { id: "update-android", platform: "android" },
47
+ { id: "update-ios", platform: "ios" },
48
+ { id: "update-web", platform: "web" },
49
+ ],
50
+ }),
51
+ );
52
+ for (const platform of ["android", "ios"]) {
53
+ writeFile(
54
+ NodePath.join(
55
+ dist,
56
+ `_expo/static/js/${platform}/index-${platform === "ios" ? "def" : "abc"}.hbc.map`,
57
+ ),
58
+ JSON.stringify({
59
+ version: 3,
60
+ sources: [`${platform}.ts`],
61
+ sourcesContent: ["secret"],
62
+ names: ["render"],
63
+ x_facebook_sources: [[null]],
64
+ mappings: "",
65
+ }),
66
+ );
67
+ }
68
+
69
+ const requests: Array<{ method?: string; url?: string; auth?: string; body: string }> = [];
70
+ const server = NodeHttp.createServer((request, response) => {
71
+ let body = "";
72
+ request.on("data", (chunk) => (body += chunk));
73
+ request.on("end", () => {
74
+ requests.push({
75
+ method: request.method,
76
+ url: request.url,
77
+ auth: request.headers.authorization,
78
+ body,
79
+ });
80
+ response.end();
81
+ });
82
+ });
83
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
84
+ try {
85
+ const address = server.address() as NodeNet.AddressInfo;
86
+ const credentials = {
87
+ endpoint: `http://127.0.0.1:${address.port}`,
88
+ bucketName: "maps",
89
+ accessKeyId: "key-id",
90
+ secretAccessKey: "secret-key",
91
+ };
92
+ for (const upload of otaSourceMapUploads(dist)) {
93
+ await uploadSourceMap(credentials, upload);
94
+ }
95
+ expect(requests).toEqual(
96
+ ["android", "ios"].map((platform) => ({
97
+ method: "PUT",
98
+ url: `/maps/${platform}/ota/update-${platform}.map`,
99
+ auth: expect.stringMatching(/^AWS4-HMAC-SHA256 Credential=key-id\/\d{8}\/auto\/s3\//),
100
+ body: JSON.stringify({ version: 3, sources: [`${platform}.ts`], mappings: "" }),
101
+ })),
102
+ );
103
+ } finally {
104
+ server.close();
105
+ }
106
+ });
107
+});
108
+
109
+describe("buildSourceMapUpload", () => {
110
+ it("names the EAS build's embedded bundle map under its build id", () => {
111
+ const project = tempDir();
112
+ const androidMap = NodePath.join(
113
+ project,
114
+ "android/app/build/generated/sourcemaps/react/release/index.android.bundle.map",
115
+ );
116
+ writeFile(androidMap, "{}");
117
+ expect(
118
+ buildSourceMapUpload({ EAS_BUILD_PLATFORM: "android", EAS_BUILD_ID: "build-1" }, project),
119
+ ).toEqual({ key: "android/build/build-1.map", path: androidMap });
120
+ expect(
121
+ buildSourceMapUpload(
122
+ { EAS_BUILD_PLATFORM: "ios", EAS_BUILD_ID: "build-2", SOURCEMAP_FILE: "/tmp/main.map" },
123
+ project,
124
+ ),
125
+ ).toEqual({ key: "ios/build/build-2.map", path: "/tmp/main.map" });
126
+ });
127
+
128
+ it("names an iOS build's dSYMs under its build id", () => {
129
+ const project = tempDir();
130
+ const dsym = NodePath.join(project, "ios/build/T3Code.app.dSYM.zip");
131
+ writeFile(dsym, "zip");
132
+ expect(
133
+ buildDsymUpload({ EAS_BUILD_PLATFORM: "ios", EAS_BUILD_ID: "build-3" }, project),
134
+ ).toEqual({ key: "ios/dsym/build-3.zip", path: dsym });
135
+ });
136
+});
apps/mobile/scripts/upload-sourcemaps.mts
new
+61
@@ -0,0 +1,61 @@
1
+#!/usr/bin/env node
2
+// Usage: `ota <dist>` after `eas update --emit-metadata`, or `build` from eas-build-on-success.
3
+// Problems are warnings, never failures: an unreadable stack is not worth failing a release over.
4
+import {
5
+ buildDsymUpload,
6
+ buildSourceMapUpload,
7
+ otaSourceMapUploads,
8
+ parseCredentials,
9
+ uploadDsym,
10
+ uploadSourceMap,
11
+ type SourceMapUpload,
12
+} from "./sourcemap-upload.mts";
13
+
14
+const warn = (message: string) => {
15
+ // GitHub Actions shows this as an annotation; elsewhere it is a log line.
16
+ console.warn(process.env.GITHUB_ACTIONS ? `::warning::${message}` : `warning: ${message}`);
17
+};
18
+
19
+const errorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error));
20
+
21
+async function main() {
22
+ const [mode, distDir] = process.argv.slice(2);
23
+ const credentials = parseCredentials(process.env.MOBILE_SOURCEMAPS_R2);
24
+ if (!credentials) {
25
+ warn("MOBILE_SOURCEMAPS_R2 is not set; crash stacks from this bundle stay minified.");
26
+ return;
27
+ }
28
+ let uploads: ReadonlyArray<SourceMapUpload>;
29
+ if (mode === "ota" && distDir) {
30
+ uploads = otaSourceMapUploads(distDir);
31
+ } else if (mode === "build") {
32
+ uploads = [buildSourceMapUpload(process.env, process.cwd())];
33
+ } else {
34
+ throw new Error("Usage: upload-sourcemaps.mts ota <dist-dir> | build");
35
+ }
36
+ if (uploads.length === 0) warn("Found no source maps to upload.");
37
+ for (const upload of uploads) {
38
+ try {
39
+ const bytes = await uploadSourceMap(credentials, upload);
40
+ console.log(`Uploaded ${upload.key} (${(bytes / 1e6).toFixed(1)} MB)`);
41
+ } catch (error) {
42
+ warn(`Could not upload ${upload.key}: ${errorMessage(error)}`);
43
+ }
44
+ }
45
+ if (mode !== "build" || process.env.EAS_BUILD_PLATFORM !== "ios") return;
46
+ const dsym = buildDsymUpload(process.env, process.cwd());
47
+ if (!dsym) {
48
+ warn("Found no dSYM under ios/build; native iOS frames from this build stay unreadable.");
49
+ return;
50
+ }
51
+ try {
52
+ const bytes = await uploadDsym(credentials, dsym);
53
+ console.log(`Uploaded ${dsym.key} (${(bytes / 1e6).toFixed(1)} MB)`);
54
+ } catch (error) {
55
+ warn(`Could not upload ${dsym.key}: ${errorMessage(error)}`);
56
+ }
57
+}
58
+
59
+await main().catch((error: unknown) => {
60
+ warn(`Source map upload skipped: ${errorMessage(error)}`);
61
+});
apps/mobile/src/components/RenderErrorBoundary.tsx
+4
-1
@@ -1,3 +1,4 @@
1
+import { Observe } from "expo-observe";
2
import { Component, type ReactNode } from "react";
3
import { ScrollView, View } from "react-native";
4
@@ -60,7 +61,9 @@ export class RenderErrorBoundary extends Component<
61
return { failed: true, error };
62
}
63
63
- override componentDidCatch(_error: unknown, info: { componentStack?: string }) {
64
+ override componentDidCatch(error: unknown, info: { componentStack?: string }) {
65
+ // A caught render error never reaches Observe's global handler.
66
+ Observe.reportError(error);
67
this.setState({ componentStack: info.componentStack });
68
}
69
apps/mobile/src/features/diagnostics/SettingsDiagnosticsRouteScreen.tsx
+55
-1
@@ -3,8 +3,9 @@ import { ScreenScrollView as ScrollView } from "../../components/ui/ScreenScroll
3
import * as Application from "expo-application";
4
import Constants from "expo-constants";
5
import * as Updates from "expo-updates";
6
-import { useAtomValue } from "@effect/atom-react";
6
+import { useAtomSet, useAtomValue } from "@effect/atom-react";
7
import * as Effect from "effect/Effect";
8
+import * as Exit from "effect/Exit";
9
import { AsyncResult, Atom } from "effect/reactivity";
10
import { useState } from "react";
11
import { View } from "react-native";
@@ -13,9 +14,12 @@ import { useSafeAreaInsets } from "react-native-safe-area-context";
14
import { SymbolView } from "../../components/ui/AppSymbol";
15
import { AppText as Text } from "../../components/ui/AppText";
16
import { tryCopyTextWithHaptic } from "../../lib/copyTextWithHaptic";
17
+import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
18
+import { crashReportingAvailable, setCrashReportsEnabled } from "../observability/observe";
19
import { SettingsActionRow } from "../settings/components/SettingsActionRow";
20
import { SettingsScreen } from "../settings/components/SettingsScreen";
21
import { SettingsSection } from "../settings/components/SettingsSection";
22
+import { SettingsSwitchRow } from "../settings/components/SettingsSwitchRow";
23
import {
24
formatStartupCrashReport,
25
parseStartupCrashRecords,
@@ -86,6 +90,8 @@ export function SettingsDiagnosticsRouteScreen() {
90
className="flex-1"
91
contentContainerClassName="gap-6 px-5 pt-4 pb-[18px]"
92
>
93
+ {crashReportingAvailable ? <CrashReportsSetting /> : null}
94
+
95
<SettingsSection title="Startup crashes">
96
{state.status === "loading" ? (
97
<View className="items-center gap-3 px-6 py-8">
@@ -131,6 +137,54 @@ export function SettingsDiagnosticsRouteScreen() {
137
);
138
}
139
140
+function CrashReportsSetting() {
141
+ const preferences = useAtomValue(mobilePreferencesAtom);
142
+ const savePreferences = useAtomSet(updateMobilePreferencesAtom, { mode: "promiseExit" });
143
+ // One change at a time, so a slow save never turns sending back on after a later opt-out.
144
+ const [saving, setSaving] = useState(false);
145
+ const ready = AsyncResult.isSuccess(preferences) && !preferences.waiting;
146
+ const enabled = AsyncResult.isSuccess(preferences)
147
+ ? preferences.value.crashReportsEnabled !== false
148
+ : true;
149
+
150
+ const change = (value: boolean) => {
151
+ setSaving(true);
152
+ // Sending stops before the save and starts only after it.
153
+ if (!value) void setCrashReportsEnabled(false);
154
+ savePreferences({ crashReportsEnabled: value })
155
+ .then((exit) => {
156
+ if (Exit.isFailure(exit)) {
157
+ console.warn("[diagnostics] could not save the crash report setting", exit.cause);
158
+ return value ? undefined : setCrashReportsEnabled(true);
159
+ }
160
+ return value ? setCrashReportsEnabled(true) : undefined;
161
+ })
162
+ .catch((error: unknown) => {
163
+ console.warn("[diagnostics] could not apply the crash report setting", error);
164
+ })
165
+ .finally(() => setSaving(false));
166
+ };
167
+
168
+ return (
169
+ <View className="gap-3">
170
+ <SettingsSection title="Crash reports">
171
+ <SettingsSwitchRow
172
+ icon="tray.and.arrow.up"
173
+ label="Send crash reports"
174
+ disabled={!ready || saving}
175
+ value={enabled}
176
+ onValueChange={change}
177
+ />
178
+ </SettingsSection>
179
+ <Text className="px-2 text-sm leading-normal text-foreground-muted">
180
+ When the app crashes or hits an error, it sends the error message, the stack trace, the app
181
+ version, and the device model and OS version to the T3 Code team. Each launch also sends
182
+ startup timings. Reports carry a random ID for this install.
183
+ </Text>
184
+ </View>
185
+ );
186
+}
187
+
188
function EmptyState(props: {
189
readonly icon: "exclamationmark.triangle" | "checkmark.circle";
190
readonly title: string;
apps/mobile/src/features/observability/observe.ts
new
+63
@@ -0,0 +1,63 @@
1
+import Constants from "expo-constants";
2
+import { Observe } from "expo-observe";
3
+
4
+// Importing expo-observe installs its global JS error handler; reports leave only through the relay.
5
+const relayUrl: unknown = Constants.expoConfig?.extra?.relay?.url;
6
+const appVariant: unknown = Constants.expoConfig?.extra?.appVariant;
7
+
8
+export const crashReportingAvailable = typeof relayUrl === "string" && relayUrl.trim() !== "";
9
+
10
+interface PromiseRejectionTracker {
11
+ readonly enablePromiseRejectionTracker?: (options: {
12
+ readonly allRejections: boolean;
13
+ readonly onUnhandled: (id: number, rejection: unknown) => void;
14
+ }) => void;
15
+}
16
+
17
+// React Native tracks unhandled rejections only in development, for LogBox; Observe has no hook.
18
+if (!__DEV__) {
19
+ const hermes: PromiseRejectionTracker | undefined = Reflect.get(globalThis, "HermesInternal");
20
+ hermes?.enablePromiseRejectionTracker?.({
21
+ allRejections: true,
22
+ onUnhandled: (_id, rejection) => Observe.reportError(rejection),
23
+ });
24
+}
25
+
26
+let latestConfiguration = 0;
27
+
28
+// While sending is off, a dispatch pass drops what it finds, so one runs before turning it back on.
29
+// Best effort: Observe skips the pass while it is backing off from a failed send.
30
+export async function setCrashReportsEnabled(enabled: boolean) {
31
+ const configuration = ++latestConfiguration;
32
+ const dispatchingEnabled = crashReportingAvailable && enabled;
33
+ if (dispatchingEnabled) {
34
+ await Observe.dispatchEvents();
35
+ if (configuration !== latestConfiguration) return;
36
+ }
37
+ Observe.configure({
38
+ dispatchingEnabled,
39
+ ...(typeof appVariant === "string" ? { environment: appVariant } : {}),
40
+ });
41
+}
42
+
43
+if (!crashReportingAvailable) {
44
+ void setCrashReportsEnabled(false);
45
+} else {
46
+ // Observe keeps its last configuration across launches; configuring it off while the setting
47
+ // loads would drop the last session's native crash. Imported lazily so the persistence graph
48
+ // never runs ahead of the imports `index.ts` orders first.
49
+ const changedSinceLoad = () => latestConfiguration !== 0;
50
+ void import("../../persistence/imperative")
51
+ .then(({ loadPreferences }) => loadPreferences())
52
+ .then(
53
+ (preferences) =>
54
+ changedSinceLoad()
55
+ ? undefined
56
+ : setCrashReportsEnabled(preferences.crashReportsEnabled !== false),
57
+ // Observe keeps the last choice it was given, which stands in for the unreadable one.
58
+ (error: unknown) => console.warn("[observe] could not read the crash report setting", error),
59
+ )
60
+ .catch((error: unknown) => {
61
+ console.warn("[observe] could not apply the crash report setting", error);
62
+ });
63
+}
apps/mobile/src/persistence/mobile-preferences.ts
+6
@@ -103,6 +103,8 @@ export interface Preferences {
103
readonly codeFontSize?: number | null;
104
readonly codeWordBreak?: boolean;
105
readonly mathRenderingEnabled?: boolean;
106
+ /** Unset means on: crash reports are sent unless the user turns them off. */
107
+ readonly crashReportsEnabled?: boolean;
108
readonly connectOnboardingOptOutAccounts?: ReadonlyArray<string>;
109
readonly collapsedProjectGroups?: readonly string[];
110
readonly dismissedDatabaseStorageNotices?: readonly string[];
@@ -183,6 +185,7 @@ function sanitizePreferences(parsed: Preferences): Preferences {
185
codeFontSize?: number | null;
186
codeWordBreak?: boolean;
187
mathRenderingEnabled?: boolean;
188
+ crashReportsEnabled?: boolean;
189
connectOnboardingOptOutAccounts?: ReadonlyArray<string>;
190
collapsedProjectGroups?: readonly string[];
191
dismissedDatabaseStorageNotices?: readonly string[];
@@ -272,6 +275,9 @@ function sanitizePreferences(parsed: Preferences): Preferences {
275
if (typeof parsed.codeWordBreak === "boolean") preferences.codeWordBreak = parsed.codeWordBreak;
276
if (typeof parsed.mathRenderingEnabled === "boolean")
277
preferences.mathRenderingEnabled = parsed.mathRenderingEnabled;
278
+ if (typeof parsed.crashReportsEnabled === "boolean") {
279
+ preferences.crashReportsEnabled = parsed.crashReportsEnabled;
280
+ }
281
if (Array.isArray(parsed.connectOnboardingOptOutAccounts)) {
282
preferences.connectOnboardingOptOutAccounts = parsed.connectOnboardingOptOutAccounts.filter(
283
(account): account is string => typeof account === "string",
docs/operations/relay-observability.md
+93
@@ -10,6 +10,11 @@ The relay Alchemy stack owns a shared Axiom trace setup:
10
- `t3-code-mobile-otel-ingest-prod`, the dataset-scoped mobile ingest token
11
- `t3-code-relay-client-otel-ingest-prod`, the dataset-scoped first-party relay-client ingest token
12
- `t3-code-relay-recent-spans-prod`, a view of recent request and endpoint spans
13
+- `t3-code-mobile-logs-prod` and `t3-code-mobile-metrics-prod`, the mobile app's crash, error, and
14
+ startup data from `expo-observe` (see [Mobile crashes and errors](#mobile-crashes-and-errors))
15
+- `t3-code-mobile-telemetry-ingest-prod`, the ingest token the Worker forwards that data with
16
+- `t3-code-mobile-sourcemaps-prod`, the R2 bucket of mobile JavaScript source maps, and
17
+ `t3-code-mobile-sourcemaps-upload-prod`, a Cloudflare token that can only write objects to it
18
19
Alchemy stages append their sanitized stage name to isolate resources, for example
20
`t3-code-relay-traces-dev-julius` for a personal stage.
@@ -159,3 +164,91 @@ see [Managed tunnel cleanup rollout](release.md#managed-tunnel-cleanup-rollout).
164
**Links degraded / database slow.** Warnings for slow degradation. Link failures above 20% over six
165
hours have so far meant provisioning running into the 9 s deadline, with Postgres latency as the
166
usual cause.
167
+
168
+## Mobile crashes and errors
169
+
170
+The mobile app reports through [`expo-observe`](https://docs.expo.dev/eas/observe/). Its
171
+`endpointUrl` points at the relay (`/v1/mobile/observe`, set in `apps/mobile/app.config.ts` from
172
+`T3CODE_RELAY_URL`), because Observe cannot send auth headers and prefixes every path with the EAS
173
+project id. The relay forwards each request to Axiom with its own token: logs (JavaScript errors,
174
+native crashes, log events) to `t3-code-mobile-logs-*`, metrics to `t3-code-mobile-metrics-*`, and
175
+spans to the shared traces dataset. Expo's Observe dashboard receives nothing. The endpoint URL is
176
+compiled into the native app, so pointing it elsewhere takes a store build.
177
+
178
+Hermes stacks from release builds are minified bytecode offsets. On the way through, the relay
179
+rewrites each JavaScript exception's `exception.stacktrace` with the bundle's source map from R2
180
+and records the outcome on the log record:
181
+
182
+- `relay.symbolication.status`: `symbolicated`, or why not. `no_bundle_id` is a development or
183
+ local build. `source_map_missing` means no map was uploaded for the bundle;
184
+ `source_map_too_large`, `source_map_invalid` and `source_map_unavailable` (R2 unreadable, retried
185
+ shortly) mean one was. `source_map_limit` is a request naming more bundles than the relay decodes
186
+ at once, and `no_bundle_frames` a stack with no frames from the bundle.
187
+- `relay.symbolication.source_map`: the R2 key used
188
+- `relay.symbolication.raw_stacktrace`: the original stack, when it was rewritten
189
+
190
+iOS frames lose their install path (`/private/var/containers/Bundle/Application/<uuid>/...`), which
191
+differs per install, so `exception.stacktrace` groups across users.
192
+
193
+A fatal JavaScript error ends in a native abort, which Observe reports as a second crash with a
194
+generic React Native stack. The relay marks that one `relay.crash.js_fatal` when both arrive in one
195
+request, as they do on the next launch. Leave it out to count real native crashes:
196
+
197
+```apl
198
+['t3-code-mobile-logs-prod']
199
+| where ['attributes.expo.error.source'] == 'nativeCrash'
200
+| where isnull(['attributes.relay.crash.js_fatal'])
201
+| summarize count() by ['attributes.exception.type'], ['attributes.exception.message']
202
+```
203
+
204
+Native crash stacks (`expo.error.source == 'nativeCrash'`) are not symbolicated. On iOS, app frames
205
+read like `main (T3Code + 19476)`: the offset into the app binary. The build hook uploads that
206
+build's dSYMs to `ios/dsym/<easBuildId>.zip`; download it with an account R2 token, then:
207
+
208
+```sh
209
+unzip <easBuildId>.zip
210
+atos -arch arm64 -o T3Code.app.dSYM/Contents/Resources/DWARF/T3Code -l 0x100000000 \
211
+ $(printf '0x%x' $((0x100000000 + 19476)))
212
+```
213
+
214
+Maps are keyed by the bundle that crashed: `<platform>/ota/<updateId>.map` for OTA updates and
215
+`<platform>/build/<easBuildId>.map` for the bundle embedded in a store build. The production
216
+mobile workflow uploads OTA maps after `eas update`, and the `eas-build-on-success` hook in
217
+`apps/mobile/package.json` uploads a build's map, both with `apps/mobile/scripts/upload-sourcemaps.mts`.
218
+They need the `MOBILE_SOURCEMAPS_R2` secret, a JSON value the stack outputs as
219
+`mobileSourceMapsUploadCredentials` from a token that can only write to the bucket. Creating that
220
+token needs **Account API Tokens: Edit** on the deploy token. Deploy output shows it redacted; read
221
+it from state, where it is under `__redacted__`:
222
+
223
+```sh
224
+cd infra/relay && vp exec alchemy state read T3CodeRelay/prod/output
225
+```
226
+
227
+Set it as a GitHub Actions secret, for OTA updates, and as an EAS environment variable with
228
+`secret` visibility in the `production` environment only, for builds. Keep it out of `preview`:
229
+PR preview builds run the PR's code in that environment. In CI only the upload step gets it, not
230
+`eas update` and the dependencies it runs. If the deploy replaces the token, update both.
231
+
232
+The `t3-code-mobile-health` dashboard is the place to look every few days: crash and handled
233
+error counts, top crashes with the installs and app versions they hit, crashes by version, whether
234
+stacks are symbolicated, and whether reports reach the relay at all. A crash counts once: the
235
+native abort after a JS crash (`relay.crash.js_fatal`) is left out. Startup times are metrics, so
236
+they are not on it; query `t3-code-mobile-metrics-*` with MPL. Prod and PR previews get the
237
+dashboard, and deployments with `RELAY_ALERT_NOTIFIER_ID` get these monitors
238
+([`infra/relay/src/mobileTelemetry/monitoring.ts`](../../infra/relay/src/mobileTelemetry/monitoring.ts)):
239
+
240
+- **crash spike** (pages): crashes per 15 min well above the same time over the last 7 days. Until
241
+ a week of data exists it has no baseline. A spike right after a release usually means a bad OTA
242
+ update; roll it back in EAS.
243
+- **one crash hitting many installs** (pages): a single crash on 25 installs within an hour.
244
+- **reports stopped** (warning): no startup metrics forwarded for an hour, so crashes are not
245
+ arriving either.
246
+- **crash stacks not symbolicated** (warning): over half of release JS stacks stayed minified for
247
+ 6 h. `relay.symbolication.status` names the cause.
248
+
249
+Thresholds are starting points, not measured baselines; revisit them after two weeks of
250
+production data.
251
+
252
+Without it, uploads are skipped with a warning and stacks stay minified. Each ingest request is one
253
+`relay.mobile_telemetry.ingest` span; `relay.mobile_telemetry.outcome` is `forwarded`,
254
+`rate_limited`, `body_too_large`, `upstream_unavailable`, or `upstream_timeout`.
docs/user/telemetry.md
+10
@@ -21,6 +21,16 @@ environment before starting it. Relay tracing also stops when `T3CODE_OTEL_SDK_D
21
The desktop app reads `T3CODE_TELEMETRY_ENABLED` from your shell profile (for example `~/.zshrc`) on macOS and
22
Linux, so export it there and restart the app. On Windows, set it as a user environment variable.
23
24
+## Mobile app crash reports
25
+
26
+When the iOS or Android app crashes or hits an error, it sends a report to T3 Code: the error
27
+message, the stack trace, the app version, and the device model and OS version. Each launch also
28
+sends startup timings, and every report carries a random ID for the install. Reports go through
29
+the T3 Connect relay to Axiom. They do not include your threads, prompts, or connection details,
30
+but an error message can quote a value the app was handling.
31
+
32
+To stop sending them, turn off **Send crash reports** in **Settings > About T3 Code > Diagnostics**.
33
+
34
## Export diagnostics to your own receiver
35
36
To send traces, metrics, or logs to an OpenTelemetry receiver, open
infra/relay/alchemy.run.ts
+7
-1
@@ -10,7 +10,7 @@ import * as Planetscale from "@alchemy.run/planetscale";
10
11
import { PublishClientConfig, tokenDigest } from "./src/clientConfig.ts";
12
import * as RelayDb from "./src/db.ts";
13
-import { RelayObservability } from "./src/observability.ts";
13
+import { MobileSourceMapsUploadCredentials, RelayObservability } from "./src/observability.ts";
14
import { isPreviewStage } from "./src/previewStages.ts";
15
import { PreviewTunnelSweep, PreviewTunnelSweepProvider } from "./src/previewTunnelSweep.ts";
16
import { ManagedEndpointZone, RelayApiZone, RelayDeploymentConfig } from "./src/zone.ts";
@@ -35,6 +35,7 @@ export default Alchemy.Stack(
35
const managedEndpointZone = yield* ManagedEndpointZone.pipe(Effect.orDie);
36
const relayApiZone = yield* RelayApiZone.pipe(Effect.orDie);
37
const observability = yield* RelayObservability;
38
+ const mobileSourceMapsUploadCredentials = yield* MobileSourceMapsUploadCredentials;
39
const api = yield* RelayWorker.Api;
40
if (isPreviewStage(stage)) {
41
const { managedEndpointZoneName } = yield* RelayDeploymentConfig;
@@ -72,6 +73,11 @@ export default Alchemy.Stack(
73
clientTracingUrl: observability.traces.otelTracesEndpoint,
74
clientTracingDataset: observability.traces.name,
75
clientTracingToken: observability.clientIngestToken.token,
76
+ mobileLogsDataset: observability.mobileLogs.name,
77
+ mobileMetricsDataset: observability.mobileMetrics.name,
78
+ mobileSourceMapsBucket: observability.mobileSourceMaps.bucketName,
79
+ // The MOBILE_SOURCEMAPS_R2 secret for GitHub Actions and EAS Build.
80
+ mobileSourceMapsUploadCredentials,
81
};
82
}).pipe(Effect.provide(RelayWorker.layer)),
83
);
infra/relay/src/mobileTelemetry/MobileTelemetry.test.ts
new
+382
@@ -0,0 +1,382 @@
1
+import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
2
+import * as NodeServices from "@effect/platform-node/NodeServices";
3
+import { describe, expect, it } from "@effect/vitest";
4
+import { RelayApi } from "@t3tools/contracts/relay";
5
+import * as Deferred from "effect/Deferred";
6
+import * as Effect from "effect/Effect";
7
+import * as Fiber from "effect/Fiber";
8
+import * as Layer from "effect/Layer";
9
+import * as Redacted from "effect/Redacted";
10
+import * as Schema from "effect/Schema";
11
+import * as Etag from "effect/http/Etag";
12
+import * as HttpClient from "effect/http/HttpClient";
13
+import type * as HttpClientError from "effect/http/HttpClientError";
14
+import type * as HttpClientRequest from "effect/http/HttpClientRequest";
15
+import * as HttpClientResponse from "effect/http/HttpClientResponse";
16
+import * as HttpEffect from "effect/http/HttpEffect";
17
+import * as HttpRouter from "effect/http/HttpRouter";
18
+import * as HttpServerRequest from "effect/http/HttpServerRequest";
19
+import * as HttpServerResponse from "effect/http/HttpServerResponse";
20
+import * as HttpApi from "effect/http-api/HttpApi";
21
+import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
22
+
23
+import { RELAY_HTTP_ROUTER_CONFIG } from "../http/Api.ts";
24
+import * as RelayHttpApi from "../http/Api.ts";
25
+import * as MobileTelemetry from "./MobileTelemetry.ts";
26
+
27
+const PROJECT_ID = "d763fcb8-d37c-41ea-a773-b54a0ab4a454";
28
+const UPDATE_ID = "0198e8a4-1111-7000-8000-000000000001";
29
+const BUILD_ID = "0198e8a4-2222-7000-8000-000000000002";
30
+const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
31
+const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
32
+
33
+const destinations = {
34
+ traces: { url: "https://axiom.test/v1/traces", dataset: "mobile-traces" },
35
+ logs: { url: "https://axiom.test/v1/logs", dataset: "mobile-logs" },
36
+ metrics: { url: "https://axiom.test/v1/metrics", dataset: "mobile-metrics" },
37
+} as const;
38
+
39
+/** Maps generated column 12 of the one-line bundle to `src/App.tsx` 1:2. */
40
+const SOURCE_MAP = encodeJson({ version: 3, sources: ["src/App.tsx"], mappings: "AAAA,UAAE" });
41
+const STACK = "Error: x\n at render (address at index.android.bundle:1:12)";
42
+
43
+interface Harness {
44
+ readonly execute?: (
45
+ request: HttpClientRequest.HttpClientRequest,
46
+ ) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>;
47
+ /** The source maps bucket, by key. */
48
+ readonly objects?: Readonly<Record<string, string>>;
49
+}
50
+
51
+const makeHarness = (options: Harness = {}) => {
52
+ const sent: Array<HttpClientRequest.HttpClientRequest> = [];
53
+ const execute =
54
+ options.execute ??
55
+ ((request: HttpClientRequest.HttpClientRequest) =>
56
+ Effect.succeed(
57
+ HttpClientResponse.fromWeb(
58
+ request,
59
+ new Response("{}", { status: 200, headers: { "content-type": "application/json" } }),
60
+ ),
61
+ ));
62
+ const layerTelemetry = MobileTelemetry.layer.pipe(
63
+ Layer.provide(
64
+ Layer.mergeAll(
65
+ Layer.succeed(MobileTelemetry.MobileTelemetrySettings, {
66
+ axiomToken: Redacted.make("axiom-ingest-token"),
67
+ destinations,
68
+ }),
69
+ Layer.succeed(MobileTelemetry.MobileTelemetryRateLimiter, {
70
+ allow: () => Effect.succeed(true),
71
+ }),
72
+ MobileTelemetry.layerSourceMaps.pipe(
73
+ Layer.provide(
74
+ Layer.succeed(MobileTelemetry.SourceMapObjects, {
75
+ get: (key) =>
76
+ Effect.sync(() => {
77
+ const object = options.objects?.[key];
78
+ return object === undefined
79
+ ? undefined
80
+ : { size: object.length, text: Effect.succeed(object) };
81
+ }),
82
+ }),
83
+ ),
84
+ ),
85
+ Layer.succeed(
86
+ HttpClient.HttpClient,
87
+ HttpClient.make((request) => {
88
+ sent.push(request);
89
+ return execute(request);
90
+ }),
91
+ ),
92
+ ),
93
+ ),
94
+ );
95
+ const httpEffect = HttpRouter.toHttpEffect(
96
+ Layer.mergeAll(
97
+ HttpApiBuilder.layer(HttpApi.make("RelayApi").add(RelayApi.groups.mobileTelemetry)).pipe(
98
+ Layer.provide(MobileTelemetry.layerApi.pipe(Layer.provide(layerTelemetry))),
99
+ Layer.provide([NodeServices.layer, NodeHttpPlatform.layer, Etag.layerWeak]),
100
+ ),
101
+ RelayHttpApi.layerNotFoundRoute,
102
+ ),
103
+ ).pipe(Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG));
104
+ const send = (request: Request) =>
105
+ Effect.gen(function* () {
106
+ const handler = yield* httpEffect;
107
+ const response = yield* Deferred.make<HttpServerResponse.HttpServerResponse>();
108
+ yield* HttpEffect.toHandled(handler, (_request, value) =>
109
+ Deferred.succeed(response, value),
110
+ ).pipe(
111
+ Effect.provideService(
112
+ HttpServerRequest.HttpServerRequest,
113
+ HttpServerRequest.fromWeb(request),
114
+ ),
115
+ );
116
+ return yield* Deferred.await(response);
117
+ });
118
+ return { sent, send };
119
+};
120
+
121
+const ingest = (signal: string, body: string) =>
122
+ new Request(`https://relay.test/v1/mobile/observe/${PROJECT_ID}/v1/${signal}`, {
123
+ method: "POST",
124
+ headers: { "content-type": "application/json", "cf-connecting-ip": "203.0.113.7" },
125
+ body,
126
+ });
127
+
128
+const sentText = (request: HttpClientRequest.HttpClientRequest) =>
129
+ request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "";
130
+
131
+const str = (key: string, value: string) => ({ key, value: { stringValue: value } });
132
+
133
+/** One resource's JS exception, as Observe reports it from the app's bundle. */
134
+const exceptionResource = (resource: ReadonlyArray<ReturnType<typeof str>>) => ({
135
+ resource: { attributes: resource },
136
+ scopeLogs: [
137
+ {
138
+ logRecords: [
139
+ {
140
+ attributes: [str("expo.error.source", "global"), str("exception.stacktrace", STACK)],
141
+ },
142
+ ],
143
+ },
144
+ ],
145
+});
146
+
147
+/** Each forwarded exception record's string attributes. */
148
+const forwardedExceptions = (request: HttpClientRequest.HttpClientRequest) => {
149
+ const payload = decodeJson(sentText(request)) as {
150
+ resourceLogs: Array<{
151
+ scopeLogs: Array<{
152
+ logRecords: Array<{ attributes: Array<{ key: string; value: { stringValue: string } }> }>;
153
+ }>;
154
+ }>;
155
+ };
156
+ return payload.resourceLogs.map((resourceLogs) =>
157
+ Object.fromEntries(
158
+ resourceLogs.scopeLogs[0]!.logRecords[0]!.attributes.map(({ key, value }) => [
159
+ key,
160
+ value.stringValue,
161
+ ]),
162
+ ),
163
+ );
164
+};
165
+
166
+describe("MobileTelemetry", () => {
167
+ it.effect("forwards each signal to its Axiom dataset with the relay's token", () =>
168
+ Effect.gen(function* () {
169
+ const harness = makeHarness();
170
+ const body = encodeJson({ resourceMetrics: [] });
171
+ for (const signal of ["metrics", "traces", "logs"] as const) {
172
+ const response = yield* harness.send(ingest(signal, body));
173
+ expect(response.status).toBe(200);
174
+ }
175
+ expect(
176
+ harness.sent.map((request) => ({
177
+ url: request.url,
178
+ authorization: request.headers.authorization,
179
+ dataset: request.headers["x-axiom-dataset"],
180
+ body: sentText(request),
181
+ })),
182
+ ).toEqual(
183
+ (["metrics", "traces", "logs"] as const).map((signal) => ({
184
+ url: destinations[signal].url,
185
+ authorization: "Bearer axiom-ingest-token",
186
+ dataset: destinations[signal].dataset,
187
+ body,
188
+ })),
189
+ );
190
+ }),
191
+ );
192
+
193
+ it.effect("passes Axiom's status and Retry-After back for Observe's retries", () =>
194
+ Effect.gen(function* () {
195
+ const harness = makeHarness({
196
+ execute: (request) =>
197
+ Effect.succeed(
198
+ HttpClientResponse.fromWeb(
199
+ request,
200
+ new Response('{"message":"slow down"}', {
201
+ status: 429,
202
+ headers: { "content-type": "application/json", "retry-after": "30" },
203
+ }),
204
+ ),
205
+ ),
206
+ });
207
+ const response = yield* harness.send(ingest("metrics", "{}"));
208
+ expect(response.status).toBe(429);
209
+ expect(response.headers["retry-after"]).toBe("30");
210
+ }),
211
+ );
212
+
213
+ it.effect("symbolicates JS exceptions with the OTA update's map, else the build's", () =>
214
+ Effect.gen(function* () {
215
+ const harness = makeHarness({
216
+ objects: {
217
+ [`android/ota/${UPDATE_ID}.map`]: SOURCE_MAP,
218
+ [`ios/build/${BUILD_ID}.map`]: SOURCE_MAP,
219
+ },
220
+ });
221
+ const body = encodeJson({
222
+ resourceLogs: [
223
+ exceptionResource([
224
+ str("os.type", "linux"),
225
+ str("expo.app.update_id", UPDATE_ID),
226
+ str("expo.eas_build.id", BUILD_ID),
227
+ ]),
228
+ exceptionResource([str("os.type", "darwin"), str("expo.eas_build.id", BUILD_ID)]),
229
+ ],
230
+ });
231
+ expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
232
+ const symbolicated = (key: string) => ({
233
+ "expo.error.source": "global",
234
+ "exception.stacktrace": "Error: x\n at render (src/App.tsx:1:3)",
235
+ "relay.symbolication.source_map": key,
236
+ "relay.symbolication.status": "symbolicated",
237
+ "relay.symbolication.raw_stacktrace": STACK,
238
+ });
239
+ expect(forwardedExceptions(harness.sent[0]!)).toEqual([
240
+ symbolicated(`android/ota/${UPDATE_ID}.map`),
241
+ symbolicated(`ios/build/${BUILD_ID}.map`),
242
+ ]);
243
+ }),
244
+ );
245
+
246
+ it.effect("forwards the stack untouched when the OTA update's map is missing", () =>
247
+ Effect.gen(function* () {
248
+ // The build's map describes the embedded bundle, not the update that threw.
249
+ const harness = makeHarness({
250
+ objects: { [`android/build/${BUILD_ID}.map`]: SOURCE_MAP },
251
+ });
252
+ const body = encodeJson({
253
+ resourceLogs: [
254
+ exceptionResource([
255
+ str("os.type", "linux"),
256
+ str("expo.app.update_id", UPDATE_ID),
257
+ str("expo.eas_build.id", BUILD_ID),
258
+ ]),
259
+ ],
260
+ });
261
+ expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
262
+ expect(forwardedExceptions(harness.sent[0]!)).toEqual([
263
+ {
264
+ "expo.error.source": "global",
265
+ "exception.stacktrace": STACK,
266
+ "relay.symbolication.status": "source_map_missing",
267
+ },
268
+ ]);
269
+ }),
270
+ );
271
+
272
+ it.effect("says why a stack stayed minified, without the iOS install path", () =>
273
+ Effect.gen(function* () {
274
+ const harness = makeHarness({ objects: { [`ios/build/${BUILD_ID}.map`]: "{}" } });
275
+ const iosStack =
276
+ "Error: x\n at render (address at /private/var/containers/Bundle/Application/" +
277
+ "0C1D/T3Code.app/main.jsbundle:1:12)";
278
+ const record = (stack: string) => ({
279
+ scopeLogs: [{ logRecords: [{ attributes: [str("exception.stacktrace", stack)] }] }],
280
+ });
281
+ const body = encodeJson({
282
+ resourceLogs: [
283
+ { resource: { attributes: [str("os.type", "linux")] }, ...record(STACK) },
284
+ {
285
+ resource: {
286
+ attributes: [str("os.type", "darwin"), str("expo.eas_build.id", BUILD_ID)],
287
+ },
288
+ ...record(iosStack),
289
+ },
290
+ ],
291
+ });
292
+ expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
293
+ expect(forwardedExceptions(harness.sent[0]!)).toEqual([
294
+ { "exception.stacktrace": STACK, "relay.symbolication.status": "no_bundle_id" },
295
+ {
296
+ "exception.stacktrace": "Error: x\n at render (address at main.jsbundle:1:12)",
297
+ "relay.symbolication.status": "source_map_invalid",
298
+ },
299
+ ]);
300
+ }),
301
+ );
302
+
303
+ it.effect("marks the native abort that follows a fatal JS error in the same session", () =>
304
+ Effect.gen(function* () {
305
+ const harness = makeHarness();
306
+ const fatal = { key: "expo.error.is_fatal", value: { boolValue: true } };
307
+ const crash = (session: string, source: string) => ({
308
+ attributes: [str("expo.error.source", source), str("session.id", session), fatal],
309
+ });
310
+ const body = encodeJson({
311
+ resourceLogs: [
312
+ {
313
+ resource: { attributes: [str("os.type", "darwin")] },
314
+ scopeLogs: [
315
+ {
316
+ logRecords: [
317
+ crash("a", "global"),
318
+ crash("a", "nativeCrash"),
319
+ crash("b", "nativeCrash"),
320
+ ],
321
+ },
322
+ ],
323
+ },
324
+ ],
325
+ });
326
+ expect((yield* harness.send(ingest("logs", body))).status).toBe(200);
327
+ const records = (
328
+ decodeJson(sentText(harness.sent[0]!)) as {
329
+ resourceLogs: Array<{
330
+ scopeLogs: Array<{ logRecords: Array<{ attributes: Array<{ key: string }> }> }>;
331
+ }>;
332
+ }
333
+ ).resourceLogs[0]!.scopeLogs[0]!.logRecords;
334
+ expect(
335
+ records.map((record) =>
336
+ record.attributes.some((attribute) => attribute.key === "relay.crash.js_fatal"),
337
+ ),
338
+ ).toEqual([false, true, false]);
339
+ }),
340
+ );
341
+
342
+ it.effect("answers 413 for a body over the cap without forwarding it", () =>
343
+ Effect.gen(function* () {
344
+ const harness = makeHarness();
345
+ const tooLarge = "x".repeat(MobileTelemetry.MOBILE_TELEMETRY_MAX_BODY_BYTES + 1);
346
+ expect((yield* harness.send(ingest("logs", tooLarge))).status).toBe(413);
347
+ expect(harness.sent).toEqual([]);
348
+ }),
349
+ );
350
+
351
+ it.effect("decodes one source map at a time", () =>
352
+ Effect.gen(function* () {
353
+ const releaseFirst = yield* Deferred.make<void>();
354
+ const firstStarted = yield* Deferred.make<void>();
355
+ let firstDone = false;
356
+ let secondSawFirstDone: boolean | undefined;
357
+ const sourceMaps = yield* MobileTelemetry.SourceMaps.pipe(
358
+ Effect.provide(MobileTelemetry.layerSourceMaps),
359
+ Effect.provideService(MobileTelemetry.SourceMapObjects, {
360
+ get: (key) =>
361
+ key === "first"
362
+ ? Deferred.succeed(firstStarted, undefined).pipe(
363
+ Effect.andThen(Deferred.await(releaseFirst)),
364
+ Effect.as({ size: SOURCE_MAP.length, text: Effect.succeed(SOURCE_MAP) }),
365
+ Effect.ensuring(Effect.sync(() => (firstDone = true))),
366
+ )
367
+ : Effect.sync(() => {
368
+ secondSawFirstDone = firstDone;
369
+ return { size: SOURCE_MAP.length, text: Effect.succeed(SOURCE_MAP) };
370
+ }),
371
+ }),
372
+ );
373
+ const first = yield* Effect.forkChild(sourceMaps.find("first"));
374
+ yield* Deferred.await(firstStarted);
375
+ const second = yield* Effect.forkChild(sourceMaps.find("second"), { startImmediately: true });
376
+ yield* Deferred.succeed(releaseFirst, undefined);
377
+ yield* Fiber.join(first);
378
+ expect(yield* Fiber.join(second)).toMatchObject({ key: "second" });
379
+ expect(secondSawFirstDone).toBe(true);
380
+ }),
381
+ );
382
+});
infra/relay/src/mobileTelemetry/MobileTelemetry.ts
new
+339
@@ -0,0 +1,339 @@
1
+import * as Cache from "effect/Cache";
2
+import * as Context from "effect/Context";
3
+import * as Duration from "effect/Duration";
4
+import * as Effect from "effect/Effect";
5
+import * as Exit from "effect/Exit";
6
+import * as Layer from "effect/Layer";
7
+import * as Option from "effect/Option";
8
+import * as Redacted from "effect/Redacted";
9
+import * as Result from "effect/Result";
10
+import * as Schema from "effect/Schema";
11
+import * as Semaphore from "effect/Semaphore";
12
+import * as Stream from "effect/Stream";
13
+import * as HttpClient from "effect/http/HttpClient";
14
+import * as HttpClientRequest from "effect/http/HttpClientRequest";
15
+import type * as HttpServerRequest from "effect/http/HttpServerRequest";
16
+import * as HttpServerResponse from "effect/http/HttpServerResponse";
17
+import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
18
+import { RelayApi, type RelayMobileTelemetrySignal } from "@t3tools/contracts/relay";
19
+
20
+import {
21
+ applySymbolication,
22
+ bundleSourceMapKey,
23
+ collectJsExceptions,
24
+ markJsFatalCrashes,
25
+ type SourceMapMiss,
26
+} from "./otlpLogs.ts";
27
+import { decodeSourceMap, type CompactSourceMap } from "./sourceMap.ts";
28
+
29
+/** Observe splits batches the endpoint answers 413 for, so this only bounds one request. */
30
+export const MOBILE_TELEMETRY_MAX_BODY_BYTES = 2 * 1024 * 1024;
31
+/** Per client IP. An app flushes a few batches per launch and background. */
32
+export const MOBILE_TELEMETRY_RATE_LIMIT = { limit: 120, periodSeconds: 60 } as const;
33
+/** Inside the relay's 9 s request deadline, so a slow Axiom answers as `upstream_timeout`. */
34
+const UPSTREAM_TIMEOUT = Duration.seconds(7);
35
+const MAX_UPSTREAM_RESPONSE_BYTES = 64 * 1024;
36
+/** A decoded production map is about 20 MB; two fit an isolate and cover iOS and Android. */
37
+const SOURCE_MAP_CACHE_CAPACITY = 2;
38
+/** Our iOS map is about 12 MB once uploads drop what the relay never reads; larger is skipped unread. */
39
+const MAX_SOURCE_MAP_BYTES = 16 * 1024 * 1024;
40
+/** A request can carry thousands of frames, each rewritten to a longer source path. */
41
+const MAX_SYMBOLICATION_GROWTH = 1024 * 1024;
42
+/** One app names one bundle, or two across an OTA update; more would evict cached maps. */
43
+const MAX_SOURCE_MAPS_PER_REQUEST = 2;
44
+/** A map can land in R2 a moment after its update ships, so a miss is retried. */
45
+const MISSING_SOURCE_MAP_TTL = Duration.minutes(5);
46
+const FAILED_SOURCE_MAP_TTL = Duration.seconds(30);
47
+
48
+const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown);
49
+const decodeJson = Schema.decodeUnknownOption(UnknownFromJsonString);
50
+const encodeJson = Schema.encodeSync(UnknownFromJsonString);
51
+const parseJson = (text: string): unknown => Option.getOrUndefined(decodeJson(text));
52
+
53
+interface AxiomDestination {
54
+ /** Axiom's OTLP/HTTP endpoint for the signal, such as `.../v1/logs`. */
55
+ readonly url: string;
56
+ readonly dataset: string;
57
+}
58
+
59
+export class MobileTelemetrySettings extends Context.Service<
60
+ MobileTelemetrySettings,
61
+ {
62
+ readonly axiomToken: Redacted.Redacted<string>;
63
+ readonly destinations: Readonly<Record<RelayMobileTelemetrySignal, AxiomDestination>>;
64
+ }
65
+>()("t3code-relay/mobileTelemetry/MobileTelemetry/MobileTelemetrySettings") {}
66
+
67
+export class MobileTelemetryRateLimiter extends Context.Service<
68
+ MobileTelemetryRateLimiter,
69
+ { readonly allow: (key: string) => Effect.Effect<boolean> }
70
+>()("t3code-relay/mobileTelemetry/MobileTelemetry/MobileTelemetryRateLimiter") {}
71
+
72
+export class SourceMapObjectError extends Schema.TaggedError<SourceMapObjectError>()(
73
+ "SourceMapObjectError",
74
+ { key: Schema.String, cause: Schema.Defect() },
75
+) {}
76
+
77
+/** An object in the source maps bucket, read only once its size is known to fit. */
78
+interface StoredSourceMap {
79
+ readonly size: number;
80
+ readonly text: Effect.Effect<string, SourceMapObjectError>;
81
+}
82
+
83
+interface FoundSourceMap {
84
+ readonly key: string;
85
+ readonly map: CompactSourceMap;
86
+}
87
+
88
+export class SourceMaps extends Context.Service<
89
+ SourceMaps,
90
+ {
91
+ readonly find: (key: string) => Effect.Effect<FoundSourceMap | SourceMapMiss>;
92
+ }
93
+>()("t3code-relay/mobileTelemetry/MobileTelemetry/SourceMaps") {}
94
+
95
+export class SourceMapObjects extends Context.Service<
96
+ SourceMapObjects,
97
+ {
98
+ readonly get: (key: string) => Effect.Effect<StoredSourceMap | undefined, SourceMapObjectError>;
99
+ }
100
+>()("t3code-relay/mobileTelemetry/MobileTelemetry/SourceMapObjects") {}
101
+
102
+const makeSourceMaps = Effect.gen(function* () {
103
+ const objects = yield* SourceMapObjects;
104
+ // The cache only joins lookups of one key; a decode holds ~40 MB, so different keys take turns.
105
+ const decoding = yield* Semaphore.make(1);
106
+ const cache = yield* Cache.makeWith(
107
+ (key: string) =>
108
+ Effect.gen(function* () {
109
+ const stored = yield* objects.get(key);
110
+ if (stored === undefined) return "source_map_missing" as const;
111
+ if (stored.size > MAX_SOURCE_MAP_BYTES) {
112
+ yield* Effect.logWarning("Stored mobile source map is too large", {
113
+ key,
114
+ size: stored.size,
115
+ });
116
+ return "source_map_too_large" as const;
117
+ }
118
+ const map = decodeSourceMap(parseJson(yield* stored.text));
119
+ if (typeof map !== "string") return { key, map } satisfies FoundSourceMap;
120
+ yield* Effect.logWarning("Stored mobile source map could not be decoded", { key, map });
121
+ return map === "too_large"
122
+ ? ("source_map_too_large" as const)
123
+ : ("source_map_invalid" as const);
124
+ }).pipe(Semaphore.withPermit(decoding)),
125
+ {
126
+ capacity: SOURCE_MAP_CACHE_CAPACITY,
127
+ timeToLive: (exit) =>
128
+ Exit.isFailure(exit)
129
+ ? FAILED_SOURCE_MAP_TTL
130
+ : typeof exit.value === "string"
131
+ ? MISSING_SOURCE_MAP_TTL
132
+ : Duration.infinity,
133
+ },
134
+ );
135
+ return SourceMaps.of({
136
+ find: Effect.fn("MobileTelemetry.findSourceMap")(function* (key: string) {
137
+ return yield* Cache.get(cache, key).pipe(
138
+ Effect.catch((error) =>
139
+ Effect.logWarning("Mobile source map lookup failed", { key: error.key }).pipe(
140
+ Effect.as("source_map_unavailable" as const),
141
+ ),
142
+ ),
143
+ );
144
+ }),
145
+ });
146
+});
147
+
148
+/** Build once per Worker isolate, not per request: decoding a production map costs ~150 ms. */
149
+export const layerSourceMaps = Layer.effect(SourceMaps, makeSourceMaps);
150
+
151
+export class MobileTelemetry extends Context.Service<
152
+ MobileTelemetry,
153
+ {
154
+ readonly handle: (
155
+ request: HttpServerRequest.HttpServerRequest,
156
+ params: { readonly projectId: string; readonly signal: RelayMobileTelemetrySignal },
157
+ ) => Effect.Effect<HttpServerResponse.HttpServerResponse>;
158
+ }
159
+>()("t3code-relay/mobileTelemetry/MobileTelemetry") {}
160
+
161
+class BodyTooLarge extends Schema.TaggedError<BodyTooLarge>()("BodyTooLarge", {}) {}
162
+
163
+const errorResponse = (status: number, error: string, headers?: Record<string, string>) =>
164
+ HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) });
165
+
166
+const readCapped = <E>(stream: Stream.Stream<Uint8Array, E>, maxBytes: number) =>
167
+ Effect.suspend(() => {
168
+ const chunks: Array<Uint8Array> = [];
169
+ let total = 0;
170
+ return stream.pipe(
171
+ Stream.runForEach((chunk) => {
172
+ total += chunk.length;
173
+ if (total > maxBytes) return Effect.fail(new BodyTooLarge());
174
+ chunks.push(chunk);
175
+ return Effect.void;
176
+ }),
177
+ Effect.map(() => {
178
+ const body = new Uint8Array(total);
179
+ let offset = 0;
180
+ for (const chunk of chunks) {
181
+ body.set(chunk, offset);
182
+ offset += chunk.length;
183
+ }
184
+ return body;
185
+ }),
186
+ );
187
+ });
188
+
189
+const hasNoBody = (request: HttpServerRequest.HttpServerRequest) =>
190
+ request.source instanceof Request && request.source.body === null;
191
+
192
+const make = Effect.gen(function* () {
193
+ const settings = yield* MobileTelemetrySettings;
194
+ const rateLimiter = yield* MobileTelemetryRateLimiter;
195
+ const sourceMaps = yield* SourceMaps;
196
+ const httpClient = yield* HttpClient.HttpClient;
197
+
198
+ const symbolicateLogs = Effect.fn("relay.mobile_telemetry.symbolicate")(function* (
199
+ body: Uint8Array,
200
+ ) {
201
+ const payload = parseJson(new TextDecoder().decode(body));
202
+ const exceptions = collectJsExceptions(payload);
203
+ const jsFatalCrashes = markJsFatalCrashes(payload);
204
+ if (exceptions.length === 0 && jsFatalCrashes === 0) return undefined;
205
+ let symbolicated = 0;
206
+ let growthLeft = MAX_SYMBOLICATION_GROWTH;
207
+ const keys = new Set<string>();
208
+ for (const exception of exceptions) {
209
+ const key = bundleSourceMapKey(exception.identity);
210
+ if (key !== undefined && keys.size < MAX_SOURCE_MAPS_PER_REQUEST) keys.add(key);
211
+ const outcome = applySymbolication(
212
+ exception,
213
+ key === undefined
214
+ ? "no_bundle_id"
215
+ : keys.has(key)
216
+ ? yield* sourceMaps.find(key)
217
+ : "source_map_limit",
218
+ growthLeft,
219
+ );
220
+ growthLeft -= outcome.growth;
221
+ if (outcome.symbolicated) symbolicated += 1;
222
+ }
223
+ yield* Effect.annotateCurrentSpan({
224
+ "relay.mobile_telemetry.exceptions": exceptions.length,
225
+ "relay.mobile_telemetry.symbolicated": symbolicated,
226
+ "relay.mobile_telemetry.js_fatal_crashes": jsFatalCrashes,
227
+ });
228
+ return new TextEncoder().encode(encodeJson(payload));
229
+ });
230
+
231
+ const forward = Effect.fn("relay.mobile_telemetry.forward")(function* (
232
+ destination: AxiomDestination,
233
+ body: Uint8Array,
234
+ ) {
235
+ const response = yield* httpClient.execute(
236
+ HttpClientRequest.post(destination.url, {
237
+ headers: {
238
+ authorization: `Bearer ${Redacted.value(settings.axiomToken)}`,
239
+ "x-axiom-dataset": destination.dataset,
240
+ },
241
+ }).pipe(HttpClientRequest.bodyUint8Array(body, "application/json")),
242
+ );
243
+ const responseBody = yield* readCapped(response.stream, MAX_UPSTREAM_RESPONSE_BYTES).pipe(
244
+ Effect.catchIf(
245
+ (error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",
246
+ () => Effect.succeed(new Uint8Array(0)),
247
+ ),
248
+ );
249
+ return { response, body: responseBody };
250
+ }, Effect.timeoutOption(UPSTREAM_TIMEOUT));
251
+
252
+ const handle = Effect.fn("relay.mobile_telemetry.ingest")(function* (
253
+ request: HttpServerRequest.HttpServerRequest,
254
+ {
255
+ projectId,
256
+ signal,
257
+ }: { readonly projectId: string; readonly signal: RelayMobileTelemetrySignal },
258
+ ) {
259
+ const outcome = (value: string) =>
260
+ Effect.annotateCurrentSpan({ "relay.mobile_telemetry.outcome": value });
261
+ yield* Effect.annotateCurrentSpan({
262
+ "relay.mobile_telemetry.project_id": projectId,
263
+ "relay.mobile_telemetry.signal": signal,
264
+ });
265
+
266
+ const clientIp = request.headers["cf-connecting-ip"] ?? "unknown";
267
+ if (!(yield* rateLimiter.allow(clientIp))) {
268
+ yield* outcome("rate_limited");
269
+ return errorResponse(429, "rate_limited", {
270
+ "retry-after": String(MOBILE_TELEMETRY_RATE_LIMIT.periodSeconds),
271
+ });
272
+ }
273
+
274
+ const read = yield* (
275
+ hasNoBody(request)
276
+ ? Effect.succeed(new Uint8Array(0))
277
+ : readCapped(request.stream, MOBILE_TELEMETRY_MAX_BODY_BYTES)
278
+ ).pipe(Effect.result);
279
+ if (Result.isFailure(read)) {
280
+ const tooLarge = read.failure._tag === "BodyTooLarge";
281
+ yield* outcome(tooLarge ? "body_too_large" : "body_unreadable");
282
+ return errorResponse(tooLarge ? 413 : 400, tooLarge ? "body_too_large" : "invalid_body");
283
+ }
284
+ yield* Effect.annotateCurrentSpan({ "relay.mobile_telemetry.bytes": read.success.length });
285
+
286
+ // Forwarding matters more than symbolication: on any defect the payload goes through as sent.
287
+ const body =
288
+ signal === "logs"
289
+ ? ((yield* symbolicateLogs(read.success).pipe(
290
+ Effect.catchCause((cause) =>
291
+ Effect.logWarning("Mobile stack symbolication failed", { cause }).pipe(
292
+ Effect.as(undefined),
293
+ ),
294
+ ),
295
+ )) ?? read.success)
296
+ : read.success;
297
+
298
+ const upstream = yield* forward(settings.destinations[signal], body).pipe(Effect.result);
299
+ if (Result.isFailure(upstream)) {
300
+ yield* outcome("upstream_unavailable");
301
+ return errorResponse(502, "upstream_unavailable");
302
+ }
303
+ if (Option.isNone(upstream.success)) {
304
+ yield* outcome("upstream_timeout");
305
+ return errorResponse(504, "upstream_timeout");
306
+ }
307
+ const { response, body: responseBody } = upstream.success.value;
308
+ yield* outcome("forwarded");
309
+ yield* Effect.annotateCurrentSpan({
310
+ "relay.mobile_telemetry.upstream_status": response.status,
311
+ });
312
+ // Observe's retry and partial-success handling reads only these from Axiom's answer.
313
+ const contentType = response.headers["content-type"];
314
+ const retryAfter = response.headers["retry-after"];
315
+ const headers = retryAfter === undefined ? undefined : { "retry-after": retryAfter };
316
+ if (responseBody.length === 0) {
317
+ return HttpServerResponse.empty({ status: response.status, ...(headers ? { headers } : {}) });
318
+ }
319
+ return HttpServerResponse.uint8Array(responseBody, {
320
+ status: response.status,
321
+ ...(headers ? { headers } : {}),
322
+ ...(contentType ? { contentType } : {}),
323
+ });
324
+ });
325
+
326
+ return MobileTelemetry.of({ handle });
327
+});
328
+
329
+export const layer = Layer.effect(MobileTelemetry, make);
330
+
331
+/** Raw so the body is read once, capped. */
332
+export const layerApi = HttpApiBuilder.group(
333
+ RelayApi,
334
+ "mobileTelemetry",
335
+ Effect.fnUntraced(function* (handlers) {
336
+ const telemetry = yield* MobileTelemetry;
337
+ return handlers.handleRaw("ingest", ({ request, params }) => telemetry.handle(request, params));
338
+ }),
339
+);
infra/relay/src/mobileTelemetry/monitoring.ts
new
+297
@@ -0,0 +1,297 @@
1
+import * as Alchemy from "alchemy";
2
+import * as Axiom from "@alchemy.run/axiom";
3
+import * as Output from "alchemy/Output";
4
+import * as Effect from "effect/Effect";
5
+import * as Option from "effect/Option";
6
+
7
+import { relayResourceNameForStage } from "../deploymentConfig.ts";
8
+import { notifierConfig } from "../monitoring.ts";
9
+import { isPreviewStage } from "../previewStages.ts";
10
+
11
+const RUNBOOK_URL =
12
+ "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#mobile-crashes-and-errors";
13
+
14
+// A new dataset has none of these fields until a report carries them, and an
15
+// unknown field fails the whole query, so every one has a default.
16
+const field = (name: string, fallback: string) => `column_ifexists('${name}', ${fallback})`;
17
+const reports = (dataset: string) =>
18
+ [
19
+ `['${dataset}']`,
20
+ `| extend source = tostring(${field("attributes.expo.error.source", "''")}),`,
21
+ ` fatal = tobool(${field("attributes.expo.error.is_fatal", "false")}),`,
22
+ ` js_fatal = tobool(${field("attributes.relay.crash.js_fatal", "false")}),`,
23
+ ` platform = tostring(${field("resource.os.name", "''")}),`,
24
+ ` install = tostring(${field("resource.expo.eas_client.id", "''")}),`,
25
+ ` error_type = tostring(${field("attributes.exception.type", "''")}),`,
26
+ ` message = substring(tostring(${field("attributes.exception.message", "''")}), 0, 140),`,
27
+ ` version = strcat(tostring(${field("service.version", "''")}), ' (', tostring(${field("resource.expo.app.build_number", "''")}), ')')`,
28
+ `| where isnotempty(source) and isnotempty(platform)`,
29
+ // A fatal JS error also ends in a native abort; counting both would double every JS crash.
30
+ `| extend kind = case(source == 'nativeCrash' and js_fatal, 'abort after JS crash', source == 'nativeCrash', 'native crash', fatal, 'JS crash', 'handled error')`,
31
+ ].join("\n");
32
+const crashes = (dataset: string) =>
33
+ `${reports(dataset)}\n| where kind in ('JS crash', 'native crash')`;
34
+const handledErrors = (dataset: string) => `${reports(dataset)}\n| where kind == 'handled error'`;
35
+const ingest = (traces: string) =>
36
+ `['${traces}']\n| where name == 'relay.mobile_telemetry.ingest'\n| extend signal = tostring(['attributes.custom']['relay.mobile_telemetry.signal']), outcome = tostring(['attributes.custom']['relay.mobile_telemetry.outcome'])`;
37
+
38
+type Severity = "page" | "warn";
39
+
40
+interface Datasets {
41
+ readonly logs: string;
42
+ readonly traces: string;
43
+}
44
+
45
+interface MobileMonitor {
46
+ readonly id: string;
47
+ readonly severity: Severity;
48
+ readonly name: string;
49
+ /** Why it fires. The runbook link is appended. */
50
+ readonly description: string;
51
+ readonly query: (datasets: Datasets) => string;
52
+ readonly props: Pick<
53
+ Axiom.MonitorProps,
54
+ | "type"
55
+ | "operator"
56
+ | "threshold"
57
+ | "rangeMinutes"
58
+ | "intervalMinutes"
59
+ | "compareDays"
60
+ | "tolerance"
61
+ | "alertOnNoData"
62
+ | "triggerFromNRuns"
63
+ >;
64
+}
65
+
66
+// No production baseline exists yet: the spike monitor learns its own, and the
67
+// thresholds below are starting points to tune after two weeks of data.
68
+const mobileMonitors: ReadonlyArray<MobileMonitor> = [
69
+ {
70
+ id: "MobileCrashSpikeMonitor",
71
+ severity: "page",
72
+ name: "T3 Code mobile: crash spike",
73
+ description:
74
+ "Crashes (JS and native, each counted once) per 15 min well above the same time over the last 7 days. Usually a bad OTA update or store build: compare app versions on the dashboard.",
75
+ query: ({ logs }) => `${crashes(logs)}\n| summarize crashes = count() by bin_auto(_time)`,
76
+ props: {
77
+ type: "AnomalyDetection",
78
+ operator: "Above",
79
+ compareDays: 7,
80
+ tolerance: 50,
81
+ rangeMinutes: 15,
82
+ intervalMinutes: 15,
83
+ },
84
+ },
85
+ {
86
+ id: "MobileCrashSpreadMonitor",
87
+ severity: "page",
88
+ name: "T3 Code mobile: one crash hitting many installs",
89
+ description:
90
+ "The most widespread crash reached this many installs in an hour. Its message and versions are at the top of the dashboard's Top crashes table.",
91
+ query: ({ logs }) =>
92
+ `${crashes(logs)}\n| summarize installs = dcount(install) by platform, error_type, message\n| summarize worst = max(installs)`,
93
+ props: {
94
+ type: "Threshold",
95
+ operator: "AboveOrEqual",
96
+ threshold: 25,
97
+ rangeMinutes: 60,
98
+ intervalMinutes: 15,
99
+ },
100
+ },
101
+ {
102
+ id: "MobileTelemetryStoppedMonitor",
103
+ severity: "warn",
104
+ name: "T3 Code mobile: reports stopped",
105
+ description:
106
+ "No startup metrics forwarded in an hour. Every launch sends some, so the relay route, the Axiom token, or the app's endpoint is broken, and crashes go unseen too.",
107
+ query: ({ traces }) =>
108
+ `${ingest(traces)}\n| where signal == 'metrics' and outcome == 'forwarded'\n| summarize forwarded = count()`,
109
+ props: {
110
+ type: "Threshold",
111
+ operator: "Below",
112
+ threshold: 1,
113
+ rangeMinutes: 60,
114
+ intervalMinutes: 15,
115
+ alertOnNoData: true,
116
+ triggerFromNRuns: 2,
117
+ },
118
+ },
119
+ {
120
+ id: "MobileSymbolicationFailingMonitor",
121
+ severity: "warn",
122
+ name: "T3 Code mobile: crash stacks not symbolicated",
123
+ description:
124
+ "Share of release JS exceptions over 6 h whose stack stayed minified. Map uploads are failing or the relay cannot read them; relay.symbolication.status says which.",
125
+ query: ({ logs }) =>
126
+ [
127
+ `['${logs}']`,
128
+ `| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})`,
129
+ `| where isnotempty(status) and status !in ('no_bundle_id', 'no_bundle_frames')`,
130
+ `| summarize total = count(), minified = countif(status != 'symbolicated')`,
131
+ `| project minified_pct = iff(total >= 20, 100.0 * minified / total, 0.0)`,
132
+ ].join("\n"),
133
+ props: {
134
+ type: "Threshold",
135
+ operator: "Above",
136
+ threshold: 50,
137
+ rangeMinutes: 360,
138
+ intervalMinutes: 60,
139
+ },
140
+ },
141
+];
142
+
143
+interface QueryChart {
144
+ readonly id: string;
145
+ readonly name: string;
146
+ readonly type: "TimeSeries" | "Table" | "Statistic";
147
+ readonly query: { readonly apl: string };
148
+ /** Grid columns out of 12. */
149
+ readonly width: 4 | 6 | 12;
150
+}
151
+
152
+const chart = (
153
+ type: QueryChart["type"],
154
+ width: QueryChart["width"],
155
+ id: string,
156
+ name: string,
157
+ apl: string,
158
+): QueryChart => ({ id, name, type, width, query: { apl } });
159
+
160
+const mobileDashboardCharts = ({ logs, traces }: Datasets): ReadonlyArray<QueryChart> => [
161
+ chart("Statistic", 4, "crashes", "Crashes", `${crashes(logs)}\n| summarize count()`),
162
+ chart(
163
+ "Statistic",
164
+ 4,
165
+ "crashed-installs",
166
+ "Installs that crashed",
167
+ `${crashes(logs)}\n| summarize dcount(install)`,
168
+ ),
169
+ chart(
170
+ "Statistic",
171
+ 4,
172
+ "handled-errors",
173
+ "Handled errors",
174
+ `${handledErrors(logs)}\n| summarize count()`,
175
+ ),
176
+ chart(
177
+ "TimeSeries",
178
+ 6,
179
+ "crashes-over-time",
180
+ "Crashes by platform",
181
+ `${crashes(logs)}\n| summarize count() by bin_auto(_time), series = strcat(platform, ' ', kind)`,
182
+ ),
183
+ chart(
184
+ "TimeSeries",
185
+ 6,
186
+ "handled-over-time",
187
+ "Handled errors by platform",
188
+ `${handledErrors(logs)}\n| summarize count() by bin_auto(_time), platform`,
189
+ ),
190
+ chart(
191
+ "Table",
192
+ 12,
193
+ "top-crashes",
194
+ "Top crashes",
195
+ `${crashes(logs)}\n| summarize installs = dcount(install), crashes = count(), first_seen = min(_time), last_seen = max(_time), versions = make_set(version, 3) by platform, kind, error_type, message\n| order by installs desc, crashes desc\n| take 15`,
196
+ ),
197
+ chart(
198
+ "Table",
199
+ 12,
200
+ "top-handled",
201
+ "Top handled errors",
202
+ `${handledErrors(logs)}\n| summarize installs = dcount(install), errors = count(), last_seen = max(_time), versions = make_set(version, 3) by platform, error_type, message\n| order by installs desc, errors desc\n| take 15`,
203
+ ),
204
+ chart(
205
+ "Table",
206
+ 6,
207
+ "by-version",
208
+ "Crashes by app version",
209
+ `${crashes(logs)}\n| summarize crashes = count(), installs = dcount(install) by platform, version\n| order by crashes desc\n| take 10`,
210
+ ),
211
+ chart(
212
+ "TimeSeries",
213
+ 6,
214
+ "symbolication",
215
+ "JS stacks by symbolication outcome",
216
+ `['${logs}']\n| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})\n| where isnotempty(status)\n| summarize count() by bin_auto(_time), status`,
217
+ ),
218
+ chart(
219
+ "TimeSeries",
220
+ 12,
221
+ "ingest",
222
+ "Reports reaching the relay",
223
+ `${ingest(traces)}\n| summarize count() by bin_auto(_time), series = strcat(signal, ' ', outcome)`,
224
+ ),
225
+];
226
+
227
+/** Lays charts left to right, wrapping at Axiom's 12-column grid. */
228
+function layout(charts: ReadonlyArray<QueryChart>) {
229
+ let x = 0;
230
+ let y = 0;
231
+ let rowHeight = 0;
232
+ return charts.map((chart) => {
233
+ const h = chart.type === "Statistic" ? 3 : 6;
234
+ if (x + chart.width > 12) {
235
+ x = 0;
236
+ y += rowHeight;
237
+ rowHeight = 0;
238
+ }
239
+ const cell = { i: chart.id, x, y, w: chart.width, h };
240
+ x += chart.width;
241
+ rowHeight = Math.max(rowHeight, h);
242
+ return cell;
243
+ });
244
+}
245
+
246
+/**
247
+ * Monitors follow the relay's: a deployment opts in by naming its paging notifier. The
248
+ * dashboard also comes up on PR previews, so a change to mobile telemetry can be checked
249
+ * there; the preview's teardown removes it.
250
+ */
251
+export const MobileMonitoring = Effect.fn("MobileMonitoring")(function* (
252
+ logs: Axiom.Dataset,
253
+ traces: Axiom.Dataset,
254
+) {
255
+ const { stage } = yield* Alchemy.Stack;
256
+ const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
257
+ if (Option.isNone(page) && !isPreviewStage(stage)) return;
258
+ const datasets = Output.map(Output.all(logs.name, traces.name), ([logs, traces]) => ({
259
+ logs,
260
+ traces,
261
+ }));
262
+
263
+ if (Option.isSome(page)) {
264
+ const notifiers: Record<Severity, Option.Option<string>> = {
265
+ page,
266
+ warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
267
+ };
268
+ for (const monitor of mobileMonitors) {
269
+ yield* Axiom.Monitor(monitor.id, {
270
+ ...monitor.props,
271
+ name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
272
+ description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
273
+ aplQuery: Output.map(datasets, monitor.query),
274
+ triggerFromNRuns: monitor.props.triggerFromNRuns ?? 1,
275
+ alertOnNoData: monitor.props.alertOnNoData ?? false,
276
+ notifierIds: Option.toArray(notifiers[monitor.severity]),
277
+ });
278
+ }
279
+ }
280
+
281
+ yield* Axiom.Dashboard("MobileHealthDashboard", {
282
+ dashboard: Output.map(datasets, (names) => {
283
+ const charts = mobileDashboardCharts(names);
284
+ return {
285
+ name: relayResourceNameForStage("t3-code-mobile-health", stage),
286
+ owner: "X-AXIOM-EVERYONE",
287
+ description: `T3 Code mobile app crashes and errors. Crashes count JS and native once each. Runbook: ${RUNBOOK_URL}`,
288
+ refreshTime: 300,
289
+ schemaVersion: 2,
290
+ timeWindowStart: "qr-now-7d",
291
+ timeWindowEnd: "qr-now",
292
+ charts: charts.map(({ width: _width, ...chart }) => chart),
293
+ layout: layout(charts),
294
+ };
295
+ }),
296
+ });
297
+});
infra/relay/src/mobileTelemetry/otlpLogs.ts
new
+178
@@ -0,0 +1,178 @@
1
+import { mobileSourceMapKey, type MobilePlatform } from "@t3tools/shared/mobileSourceMaps";
2
+
3
+import { stripBundlePaths, symbolicateStack, type CompactSourceMap } from "./sourceMap.ts";
4
+
5
+/** The JS bundle a record came from, read from its OTLP resource. */
6
+interface BundleIdentity {
7
+ readonly platform: MobilePlatform;
8
+ /** Observe reports one only for an OTA launch. */
9
+ readonly updateId: string | undefined;
10
+ readonly easBuildId: string | undefined;
11
+}
12
+
13
+const BUNDLE_ID_PATTERN = /^[A-Za-z0-9-]{1,64}$/u;
14
+
15
+/**
16
+ * The map for the bundle that threw. A build map never stands in for a
17
+ * missing OTA map: its offsets belong to a different bundle.
18
+ */
19
+export function bundleSourceMapKey(identity: BundleIdentity): string | undefined {
20
+ if (identity.updateId !== undefined) {
21
+ return BUNDLE_ID_PATTERN.test(identity.updateId)
22
+ ? mobileSourceMapKey(identity.platform, "ota", identity.updateId)
23
+ : undefined;
24
+ }
25
+ if (identity.easBuildId !== undefined && BUNDLE_ID_PATTERN.test(identity.easBuildId)) {
26
+ return mobileSourceMapKey(identity.platform, "build", identity.easBuildId);
27
+ }
28
+ return undefined;
29
+}
30
+
31
+interface JsException {
32
+ readonly identity: BundleIdentity;
33
+ readonly stack: string;
34
+ /** The record's attribute list, updated in place by `applySymbolication`. */
35
+ readonly attributes: Array<unknown>;
36
+}
37
+
38
+const isRecord = (value: unknown): value is Record<string, unknown> =>
39
+ typeof value === "object" && value !== null && !Array.isArray(value);
40
+
41
+const arrayAt = (value: unknown, key: string): Array<unknown> | undefined => {
42
+ const child = isRecord(value) ? value[key] : undefined;
43
+ return Array.isArray(child) ? child : undefined;
44
+};
45
+
46
+const stringAttribute = (attributes: ReadonlyArray<unknown>, key: string): string | undefined => {
47
+ for (const attribute of attributes) {
48
+ if (!isRecord(attribute) || attribute.key !== key) continue;
49
+ const value = isRecord(attribute.value) ? attribute.value.stringValue : undefined;
50
+ return typeof value === "string" ? value : undefined;
51
+ }
52
+ return undefined;
53
+};
54
+
55
+const platformOf = (attributes: ReadonlyArray<unknown>): MobilePlatform | undefined => {
56
+ const osType = stringAttribute(attributes, "os.type");
57
+ if (osType === "darwin") return "ios";
58
+ if (osType === "linux") return "android";
59
+ return undefined;
60
+};
61
+
62
+/** JavaScript exception records with a stack; native crash frames are not in the bundle. */
63
+export function collectJsExceptions(payload: unknown): ReadonlyArray<JsException> {
64
+ const exceptions: Array<JsException> = [];
65
+ for (const resourceLogs of arrayAt(payload, "resourceLogs") ?? []) {
66
+ const resourceAttributes =
67
+ arrayAt(isRecord(resourceLogs) ? resourceLogs.resource : undefined, "attributes") ?? [];
68
+ const platform = platformOf(resourceAttributes);
69
+ if (platform === undefined) continue;
70
+ const identity: BundleIdentity = {
71
+ platform,
72
+ updateId:
73
+ stringAttribute(resourceAttributes, "expo.app.updates.id") ??
74
+ stringAttribute(resourceAttributes, "expo.app.update_id"),
75
+ easBuildId: stringAttribute(resourceAttributes, "expo.eas_build.id"),
76
+ };
77
+ for (const scopeLogs of arrayAt(resourceLogs, "scopeLogs") ?? []) {
78
+ for (const logRecord of arrayAt(scopeLogs, "logRecords") ?? []) {
79
+ const attributes = arrayAt(logRecord, "attributes");
80
+ if (!attributes) continue;
81
+ const stack = stringAttribute(attributes, "exception.stacktrace");
82
+ if (stack === undefined) continue;
83
+ if (stringAttribute(attributes, "expo.error.source") === "nativeCrash") continue;
84
+ exceptions.push({ identity, stack, attributes });
85
+ }
86
+ }
87
+ }
88
+ return exceptions;
89
+}
90
+
91
+const stringValue = (key: string, value: string) => ({ key, value: { stringValue: value } });
92
+
93
+const setAttribute = (attributes: Array<unknown>, key: string, value: string) => {
94
+ const index = attributes.findIndex((attribute) => isRecord(attribute) && attribute.key === key);
95
+ if (index === -1) attributes.push(stringValue(key, value));
96
+ else attributes[index] = stringValue(key, value);
97
+};
98
+
99
+/** Why a stack stayed minified, recorded as `relay.symbolication.status`. */
100
+export type SourceMapMiss =
101
+ /** A development or local build: no update or EAS build to name a map. */
102
+ | "no_bundle_id"
103
+ | "source_map_missing"
104
+ | "source_map_too_large"
105
+ | "source_map_invalid"
106
+ /** The bucket could not be read; retried shortly. */
107
+ | "source_map_unavailable"
108
+ /** The request named more bundles than the relay decodes for one request. */
109
+ | "source_map_limit";
110
+
111
+/**
112
+ * Rewrites one exception's stack, growing it by at most `maxGrowth`, and
113
+ * records the outcome on the record so a stack still minified in Axiom
114
+ * explains itself.
115
+ */
116
+export function applySymbolication(
117
+ exception: JsException,
118
+ found: { readonly key: string; readonly map: CompactSourceMap } | SourceMapMiss,
119
+ maxGrowth: number,
120
+): { readonly symbolicated: boolean; readonly growth: number } {
121
+ const stack = stripBundlePaths(exception.stack);
122
+ if (stack !== exception.stack) setAttribute(exception.attributes, "exception.stacktrace", stack);
123
+ if (typeof found === "string") {
124
+ setAttribute(exception.attributes, "relay.symbolication.status", found);
125
+ return { symbolicated: false, growth: 0 };
126
+ }
127
+ const result = symbolicateStack(stack, found.map, maxGrowth);
128
+ setAttribute(exception.attributes, "relay.symbolication.source_map", found.key);
129
+ if (result.mappedFrames === 0) {
130
+ setAttribute(exception.attributes, "relay.symbolication.status", "no_bundle_frames");
131
+ return { symbolicated: false, growth: 0 };
132
+ }
133
+ setAttribute(exception.attributes, "relay.symbolication.status", "symbolicated");
134
+ setAttribute(exception.attributes, "relay.symbolication.raw_stacktrace", stack);
135
+ setAttribute(exception.attributes, "exception.stacktrace", result.stack);
136
+ return { symbolicated: true, growth: result.growth };
137
+}
138
+
139
+const isFatal = (attributes: ReadonlyArray<unknown>) =>
140
+ attributes.some(
141
+ (attribute) =>
142
+ isRecord(attribute) &&
143
+ attribute.key === "expo.error.is_fatal" &&
144
+ isRecord(attribute.value) &&
145
+ attribute.value.boolValue === true,
146
+ );
147
+
148
+/**
149
+ * A fatal JS error ends in a native abort, which Observe reports as a second crash with the
150
+ * same generic stack in the same session. Marks those `relay.crash.js_fatal` so they don't
151
+ * bury real native crashes. Both arrive together on the next launch.
152
+ */
153
+export function markJsFatalCrashes(payload: unknown): number {
154
+ const records: Array<Array<unknown>> = [];
155
+ const fatalSessions = new Set<string>();
156
+ for (const resourceLogs of arrayAt(payload, "resourceLogs") ?? []) {
157
+ for (const scopeLogs of arrayAt(resourceLogs, "scopeLogs") ?? []) {
158
+ for (const logRecord of arrayAt(scopeLogs, "logRecords") ?? []) {
159
+ const attributes = arrayAt(logRecord, "attributes");
160
+ if (!attributes || !isFatal(attributes)) continue;
161
+ const session = stringAttribute(attributes, "session.id");
162
+ if (session === undefined) continue;
163
+ if (stringAttribute(attributes, "expo.error.source") === "nativeCrash") {
164
+ records.push(attributes);
165
+ } else {
166
+ fatalSessions.add(session);
167
+ }
168
+ }
169
+ }
170
+ }
171
+ let marked = 0;
172
+ for (const attributes of records) {
173
+ if (!fatalSessions.has(stringAttribute(attributes, "session.id")!)) continue;
174
+ attributes.push({ key: "relay.crash.js_fatal", value: { boolValue: true } });
175
+ marked += 1;
176
+ }
177
+ return marked;
178
+}
infra/relay/src/mobileTelemetry/sourceMap.test.ts
new
+60
@@ -0,0 +1,60 @@
1
+import { describe, expect, it } from "@effect/vitest";
2
+
3
+import { decodeSourceMap, symbolicateStack, type CompactSourceMap } from "./sourceMap.ts";
4
+
5
+// Real `hermesc -O -output-source-map` output and the stack its VM printed: a frame's column is the
6
+// map's 0-based generated column. In `t.js`, alpha throws at 2:18, beta calls it at 5:19, gamma calls
7
+// beta at 8:13, and the top level calls gamma at 10:6.
8
+const HERMES_MAP = {
9
+ version: 3,
10
+ sources: ["t.js"],
11
+ mappings:
12
+ "AAAA,iBAAA,OAAA,MAAA,KAAA,MAAA,KAAA,MASA,OAAK,IAAE,EATP,EACY,MAAK,KAAA,WAAA,IAAA,IAAf,EAEF,EACa,OAAK,OAAT,IAAP,EAEF,AACE,EAAM,OAAI,IAAI,EAAE,IAAY,MAAO,OAAF,KAAW,EAC9C;",
13
+ x_hermes_function_offsets: { "0": [0, 65, 99, 121] },
14
+};
15
+const HERMES_STACK = [
16
+ "Error: boom",
17
+ " at alpha (address at t.hbc:1:89)",
18
+ " at beta (address at t.hbc:1:108)",
19
+ " at gamma (address at t.hbc:1:130)",
20
+ " at global (address at t.hbc:1:59)",
21
+].join("\n");
22
+
23
+describe("symbolicateStack", () => {
24
+ it("rewrites Hermes bundle frames to their original locations and keeps the rest", () => {
25
+ const map = decodeSourceMap(HERMES_MAP) as CompactSourceMap;
26
+ const result = symbolicateStack(
27
+ `${HERMES_STACK}\n at apply (native)\n at tryCallOne (address at InternalBytecode.js:1:89)`,
28
+ map,
29
+ Number.POSITIVE_INFINITY,
30
+ );
31
+ expect(result).toEqual({
32
+ stack: [
33
+ "Error: boom",
34
+ " at alpha (t.js:2:18)",
35
+ " at beta (t.js:5:19)",
36
+ " at gamma (t.js:8:13)",
37
+ " at global (t.js:10:6)",
38
+ " at apply (native)",
39
+ " at tryCallOne (address at InternalBytecode.js:1:89)",
40
+ ].join("\n"),
41
+ bundleFrames: 4,
42
+ mappedFrames: 4,
43
+ growth: -50,
44
+ });
45
+ });
46
+});
47
+
48
+describe("decodeSourceMap", () => {
49
+ it("fits a map as large as our iOS release map, read from an OTA bundle's path", () => {
50
+ // 2.1M segments, every other one repeating its predecessor's position, like Hermes output.
51
+ const mappings = `AAAA,${"CAAC,CAAA,".repeat(1_050_000)}CAAC`;
52
+ const map = decodeSourceMap({ version: 3, sources: ["src/App.tsx"], mappings });
53
+ const frame =
54
+ " at render (address at /var/mobile/Containers/Data/Application/A1/Library/" +
55
+ "Application Support/.expo-internal/bundle:1:4)";
56
+ expect(symbolicateStack(frame, map as CompactSourceMap, Number.POSITIVE_INFINITY).stack).toBe(
57
+ " at render (src/App.tsx:1:3)",
58
+ );
59
+ });
60
+});
infra/relay/src/mobileTelemetry/sourceMap.ts
new
+287
@@ -0,0 +1,287 @@
1
+// A production Hermes map has over a million segments. General-purpose decoders hold
2
+// each as its own array, past a Worker isolate's memory budget; typed arrays fit.
3
+
4
+const BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
5
+const BASE64_VALUES = new Int8Array(128).fill(-1);
6
+for (let index = 0; index < BASE64.length; index += 1) {
7
+ BASE64_VALUES[BASE64.charCodeAt(index)] = index;
8
+}
9
+const COMMA = 44;
10
+const SEMICOLON = 59;
11
+/** The relay caches two decoded maps in a 128 MB isolate. */
12
+const MAX_DECODED_BYTES = 32 * 1024 * 1024;
13
+
14
+export interface CompactSourceMap {
15
+ readonly sources: ReadonlyArray<string>;
16
+ /** Segment index where each generated line starts, plus one past the last line. */
17
+ readonly lineStarts: Int32Array;
18
+ readonly generatedColumns: Int32Array;
19
+ /** -1 for segments that map to no source. */
20
+ readonly sourceIndexes: Int32Array;
21
+ readonly originalLines: Int32Array;
22
+ readonly originalColumns: Int32Array;
23
+}
24
+
25
+interface OriginalPosition {
26
+ readonly source: string;
27
+ /** 1-based. */
28
+ readonly line: number;
29
+ /** 0-based, as source maps store it. */
30
+ readonly column: number;
31
+}
32
+
33
+const isStringArray = (value: unknown): value is ReadonlyArray<string> =>
34
+ Array.isArray(value) && value.every((item) => typeof item === "string");
35
+
36
+/**
37
+ * Calls `visit` per segment in order; `sourceIndex` is -1 for a segment that maps to no source.
38
+ * False if a segment is malformed.
39
+ */
40
+function walkSegments(
41
+ mappings: string,
42
+ visit: (
43
+ line: number,
44
+ generatedColumn: number,
45
+ sourceIndex: number,
46
+ originalLine: number,
47
+ originalColumn: number,
48
+ ) => void,
49
+): boolean {
50
+ // Names keep the running offsets right but are not stored: they name an identifier, not the function.
51
+ const fields = [0, 0, 0, 0, 0];
52
+ let line = 0;
53
+ let generatedColumn = 0;
54
+ let sourceIndex = 0;
55
+ let originalLine = 0;
56
+ let originalColumn = 0;
57
+ let index = 0;
58
+
59
+ while (index < mappings.length) {
60
+ const code = mappings.charCodeAt(index);
61
+ if (code === SEMICOLON) {
62
+ line += 1;
63
+ generatedColumn = 0;
64
+ index += 1;
65
+ continue;
66
+ }
67
+ if (code === COMMA) {
68
+ index += 1;
69
+ continue;
70
+ }
71
+
72
+ let fieldCount = 0;
73
+ while (index < mappings.length) {
74
+ const next = mappings.charCodeAt(index);
75
+ if (next === COMMA || next === SEMICOLON) break;
76
+ let value = 0;
77
+ let shift = 0;
78
+ let digit: number;
79
+ do {
80
+ const char = mappings.charCodeAt(index);
81
+ digit = char < 128 ? BASE64_VALUES[char]! : -1;
82
+ if (digit === -1) return false;
83
+ value += (digit & 31) * 2 ** shift;
84
+ shift += 5;
85
+ index += 1;
86
+ } while (digit & 32);
87
+ const negative = value % 2 === 1;
88
+ value = Math.floor(value / 2);
89
+ if (fieldCount < fields.length) fields[fieldCount] = negative ? -value : value;
90
+ fieldCount += 1;
91
+ }
92
+ if (fieldCount !== 1 && fieldCount !== 4 && fieldCount !== 5) return false;
93
+
94
+ generatedColumn += fields[0]!;
95
+ if (fieldCount === 1) {
96
+ visit(line, generatedColumn, -1, 0, 0);
97
+ } else {
98
+ sourceIndex += fields[1]!;
99
+ originalLine += fields[2]!;
100
+ originalColumn += fields[3]!;
101
+ visit(line, generatedColumn, sourceIndex, originalLine, originalColumn);
102
+ }
103
+ }
104
+ return true;
105
+}
106
+
107
+/**
108
+ * `too_large` past the decode budget; `invalid` for anything but a version 3 map, including
109
+ * index maps, which React Native never emits.
110
+ */
111
+export function decodeSourceMap(raw: unknown): CompactSourceMap | "too_large" | "invalid" {
112
+ if (typeof raw !== "object" || raw === null) return "invalid";
113
+ const version = Reflect.get(raw, "version");
114
+ const mappings = Reflect.get(raw, "mappings");
115
+ const sources = Reflect.get(raw, "sources");
116
+ const sourceRoot = Reflect.get(raw, "sourceRoot");
117
+ if (version !== 3 || typeof mappings !== "string" || !isStringArray(sources)) return "invalid";
118
+
119
+ // Hermes maps most bytecode instructions to the same position as the one before; a lookup
120
+ // lands on the earlier segment instead with the same answer. Dropping them shrinks our iOS
121
+ // map from 2.1M segments to 1.2M, so the first pass counts what the second keeps.
122
+ let lineCount = 1;
123
+ for (let index = 0; index < mappings.length; index += 1) {
124
+ if (mappings.charCodeAt(index) === SEMICOLON) lineCount += 1;
125
+ }
126
+ let kept = 0;
127
+ let lastLine = -1;
128
+ let lastSource = -2;
129
+ let lastOriginalLine = 0;
130
+ let lastOriginalColumn = 0;
131
+ const repeats = (line: number, source: number, originalLine: number, originalColumn: number) => {
132
+ const same =
133
+ line === lastLine &&
134
+ source === lastSource &&
135
+ (source === -1 ||
136
+ (originalLine === lastOriginalLine && originalColumn === lastOriginalColumn));
137
+ lastLine = line;
138
+ lastSource = source;
139
+ lastOriginalLine = originalLine;
140
+ lastOriginalColumn = originalColumn;
141
+ return same;
142
+ };
143
+ const counted = walkSegments(mappings, (line, _column, source, originalLine, originalColumn) => {
144
+ if (!repeats(line, source, originalLine, originalColumn)) kept += 1;
145
+ });
146
+ if (!counted) return "invalid";
147
+
148
+ const prefixLength = typeof sourceRoot === "string" ? sourceRoot.length + 1 : 0;
149
+ let sourceChars = 0;
150
+ for (const source of sources) sourceChars += source.length + prefixLength;
151
+ // A path takes up to two bytes a character, plus its string header and array slot.
152
+ const decodedBytes = kept * 16 + (lineCount + 1) * 4 + sourceChars * 2 + sources.length * 32;
153
+ if (decodedBytes > MAX_DECODED_BYTES) return "too_large";
154
+
155
+ const lineStarts = new Int32Array(lineCount + 1);
156
+ const generatedColumns = new Int32Array(kept);
157
+ const sourceIndexes = new Int32Array(kept);
158
+ const originalLines = new Int32Array(kept);
159
+ const originalColumns = new Int32Array(kept);
160
+ let segment = 0;
161
+ let currentLine = 0;
162
+ lastLine = -1;
163
+ lastSource = -2;
164
+ walkSegments(mappings, (line, generatedColumn, source, originalLine, originalColumn) => {
165
+ if (repeats(line, source, originalLine, originalColumn)) return;
166
+ while (currentLine < line) {
167
+ currentLine += 1;
168
+ lineStarts[currentLine] = segment;
169
+ }
170
+ generatedColumns[segment] = generatedColumn;
171
+ sourceIndexes[segment] = source;
172
+ originalLines[segment] = originalLine;
173
+ originalColumns[segment] = originalColumn;
174
+ segment += 1;
175
+ });
176
+ while (currentLine < lineCount) {
177
+ currentLine += 1;
178
+ lineStarts[currentLine] = segment;
179
+ }
180
+
181
+ const prefix = typeof sourceRoot === "string" && sourceRoot !== "" ? sourceRoot : undefined;
182
+ return {
183
+ sources: prefix ? sources.map((source) => `${prefix.replace(/\/?$/u, "/")}${source}`) : sources,
184
+ lineStarts,
185
+ generatedColumns,
186
+ sourceIndexes,
187
+ originalLines,
188
+ originalColumns,
189
+ };
190
+}
191
+
192
+/** `line` is 1-based and `column` 0-based, as in `@jridgewell/trace-mapping`. */
193
+function originalPositionFor(
194
+ map: CompactSourceMap,
195
+ line: number,
196
+ column: number,
197
+): OriginalPosition | undefined {
198
+ const lineIndex = line - 1;
199
+ if (lineIndex < 0 || lineIndex >= map.lineStarts.length - 1) return undefined;
200
+ const lineStart = map.lineStarts[lineIndex]!;
201
+ // An exact hit takes the first segment at `column`: Hermes maps a function's first
202
+ // instruction twice (declaration, then first statement), and other libraries take the first.
203
+ let low = lineStart;
204
+ let high = map.lineStarts[lineIndex + 1]!;
205
+ while (low < high) {
206
+ const middle = (low + high) >>> 1;
207
+ if (map.generatedColumns[middle]! < column) low = middle + 1;
208
+ else high = middle;
209
+ }
210
+ const exact = low < map.lineStarts[lineIndex + 1]! && map.generatedColumns[low] === column;
211
+ const found = exact ? low : low - 1;
212
+ if (found < lineStart) return undefined;
213
+ const sourceIndex = map.sourceIndexes[found]!;
214
+ const source = sourceIndex === -1 ? undefined : map.sources[sourceIndex];
215
+ if (source === undefined) return undefined;
216
+ return {
217
+ source,
218
+ line: map.originalLines[found]! + 1,
219
+ column: map.originalColumns[found]!,
220
+ };
221
+}
222
+
223
+/**
224
+ * `at render (address at index.android.bundle:1:48213)`: Hermes prints the bytecode offset,
225
+ * the composed map's generated column. On iOS the path is absolute, and an OTA bundle's
226
+ * contains a space (`Application Support`).
227
+ */
228
+const BUNDLE_FRAME = /address at (.+?):(\d+):(\d+)(?=\)?$)/u;
229
+/** Hermes's own runtime, not in the map. */
230
+const HERMES_RUNTIME = "InternalBytecode.js";
231
+
232
+const baseName = (path: string) => path.slice(path.lastIndexOf("/") + 1);
233
+
234
+/** iOS bundle paths carry per-install container UUIDs, which split one crash into many groups. */
235
+export function stripBundlePaths(stack: string): string {
236
+ return stack
237
+ .split("\n")
238
+ .map((line) => {
239
+ const match = BUNDLE_FRAME.exec(line);
240
+ if (!match || !match[1]!.includes("/")) return line;
241
+ const start = match.index + "address at ".length;
242
+ return line.slice(0, start) + baseName(match[1]!) + line.slice(start + match[1]!.length);
243
+ })
244
+ .join("\n");
245
+}
246
+
247
+interface SymbolicatedStack {
248
+ readonly stack: string;
249
+ /** Frames in the app bundle. */
250
+ readonly bundleFrames: number;
251
+ /** Bundle frames the map resolved. */
252
+ readonly mappedFrames: number;
253
+ /** How much longer the stack got. */
254
+ readonly growth: number;
255
+}
256
+
257
+/** Once the stack would grow by more than `maxGrowth`, the remaining frames keep their offsets. */
258
+export function symbolicateStack(
259
+ stack: string,
260
+ map: CompactSourceMap,
261
+ maxGrowth: number,
262
+): SymbolicatedStack {
263
+ let bundleFrames = 0;
264
+ let mappedFrames = 0;
265
+ let growth = 0;
266
+ let exhausted = false;
267
+ const lines = stack.split("\n").map((line) => {
268
+ const match = BUNDLE_FRAME.exec(line);
269
+ if (!match || baseName(match[1]!) === HERMES_RUNTIME) return line;
270
+ bundleFrames += 1;
271
+ if (exhausted) return line;
272
+ const position = originalPositionFor(map, Number(match[2]), Number(match[3]));
273
+ if (!position) return line;
274
+ const lineAndColumn = `:${position.line}:${position.column + 1}`;
275
+ const added = position.source.length + lineAndColumn.length - match[0].length;
276
+ if (growth + added > maxGrowth) {
277
+ exhausted = true;
278
+ return line;
279
+ }
280
+ growth += added;
281
+ mappedFrames += 1;
282
+ // Spliced, not `replace`d: a replacement string would expand `$&` and `$'` in the path.
283
+ const end = match.index + match[0].length;
284
+ return line.slice(0, match.index) + position.source + lineAndColumn + line.slice(end);
285
+ });
286
+ return { stack: lines.join("\n"), bundleFrames, mappedFrames, growth };
287
+}
infra/relay/src/monitoring.ts
+1
-1
@@ -343,7 +343,7 @@ const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
343
];
344
};
345
346
-const notifierConfig = (name: string) =>
346
+export const notifierConfig = (name: string) =>
347
Config.String(name).pipe(
348
Config.option,
349
Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
infra/relay/src/observability.ts
+83
-1
@@ -1,14 +1,20 @@
1
+// @effect-diagnostics nodeBuiltinImport:off - one sha256 inside a synchronous Output.map.
2
+import * as NodeCrypto from "node:crypto";
3
+
4
import * as Alchemy from "alchemy";
5
import * as Axiom from "@alchemy.run/axiom";
6
+import * as Cloudflare from "@alchemy.run/cloudflare";
7
import * as Output from "alchemy/Output";
8
import * as Cause from "effect/Cause";
9
import * as Effect from "effect/Effect";
10
import * as Exit from "effect/Exit";
11
import * as Option from "effect/Option";
12
+import * as Redacted from "effect/Redacted";
13
import * as Schema from "effect/Schema";
14
import * as Tracer from "effect/Tracer";
15
16
import { relayResourceNameForStage } from "./deploymentConfig.ts";
17
+import { MobileMonitoring } from "./mobileTelemetry/monitoring.ts";
18
import { RelayMonitoring } from "./monitoring.ts";
19
20
const relayRecentSpansQuery = (dataset: string) =>
@@ -56,6 +62,37 @@ export const RelayObservability = Effect.gen(function* () {
62
})),
63
});
64
65
+ // expo-observe data, forwarded by src/mobileTelemetry. Mobile spans share the traces dataset.
66
+ const mobileLogs = yield* Axiom.Dataset("MobileLogsDataset", {
67
+ name: relayResourceNameForStage("t3-code-mobile-logs", stage),
68
+ kind: "otel:logs:v1",
69
+ description: "T3 Code mobile app errors, crashes, and log events from expo-observe.",
70
+ retentionDays: 30,
71
+ useRetentionPeriod: true,
72
+ });
73
+ const mobileMetrics = yield* Axiom.Dataset("MobileMetricsDataset", {
74
+ name: relayResourceNameForStage("t3-code-mobile-metrics", stage),
75
+ kind: "otel:metrics:v1",
76
+ description: "T3 Code mobile app startup and update metrics from expo-observe.",
77
+ retentionDays: 30,
78
+ useRetentionPeriod: true,
79
+ });
80
+ const mobileTelemetryIngestToken = yield* Axiom.ApiToken("MobileTelemetryAxiomIngestToken", {
81
+ name: relayResourceNameForStage("t3-code-mobile-telemetry-ingest", stage),
82
+ description: "Owned by Alchemy. Scoped ingest token the relay forwards mobile telemetry with.",
83
+ datasetCapabilities: Output.map(
84
+ Output.all(traces.name, mobileLogs.name, mobileMetrics.name),
85
+ (datasets) =>
86
+ Object.fromEntries(datasets.map((dataset) => [dataset, { ingest: ["create" as const] }])),
87
+ ),
88
+ });
89
+
90
+ const mobileSourceMaps = yield* Cloudflare.R2.Bucket("MobileSourceMapsBucket", {
91
+ name: mobileSourceMapsBucketName(stage),
92
+ // R2 refuses to delete a non-empty bucket, which would half tear down every non-prod stage.
93
+ forceDestroy: stage !== "prod",
94
+ });
95
+
96
yield* Axiom.View("RelayRecentSpansView", {
97
name: relayResourceNameForStage("t3-code-relay-recent-spans", stage),
98
description: "Recent relay HTTP request spans.",
@@ -64,8 +101,53 @@ export const RelayObservability = Effect.gen(function* () {
101
});
102
103
yield* RelayMonitoring(traces);
104
+ yield* MobileMonitoring(mobileLogs, traces);
105
+
106
+ return {
107
+ traces,
108
+ workerIngestToken,
109
+ mobileIngestToken,
110
+ clientIngestToken,
111
+ mobileLogs,
112
+ mobileMetrics,
113
+ mobileTelemetryIngestToken,
114
+ mobileSourceMaps,
115
+ } as const;
116
+});
117
68
- return { traces, workerIngestToken, mobileIngestToken, clientIngestToken } as const;
118
+const mobileSourceMapsBucketName = (stage: string) =>
119
+ relayResourceNameForStage("t3-code-mobile-sourcemaps", stage);
120
+
121
+/**
122
+ * Write-only S3 credentials for the source maps bucket: the `MOBILE_SOURCEMAPS_R2` secret.
123
+ * R2's S3 access key id is the API token's id and the secret the SHA-256 of its value.
124
+ */
125
+export const MobileSourceMapsUploadCredentials = Effect.gen(function* () {
126
+ const { stage } = yield* Alchemy.Stack;
127
+ const { accountId } = yield* yield* Cloudflare.CloudflareEnvironment;
128
+ const bucketName = mobileSourceMapsBucketName(stage);
129
+ const token = yield* Cloudflare.ApiToken.AccountApiToken("MobileSourceMapsUploadToken", {
130
+ name: relayResourceNameForStage("t3-code-mobile-sourcemaps-upload", stage),
131
+ policies: [
132
+ {
133
+ effect: "allow",
134
+ permissionGroups: ["Workers R2 Storage Bucket Item Write"],
135
+ resources: { [`com.cloudflare.edge.r2.bucket.${accountId}_default_${bucketName}`]: "*" },
136
+ },
137
+ ],
138
+ });
139
+ return Output.map(Output.all(token.tokenId, token.value), ([tokenId, value]) =>
140
+ Redacted.make(
141
+ JSON.stringify({
142
+ endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
143
+ bucketName,
144
+ accessKeyId: tokenId,
145
+ secretAccessKey: NodeCrypto.createHash("sha256")
146
+ .update(Redacted.value(value))
147
+ .digest("hex"),
148
+ }),
149
+ ),
150
+ );
151
});
152
153
export const withSpanAttributes =
infra/relay/src/worker.ts
+78
-8
@@ -69,6 +69,7 @@ import * as HookForwarder from "./hooks/HookForwarder.ts";
69
import * as HeldHooks from "./hooks/HeldHooks.ts";
70
import * as HookInbox from "./hooks/HookInbox.ts";
71
import * as HookInboxObject from "./hooks/HookInboxObject.ts";
72
+import * as MobileTelemetry from "./mobileTelemetry/MobileTelemetry.ts";
73
74
const layerWebcrypto = Layer.succeed(
75
Crypto.Crypto,
@@ -140,7 +141,7 @@ export const layer = Api.make(
141
const relayApiZone = yield* RelayApiZone;
142
const managedEndpointZone = yield* ManagedEndpointZone;
143
const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;
143
- yield* RelayObservability;
144
+ const observability = yield* RelayObservability;
145
146
const apnsEnabled = yield* Config.Boolean("APNS_ENABLED").pipe(Config.withDefault(true));
147
const apnsCredentials = apnsEnabled
@@ -201,6 +202,21 @@ export const layer = Api.make(
202
},
203
});
204
const hookInboxes = yield* HookInboxObject.HookInboxObject;
205
+ const mobileTelemetryRateLimit = yield* Cloudflare.RateLimit("MOBILE_TELEMETRY_RATE_LIMIT", {
206
+ namespaceId: 1003,
207
+ simple: {
208
+ limit: MobileTelemetry.MOBILE_TELEMETRY_RATE_LIMIT.limit,
209
+ period: MobileTelemetry.MOBILE_TELEMETRY_RATE_LIMIT.periodSeconds,
210
+ },
211
+ });
212
+ const mobileSourceMaps = yield* Cloudflare.R2.ReadBucket(observability.mobileSourceMaps);
213
+ const mobileTelemetryToken = yield* observability.mobileTelemetryIngestToken.token;
214
+ const mobileTracesUrl = yield* observability.traces.otelTracesEndpoint;
215
+ const mobileTracesDataset = yield* observability.traces.name;
216
+ const mobileLogsUrl = yield* observability.mobileLogs.otelLogsEndpoint;
217
+ const mobileLogsDataset = yield* observability.mobileLogs.name;
218
+ const mobileMetricsUrl = yield* observability.mobileMetrics.otelMetricsEndpoint;
219
+ const mobileMetricsDataset = yield* observability.mobileMetrics.name;
220
221
const alchemyRuntimeContext: Alchemy.BaseRuntimeContext = yield* Cloudflare.Worker;
222
@@ -325,18 +341,19 @@ export const layer = Api.make(
341
Layer.provideMerge(layerWebcrypto),
342
);
343
328
- // Fails open: a limiter outage must not drop webhooks the environment would accept.
344
+ // Fails open: a limiter outage must not drop webhooks or app telemetry.
345
const allowWith =
330
- (limiter: typeof hookRateLimit) =>
346
+ (
347
+ limiter: typeof hookRateLimit,
348
+ failedOpenAttribute = "relay.hook.rate_limiter_failed_open",
349
+ ) =>
350
(key: string): Effect.Effect<boolean> =>
351
limiter.limit({ key }).pipe(
352
Effect.map((result) => result.success),
353
Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
354
Effect.catch((error) =>
336
- Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe(
337
- Effect.andThen(
338
- Effect.annotateCurrentSpan({ "relay.hook.rate_limiter_failed_open": true }),
339
- ),
355
+ Effect.logWarning("Rate limiter unavailable", { error: error.message }).pipe(
356
+ Effect.andThen(Effect.annotateCurrentSpan({ [failedOpenAttribute]: true })),
357
Effect.as(true),
358
),
359
),
@@ -346,12 +363,64 @@ export const layer = Api.make(
363
allowEndpoint: allowWith(hookEndpointRateLimit),
364
});
365
349
- const layerApp = Layer.merge(
366
+ // `fetch` rebuilds its layers per request; the source map cache must outlive one.
367
+ const mobileSourceMapLookups = yield* MobileTelemetry.SourceMaps.pipe(
368
+ Effect.provide(MobileTelemetry.layerSourceMaps),
369
+ Effect.provideService(MobileTelemetry.SourceMapObjects, {
370
+ get: (key) =>
371
+ mobileSourceMaps.get(key).pipe(
372
+ Effect.map((object) =>
373
+ object === null
374
+ ? undefined
375
+ : {
376
+ size: object.size,
377
+ text: object.text().pipe(
378
+ Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
379
+ Effect.mapError(
380
+ (cause) => new MobileTelemetry.SourceMapObjectError({ key, cause }),
381
+ ),
382
+ ),
383
+ },
384
+ ),
385
+ Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
386
+ Effect.mapError((cause) => new MobileTelemetry.SourceMapObjectError({ key, cause })),
387
+ ),
388
+ }),
389
+ );
390
+ const layerMobileTelemetry = MobileTelemetry.layer.pipe(
391
+ Layer.provide(
392
+ Layer.mergeAll(
393
+ Layer.effect(
394
+ MobileTelemetry.MobileTelemetrySettings,
395
+ Effect.gen(function* () {
396
+ return MobileTelemetry.MobileTelemetrySettings.of({
397
+ axiomToken: yield* mobileTelemetryToken,
398
+ destinations: {
399
+ traces: { url: yield* mobileTracesUrl, dataset: yield* mobileTracesDataset },
400
+ logs: { url: yield* mobileLogsUrl, dataset: yield* mobileLogsDataset },
401
+ metrics: { url: yield* mobileMetricsUrl, dataset: yield* mobileMetricsDataset },
402
+ },
403
+ });
404
+ }),
405
+ ),
406
+ Layer.succeed(MobileTelemetry.MobileTelemetryRateLimiter, {
407
+ allow: allowWith(
408
+ mobileTelemetryRateLimit,
409
+ "relay.mobile_telemetry.rate_limiter_failed_open",
410
+ ),
411
+ }),
412
+ Layer.succeed(MobileTelemetry.SourceMaps, mobileSourceMapLookups),
413
+ ),
414
+ ),
415
+ );
416
+
417
+ const layerApp = Layer.mergeAll(
418
layerRelayApi,
419
HookForwarder.layerApi.pipe(
420
Layer.provide(HookForwarder.layer),
421
Layer.provide(layerHookRateLimiter),
422
),
423
+ MobileTelemetry.layerApi.pipe(Layer.provide(layerMobileTelemetry)),
424
).pipe(
425
Layer.provideMerge(RelayHttpApi.layerClientAuth),
426
Layer.provideMerge(RelayHttpApi.layerDpopClientAuth),
@@ -481,6 +550,7 @@ export const layer = Api.make(
550
Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding),
551
Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
552
Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
553
+ Layer.provideMerge(Cloudflare.R2.ReadBucketBinding),
554
Layer.provideMerge(HookInboxObject.layer),
555
// Disable the duplicate runtime span; ours applies route/header/token redaction and rejects sender traceparent.
556
Layer.provideMerge(
packages/contracts/src/relay.ts
+17
@@ -1250,6 +1250,22 @@ const RelayHooksGroup = HttpApiGroup.make("hooks")
1250
)
1251
.annotate(OpenApi.Description, "Forward webhook requests to an environment.");
1252
1253
+export const RelayMobileTelemetrySignal = Schema.Literals(["logs", "metrics", "traces"]);
1254
+export type RelayMobileTelemetrySignal = typeof RelayMobileTelemetrySignal.Type;
1255
+
1256
+// expo-observe appends `<easProjectId>/v1/<signal>` to its endpoint URL and cannot send credentials.
1257
+const RelayMobileTelemetryParams = Schema.Struct({
1258
+ projectId: Schema.String.check(Schema.isGUID()),
1259
+ signal: RelayMobileTelemetrySignal,
1260
+});
1261
+const RelayMobileTelemetryGroup = HttpApiGroup.make("mobileTelemetry")
1262
+ .add(
1263
+ HttpApiEndpoint.post("ingest", "/v1/mobile/observe/:projectId/v1/:signal", {
1264
+ params: RelayMobileTelemetryParams,
1265
+ }),
1266
+ )
1267
+ .annotate(OpenApi.Description, "Receive mobile app telemetry from expo-observe.");
1268
+
1269
export const RelayApi = HttpApi.make("RelayApi")
1270
.add(
1271
RelayHealthGroup,
@@ -1260,6 +1276,7 @@ export const RelayApi = HttpApi.make("RelayApi")
1276
RelayDpopClientGroup,
1277
RelayServerGroup,
1278
RelayHooksGroup,
1279
+ RelayMobileTelemetryGroup,
1280
)
1281
.annotate(OpenApi.Title, "T3 Code Relay API")
1282
.annotate(OpenApi.Version, "1.0.0")
packages/shared/package.json
+4
@@ -35,6 +35,10 @@
35
"types": "./src/projectFavicon.ts",
36
"import": "./src/projectFavicon.ts"
37
},
38
+ "./mobileSourceMaps": {
39
+ "types": "./src/mobileSourceMaps.ts",
40
+ "import": "./src/mobileSourceMaps.ts"
41
+ },
42
"./model": {
43
"types": "./src/model.ts",
44
"import": "./src/model.ts"
packages/shared/src/mobileSourceMaps.ts
new
+10
@@ -0,0 +1,10 @@
1
+export type MobilePlatform = "ios" | "android";
2
+
3
+export type MobileSourceMapKind = "ota" | "build";
4
+
5
+/** The relay's R2 key for a bundle's source map: an OTA update's, or an EAS build's embedded one. */
6
+export const mobileSourceMapKey = (
7
+ platform: MobilePlatform,
8
+ kind: MobileSourceMapKind,
9
+ id: string,
10
+) => `${platform}/${kind}/${id}.map`;
pnpm-lock.yaml
+49
-2
@@ -680,6 +680,9 @@ importers:
680
expo-notifications:
681
specifier: ~58.1.5
682
version: 58.1.5(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)
683
+ expo-observe:
684
+ specifier: ~58.0.15
685
+ version: 58.0.16(677910f8cba89c74801637ca226a8913)
686
expo-paste-input:
687
specifier: ^0.1.15
688
version: 0.1.15(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
@@ -801,6 +804,9 @@ importers:
804
'@types/react-dom':
805
specifier: 'catalog:'
806
version: 19.3.0(@types/react@19.3.0)
807
+ aws4fetch:
808
+ specifier: 1.0.20
809
+ version: 1.0.20
810
babel-preset-expo:
811
specifier: ~58.0.17
812
version: 58.0.17(@babel/core@7.29.7(supports-color@8.1.1))(@babel/runtime@7.29.7)(expo-widgets@58.0.20)(expo@58.0.14)(react-refresh@0.14.2)(supports-color@8.1.1)
@@ -8568,6 +8574,13 @@ packages:
8574
resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==}
8575
engines: {node: '>=12.0.0'}
8576
8577
+ expo-app-metrics@58.0.14:
8578
+ resolution: {integrity: sha512-Bj/Bx3s5woOKThUcqucmDOdOAxmyHr232KbnOBD8SoA/iUo7E4RBWvRT8DwZOmLJ7H1wFK2ckFTiNoDFB+Ag+Q==}
8579
+ peerDependencies:
8580
+ expo: '*'
8581
+ react: '*'
8582
+ react-native: '*'
8583
+
8584
expo-application@58.1.2:
8585
resolution: {integrity: sha512-WYFdNHGaxe/j69oYREhK0cKsKYszuIuz399WXYCP+M1grfO+FCWB/RbuMgaXJsLWlzqhNjA/zLgfJ+33FwrQvw==}
8586
peerDependencies:
@@ -8779,6 +8792,23 @@ packages:
8792
react: '*'
8793
react-native: '*'
8794
8795
+ expo-observe@58.0.16:
8796
+ resolution: {integrity: sha512-STG1qp5VKA3f0jFQuf+waoO8QNIG6fpog8Hqi2UnklA08ec8a+w+hJm0wL4DgyO8dy5oDdiTVUnpqaloeUC2lQ==}
8797
+ peerDependencies:
8798
+ '@react-navigation/native': ^7.0.0
8799
+ expo: '*'
8800
+ expo-router: 58.0.20
8801
+ react: '*'
8802
+ react-native: '*'
8803
+ react-native-reanimated: 4.7.1
8804
+ peerDependenciesMeta:
8805
+ '@react-navigation/native':
8806
+ optional: true
8807
+ expo-router:
8808
+ optional: true
8809
+ react-native-reanimated:
8810
+ optional: true
8811
+
8812
expo-paste-input@0.1.15:
8813
resolution: {integrity: sha512-wV0z1s0/Q4MB+lNiSwbMfck+1Im6sDIcgyCi5Y0aF35n5kcM4FHSOjmRfzn3UbkMlvxmYfDiHeHvIqn22qatqA==}
8814
peerDependencies:
@@ -17192,8 +17222,7 @@ snapshots:
17222
17223
aws4@1.13.2: {}
17224
17195
- aws4fetch@1.0.20:
17196
- optional: true
17225
+ aws4fetch@1.0.20: {}
17226
17227
axobject-query@4.1.0: {}
17228
@@ -18201,6 +18230,13 @@ snapshots:
18230
18231
expect-type@1.4.0: {}
18232
18233
+ expo-app-metrics@58.0.14(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18234
+ dependencies:
18235
+ expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
18236
+ expo-updates-interface: 58.0.1(expo@58.0.14)
18237
+ react: 19.3.0
18238
+ react-native: 0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6)
18239
+
18240
expo-application@58.1.2(expo@58.0.14):
18241
dependencies:
18242
expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
@@ -18436,6 +18472,17 @@ snapshots:
18472
- supports-color
18473
- typescript
18474
18475
+ expo-observe@58.0.16(677910f8cba89c74801637ca226a8913):
18476
+ dependencies:
18477
+ expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
18478
+ expo-app-metrics: 58.0.14(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
18479
+ expo-eas-client: 58.0.2
18480
+ react: 19.3.0
18481
+ react-native: 0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6)
18482
+ optionalDependencies:
18483
+ '@react-navigation/native': 7.5.1(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
18484
+ react-native-reanimated: 4.7.1(patch_hash=95ee26e5fcab5f50bc4b4c47b3f7e6278a0b20850205efb9f6ee7e70a22d2b59)(react-native-worklets@0.13.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1))(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
18485
+
18486
expo-paste-input@0.1.15(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18487
dependencies:
18488
expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)