@devries-cameron20 / t3code-1 / commits / cb7e3814d7
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

chore(mobile): e2e runner retries pool failures and reports them apart from app failures

Pool errors (a worker off the gateway, a closed daemon connection, xcrun timeouts) now retry the pose twice wherever they happen, and a pose the pool never let finish is reported as blocked rather than failed. Error dialogs while pairing are recorded and dismissed like those at launch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 27, 2026 at 16:06 UTC cb7e3814d71c8fcdc5b803c660c29b7d7e39be96
4 files changed +75 -28
scripts/mobile-e2e.ts
+26 -9
@@ -8,10 +8,16 @@ import * as NodePath from "node:path";
8 import * as NodeProcess from "node:process";
9 import * as NodeURL from "node:url";
10
11 -import { Device, type Pool, run } from "./mobile-e2e/device.ts";
11 +import { Device, type Pool, poolError, run } from "./mobile-e2e/device.ts";
12 import { runJourney, type StepResult } from "./mobile-e2e/journey.ts";
13 import { POSES, type Pose } from "./mobile-e2e/poses.ts";
14 -import { failedChecks, poseFailed, type PoseResult, renderReport } from "./mobile-e2e/report.ts";
14 +import {
15 + failedChecks,
16 + poolBlocked,
17 + poseFailed,
18 + type PoseResult,
19 + renderReport,
20 +} from "./mobile-e2e/report.ts";
21 import { reviewPose } from "./mobile-e2e/review.ts";
22 import {
23 reserveAvailablePort,
@@ -247,11 +253,17 @@ async function main() {
253 issuePairUrl(environment, host),
254 );
255 const runOne = async (pose: Pose) => {
250 - const first = await attempt(pose, runId);
256 + let result = await attempt(pose, runId);
257 // A pose that failed before its first screen usually met the pool, not the app.
252 - if (!first.error || first.steps.length > 0) return first;
253 - log(pose.id, `retrying after: ${first.error.split("\n")[0]}`);
254 - return attempt(pose, `${runId}-retry`);
258 + for (
259 + let retry = 1;
260 + retry <= 2 && result.error && (result.steps.length === 0 || poolError(result.error));
261 + retry++
262 + ) {
263 + log(pose.id, `retrying after: ${result.error.split("\n")[0]}`);
264 + result = await attempt(pose, `${runId}-retry${retry}`);
265 + }
266 + return result;
267 };
268 // Each platform has its own workers, so each gets its own limit; one Mac runs every
269 // iOS simulator, and recording several at once on a shared pool times it out.
@@ -289,20 +301,25 @@ async function main() {
301 );
302 log(
303 result.pose.id,
292 - [result.error, ...failures].filter(Boolean).join("; ") ||
304 + [poolBlocked(result) ? `blocked by the pool: ${result.error}` : result.error, ...failures]
305 + .filter(Boolean)
306 + .join("; ") ||
307 `passed${result.findings?.length ? `, ${result.findings.length} review findings` : ""}`,
308 );
309 }
296 - const failed = reviewed.filter(poseFailed).length;
310 + const failed = reviewed.filter((result) => poseFailed(result) && !poolBlocked(result)).length;
311 + const blocked = reviewed.filter(poolBlocked).length;
312 NodeProcess.stdout.write(
313 `${JSON.stringify({
314 report,
315 poses: reviewed.length,
316 failed,
317 + blocked,
318 findings: reviewed.reduce((count, result) => count + (result.findings?.length ?? 0), 0),
319 })}\n`,
320 );
305 - process.exitCode = failed ? 1 : 0;
321 + // A pose the pool blocked was not tested, so it fails the run too.
322 + process.exitCode = failed || blocked ? 1 : 0;
323 } finally {
324 if (metro?.pid) {
325 try {
scripts/mobile-e2e/device.ts
+7
@@ -67,6 +67,13 @@ export class AgentDeviceError extends Error {}
67 const TRANSIENT =
68 /Remote daemon is unavailable|not connected to the gateway|still finishing a previous command|Invalid daemon response|Daemon request timed out|no worker connected/i;
69
70 +/** Failures that say the pool could not serve the pose, not that the app misbehaved. */
71 +export const poolError = (message: string) =>
72 + TRANSIENT.test(message) ||
73 + /Daemon connection closed|WORKER_UNREACHABLE|xcrun timed out|refused by simlock|is not available on this simlock/i.test(
74 + message,
75 + );
76 +
77 export class Device {
78 readonly pose: Pose;
79 readonly session: string;
scripts/mobile-e2e/journey.ts
+33 -16
@@ -113,26 +113,43 @@ export async function runJourney(
113 findLabel(snapshot, /^(Dismiss|OK|Close)$/) !== undefined
114 ? findLabel(snapshot, /^Could not|failed|went wrong/i)
115 : undefined;
116 - for (let dialogs = 0; ; dialogs++) {
117 - const launch = await device.waitFor(
118 - (snapshot) => loaded(snapshot) || errorDialog(snapshot) !== undefined,
119 - "the app to load",
120 - 180_000,
121 - );
122 - const dialog = errorDialog(launch);
123 - if (!dialog || dialogs >= 3) break;
124 - await capture("launch error", launch, [
125 - check("no error dialog on launch", false, dialog.label ?? "error dialog"),
126 - ]);
127 - await device.agent(["press", `label="${findLabel(launch, /^(Dismiss|OK|Close)$/)!.label}"`]);
128 - await delay(1_000);
129 - }
116 + let dialogs = 0;
117 + /** Waits for `ready`, recording and dismissing up to three error dialogs on the way. */
118 + const settle = async (
119 + ready: (snapshot: Snapshot) => boolean,
120 + label: string,
121 + timeoutMs: number,
122 + ) => {
123 + for (;;) {
124 + const snapshot = await device.waitFor(
125 + (current) => ready(current) || errorDialog(current) !== undefined,
126 + label,
127 + timeoutMs,
128 + );
129 + const dialog = errorDialog(snapshot);
130 + if (!dialog || dialogs >= 3) return snapshot;
131 + dialogs++;
132 + await capture("launch error", snapshot, [
133 + check("no error dialog on launch", false, dialog.label ?? "error dialog"),
134 + ]);
135 + await device.agent([
136 + "press",
137 + `label="${findLabel(snapshot, /^(Dismiss|OK|Close)$/)!.label}"`,
138 + ]);
139 + await delay(1_000);
140 + }
141 + };
142 + await settle(loaded, "the app to load", 180_000);
143 let threads: Snapshot | null = null;
144 for (let attempt = 1; attempt <= 2 && !threads; attempt++) {
145 const pairing = new URLSearchParams({ pairingUrl: await pair(), autoConnect: "1" });
146 await device.openUrl(`t3code-dev://connections/new?${pairing}`);
134 - threads = await device
135 - .waitFor((snapshot) => threadRow(snapshot) !== undefined, "the paired thread list", 60_000)
147 + threads = await settle(
148 + (snapshot) => threadRow(snapshot) !== undefined,
149 + "the paired thread list",
150 + 60_000,
151 + )
152 + .then((snapshot) => (threadRow(snapshot) ? snapshot : null))
153 .catch(() => null);
154 }
155 if (!threads) throw new Error("the app did not show the fixture threads after pairing");
scripts/mobile-e2e/report.ts
+9 -3
@@ -2,6 +2,7 @@
2
3 import * as NodePath from "node:path";
4
5 +import { poolError } from "./device.ts";
6 import type { StepResult } from "./journey.ts";
7 import type { Pose } from "./poses.ts";
8
@@ -28,6 +29,10 @@ export const failedChecks = (result: PoseResult) =>
29 export const poseFailed = (result: PoseResult) =>
30 result.error !== undefined || failedChecks(result).length > 0;
31
32 +/** The pool never let the pose finish, so it says nothing about the app. */
33 +export const poolBlocked = (result: PoseResult) =>
34 + result.error !== undefined && poolError(result.error) && failedChecks(result).length === 0;
35 +
36 const escapeHtml = (value: string) =>
37 value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
38
@@ -75,16 +80,17 @@ export function renderReport(results: ReadonlyArray<PoseResult>, outputDirectory
80 })
81 .join("");
82 const error = result.error
78 - ? `<td class="bad"><h3>stopped</h3>${result.stopped ? `<a href="${relative(result.stopped)}"><img src="${relative(result.stopped)}"></a>` : ""}<p class="fail">${escapeHtml(result.error)}</p></td>`
83 + ? `<td class="bad"><h3>${poolBlocked(result) ? "blocked by the pool" : "stopped"}</h3>${result.stopped ? `<a href="${relative(result.stopped)}"><img src="${relative(result.stopped)}"></a>` : ""}<p class="fail">${escapeHtml(result.error)}</p></td>`
84 : "";
85 return `<tr><th>${escapeHtml(result.pose.id)}<br><small>${escapeHtml(describePose(result.pose))}</small></th>${cells}${error}</tr>`;
86 })
87 .join("");
83 - const failed = results.filter(poseFailed).length;
88 + const failed = results.filter((result) => poseFailed(result) && !poolBlocked(result)).length;
89 + const blocked = results.filter(poolBlocked).length;
90 const findings = results.reduce((count, result) => count + (result.findings?.length ?? 0), 0);
91 return `<!doctype html><meta charset="utf-8"><title>Mobile E2E</title>
92 <style>body{font:14px system-ui;margin:24px}table{border-collapse:collapse}td,th{border:1px solid #ddd;padding:8px;vertical-align:top;text-align:left}td{width:190px}td.bad{background:#fff5f4}h3{margin:0 0 6px;font-size:13px}img{width:180px;display:block;margin:0 0 6px}img.lag{outline:3px solid #b42318}.pass{color:#16794a}.fail{color:#b42318}.review{color:#7a4b00}.review.high{color:#b42318}ul{padding-left:16px;margin:4px 0}small{color:#666}</style>
87 -<h1>Mobile E2E · ${results.length} poses · ${failed === 0 ? "all checks passed" : `${failed} with failing checks`} · ${findings} review findings</h1>
93 +<h1>Mobile E2E · ${results.length} poses · ${failed === 0 ? "all checks passed" : `${failed} with failing checks`}${blocked ? ` · ${blocked} blocked by the pool` : ""} · ${findings} review findings</h1>
94 <p><span class="fail">✗</span> failed check <span class="review">◆</span> agent review finding</p>
95 <table>${rows}</table>`;
96 }