The best way to control Claude Code, Codex and more from any computer or phone
fix(server): a slow server update download no longer fails at a 10-minute cap (#19649)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Ibrahim Elkamali committed
Oct 20, 2026 at 08:09 UTC
a3635a14e8eada66bae959a78cadcff04aee0c81
2 files changed
+180
-11
apps/server/src/cloud/pinnedRuntime.test.ts
+161
@@ -1,10 +1,12 @@
1
import * as NodeServices from "@effect/platform-node/NodeServices";
2
import { assert, it } from "@effect/vitest";
3
import * as Deferred from "effect/Deferred";
4
+import * as Duration from "effect/Duration";
5
import * as Effect from "effect/Effect";
6
import * as FileSystem from "effect/FileSystem";
7
import * as Fiber from "effect/Fiber";
8
import * as Path from "effect/Path";
9
+import * as TestClock from "effect/testing/TestClock";
10
import { HttpClient, HttpClientResponse } from "effect/http";
11
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
12
@@ -215,6 +217,165 @@ it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => {
217
}),
218
);
219
220
+ it.effect.each([true, false])(
221
+ "installs a download receiving bytes for over ten minutes (progress: %s)",
222
+ (reportProgress) =>
223
+ Effect.gen(function* () {
224
+ const fs = yield* FileSystem.FileSystem;
225
+ const path = yield* Path.Path;
226
+ const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-slow-" });
227
+ const checksums = yield* validChecksums;
228
+ const reads = yield* Effect.forEach(archiveBytes, () => Deferred.make<void>());
229
+ let controller: ReadableStreamDefaultController<Uint8Array>;
230
+ let offset = 0;
231
+ const body = new ReadableStream<Uint8Array>(
232
+ {
233
+ start(value) {
234
+ controller = value;
235
+ },
236
+ pull() {
237
+ Deferred.doneUnsafe(reads[offset++]!, Effect.void);
238
+ },
239
+ },
240
+ { highWaterMark: 0 },
241
+ );
242
+ const client = HttpClient.make((request) =>
243
+ Effect.succeed(
244
+ HttpClientResponse.fromWeb(
245
+ request,
246
+ new Response(request.url.endsWith("/SHA256SUMS") ? checksums : body),
247
+ ),
248
+ ),
249
+ );
250
+ const progress: PinnedRuntimeProgress[] = [];
251
+ yield* Effect.gen(function* () {
252
+ // Each read is demanded by the consumer, with no prefetched chunks.
253
+ for (let index = 0; index < archiveBytes.length; index++) {
254
+ yield* Deferred.await(reads[index]!);
255
+ yield* TestClock.adjust(Duration.seconds(40));
256
+ controller.enqueue(archiveBytes.slice(index, index + 1));
257
+ }
258
+ controller.close();
259
+ }).pipe(Effect.forkScoped);
260
+ const installed = yield* ensurePinnedRuntimeInstalled({
261
+ baseDir,
262
+ version,
263
+ fs,
264
+ path,
265
+ platform: "linux",
266
+ arch: "x64",
267
+ httpClient: client,
268
+ runner: extractingRunner(fs, path),
269
+ validate: () => Effect.void,
270
+ ...(reportProgress
271
+ ? { onProgress: (event: PinnedRuntimeProgress) => progress.push(event) }
272
+ : {}),
273
+ });
274
+ assert.equal(yield* fs.readFileString(installed.sentinelPath), `${version}\n`);
275
+ if (reportProgress) {
276
+ assert.deepEqual(progress.slice(-4), [
277
+ { stage: "download", received: archiveBytes.length, total: archiveBytes.length },
278
+ { stage: "verify" },
279
+ { stage: "extract" },
280
+ { stage: "validate" },
281
+ ]);
282
+ }
283
+ }),
284
+ );
285
+
286
+ it.effect.each([
287
+ { stall: "response", reportProgress: false },
288
+ { stall: "body", reportProgress: false },
289
+ { stall: "body", reportProgress: true },
290
+ ])(
291
+ "fails a one-minute $stall stall, cleans up and releases the lock (progress: $reportProgress)",
292
+ ({ stall, reportProgress }) =>
293
+ Effect.gen(function* () {
294
+ const fs = yield* FileSystem.FileSystem;
295
+ const path = yield* Path.Path;
296
+ const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-stalled-" });
297
+ const checksums = yield* validChecksums;
298
+ const waiting = yield* Deferred.make<void>();
299
+ let cancelled = false;
300
+ let sentChunk = false;
301
+ const client = HttpClient.make((request) => {
302
+ if (request.url.endsWith("/SHA256SUMS")) {
303
+ return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(checksums)));
304
+ }
305
+ if (stall === "response") {
306
+ return Deferred.succeed(waiting, undefined).pipe(
307
+ Effect.andThen(Effect.never),
308
+ Effect.ensuring(
309
+ Effect.sync(() => {
310
+ cancelled = true;
311
+ }),
312
+ ),
313
+ );
314
+ }
315
+ return Effect.succeed(
316
+ HttpClientResponse.fromWeb(
317
+ request,
318
+ new Response(
319
+ new ReadableStream<Uint8Array>(
320
+ {
321
+ pull(controller) {
322
+ if (!sentChunk) {
323
+ sentChunk = true;
324
+ controller.enqueue(archiveBytes.slice(0, 4));
325
+ } else {
326
+ Deferred.doneUnsafe(waiting, Effect.void);
327
+ }
328
+ },
329
+ cancel() {
330
+ cancelled = true;
331
+ },
332
+ },
333
+ { highWaterMark: 0 },
334
+ ),
335
+ ),
336
+ ),
337
+ );
338
+ });
339
+ const progress: PinnedRuntimeProgress[] = [];
340
+ const commands: string[] = [];
341
+ const install = yield* ensurePinnedRuntimeInstalled({
342
+ baseDir,
343
+ version,
344
+ fs,
345
+ path,
346
+ platform: "linux",
347
+ arch: "x64",
348
+ httpClient: client,
349
+ runner: extractingRunner(fs, path, commands),
350
+ validate: () => Effect.die("must not validate a stalled archive"),
351
+ ...(reportProgress
352
+ ? { onProgress: (event: PinnedRuntimeProgress) => progress.push(event) }
353
+ : {}),
354
+ }).pipe(Effect.forkScoped);
355
+ yield* Deferred.await(waiting);
356
+ yield* TestClock.adjust(Duration.minutes(1));
357
+ assert.isTrue(cancelled, "the stalled transfer must be cancelled after one minute");
358
+ const error = yield* Fiber.join(install).pipe(Effect.flip);
359
+ assert.instanceOf(error, PinnedRuntimeInstallError);
360
+ assert.equal(error.step, "downloading the t3 release archive (timed out)");
361
+ assert.deepEqual(commands, []);
362
+ assert.isTrue(progress.every((event) => event.stage === "download"));
363
+ assert.deepEqual(yield* fs.readDirectory(path.join(baseDir, "runtime", "versions")), []);
364
+ const installed = yield* ensurePinnedRuntimeInstalled({
365
+ baseDir,
366
+ version,
367
+ fs,
368
+ path,
369
+ platform: "linux",
370
+ arch: "x64",
371
+ httpClient: releaseHttpClient(checksums),
372
+ runner: extractingRunner(fs, path),
373
+ validate: () => Effect.void,
374
+ });
375
+ assert.equal(yield* fs.readFileString(installed.sentinelPath), `${version}\n`);
376
+ }),
377
+ );
378
+
379
it.effect("refuses an archive whose checksum does not match the release", () =>
380
Effect.gen(function* () {
381
const fs = yield* FileSystem.FileSystem;
apps/server/src/cloud/pinnedRuntime.ts
+19
-11
@@ -1,3 +1,4 @@
1
+import * as Cause from "effect/Cause";
2
import * as Duration from "effect/Duration";
3
import * as Effect from "effect/Effect";
4
import * as Hex from "effect/encoding/Hex";
@@ -22,6 +23,7 @@ import * as ProcessRunner from "../processRunner.ts";
23
24
const PINNED_RUNTIME_DIR = "runtime";
25
const PINNED_RUNTIME_INSTALL_TIMEOUT = Duration.minutes(10);
26
+const PINNED_RUNTIME_DOWNLOAD_STALL_TIMEOUT = Duration.minutes(1);
27
const PINNED_RUNTIME_ARCHIVE_FILE = "t3-runtime-archive";
28
// Boot-service setup and remote update can construct separate layers. Serialize
29
// the complete install transaction across every caller in this process.
@@ -119,28 +121,32 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")(
121
step: string,
122
onProgress?: (progress: PinnedRuntimeProgress) => void,
123
) {
122
- // The install lock is held for the whole transaction, so a stalled download
123
- // must fail rather than block every other caller.
124
+ // Bound stalls while holding the install lock, but allow transfers that keep
125
+ // receiving bytes to finish regardless of their total duration.
126
return yield* httpClient.execute(HttpClientRequest.get(url)).pipe(
127
Effect.flatMap(HttpClientResponse.filterStatusOk),
128
+ Effect.timeout(PINNED_RUNTIME_DOWNLOAD_STALL_TIMEOUT),
129
Effect.flatMap(
130
Effect.fn(function* (response) {
128
- if (onProgress === undefined) return new Uint8Array(yield* response.arrayBuffer);
131
const length = Number(response.headers["content-length"]);
132
const total = Number.isFinite(length) && length > 0 ? length : undefined;
133
let received = 0;
132
- onProgress({ stage: "download", received, total });
134
+ onProgress?.({ stage: "download", received, total });
135
const chunks = yield* response.stream.pipe(
136
+ Stream.timeoutOrElse({
137
+ duration: PINNED_RUNTIME_DOWNLOAD_STALL_TIMEOUT,
138
+ orElse: () => Stream.fail(new Cause.TimeoutError()),
139
+ }),
140
Stream.tap((chunk) =>
141
Effect.sync(() => {
142
received += chunk.byteLength;
137
- onProgress({ stage: "download", received, total });
143
+ onProgress?.({ stage: "download", received, total });
144
}),
145
),
146
Stream.runCollect,
147
);
148
if (total === undefined && received > 0) {
143
- onProgress({ stage: "download", received, total: received });
149
+ onProgress?.({ stage: "download", received, total: received });
150
}
151
const bytes = new Uint8Array(received);
152
let offset = 0;
@@ -151,11 +157,13 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")(
157
return bytes;
158
}),
159
),
154
- Effect.mapError((cause) => new PinnedRuntimeInstallError({ step, cause })),
155
- Effect.timeoutOrElse({
156
- duration: PINNED_RUNTIME_INSTALL_TIMEOUT,
157
- orElse: () => Effect.fail(new PinnedRuntimeInstallError({ step: `${step} (timed out)` })),
158
- }),
160
+ Effect.mapError(
161
+ (cause) =>
162
+ new PinnedRuntimeInstallError({
163
+ step: cause._tag === "TimeoutError" ? `${step} (timed out)` : step,
164
+ cause,
165
+ }),
166
+ ),
167
);
168
});
169