The best way to control Claude Code, Codex and more from any computer or phone
fix(ssh): reconnect stops waiting on a remote server that is shutting down (#39807)
Julius Marminge committed
Nov 20, 2026 at 09:01 UTC
ccc1feb15572c422fbdce26ba6077b91459cfd47
3 files changed
+78
-12
apps/server/src/cli/sshHelper.ts
+12
-6
@@ -63,22 +63,24 @@ const pickPort = Command.make("pick-port", {
63
);
64
65
const probe = (port: number, probeTimeoutMs: number) =>
66
- new Promise<boolean>((resolve) => {
66
+ new Promise<"ready" | "refused" | "not-ready">((resolve) => {
67
const request = NodeHttp.get(
68
{ hostname: "127.0.0.1", port, path: "/", timeout: probeTimeoutMs },
69
(response) => {
70
response.resume();
71
response.once("end", () => {
72
const status = response.statusCode ?? 0;
73
- resolve(status >= 200 && status < 300);
73
+ resolve(status >= 200 && status < 300 ? "ready" : "not-ready");
74
});
75
},
76
);
77
request.once("timeout", () => {
78
request.destroy();
79
- resolve(false);
79
+ resolve("not-ready");
80
});
81
- request.once("error", () => resolve(false));
81
+ request.once("error", (error: NodeJS.ErrnoException) =>
82
+ resolve(error.code === "ECONNREFUSED" ? "refused" : "not-ready"),
83
+ );
84
});
85
86
const waitReady = Command.make("wait-ready", {
@@ -89,11 +91,15 @@ const waitReady = Command.make("wait-ready", {
91
Command.withHandler(({ port, timeoutMs, probeTimeoutMs }) =>
92
Effect.promise(async () => {
93
const deadline = Date.now() + timeoutMs;
94
+ let refusedOnly = true;
95
while (Date.now() < deadline) {
93
- if (await probe(port, probeTimeoutMs)) return;
96
+ const result = await probe(port, probeTimeoutMs);
97
+ if (result === "ready") return;
98
+ refusedOnly &&= result === "refused";
99
await new Promise((resolve) => setTimeout(resolve, 100));
100
}
96
- process.exitCode = 1;
101
+ // The launcher reads 2 as "nothing listens on this port".
102
+ process.exitCode = refusedOnly ? 2 : 1;
103
}),
104
),
105
);
packages/ssh/src/tunnel.test.ts
+40
-1
@@ -26,6 +26,7 @@ import * as Sink from "effect/Sink";
26
import * as Stream from "effect/Stream";
27
import { TestClock } from "effect/testing";
28
import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
29
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
30
import { ChildProcess, ChildProcessSpawner } from "effect/process";
31
32
import { captureLauncher, captureLauncherFor, runLauncher } from "./launcherTestHarness.ts";
@@ -1363,7 +1364,15 @@ const http = require("node:http");
1364
const path = require("node:path");
1365
const args = process.argv.slice(2);
1366
const flag = (name) => args[args.indexOf(name) + 1];
1366
-const server = http.createServer((_request, response) => response.end("ok"));
1367
+const server = http.createServer((request, response) => {
1368
+ // Closes the listener but stays up, like a server stuck in its teardown.
1369
+ if (request.url === "/stop") {
1370
+ server.close();
1371
+ setInterval(() => {}, 1 << 30);
1372
+ response.setHeader("connection", "close");
1373
+ }
1374
+ response.end("ok");
1375
+});
1376
server.listen(Number(flag("--port") ?? 0), "127.0.0.1", () => {
1377
const port = server.address().port;
1378
const base = flag("--base-dir");
@@ -1485,6 +1494,36 @@ server.listen(Number(flag("--port") ?? 0), "127.0.0.1", () => {
1494
}).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1495
);
1496
1497
+ it.live("a reconnect names the previous server while it is still shutting down", () =>
1498
+ Effect.gen(function* () {
1499
+ const { fs, fakeServer, sshAsLocalShell, stateDir } = yield* prepare("an unrelated process");
1500
+ const launch = SshTunnel.launchOrReuseRemoteServer(target, undefined, {
1501
+ nodeScriptPath: fakeServer,
1502
+ }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, sshAsLocalShell));
1503
+
1504
+ const first = yield* launch;
1505
+ const pid = Number((yield* fs.readFileString(`${stateDir}/pid`)).trim());
1506
+ yield* Effect.addFinalizer(() =>
1507
+ Effect.sync(() => {
1508
+ try {
1509
+ process.kill(pid, "SIGKILL");
1510
+ } catch {}
1511
+ }),
1512
+ );
1513
+ yield* HttpClient.get(`http://127.0.0.1:${first.remotePort}/stop`).pipe(
1514
+ Effect.flatMap((response) => response.text),
1515
+ Effect.provide(FetchHttpClient.layer),
1516
+ );
1517
+
1518
+ const error = yield* Effect.flip(launch);
1519
+ assert.include(
1520
+ error.message,
1521
+ `The previous remote T3 server (PID ${pid}) is still shutting down.`,
1522
+ );
1523
+ assert.isTrue(yield* isRunning(pid), "the launcher signalled the stopping server");
1524
+ }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1525
+ );
1526
+
1527
it.live("an update replaces the managed server after its connection is renamed", () =>
1528
Effect.gen(function* () {
1529
const { fs, home, fakeServer, sshAsLocalShell, stateDir } =
packages/ssh/src/tunnel.ts
+26
-5
@@ -346,26 +346,29 @@ function probe() {
346
(response) => {
347
response.resume();
348
response.once("end", () => {
349
- resolve(response.statusCode >= 200 && response.statusCode < 300);
349
+ resolve(response.statusCode >= 200 && response.statusCode < 300 ? "ready" : "not-ready");
350
});
351
},
352
);
353
request.once("timeout", () => {
354
request.destroy();
355
- resolve(false);
355
+ resolve("not-ready");
356
});
357
- request.once("error", () => resolve(false));
357
+ request.once("error", (error) => resolve(error?.code === "ECONNREFUSED" ? "refused" : "not-ready"));
358
});
359
}
360
361
(async () => {
362
+ let refusedOnly = true;
363
while (Date.now() < deadline) {
363
- if (await probe()) {
364
+ const result = await probe();
365
+ if (result === "ready") {
366
process.exit(0);
367
}
368
+ refusedOnly &&= result === "refused";
369
await sleep(100);
370
}
368
- process.exit(1);
371
+ process.exit(refusedOnly ? 2 : 1);
372
})().catch(() => process.exit(1));
373
`;
374
@@ -740,6 +743,15 @@ keeps_running_server() {
743
"$RUNNER_FILE" __ssh-helper keep-server "$DEFAULT_RUNTIME_FILE" "$REMOTE_PID" >/dev/null 2>&1
744
}
745
@@T3_MANAGED_SERVER_RUNNING_SCRIPT@@
746
+# The server publishes its runtime record only after it answers, so one that
747
+# has published and now refuses every connection has closed its listener to
748
+# shut down; a starting server has not published yet. Exit 2 means refused.
749
+managed_server_stopping() {
750
+ [ "$DEFAULT_RUNTIME_PID" = "$REMOTE_PID" ] || return 1
751
+ READY_STATUS=0
752
+ wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@" || READY_STATUS=$?
753
+ [ "$READY_STATUS" -eq 2 ]
754
+}
755
# Both are SQLite locks the OS drops with their holder: a server holds
756
# server-owner.sqlite, and a boot service holds service-serving.lock from
757
# before it starts a server, between two of them during its own update, until
@@ -909,6 +921,15 @@ elif [ -n "$REMOTE_PORT" ] && managed_server_running "$REMOTE_PID"; then
921
REMOTE_PID=""
922
REMOTE_PORT=""
923
REMOTE_MANAGED=""
924
+ elif managed_server_stopping; then
925
+ wait_for_pid_exit "$REMOTE_PID"
926
+ if managed_server_running "$REMOTE_PID"; then
927
+ printf 'The previous remote T3 server (PID %s) is still shutting down. Retry once it has exited.\\n' "$REMOTE_PID" >&2
928
+ exit 1
929
+ fi
930
+ REMOTE_PID=""
931
+ REMOTE_PORT=""
932
+ REMOTE_MANAGED=""
933
elif ! wait_ready "@@T3_READY_TIMEOUT_MS@@"; then
934
printf 'Remote T3 server is still running but did not become ready. Retry the connection; the server was left running.\\n' >&2
935
exit 1