The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 00:45 UTC
fix(ssh): concurrent launches on one remote home share one server (#32587)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Nov 10, 2026 at 07:20 UTC
20c3dd3689782bfcb1a04332086d08017028c718
2 files changed
+250
-13
packages/ssh/src/tunnel.test.ts
+138
@@ -13,7 +13,9 @@ import * as Deferred from "effect/Deferred";
13
import * as Duration from "effect/Duration";
14
import * as Effect from "effect/Effect";
15
import * as Fiber from "effect/Fiber";
16
+import * as FileSystem from "effect/FileSystem";
17
import * as Layer from "effect/Layer";
18
+import * as Path from "effect/Path";
19
import * as PlatformError from "effect/PlatformError";
20
import * as Schema from "effect/Schema";
21
import * as Result from "effect/Result";
@@ -24,6 +26,7 @@ import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
26
import { ChildProcess, ChildProcessSpawner } from "effect/process";
27
28
import * as SshAuth from "./auth.ts";
29
+import { remoteStateKey } from "./command.ts";
30
import { SshCommandError, SshPasswordPromptError, SshReadinessError } from "./errors.ts";
31
import * as SshTunnel from "./tunnel.ts";
32
@@ -92,6 +95,73 @@ function commandArgs(command: ChildProcess.Command): ReadonlyArray<string> {
95
96
const ARCHIVE = { archiveVersion: "1.2.3-preview.20260911.4" } as const;
97
98
+// Emulates `t3 serve`: it takes the home owner lock at startup, then binds
99
+// once a second server has tried the same home (or after 2s), so a concurrent
100
+// launch always sees the home still unclaimed. Like the real server, it
101
+// publishes server-runtime.json after it starts answering HTTP, if at all.
102
+const fakeT3Serve = (publishDelayMs: number | null) => `const fs = require("node:fs");
103
+const http = require("node:http");
104
+const path = require("node:path");
105
+const args = process.argv.slice(2);
106
+const port = Number(args[args.indexOf("--port") + 1]);
107
+const userdata = path.join(args[args.indexOf("--base-dir") + 1], "userdata");
108
+const servers = path.join(process.env.HOME, "servers");
109
+fs.mkdirSync(userdata, { recursive: true });
110
+fs.appendFileSync(servers, process.pid + "\\n");
111
+try {
112
+ fs.writeFileSync(path.join(userdata, "server.lock"), String(process.pid), { flag: "wx" });
113
+} catch {
114
+ console.error("ServerAlreadyRunningError: another server owns " + userdata);
115
+ process.exit(1);
116
+}
117
+setTimeout(() => process.exit(0), 60_000);
118
+const serve = (tries) => {
119
+ if (tries > 0 && fs.readFileSync(servers, "utf8").trim().split("\\n").length < 2) {
120
+ setTimeout(() => serve(tries - 1), 50);
121
+ return;
122
+ }
123
+ http.createServer((_, response) => response.end()).listen(port, "127.0.0.1", () => {
124
+ const origin = "http://127.0.0.1:" + port;
125
+ ${publishDelayMs === null ? "return;" : ""}
126
+ setTimeout(() => {
127
+ fs.writeFileSync(path.join(userdata, "server-runtime.json"), JSON.stringify({ pid: process.pid, port, origin }));
128
+ }, ${publishDelayMs});
129
+ });
130
+};
131
+serve(40);
132
+`;
133
+
134
+// Runs the remote half of an SSH command (`sh -l -s -- <key>`) as a local shell
135
+// whose HOME is the given directory. Every launch after the first starts once
136
+// a server is running (or after 10s), which lines up the race deterministically.
137
+const localRemoteShellLayer = (home: string) =>
138
+ Layer.effect(
139
+ ChildProcessSpawner.ChildProcessSpawner,
140
+ Effect.gen(function* () {
141
+ const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
142
+ let launches = 0;
143
+ return ChildProcessSpawner.make((command) => {
144
+ if (command._tag !== "StandardCommand") return spawner.spawn(command);
145
+ const gate =
146
+ launches++ === 0
147
+ ? ""
148
+ : 'i=0; while [ ! -s "$HOME/servers" ] && [ "$i" -lt 200 ]; do sleep 0.05; i=$((i+1)); done; ';
149
+ return spawner.spawn(
150
+ ChildProcess.make(
151
+ "sh",
152
+ ["-c", `${gate}exec sh -s -- "$1"`, "sh", ...commandArgs(command).slice(-1)],
153
+ {
154
+ ...command.options,
155
+ env: { ...command.options.env, HOME: home },
156
+ // Like an SSH session ending, the shell's exit leaves its nohup'd server running.
157
+ detached: false,
158
+ },
159
+ ),
160
+ );
161
+ });
162
+ }),
163
+ );
164
+
165
describe("ssh tunnel scripts", () => {
166
it("rejects archive versions that are not a single exact version segment", () => {
167
for (const archiveVersion of [
@@ -630,6 +700,74 @@ describe("ssh tunnel scripts", () => {
700
);
701
}),
702
);
703
+
704
+ it.live("concurrent launches on one remote home record the server that owns it", () =>
705
+ // The second row's server never publishes server-runtime.json.
706
+ Effect.forEach([1_500, null], (publishDelayMs) =>
707
+ Effect.gen(function* () {
708
+ const fs = yield* FileSystem.FileSystem;
709
+ const path = yield* Path.Path;
710
+ const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
711
+ const serversFile = path.join(home, "servers");
712
+ yield* Effect.addFinalizer(() =>
713
+ fs.readFileString(serversFile).pipe(
714
+ Effect.map((pids) => {
715
+ for (const pid of pids.split("\n").filter(Boolean)) {
716
+ try {
717
+ process.kill(Number(pid), "SIGKILL");
718
+ } catch {}
719
+ }
720
+ }),
721
+ Effect.ignore,
722
+ ),
723
+ );
724
+ const serverScript = path.join(home, "fake-t3.cjs");
725
+ yield* fs.writeFileString(serverScript, fakeT3Serve(publishDelayMs));
726
+ // A launch that died holding the lock must not block or double-admit later ones.
727
+ const launchLock = path.join(home, ".t3", "ssh-launch", ".launch.lock");
728
+ yield* fs.makeDirectory(path.dirname(launchLock), { recursive: true });
729
+ yield* fs.writeFileString(launchLock, "999999999\n");
730
+ const targets = [
731
+ { alias: "devbox", hostname: "devbox", username: null, port: null },
732
+ { alias: "devbox-tailnet", hostname: "devbox-tailnet", username: null, port: null },
733
+ ];
734
+
735
+ const launches = yield* Effect.forEach(
736
+ targets,
737
+ (target) =>
738
+ SshTunnel.launchOrReuseRemoteServer(target, undefined, {
739
+ nodeScriptPath: serverScript,
740
+ }),
741
+ { concurrency: "unbounded" },
742
+ ).pipe(Effect.provide(localRemoteShellLayer(home)));
743
+
744
+ const ownerPid = Number(
745
+ yield* fs.readFileString(path.join(home, ".t3", "userdata", "server.lock")),
746
+ );
747
+ assert.sameMembers(
748
+ launches.map((launch) => launch.remoteServerKind),
749
+ ["managed", "external"],
750
+ );
751
+ assert.equal(launches[0]!.remotePort, launches[1]!.remotePort);
752
+ assert.deepEqual((yield* fs.readFileString(serversFile)).trim().split("\n"), [
753
+ String(ownerPid),
754
+ ]);
755
+ for (const [index, launch] of launches.entries()) {
756
+ if (launch.remoteServerKind !== "managed") continue;
757
+ const stateKey = yield* remoteStateKey(targets[index]!);
758
+ const recordedPid = yield* fs.readFileString(
759
+ path.join(home, ".t3", "ssh-launch", stateKey, "pid"),
760
+ );
761
+ assert.equal(
762
+ Number(recordedPid.trim()),
763
+ ownerPid,
764
+ `${targets[index]!.alias} reported a managed server, but its recorded PID is not the server that owns the remote home`,
765
+ );
766
+ }
767
+ assert.isFalse(yield* fs.exists(launchLock));
768
+ }).pipe(Effect.scoped),
769
+ ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
770
+ );
771
});
772
773
// On macOS, Node can bind 127.0.0.1:<port> while another process (such as the
packages/ssh/src/tunnel.ts
+112
-13
@@ -53,9 +53,9 @@ const SSH_READY_PROBE_TIMEOUT_MS = 1_000;
53
const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000;
54
const TUNNEL_STDERR_TAIL_CHARS = 16_384;
55
const REMOTE_READY_TIMEOUT_MS = 60_000;
56
-const REMOTE_LAUNCH_TIMEOUT_MS = 90_000;
57
-// Matches the node-script launch command's whole ssh run against the same host.
58
-const SSH_TUNNEL_CONNECT_TIMEOUT_MS = REMOTE_LAUNCH_TIMEOUT_MS;
56
+const REMOTE_LAUNCH_BASE_TIMEOUT_MS = 90_000;
57
+// Matches the node-script launch command's ssh run without the launch lock wait.
58
+const SSH_TUNNEL_CONNECT_TIMEOUT_MS = REMOTE_LAUNCH_BASE_TIMEOUT_MS;
59
const SSH_TUNNEL_FORWARD_PROBE_INTERVAL_MS = 100;
60
// Cold archive launches download and extract ~70 MB and may wait on an install lock.
61
// Lock waits must outlast both downloads and extraction; SSH must outlast install or wait plus readiness.
@@ -65,6 +65,23 @@ const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240;
65
const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360;
66
const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000;
67
const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
68
+const REMOTE_PID_EXIT_WAIT_MS = 2_000;
69
+// The server answers HTTP before it writes server-runtime.json, which is how
70
+// a launch for another alias finds it.
71
+const REMOTE_RUNTIME_RECORD_WAIT_SECONDS = 10;
72
+// A launch holds the home lock through up to two reuse probes, readiness, two
73
+// server exit waits, and the runtime record wait; each probe may overrun its
74
+// deadline by one request.
75
+const REMOTE_LAUNCH_LOCK_WAIT_SECONDS = Math.ceil(
76
+ (REMOTE_READY_TIMEOUT_MS +
77
+ 2 * REMOTE_REUSE_READY_TIMEOUT_MS +
78
+ 2 * REMOTE_PID_EXIT_WAIT_MS +
79
+ REMOTE_RUNTIME_RECORD_WAIT_SECONDS * 1000 +
80
+ 4 * SSH_READY_PROBE_TIMEOUT_MS) /
81
+ 1000,
82
+);
83
+const REMOTE_LAUNCH_TIMEOUT_MS =
84
+ REMOTE_LAUNCH_LOCK_WAIT_SECONDS * 1000 + REMOTE_LAUNCH_BASE_TIMEOUT_MS;
85
86
export interface RemoteT3RunnerOptions {
87
readonly nodeScriptPath?: string | null;
@@ -537,30 +554,82 @@ MANAGED_FILE="$STATE_DIR/managed"
554
LOG_FILE="$STATE_DIR/server.log"
555
RUNNER_FILE="$STATE_DIR/run-t3.sh"
556
RUNNER_NEXT="$STATE_DIR/run-t3.next.$$"
557
+LAUNCH_LOCK="$DEFAULT_SERVER_HOME/ssh-launch/.launch.lock"
558
+LAUNCH_BREAK_LOCK="$LAUNCH_LOCK.break"
559
mkdir -p "$STATE_DIR"
541
-cleanup_runner_next() {
560
+cleanup_launch() {
561
rm -f "$RUNNER_NEXT"
562
+ for OWNED_LOCK in "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK"; do
563
+ if [ "$(cat "$OWNED_LOCK" 2>/dev/null || true)" = "$$" ]; then
564
+ rm -f "$OWNED_LOCK"
565
+ fi
566
+ done
567
}
544
-trap cleanup_runner_next EXIT
568
+trap cleanup_launch EXIT
569
+# A dropped SSH session signals the launch (PIPE on its next write); exit so
570
+# the lock is released.
571
+trap 'exit 1' HUP INT PIPE TERM
572
cat >"$RUNNER_NEXT" <<'SH'
573
@@T3_RUNNER_SCRIPT@@
574
SH
548
-RUNNER_CHANGED=0
549
-if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
550
- RUNNER_CHANGED=1
551
-fi
552
-mv "$RUNNER_NEXT" "$RUNNER_FILE"
553
-chmod 700 "$RUNNER_FILE"
575
+chmod 700 "$RUNNER_NEXT"
576
T3_ARCHIVE_MODE=@@T3_ARCHIVE_MODE@@
577
if [ "$T3_ARCHIVE_MODE" = "1" ]; then
578
# The archive ships the helpers below inside the executable; the remote
579
# needs no Node at all. Resolving the runner once here also downloads the
580
# archive before the port and readiness probes rely on it.
559
- "$RUNNER_FILE" --version >/dev/null
581
+ "$RUNNER_NEXT" --version >/dev/null
582
elif ! ensure_remote_node_path; then
583
printf 'Remote host is missing node on PATH. Install Node or configure a supported version manager for non-interactive shells.\\n' >&2
584
exit 1
585
fi
586
+# Recovery runs under its own lock and re-reads the owner there, so a waiter
587
+# that judged an earlier lock stale cannot remove its successor's lock. No
588
+# launch removes another's recovery lock: only SIGKILL inside these lines
589
+# leaves one behind, and the wait below then fails naming it.
590
+discard_stale_launch_lock() {
591
+ (set -C; printf '%s\\n' "$$" >"$LAUNCH_BREAK_LOCK") 2>/dev/null || return 1
592
+ if [ "$(cat "$LAUNCH_LOCK" 2>/dev/null || true)" = "$1" ]; then
593
+ rm -f "$LAUNCH_LOCK"
594
+ fi
595
+ rm -f "$LAUNCH_BREAK_LOCK"
596
+}
597
+# Concurrent launches share this home's owner lock and may share this record.
598
+# Hold the lock from comparing the runner until the server is recorded and
599
+# ready, so a waiting launch reuses that server instead of starting or
600
+# replacing one.
601
+LAUNCH_LOCK_WAITED=0
602
+LAUNCH_LOCK_UNOWNED=0
603
+while ! (set -C; printf '%s\\n' "$$" >"$LAUNCH_LOCK") 2>/dev/null; do
604
+ LAUNCH_LOCK_OWNER="$(cat "$LAUNCH_LOCK" 2>/dev/null || true)"
605
+ if [ -n "$LAUNCH_LOCK_OWNER" ]; then
606
+ LAUNCH_LOCK_UNOWNED=0
607
+ if ! kill -0 "$LAUNCH_LOCK_OWNER" 2>/dev/null && discard_stale_launch_lock "$LAUNCH_LOCK_OWNER"; then
608
+ continue
609
+ fi
610
+ else
611
+ LAUNCH_LOCK_UNOWNED=$((LAUNCH_LOCK_UNOWNED + 1))
612
+ if [ "$LAUNCH_LOCK_UNOWNED" -ge 5 ] && discard_stale_launch_lock ""; then
613
+ LAUNCH_LOCK_UNOWNED=0
614
+ continue
615
+ fi
616
+ fi
617
+ if [ "$LAUNCH_LOCK_WAITED" -ge @@T3_LAUNCH_LOCK_WAIT_SECONDS@@ ]; then
618
+ if [ -e "$LAUNCH_BREAK_LOCK" ]; then
619
+ printf 'A T3 server launch stopped while recovering %s. If no launch is running, delete %s.\\n' "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK" >&2
620
+ else
621
+ printf 'Another T3 server launch has held %s for too long.\\n' "$LAUNCH_LOCK" >&2
622
+ fi
623
+ exit 1
624
+ fi
625
+ sleep 1
626
+ LAUNCH_LOCK_WAITED=$((LAUNCH_LOCK_WAITED + 1))
627
+done
628
+RUNNER_CHANGED=0
629
+if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
630
+ RUNNER_CHANGED=1
631
+fi
632
+mv "$RUNNER_NEXT" "$RUNNER_FILE"
633
pick_port() {
634
if [ "$T3_ARCHIVE_MODE" = "1" ]; then
635
"$RUNNER_FILE" __ssh-helper pick-port "$PORT_FILE" "@@T3_DEFAULT_REMOTE_PORT@@" "@@T3_REMOTE_PORT_SCAN_WINDOW@@"
@@ -582,7 +651,7 @@ NODE
651
wait_for_pid_exit() {
652
PID_TO_WAIT="$1"
653
WAIT_COUNT=0
585
- while kill -0 "$PID_TO_WAIT" 2>/dev/null && [ "$WAIT_COUNT" -lt 20 ]; do
654
+ while kill -0 "$PID_TO_WAIT" 2>/dev/null && [ "$WAIT_COUNT" -lt @@T3_PID_EXIT_WAIT_TICKS@@ ]; do
655
WAIT_COUNT=$((WAIT_COUNT + 1))
656
sleep 0.1
657
done
@@ -693,6 +762,25 @@ else
762
REMOTE_PORT=""
763
REMOTE_MANAGED=""
764
fi
765
+# The server may answer long before it publishes its runtime record, or never
766
+# publish one; another alias's launch record still names it. Only one server
767
+# owns the home, so the first live one decides.
768
+if [ -z "$REMOTE_PORT" ] && [ -z "$DEFAULT_RUNTIME_INFO" ]; then
769
+ for SIBLING_DIR in "$DEFAULT_SERVER_HOME"/ssh-launch/*; do
770
+ [ "$SIBLING_DIR" != "$STATE_DIR" ] && [ "$(cat "$SIBLING_DIR/managed" 2>/dev/null || true)" = "managed" ] || continue
771
+ server_running "$(cat "$SIBLING_DIR/pid" 2>/dev/null || true)" || continue
772
+ REMOTE_PORT="$(cat "$SIBLING_DIR/port" 2>/dev/null || true)"
773
+ if [ -n "$REMOTE_PORT" ] && wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
774
+ REMOTE_MANAGED="external"
775
+ rm -f "$PID_FILE"
776
+ printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
777
+ printf 'external\\n' >"$MANAGED_FILE"
778
+ else
779
+ REMOTE_PORT=""
780
+ fi
781
+ break
782
+ done
783
+fi
784
if [ -z "$REMOTE_PORT" ]; then
785
REMOTE_PORT="$(pick_port)" || true
786
if [ -z "$REMOTE_PORT" ]; then
@@ -720,6 +808,14 @@ if [ -z "$REMOTE_PORT" ]; then
808
rm -f "$PID_FILE" "$PORT_FILE" "$MANAGED_FILE"
809
exit 1
810
fi
811
+ # Keep the lock until another alias's launch can find this server, or it
812
+ # would start a second one that loses the home's owner lock.
813
+ RECORD_DEADLINE=$(($(date +%s) + @@T3_RUNTIME_RECORD_WAIT_SECONDS@@))
814
+ while [ "$(date +%s)" -lt "$RECORD_DEADLINE" ]; do
815
+ DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
816
+ [ "\${DEFAULT_RUNTIME_INFO%% *}" = "$REMOTE_PID" ] && break
817
+ sleep 0.1
818
+ done
819
fi
820
printf '{"remotePort":%s,"serverKind":"%s"}\\n' "$REMOTE_PORT" "\${REMOTE_MANAGED:-managed}"
821
`;
@@ -839,6 +935,9 @@ function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
935
T3_READY_TIMEOUT_MS: String(REMOTE_READY_TIMEOUT_MS),
936
T3_REUSE_READY_TIMEOUT_MS: String(REMOTE_REUSE_READY_TIMEOUT_MS),
937
T3_READY_PROBE_TIMEOUT_MS: String(SSH_READY_PROBE_TIMEOUT_MS),
938
+ T3_PID_EXIT_WAIT_TICKS: String(REMOTE_PID_EXIT_WAIT_MS / 100),
939
+ T3_LAUNCH_LOCK_WAIT_SECONDS: String(REMOTE_LAUNCH_LOCK_WAIT_SECONDS),
940
+ T3_RUNTIME_RECORD_WAIT_SECONDS: String(REMOTE_RUNTIME_RECORD_WAIT_SECONDS),
941
});
942
}
943