@devries-cameron20 / t3code Mirror · read-only / commits / 20c3dd3689
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 00:45 UTC

fix(ssh): concurrent launches on one remote home share one server (#32587)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 10, 2026 at 07:20 UTC 20c3dd3689782bfcb1a04332086d08017028c718
2 files changed +250 -13
packages/ssh/src/tunnel.test.ts
+138
@@ -13,7 +13,9 @@ import * as Deferred from "effect/Deferred";
13 import * as Duration from "effect/Duration";
14 import * as Effect from "effect/Effect";
15 import * as Fiber from "effect/Fiber";
16 +import * as FileSystem from "effect/FileSystem";
17 import * as Layer from "effect/Layer";
18 +import * as Path from "effect/Path";
19 import * as PlatformError from "effect/PlatformError";
20 import * as Schema from "effect/Schema";
21 import * as Result from "effect/Result";
@@ -24,6 +26,7 @@ import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
26 import { ChildProcess, ChildProcessSpawner } from "effect/process";
27
28 import * as SshAuth from "./auth.ts";
29 +import { remoteStateKey } from "./command.ts";
30 import { SshCommandError, SshPasswordPromptError, SshReadinessError } from "./errors.ts";
31 import * as SshTunnel from "./tunnel.ts";
32
@@ -92,6 +95,73 @@ function commandArgs(command: ChildProcess.Command): ReadonlyArray<string> {
95
96 const ARCHIVE = { archiveVersion: "1.2.3-preview.20260911.4" } as const;
97
98 +// Emulates `t3 serve`: it takes the home owner lock at startup, then binds
99 +// once a second server has tried the same home (or after 2s), so a concurrent
100 +// launch always sees the home still unclaimed. Like the real server, it
101 +// publishes server-runtime.json after it starts answering HTTP, if at all.
102 +const fakeT3Serve = (publishDelayMs: number | null) => `const fs = require("node:fs");
103 +const http = require("node:http");
104 +const path = require("node:path");
105 +const args = process.argv.slice(2);
106 +const port = Number(args[args.indexOf("--port") + 1]);
107 +const userdata = path.join(args[args.indexOf("--base-dir") + 1], "userdata");
108 +const servers = path.join(process.env.HOME, "servers");
109 +fs.mkdirSync(userdata, { recursive: true });
110 +fs.appendFileSync(servers, process.pid + "\\n");
111 +try {
112 + fs.writeFileSync(path.join(userdata, "server.lock"), String(process.pid), { flag: "wx" });
113 +} catch {
114 + console.error("ServerAlreadyRunningError: another server owns " + userdata);
115 + process.exit(1);
116 +}
117 +setTimeout(() => process.exit(0), 60_000);
118 +const serve = (tries) => {
119 + if (tries > 0 && fs.readFileSync(servers, "utf8").trim().split("\\n").length < 2) {
120 + setTimeout(() => serve(tries - 1), 50);
121 + return;
122 + }
123 + http.createServer((_, response) => response.end()).listen(port, "127.0.0.1", () => {
124 + const origin = "http://127.0.0.1:" + port;
125 + ${publishDelayMs === null ? "return;" : ""}
126 + setTimeout(() => {
127 + fs.writeFileSync(path.join(userdata, "server-runtime.json"), JSON.stringify({ pid: process.pid, port, origin }));
128 + }, ${publishDelayMs});
129 + });
130 +};
131 +serve(40);
132 +`;
133 +
134 +// Runs the remote half of an SSH command (`sh -l -s -- <key>`) as a local shell
135 +// whose HOME is the given directory. Every launch after the first starts once
136 +// a server is running (or after 10s), which lines up the race deterministically.
137 +const localRemoteShellLayer = (home: string) =>
138 + Layer.effect(
139 + ChildProcessSpawner.ChildProcessSpawner,
140 + Effect.gen(function* () {
141 + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
142 + let launches = 0;
143 + return ChildProcessSpawner.make((command) => {
144 + if (command._tag !== "StandardCommand") return spawner.spawn(command);
145 + const gate =
146 + launches++ === 0
147 + ? ""
148 + : 'i=0; while [ ! -s "$HOME/servers" ] && [ "$i" -lt 200 ]; do sleep 0.05; i=$((i+1)); done; ';
149 + return spawner.spawn(
150 + ChildProcess.make(
151 + "sh",
152 + ["-c", `${gate}exec sh -s -- "$1"`, "sh", ...commandArgs(command).slice(-1)],
153 + {
154 + ...command.options,
155 + env: { ...command.options.env, HOME: home },
156 + // Like an SSH session ending, the shell's exit leaves its nohup'd server running.
157 + detached: false,
158 + },
159 + ),
160 + );
161 + });
162 + }),
163 + );
164 +
165 describe("ssh tunnel scripts", () => {
166 it("rejects archive versions that are not a single exact version segment", () => {
167 for (const archiveVersion of [
@@ -630,6 +700,74 @@ describe("ssh tunnel scripts", () => {
700 );
701 }),
702 );
703 +
704 + it.live("concurrent launches on one remote home record the server that owns it", () =>
705 + // The second row's server never publishes server-runtime.json.
706 + Effect.forEach([1_500, null], (publishDelayMs) =>
707 + Effect.gen(function* () {
708 + const fs = yield* FileSystem.FileSystem;
709 + const path = yield* Path.Path;
710 + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
711 + const serversFile = path.join(home, "servers");
712 + yield* Effect.addFinalizer(() =>
713 + fs.readFileString(serversFile).pipe(
714 + Effect.map((pids) => {
715 + for (const pid of pids.split("\n").filter(Boolean)) {
716 + try {
717 + process.kill(Number(pid), "SIGKILL");
718 + } catch {}
719 + }
720 + }),
721 + Effect.ignore,
722 + ),
723 + );
724 + const serverScript = path.join(home, "fake-t3.cjs");
725 + yield* fs.writeFileString(serverScript, fakeT3Serve(publishDelayMs));
726 + // A launch that died holding the lock must not block or double-admit later ones.
727 + const launchLock = path.join(home, ".t3", "ssh-launch", ".launch.lock");
728 + yield* fs.makeDirectory(path.dirname(launchLock), { recursive: true });
729 + yield* fs.writeFileString(launchLock, "999999999\n");
730 + const targets = [
731 + { alias: "devbox", hostname: "devbox", username: null, port: null },
732 + { alias: "devbox-tailnet", hostname: "devbox-tailnet", username: null, port: null },
733 + ];
734 +
735 + const launches = yield* Effect.forEach(
736 + targets,
737 + (target) =>
738 + SshTunnel.launchOrReuseRemoteServer(target, undefined, {
739 + nodeScriptPath: serverScript,
740 + }),
741 + { concurrency: "unbounded" },
742 + ).pipe(Effect.provide(localRemoteShellLayer(home)));
743 +
744 + const ownerPid = Number(
745 + yield* fs.readFileString(path.join(home, ".t3", "userdata", "server.lock")),
746 + );
747 + assert.sameMembers(
748 + launches.map((launch) => launch.remoteServerKind),
749 + ["managed", "external"],
750 + );
751 + assert.equal(launches[0]!.remotePort, launches[1]!.remotePort);
752 + assert.deepEqual((yield* fs.readFileString(serversFile)).trim().split("\n"), [
753 + String(ownerPid),
754 + ]);
755 + for (const [index, launch] of launches.entries()) {
756 + if (launch.remoteServerKind !== "managed") continue;
757 + const stateKey = yield* remoteStateKey(targets[index]!);
758 + const recordedPid = yield* fs.readFileString(
759 + path.join(home, ".t3", "ssh-launch", stateKey, "pid"),
760 + );
761 + assert.equal(
762 + Number(recordedPid.trim()),
763 + ownerPid,
764 + `${targets[index]!.alias} reported a managed server, but its recorded PID is not the server that owns the remote home`,
765 + );
766 + }
767 + assert.isFalse(yield* fs.exists(launchLock));
768 + }).pipe(Effect.scoped),
769 + ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
770 + );
771 });
772
773 // On macOS, Node can bind 127.0.0.1:<port> while another process (such as the
packages/ssh/src/tunnel.ts
+112 -13
@@ -53,9 +53,9 @@ const SSH_READY_PROBE_TIMEOUT_MS = 1_000;
53 const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000;
54 const TUNNEL_STDERR_TAIL_CHARS = 16_384;
55 const REMOTE_READY_TIMEOUT_MS = 60_000;
56 -const REMOTE_LAUNCH_TIMEOUT_MS = 90_000;
57 -// Matches the node-script launch command's whole ssh run against the same host.
58 -const SSH_TUNNEL_CONNECT_TIMEOUT_MS = REMOTE_LAUNCH_TIMEOUT_MS;
56 +const REMOTE_LAUNCH_BASE_TIMEOUT_MS = 90_000;
57 +// Matches the node-script launch command's ssh run without the launch lock wait.
58 +const SSH_TUNNEL_CONNECT_TIMEOUT_MS = REMOTE_LAUNCH_BASE_TIMEOUT_MS;
59 const SSH_TUNNEL_FORWARD_PROBE_INTERVAL_MS = 100;
60 // Cold archive launches download and extract ~70 MB and may wait on an install lock.
61 // Lock waits must outlast both downloads and extraction; SSH must outlast install or wait plus readiness.
@@ -65,6 +65,23 @@ const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240;
65 const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360;
66 const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000;
67 const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
68 +const REMOTE_PID_EXIT_WAIT_MS = 2_000;
69 +// The server answers HTTP before it writes server-runtime.json, which is how
70 +// a launch for another alias finds it.
71 +const REMOTE_RUNTIME_RECORD_WAIT_SECONDS = 10;
72 +// A launch holds the home lock through up to two reuse probes, readiness, two
73 +// server exit waits, and the runtime record wait; each probe may overrun its
74 +// deadline by one request.
75 +const REMOTE_LAUNCH_LOCK_WAIT_SECONDS = Math.ceil(
76 + (REMOTE_READY_TIMEOUT_MS +
77 + 2 * REMOTE_REUSE_READY_TIMEOUT_MS +
78 + 2 * REMOTE_PID_EXIT_WAIT_MS +
79 + REMOTE_RUNTIME_RECORD_WAIT_SECONDS * 1000 +
80 + 4 * SSH_READY_PROBE_TIMEOUT_MS) /
81 + 1000,
82 +);
83 +const REMOTE_LAUNCH_TIMEOUT_MS =
84 + REMOTE_LAUNCH_LOCK_WAIT_SECONDS * 1000 + REMOTE_LAUNCH_BASE_TIMEOUT_MS;
85
86 export interface RemoteT3RunnerOptions {
87 readonly nodeScriptPath?: string | null;
@@ -537,30 +554,82 @@ MANAGED_FILE="$STATE_DIR/managed"
554 LOG_FILE="$STATE_DIR/server.log"
555 RUNNER_FILE="$STATE_DIR/run-t3.sh"
556 RUNNER_NEXT="$STATE_DIR/run-t3.next.$$"
557 +LAUNCH_LOCK="$DEFAULT_SERVER_HOME/ssh-launch/.launch.lock"
558 +LAUNCH_BREAK_LOCK="$LAUNCH_LOCK.break"
559 mkdir -p "$STATE_DIR"
541 -cleanup_runner_next() {
560 +cleanup_launch() {
561 rm -f "$RUNNER_NEXT"
562 + for OWNED_LOCK in "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK"; do
563 + if [ "$(cat "$OWNED_LOCK" 2>/dev/null || true)" = "$$" ]; then
564 + rm -f "$OWNED_LOCK"
565 + fi
566 + done
567 }
544 -trap cleanup_runner_next EXIT
568 +trap cleanup_launch EXIT
569 +# A dropped SSH session signals the launch (PIPE on its next write); exit so
570 +# the lock is released.
571 +trap 'exit 1' HUP INT PIPE TERM
572 cat >"$RUNNER_NEXT" <<'SH'
573 @@T3_RUNNER_SCRIPT@@
574 SH
548 -RUNNER_CHANGED=0
549 -if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
550 - RUNNER_CHANGED=1
551 -fi
552 -mv "$RUNNER_NEXT" "$RUNNER_FILE"
553 -chmod 700 "$RUNNER_FILE"
575 +chmod 700 "$RUNNER_NEXT"
576 T3_ARCHIVE_MODE=@@T3_ARCHIVE_MODE@@
577 if [ "$T3_ARCHIVE_MODE" = "1" ]; then
578 # The archive ships the helpers below inside the executable; the remote
579 # needs no Node at all. Resolving the runner once here also downloads the
580 # archive before the port and readiness probes rely on it.
559 - "$RUNNER_FILE" --version >/dev/null
581 + "$RUNNER_NEXT" --version >/dev/null
582 elif ! ensure_remote_node_path; then
583 printf 'Remote host is missing node on PATH. Install Node or configure a supported version manager for non-interactive shells.\\n' >&2
584 exit 1
585 fi
586 +# Recovery runs under its own lock and re-reads the owner there, so a waiter
587 +# that judged an earlier lock stale cannot remove its successor's lock. No
588 +# launch removes another's recovery lock: only SIGKILL inside these lines
589 +# leaves one behind, and the wait below then fails naming it.
590 +discard_stale_launch_lock() {
591 + (set -C; printf '%s\\n' "$$" >"$LAUNCH_BREAK_LOCK") 2>/dev/null || return 1
592 + if [ "$(cat "$LAUNCH_LOCK" 2>/dev/null || true)" = "$1" ]; then
593 + rm -f "$LAUNCH_LOCK"
594 + fi
595 + rm -f "$LAUNCH_BREAK_LOCK"
596 +}
597 +# Concurrent launches share this home's owner lock and may share this record.
598 +# Hold the lock from comparing the runner until the server is recorded and
599 +# ready, so a waiting launch reuses that server instead of starting or
600 +# replacing one.
601 +LAUNCH_LOCK_WAITED=0
602 +LAUNCH_LOCK_UNOWNED=0
603 +while ! (set -C; printf '%s\\n' "$$" >"$LAUNCH_LOCK") 2>/dev/null; do
604 + LAUNCH_LOCK_OWNER="$(cat "$LAUNCH_LOCK" 2>/dev/null || true)"
605 + if [ -n "$LAUNCH_LOCK_OWNER" ]; then
606 + LAUNCH_LOCK_UNOWNED=0
607 + if ! kill -0 "$LAUNCH_LOCK_OWNER" 2>/dev/null && discard_stale_launch_lock "$LAUNCH_LOCK_OWNER"; then
608 + continue
609 + fi
610 + else
611 + LAUNCH_LOCK_UNOWNED=$((LAUNCH_LOCK_UNOWNED + 1))
612 + if [ "$LAUNCH_LOCK_UNOWNED" -ge 5 ] && discard_stale_launch_lock ""; then
613 + LAUNCH_LOCK_UNOWNED=0
614 + continue
615 + fi
616 + fi
617 + if [ "$LAUNCH_LOCK_WAITED" -ge @@T3_LAUNCH_LOCK_WAIT_SECONDS@@ ]; then
618 + if [ -e "$LAUNCH_BREAK_LOCK" ]; then
619 + printf 'A T3 server launch stopped while recovering %s. If no launch is running, delete %s.\\n' "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK" >&2
620 + else
621 + printf 'Another T3 server launch has held %s for too long.\\n' "$LAUNCH_LOCK" >&2
622 + fi
623 + exit 1
624 + fi
625 + sleep 1
626 + LAUNCH_LOCK_WAITED=$((LAUNCH_LOCK_WAITED + 1))
627 +done
628 +RUNNER_CHANGED=0
629 +if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
630 + RUNNER_CHANGED=1
631 +fi
632 +mv "$RUNNER_NEXT" "$RUNNER_FILE"
633 pick_port() {
634 if [ "$T3_ARCHIVE_MODE" = "1" ]; then
635 "$RUNNER_FILE" __ssh-helper pick-port "$PORT_FILE" "@@T3_DEFAULT_REMOTE_PORT@@" "@@T3_REMOTE_PORT_SCAN_WINDOW@@"
@@ -582,7 +651,7 @@ NODE
651 wait_for_pid_exit() {
652 PID_TO_WAIT="$1"
653 WAIT_COUNT=0
585 - while kill -0 "$PID_TO_WAIT" 2>/dev/null && [ "$WAIT_COUNT" -lt 20 ]; do
654 + while kill -0 "$PID_TO_WAIT" 2>/dev/null && [ "$WAIT_COUNT" -lt @@T3_PID_EXIT_WAIT_TICKS@@ ]; do
655 WAIT_COUNT=$((WAIT_COUNT + 1))
656 sleep 0.1
657 done
@@ -693,6 +762,25 @@ else
762 REMOTE_PORT=""
763 REMOTE_MANAGED=""
764 fi
765 +# The server may answer long before it publishes its runtime record, or never
766 +# publish one; another alias's launch record still names it. Only one server
767 +# owns the home, so the first live one decides.
768 +if [ -z "$REMOTE_PORT" ] && [ -z "$DEFAULT_RUNTIME_INFO" ]; then
769 + for SIBLING_DIR in "$DEFAULT_SERVER_HOME"/ssh-launch/*; do
770 + [ "$SIBLING_DIR" != "$STATE_DIR" ] && [ "$(cat "$SIBLING_DIR/managed" 2>/dev/null || true)" = "managed" ] || continue
771 + server_running "$(cat "$SIBLING_DIR/pid" 2>/dev/null || true)" || continue
772 + REMOTE_PORT="$(cat "$SIBLING_DIR/port" 2>/dev/null || true)"
773 + if [ -n "$REMOTE_PORT" ] && wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
774 + REMOTE_MANAGED="external"
775 + rm -f "$PID_FILE"
776 + printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
777 + printf 'external\\n' >"$MANAGED_FILE"
778 + else
779 + REMOTE_PORT=""
780 + fi
781 + break
782 + done
783 +fi
784 if [ -z "$REMOTE_PORT" ]; then
785 REMOTE_PORT="$(pick_port)" || true
786 if [ -z "$REMOTE_PORT" ]; then
@@ -720,6 +808,14 @@ if [ -z "$REMOTE_PORT" ]; then
808 rm -f "$PID_FILE" "$PORT_FILE" "$MANAGED_FILE"
809 exit 1
810 fi
811 + # Keep the lock until another alias's launch can find this server, or it
812 + # would start a second one that loses the home's owner lock.
813 + RECORD_DEADLINE=$(($(date +%s) + @@T3_RUNTIME_RECORD_WAIT_SECONDS@@))
814 + while [ "$(date +%s)" -lt "$RECORD_DEADLINE" ]; do
815 + DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
816 + [ "\${DEFAULT_RUNTIME_INFO%% *}" = "$REMOTE_PID" ] && break
817 + sleep 0.1
818 + done
819 fi
820 printf '{"remotePort":%s,"serverKind":"%s"}\\n' "$REMOTE_PORT" "\${REMOTE_MANAGED:-managed}"
821 `;
@@ -839,6 +935,9 @@ function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
935 T3_READY_TIMEOUT_MS: String(REMOTE_READY_TIMEOUT_MS),
936 T3_REUSE_READY_TIMEOUT_MS: String(REMOTE_REUSE_READY_TIMEOUT_MS),
937 T3_READY_PROBE_TIMEOUT_MS: String(SSH_READY_PROBE_TIMEOUT_MS),
938 + T3_PID_EXIT_WAIT_TICKS: String(REMOTE_PID_EXIT_WAIT_MS / 100),
939 + T3_LAUNCH_LOCK_WAIT_SECONDS: String(REMOTE_LAUNCH_LOCK_WAIT_SECONDS),
940 + T3_RUNTIME_RECORD_WAIT_SECONDS: String(REMOTE_RUNTIME_RECORD_WAIT_SECONDS),
941 });
942 }
943