The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import { |
| 2 | type DeviceHostSummary, |
| 3 | DevicePlatformAvailability, |
| 4 | DeviceToolVersions, |
| 5 | deviceToolInstallMessage, |
| 6 | type SshDeviceHostConfig, |
| 7 | } from "@t3tools/contracts"; |
| 8 | import { runSshCommand, baseSshArgs, resolveSshCommand } from "@t3tools/ssh/command"; |
| 9 | import { SshCommandError } from "@t3tools/ssh/errors"; |
| 10 | import * as NetService from "@t3tools/shared/Net"; |
| 11 | import { waitForHttpReady } from "@t3tools/shared/httpReadiness"; |
| 12 | import * as Digest from "effect/crypto/Digest"; |
| 13 | import * as Exit from "effect/Exit"; |
| 14 | import * as Effect from "effect/Effect"; |
| 15 | import * as FileSystem from "effect/FileSystem"; |
| 16 | import * as Path from "effect/Path"; |
| 17 | import * as Scope from "effect/Scope"; |
| 18 | import * as Semaphore from "effect/Semaphore"; |
| 19 | import * as Schema from "effect/Schema"; |
| 20 | import * as Stream from "effect/Stream"; |
| 21 | import * as Hex from "effect/encoding/Hex"; |
| 22 | import * as HttpClient from "effect/http/HttpClient"; |
| 23 | import * as ChildProcess from "effect/process/ChildProcess"; |
| 24 | import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; |
| 25 | import * as ServerConfig from "../config.ts"; |
| 26 | import { releaseAgentDeviceSession } from "./AgentDeviceTarget.ts"; |
| 27 | import * as DeviceHost from "./DeviceHost.ts"; |
| 28 | import { makeDeviceHubOperations } from "./DeviceHubOperations.ts"; |
| 29 | import { quoteRemoteArg, remoteDeviceEnvironment, remoteDeviceScript } from "./sshDeviceScript.ts"; |
| 30 | |
| 31 | const UNHEALTHY_PROBE_LIMIT = 3; |
| 32 | |
| 33 | /** The host bootstrap failed on a host identified as Windows. */ |
| 34 | export class SshWindowsHostError extends Schema.TaggedError<SshWindowsHostError>()( |
| 35 | "SshWindowsHostError", |
| 36 | { cause: Schema.Defect() }, |
| 37 | ) { |
| 38 | override get message(): string { |
| 39 | return "Connected over SSH, but this host runs Windows, and SSH device hosts currently need macOS or Linux. To use this machine's devices, run T3 Code on it and connect to that environment."; |
| 40 | } |
| 41 | } |
| 42 | export const isSshWindowsHostError = Schema.is(SshWindowsHostError); |
| 43 | |
| 44 | /** ssh itself refused the connection before the host bootstrap ran. */ |
| 45 | export class SshConnectError extends Schema.TaggedError<SshConnectError>()("SshConnectError", { |
| 46 | reason: Schema.Literals(["host_key_unknown", "host_key_changed", "auth", "resolve", "connect"]), |
| 47 | target: Schema.String, |
| 48 | /** The ssh diagnostic line that identified the reason. */ |
| 49 | detail: Schema.String, |
| 50 | cause: Schema.Defect(), |
| 51 | }) { |
| 52 | override get message(): string { |
| 53 | const here = "the machine running this T3 Code server"; |
| 54 | const detail = this.detail.replace(/\.$/u, ""); |
| 55 | switch (this.reason) { |
| 56 | case "host_key_unknown": |
| 57 | return `SSH encountered an untrusted host key while connecting to ${this.target} from ${here}. Connect once with ssh from there to identify the host and verify its key before accepting it.`; |
| 58 | case "host_key_changed": |
| 59 | return `SSH encountered a changed host key while connecting to ${this.target} from ${here}. Connect once with ssh from there to identify the affected host. If the change is expected, remove that host's old key with \`ssh-keygen -R\`, then verify and accept its new key.`; |
| 60 | case "auth": |
| 61 | return `SSH login failed while connecting to ${this.target}: ${detail}. SSH from ${here} must log in without a prompt.`; |
| 62 | case "resolve": |
| 63 | return `Could not resolve the host name ${this.target} from ${here}: ${detail}.`; |
| 64 | case "connect": |
| 65 | return `Could not reach ${this.target} from ${here}: ${detail}. Check that the host is on, its SSH server is running, and the address and port are right.`; |
| 66 | } |
| 67 | } |
| 68 | } |
| 69 | export const isSshConnectError = Schema.is(SshConnectError); |
| 70 | |
| 71 | const Probe = Schema.Struct({ |
| 72 | nodePath: Schema.String, |
| 73 | tools: Schema.optional(DeviceToolVersions), |
| 74 | platforms: Schema.Array(DevicePlatformAvailability), |
| 75 | }); |
| 76 | const Started = Schema.Struct({ |
| 77 | ...Probe.fields, |
| 78 | hubPort: Schema.Int, |
| 79 | daemonPort: Schema.optionalKey(Schema.Int), |
| 80 | token: Schema.optionalKey(Schema.String), |
| 81 | entryPath: Schema.optionalKey(Schema.String), |
| 82 | helpers: Schema.Struct({ |
| 83 | serveSimAxSettings: Schema.NullOr(Schema.String), |
| 84 | serveSimCli: Schema.NullOr(Schema.String), |
| 85 | }), |
| 86 | }); |
| 87 | const decodeProbe = Schema.decodeUnknownEffect(Schema.fromJsonString(Probe)); |
| 88 | const decodeStarted = Schema.decodeUnknownEffect(Schema.fromJsonString(Started)); |
| 89 | const targetFor = (config: SshDeviceHostConfig) => ({ |
| 90 | alias: config.target, |
| 91 | hostname: config.target, |
| 92 | username: null, |
| 93 | port: config.port ?? null, |
| 94 | }); |
| 95 | const identityArgs = (config: SshDeviceHostConfig) => |
| 96 | config.identityFile ? ["-i", config.identityFile] : []; |
| 97 | const commandArgs = (script: string) => [ |
| 98 | "sh", |
| 99 | "-c", |
| 100 | quoteRemoteArg(remoteDeviceEnvironment + script), |
| 101 | ]; |
| 102 | const bootstrap = ( |
| 103 | config: SshDeviceHostConfig, |
| 104 | owner: string, |
| 105 | mode: "probe" | "start" | "agent-start" | "stop-agent" | "stop", |
| 106 | ) => |
| 107 | runSshCommand(targetFor(config), { |
| 108 | preHostArgs: identityArgs(config), |
| 109 | remoteCommandArgs: commandArgs( |
| 110 | 'command -v node >/dev/null 2>&1 || { echo "Node is missing from the non-interactive SSH PATH" >&2; exit 1; }; exec node', |
| 111 | ), |
| 112 | stdin: remoteDeviceScript(owner, mode), |
| 113 | timeoutMs: mode === "start" || mode === "agent-start" ? 1_300_000 : 45_000, |
| 114 | }).pipe( |
| 115 | Effect.mapError( |
| 116 | (cause) => new DeviceHost.DeviceHostError({ hostId: config.id, step: mode, cause }), |
| 117 | ), |
| 118 | ); |
| 119 | |
| 120 | const isSshCommandError = Schema.is(SshCommandError); |
| 121 | |
| 122 | // ssh exits 255 for its own failures; the remote bootstrap never does. |
| 123 | const SSH_CONNECT_FAILURES: ReadonlyArray<readonly [SshConnectError["reason"], RegExp]> = [ |
| 124 | ["host_key_changed", /REMOTE HOST IDENTIFICATION HAS CHANGED/u], |
| 125 | ["host_key_unknown", /^Host key verification failed\.$/u], |
| 126 | [ |
| 127 | "auth", |
| 128 | /^\S+: (?:Permission denied \(|No supported authentication methods)|: Too many authentication failures$/u, |
| 129 | ], |
| 130 | ["resolve", /^ssh: Could not resolve hostname /u], |
| 131 | ["connect", /^ssh: connect to host /u], |
| 132 | ]; |
| 133 | const explainSshConnectFailure = ( |
| 134 | config: SshDeviceHostConfig, |
| 135 | error: DeviceHost.DeviceHostError, |
| 136 | ): DeviceHost.DeviceHostError => { |
| 137 | const cause = error.cause; |
| 138 | if (!isSshCommandError(cause) || cause.exitCode !== 255) return error; |
| 139 | const lines = cause.stderr.split(/\r?\n/u).map((line) => line.trim()); |
| 140 | for (const [reason, pattern] of SSH_CONNECT_FAILURES) { |
| 141 | const detail = lines.find((line) => pattern.test(line)); |
| 142 | if (detail === undefined) continue; |
| 143 | return new DeviceHost.DeviceHostError({ |
| 144 | hostId: config.id, |
| 145 | step: error.step, |
| 146 | cause: new SshConnectError({ |
| 147 | reason, |
| 148 | target: config.target, |
| 149 | detail, |
| 150 | cause, |
| 151 | }), |
| 152 | }); |
| 153 | } |
| 154 | return error; |
| 155 | }; |
| 156 | |
| 157 | // cmd expands only %OS% and PowerShell only $env:OS; POSIX shells expand neither to Windows_NT. |
| 158 | const explainWindowsHost = Effect.fn("SshDeviceHost.explainWindowsHost")(function* ( |
| 159 | config: SshDeviceHostConfig, |
| 160 | error: DeviceHost.DeviceHostError, |
| 161 | ) { |
| 162 | const cause = error.cause; |
| 163 | // Windows ssh.exe preserves full remote exit statuses such as 9009. |
| 164 | if ( |
| 165 | !isSshCommandError(cause) || |
| 166 | cause.exitCode === null || |
| 167 | cause.exitCode < 1 || |
| 168 | (cause.exitCode > 254 && cause.exitCode !== 9009) |
| 169 | ) |
| 170 | return yield* error; |
| 171 | let windows = yield* runSshCommand(targetFor(config), { |
| 172 | preHostArgs: identityArgs(config), |
| 173 | remoteCommandArgs: ["echo %OS% $env:OS"], |
| 174 | timeoutMs: 15_000, |
| 175 | }).pipe( |
| 176 | Effect.map(({ stdout }) => /(?:^|\s)Windows_NT(?:\s|$)/u.test(stdout)), |
| 177 | Effect.orElseSucceed(() => false), |
| 178 | ); |
| 179 | if (!windows) { |
| 180 | // Git Bash does not expand Windows shell variables, but its Node still reports win32. |
| 181 | windows = yield* runSshCommand(targetFor(config), { |
| 182 | preHostArgs: identityArgs(config), |
| 183 | remoteCommandArgs: ["node", "-p", "process.platform"], |
| 184 | timeoutMs: 15_000, |
| 185 | }).pipe( |
| 186 | Effect.map(({ stdout }) => stdout.trim() === "win32"), |
| 187 | Effect.orElseSucceed(() => false), |
| 188 | ); |
| 189 | } |
| 190 | if (!windows) return yield* error; |
| 191 | return yield* new DeviceHost.DeviceHostError({ |
| 192 | hostId: config.id, |
| 193 | step: error.step, |
| 194 | cause: new SshWindowsHostError({ cause }), |
| 195 | }); |
| 196 | }); |
| 197 | |
| 198 | const ownerFor = Effect.fn("SshDeviceHost.ownerFor")(function* (hostId: string) { |
| 199 | const fs = yield* FileSystem.FileSystem; |
| 200 | const server = yield* ServerConfig.ServerConfig; |
| 201 | const environmentId = yield* fs |
| 202 | .readFileString(server.environmentIdPath) |
| 203 | .pipe(Effect.orElseSucceed(() => server.stateDir)); |
| 204 | const digest = yield* Digest.Digest; |
| 205 | const owner = yield* digest |
| 206 | .digest( |
| 207 | Digest.sha256, |
| 208 | new TextEncoder().encode(`${environmentId}\0${server.stateDir}\0${hostId}`), |
| 209 | ) |
| 210 | .pipe(Effect.map(Hex.encode), Effect.orDie); |
| 211 | return owner.slice(0, 24); |
| 212 | }); |
| 213 | |
| 214 | export const probe = Effect.fn("SshDeviceHost.probe")(function* ( |
| 215 | config: SshDeviceHostConfig, |
| 216 | owner?: string, |
| 217 | ) { |
| 218 | const result = yield* bootstrap(config, owner ?? (yield* ownerFor(config.id)), "probe").pipe( |
| 219 | Effect.catchTags({ |
| 220 | DeviceHostError: (error) => |
| 221 | explainWindowsHost(config, explainSshConnectFailure(config, error)), |
| 222 | }), |
| 223 | ); |
| 224 | const value = yield* decodeProbe(result.stdout.trim()).pipe( |
| 225 | Effect.mapError( |
| 226 | (cause) => |
| 227 | new DeviceHost.DeviceHostError({ hostId: config.id, step: "reading probe result", cause }), |
| 228 | ), |
| 229 | ); |
| 230 | return { |
| 231 | id: config.id, |
| 232 | label: config.label, |
| 233 | kind: "ssh", |
| 234 | tools: value.tools, |
| 235 | hubInstalled: |
| 236 | value.tools?.hub.installedVersions.includes(value.tools.hub.requiredVersion) ?? false, |
| 237 | agentDeviceInstalled: |
| 238 | value.tools?.agent.installedVersions.includes(value.tools.agent.requiredVersion) ?? false, |
| 239 | platforms: value.platforms, |
| 240 | } satisfies DeviceHostSummary; |
| 241 | }); |
| 242 | |
| 243 | export const make = Effect.fn("SshDeviceHost.make")(function* ( |
| 244 | config: SshDeviceHostConfig, |
| 245 | onReady: ( |
| 246 | ready: DeviceHost.DeviceHostAgentReady, |
| 247 | ) => Effect.Effect<void, DeviceHost.DeviceHostError> = () => Effect.void, |
| 248 | onStatus: ( |
| 249 | status: "starting" | "ready" | "failed", |
| 250 | detail?: string, |
| 251 | ) => Effect.Effect<void> = () => Effect.void, |
| 252 | ) { |
| 253 | const fs = yield* FileSystem.FileSystem; |
| 254 | const path = yield* Path.Path; |
| 255 | const server = yield* ServerConfig.ServerConfig; |
| 256 | const net = yield* NetService.NetService; |
| 257 | const http = yield* HttpClient.HttpClient; |
| 258 | const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; |
| 259 | const digest = yield* Digest.Digest; |
| 260 | const parentScope = yield* Scope.Scope; |
| 261 | const ssh = yield* resolveSshCommand; |
| 262 | const owner = yield* ownerFor(config.id); |
| 263 | const provide = <A, E>( |
| 264 | effect: Effect.Effect< |
| 265 | A, |
| 266 | E, |
| 267 | | FileSystem.FileSystem |
| 268 | | Path.Path |
| 269 | | ChildProcessSpawner.ChildProcessSpawner |
| 270 | | Digest.Digest |
| 271 | | ServerConfig.ServerConfig |
| 272 | >, |
| 273 | ) => |
| 274 | effect.pipe( |
| 275 | Effect.provideService(FileSystem.FileSystem, fs), |
| 276 | Effect.provideService(Path.Path, path), |
| 277 | Effect.provideService(ServerConfig.ServerConfig, server), |
| 278 | Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), |
| 279 | Effect.provideService(Digest.Digest, digest), |
| 280 | ); |
| 281 | const lock = yield* Semaphore.make(1); |
| 282 | let stopped = false; |
| 283 | let activated = false; |
| 284 | let wantsAgent = false; |
| 285 | let ready: |
| 286 | | (DeviceHost.DeviceHostReady & { |
| 287 | agentDevice?: DeviceHost.DeviceHostAgentReady["agentDevice"]; |
| 288 | }) |
| 289 | | null = null; |
| 290 | let connectionScope: Scope.Closeable | null = null; |
| 291 | let summary: DeviceHostSummary = { |
| 292 | id: config.id, |
| 293 | label: config.label, |
| 294 | kind: "ssh", |
| 295 | hubInstalled: false, |
| 296 | agentDeviceInstalled: false, |
| 297 | platforms: [], |
| 298 | }; |
| 299 | |
| 300 | const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => |
| 301 | provide( |
| 302 | runSshCommand(targetFor(config), { |
| 303 | preHostArgs: identityArgs(config), |
| 304 | // Match the hub cwd so relative executables and PATH entries resolve identically. |
| 305 | remoteCommandArgs: commandArgs( |
| 306 | `cd "$HOME/.t3/device/hosts/${owner}" && exec ${[command, ...args].map(quoteRemoteArg).join(" ")}`, |
| 307 | ), |
| 308 | keepErrorOutputTail: true, |
| 309 | ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), |
| 310 | ...(options?.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), |
| 311 | }), |
| 312 | ).pipe( |
| 313 | Effect.map((result) => ({ ...result, code: 0 })), |
| 314 | Effect.catch((error) => |
| 315 | Effect.succeed({ |
| 316 | stdout: "stdout" in error ? (error.stdout ?? "") : "", |
| 317 | stderr: error.message, |
| 318 | code: "exitCode" in error ? (error.exitCode ?? 127) : 127, |
| 319 | }), |
| 320 | ), |
| 321 | ); |
| 322 | |
| 323 | const platformAvailability: DeviceHost.DeviceHost["Service"]["platformAvailability"] = ( |
| 324 | platform, |
| 325 | ) => |
| 326 | provide(probe(config, owner)).pipe( |
| 327 | Effect.map((value) => { |
| 328 | summary = { ...summary, platforms: value.platforms }; |
| 329 | return value.platforms.find((p) => p.platform === platform)!; |
| 330 | }), |
| 331 | Effect.orElseSucceed(() => ({ |
| 332 | platform, |
| 333 | available: false, |
| 334 | reason: "Cannot reach device host. Test its SSH connection in Settings.", |
| 335 | })), |
| 336 | ); |
| 337 | |
| 338 | const connectOnce = Effect.fn("SshDeviceHost.connectOnce")(function* (): Effect.fn.Return< |
| 339 | DeviceHost.DeviceHostReady & { agentDevice?: DeviceHost.DeviceHostAgentReady["agentDevice"] }, |
| 340 | DeviceHost.DeviceHostError |
| 341 | > { |
| 342 | activated = true; |
| 343 | const result = yield* provide(bootstrap(config, owner, wantsAgent ? "agent-start" : "start")); |
| 344 | yield* onStatus("starting"); |
| 345 | const remote = yield* decodeStarted(result.stdout.trim()).pipe( |
| 346 | Effect.mapError( |
| 347 | (cause) => |
| 348 | new DeviceHost.DeviceHostError({ |
| 349 | hostId: config.id, |
| 350 | step: "reading host endpoints", |
| 351 | cause, |
| 352 | }), |
| 353 | ), |
| 354 | ); |
| 355 | summary = { |
| 356 | ...summary, |
| 357 | platforms: remote.platforms, |
| 358 | tools: remote.tools, |
| 359 | hubInstalled: true, |
| 360 | agentDeviceInstalled: wantsAgent || summary.agentDeviceInstalled, |
| 361 | }; |
| 362 | const hubPort = yield* net.reserveLoopbackPort("127.0.0.1").pipe( |
| 363 | Effect.mapError( |
| 364 | (cause) => |
| 365 | new DeviceHost.DeviceHostError({ |
| 366 | hostId: config.id, |
| 367 | step: "reserving hub port", |
| 368 | cause, |
| 369 | }), |
| 370 | ), |
| 371 | ); |
| 372 | const daemonPort = yield* net.reserveLoopbackPort("127.0.0.1").pipe( |
| 373 | Effect.mapError( |
| 374 | (cause) => |
| 375 | new DeviceHost.DeviceHostError({ |
| 376 | hostId: config.id, |
| 377 | step: "reserving daemon port", |
| 378 | cause, |
| 379 | }), |
| 380 | ), |
| 381 | ); |
| 382 | const scope = yield* Scope.make(); |
| 383 | connectionScope = scope; |
| 384 | const child = yield* spawner |
| 385 | .spawn( |
| 386 | ChildProcess.make( |
| 387 | ssh, |
| 388 | [ |
| 389 | ...baseSshArgs(targetFor(config), { batchMode: "yes" }), |
| 390 | ...identityArgs(config), |
| 391 | "-o", |
| 392 | "ExitOnForwardFailure=yes", |
| 393 | // Multiplexed clients exit after handing forwards to the master; own this connection. |
| 394 | "-o", |
| 395 | "ControlMaster=no", |
| 396 | "-o", |
| 397 | "ControlPath=none", |
| 398 | "-o", |
| 399 | "ControlPersist=no", |
| 400 | "-o", |
| 401 | "ServerAliveInterval=10", |
| 402 | "-o", |
| 403 | "ServerAliveCountMax=3", |
| 404 | "-T", |
| 405 | "-L", |
| 406 | `127.0.0.1:${hubPort}:127.0.0.1:${remote.hubPort}`, |
| 407 | ...(remote.daemonPort === undefined |
| 408 | ? [] |
| 409 | : ["-L", `127.0.0.1:${daemonPort}:127.0.0.1:${remote.daemonPort}`]), |
| 410 | config.target, |
| 411 | // The server holds the only write end of this stdin, so when it exits for any |
| 412 | // reason the remote `cat` sees EOF and ssh exits once its forwarded connections |
| 413 | // close, instead of being orphaned. |
| 414 | "sh", |
| 415 | "-c", |
| 416 | quoteRemoteArg("exec cat >/dev/null"), |
| 417 | ], |
| 418 | { stdin: "pipe", stdout: "ignore", stderr: "pipe" }, |
| 419 | ), |
| 420 | ) |
| 421 | .pipe( |
| 422 | Effect.provideService(Scope.Scope, scope), |
| 423 | Effect.mapError( |
| 424 | (cause) => |
| 425 | new DeviceHost.DeviceHostError({ hostId: config.id, step: "forwarding ports", cause }), |
| 426 | ), |
| 427 | ); |
| 428 | let stderr = ""; |
| 429 | yield* child.stderr.pipe( |
| 430 | Stream.decodeText(), |
| 431 | Stream.runForEach((chunk) => |
| 432 | Effect.sync(() => { |
| 433 | stderr = (stderr + chunk).slice(-2000); |
| 434 | }), |
| 435 | ), |
| 436 | Effect.forkIn(scope), |
| 437 | ); |
| 438 | const hubOrigin = `http://127.0.0.1:${hubPort}`; |
| 439 | const next = { |
| 440 | nodePath: remote.nodePath, |
| 441 | hub: { origin: hubOrigin }, |
| 442 | devices: makeDeviceHubOperations({ |
| 443 | hostId: config.id, |
| 444 | origin: hubOrigin, |
| 445 | http, |
| 446 | run, |
| 447 | // Keep probing negative reports so tooling installed after connecting is still found. |
| 448 | platformAvailable: (platform) => |
| 449 | Effect.suspend(() => |
| 450 | summary.platforms.some((p) => p.platform === platform && p.available) |
| 451 | ? Effect.succeed(true) |
| 452 | : platformAvailability(platform).pipe( |
| 453 | Effect.map((availability) => availability.available), |
| 454 | ), |
| 455 | ), |
| 456 | }), |
| 457 | ...(remote.daemonPort !== undefined && |
| 458 | remote.token !== undefined && |
| 459 | remote.entryPath !== undefined |
| 460 | ? { |
| 461 | agentDevice: { |
| 462 | baseUrl: `http://127.0.0.1:${daemonPort}`, |
| 463 | token: remote.token, |
| 464 | entryPath: remote.entryPath, |
| 465 | }, |
| 466 | } |
| 467 | : {}), |
| 468 | helpers: remote.helpers, |
| 469 | run, |
| 470 | }; |
| 471 | for (const [baseUrl, route] of [ |
| 472 | [next.hub.origin, "/readyz"], |
| 473 | ...(next.agentDevice ? [[next.agentDevice.baseUrl, "/health"]] : []), |
| 474 | ]) { |
| 475 | yield* waitForHttpReady({ |
| 476 | baseUrl: baseUrl!, |
| 477 | path: route!, |
| 478 | timeoutMs: 15000, |
| 479 | makeError: () => |
| 480 | new DeviceHost.DeviceHostError({ |
| 481 | hostId: config.id, |
| 482 | step: "waiting for SSH forward", |
| 483 | cause: new Error(stderr || "Forwarded endpoint did not answer."), |
| 484 | }), |
| 485 | }).pipe(Effect.provideService(HttpClient.HttpClient, http)); |
| 486 | } |
| 487 | if (next.agentDevice) yield* onReady({ ...next, agentDevice: next.agentDevice }); |
| 488 | ready = next.agentDevice |
| 489 | ? { ...next, releaseAgentSession: releaseAgentDeviceSession(http, next.agentDevice) } |
| 490 | : next; |
| 491 | yield* onStatus("ready"); |
| 492 | // Reconnect also repairs helpers that died while SSH itself stayed connected. |
| 493 | // A loaded host can miss a probe or two while still working, and reconnecting |
| 494 | // then drops the forward in use, so only consecutive misses count. |
| 495 | const unhealthy = Effect.gen(function* () { |
| 496 | let misses = 0; |
| 497 | while (true) { |
| 498 | yield* Effect.sleep("10 seconds"); |
| 499 | const alive = yield* http.get(`${next.hub.origin}/readyz`).pipe( |
| 500 | Effect.timeout("5 seconds"), |
| 501 | Effect.map((r) => r.status === 200), |
| 502 | Effect.orElseSucceed(() => false), |
| 503 | ); |
| 504 | const daemonAlive = next.agentDevice |
| 505 | ? yield* http.get(`${next.agentDevice!.baseUrl}/health`).pipe( |
| 506 | Effect.timeout("5 seconds"), |
| 507 | Effect.map((r) => r.status === 200), |
| 508 | Effect.orElseSucceed(() => false), |
| 509 | ) |
| 510 | : true; |
| 511 | misses = alive && daemonAlive ? 0 : misses + 1; |
| 512 | if (misses >= UNHEALTHY_PROBE_LIMIT) return; |
| 513 | } |
| 514 | }); |
| 515 | yield* Effect.gen(function* () { |
| 516 | yield* Effect.raceFirst(child.exitCode.pipe(Effect.ignore), unhealthy); |
| 517 | if (stopped || connectionScope !== scope) return; |
| 518 | ready = null; |
| 519 | yield* onStatus("starting", "Reconnecting to device host…"); |
| 520 | yield* Scope.close(scope, Exit.void); |
| 521 | let delay = 1000; |
| 522 | while (true) { |
| 523 | if (stopped || connectionScope !== scope) return; |
| 524 | yield* Effect.sleep(delay); |
| 525 | const result = yield* lock |
| 526 | .withPermit( |
| 527 | Effect.suspend(() => (stopped || ready ? Effect.void : connect().pipe(Effect.asVoid))), |
| 528 | ) |
| 529 | .pipe(Effect.result); |
| 530 | if (result._tag === "Success") return; |
| 531 | yield* onStatus("failed", result.failure.message); |
| 532 | if (connectionScope && connectionScope !== scope) |
| 533 | yield* Scope.close(connectionScope, Exit.void); |
| 534 | connectionScope = scope; |
| 535 | delay = Math.min(delay * 2, 30000); |
| 536 | } |
| 537 | }).pipe(Effect.forkIn(parentScope)); |
| 538 | return next; |
| 539 | }); |
| 540 | |
| 541 | const closeConnection = Effect.suspend(() => { |
| 542 | const failedScope = connectionScope; |
| 543 | connectionScope = null; |
| 544 | ready = null; |
| 545 | return failedScope ? Scope.close(failedScope, Exit.void) : Effect.void; |
| 546 | }); |
| 547 | |
| 548 | const connect = Effect.fn("SshDeviceHost.connect")(function* () { |
| 549 | for (let attempt = 0; ; attempt++) { |
| 550 | const result = yield* connectOnce().pipe( |
| 551 | Effect.onInterrupt(() => closeConnection), |
| 552 | Effect.result, |
| 553 | ); |
| 554 | if (result._tag === "Success") return result.success; |
| 555 | yield* closeConnection; |
| 556 | // SSH binds after the reservation is released, so a competing bind needs fresh ports. |
| 557 | if ( |
| 558 | attempt >= 2 || |
| 559 | !["forwarding ports", "waiting for SSH forward"].includes(result.failure.step) |
| 560 | ) |
| 561 | return yield* result.failure; |
| 562 | } |
| 563 | }); |
| 564 | |
| 565 | const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => |
| 566 | lock.withPermit( |
| 567 | Effect.gen(function* () { |
| 568 | stopped = false; |
| 569 | if (ready) return ready; |
| 570 | summary = yield* provide(probe(config, owner)); |
| 571 | yield* onPhase( |
| 572 | summary.hubInstalled ? "starting" : "installing", |
| 573 | summary.hubInstalled |
| 574 | ? undefined |
| 575 | : deviceToolInstallMessage("device hub", summary.tools?.hub), |
| 576 | ); |
| 577 | return yield* connect().pipe( |
| 578 | Effect.tapError(() => |
| 579 | connectionScope ? Scope.close(connectionScope, Exit.void) : Effect.void, |
| 580 | ), |
| 581 | ); |
| 582 | }), |
| 583 | ); |
| 584 | const stop = lock.withPermit( |
| 585 | Effect.gen(function* () { |
| 586 | stopped = true; |
| 587 | ready = null; |
| 588 | if (connectionScope) yield* Scope.close(connectionScope, Exit.void); |
| 589 | connectionScope = null; |
| 590 | if (activated) yield* provide(bootstrap(config, owner, "stop")).pipe(Effect.ignore); |
| 591 | activated = false; |
| 592 | wantsAgent = false; |
| 593 | }), |
| 594 | ); |
| 595 | const changeAgent = (enabled: boolean) => |
| 596 | lock.withPermit( |
| 597 | Effect.gen(function* () { |
| 598 | wantsAgent = enabled; |
| 599 | if (enabled && ready?.agentDevice) return { ...ready, agentDevice: ready.agentDevice }; |
| 600 | if (!enabled && !ready?.agentDevice) return null; |
| 601 | ready = null; |
| 602 | const previousScope = connectionScope; |
| 603 | connectionScope = null; |
| 604 | if (previousScope) yield* Scope.close(previousScope, Exit.void); |
| 605 | if (!enabled) yield* provide(bootstrap(config, owner, "stop-agent")); |
| 606 | return yield* connect().pipe( |
| 607 | Effect.onError(() => |
| 608 | Effect.gen(function* () { |
| 609 | const failedScope = connectionScope; |
| 610 | connectionScope = null; |
| 611 | if (failedScope) yield* Scope.close(failedScope, Exit.void); |
| 612 | if (enabled) |
| 613 | yield* provide(bootstrap(config, owner, "stop-agent")).pipe(Effect.ignore); |
| 614 | }), |
| 615 | ), |
| 616 | ); |
| 617 | }), |
| 618 | ); |
| 619 | yield* Effect.addFinalizer(() => stop); |
| 620 | return { |
| 621 | id: config.id, |
| 622 | label: config.label, |
| 623 | summary: Effect.sync(() => summary), |
| 624 | inspect: provide(probe(config, owner)).pipe( |
| 625 | Effect.tap((value) => |
| 626 | Effect.sync(() => { |
| 627 | summary = value; |
| 628 | }), |
| 629 | ), |
| 630 | ), |
| 631 | current: Effect.sync(() => ready), |
| 632 | ensureReady, |
| 633 | ensureAgentReady: (onPhase) => |
| 634 | ensureReady(onPhase).pipe( |
| 635 | Effect.flatMap(() => |
| 636 | onPhase( |
| 637 | summary.agentDeviceInstalled ? "starting" : "installing", |
| 638 | summary.agentDeviceInstalled |
| 639 | ? undefined |
| 640 | : deviceToolInstallMessage("agent tools", summary.tools?.agent), |
| 641 | ), |
| 642 | ), |
| 643 | Effect.flatMap(() => changeAgent(true)), |
| 644 | Effect.flatMap((value) => |
| 645 | value?.agentDevice |
| 646 | ? Effect.succeed({ ...value, agentDevice: value.agentDevice }) |
| 647 | : Effect.fail( |
| 648 | new DeviceHost.DeviceHostError({ |
| 649 | hostId: config.id, |
| 650 | step: "starting agent tools", |
| 651 | cause: new Error("Daemon endpoint missing"), |
| 652 | }), |
| 653 | ), |
| 654 | ), |
| 655 | ), |
| 656 | stopAgent: changeAgent(false).pipe(Effect.asVoid, Effect.ignore), |
| 657 | stop, |
| 658 | platformAvailability, |
| 659 | } satisfies DeviceHost.DeviceHost["Service"]; |
| 660 | }); |