The best way to control Claude Code, Codex and more from any computer or phone
| 1 | const assert = require("node:assert/strict"); |
| 2 | const test = require("node:test"); |
| 3 | |
| 4 | const { |
| 5 | renderComment, |
| 6 | resolve, |
| 7 | upsertCommentForCurrentHead, |
| 8 | validateResult, |
| 9 | validateScenarios, |
| 10 | } = require("./thread-transfer-report.cjs"); |
| 11 | |
| 12 | function result(overrides = {}) { |
| 13 | const observed = { |
| 14 | totalWireBytes: 2_200_000, |
| 15 | threadSnapshotWireBytes: 1_950_000, |
| 16 | threadSnapshotDecodedBytes: 9_100_000, |
| 17 | measuredTurnWebSocketWireBytes: 250_000, |
| 18 | measuredTurnWebSocketDecodedBytes: 1_150_000, |
| 19 | measuredTurnWebSocketMessages: 15, |
| 20 | }; |
| 21 | const ceiling = { |
| 22 | totalWireBytes: 2_900_000, |
| 23 | threadSnapshotWireBytes: 2_600_000, |
| 24 | measuredTurnWebSocketWireBytes: 320_000, |
| 25 | measuredTurnWebSocketDecodedBytes: 1_550_000, |
| 26 | measuredTurnWebSocketMessages: 20, |
| 27 | }; |
| 28 | return { |
| 29 | schemaVersion: 1, |
| 30 | scenario: { |
| 31 | id: "thread-transfer-v1", |
| 32 | historyTurns: 10, |
| 33 | historyCommandToolsPerTurn: 5, |
| 34 | historyMcpResultBytes: 900_000, |
| 35 | measuredCommandTools: 20, |
| 36 | measuredMcpResultBytes: 1_100_000, |
| 37 | }, |
| 38 | providers: { |
| 39 | codex: { observed: { ...observed, ...overrides }, ceiling }, |
| 40 | claudeAgent: { observed, ceiling }, |
| 41 | }, |
| 42 | }; |
| 43 | } |
| 44 | |
| 45 | function scenarios(overrides = {}) { |
| 46 | const rows = [ |
| 47 | { |
| 48 | phase: "long turn finished, cold open", |
| 49 | metric: "HTTP decoded", |
| 50 | unit: "bytes", |
| 51 | observed: 390_000, |
| 52 | ceiling: 420_000, |
| 53 | }, |
| 54 | { |
| 55 | phase: "idle thread resume", |
| 56 | metric: "WebSocket messages", |
| 57 | unit: "count", |
| 58 | observed: 1, |
| 59 | ceiling: 1, |
| 60 | }, |
| 61 | ]; |
| 62 | return { |
| 63 | schemaVersion: 1, |
| 64 | providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows }, |
| 65 | }; |
| 66 | } |
| 67 | |
| 68 | test("validates the fixed artifact schema", () => { |
| 69 | assert.equal(validateResult(result()).schemaVersion, 1); |
| 70 | assert.throws( |
| 71 | () => validateResult({ ...result(), injectedMarkdown: "@everyone" }), |
| 72 | /unexpected fields/, |
| 73 | ); |
| 74 | assert.throws( |
| 75 | () => validateResult(result({ totalWireBytes: "lots" })), |
| 76 | /non-negative safe integer/, |
| 77 | ); |
| 78 | |
| 79 | assert.equal(validateScenarios(scenarios()).schemaVersion, 1); |
| 80 | for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "<img>", ""]) { |
| 81 | assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/); |
| 82 | } |
| 83 | assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/); |
| 84 | assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/); |
| 85 | assert.throws( |
| 86 | () => |
| 87 | validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })), |
| 88 | /repeats an earlier row/, |
| 89 | ); |
| 90 | }); |
| 91 | |
| 92 | test("lists scenario rows that changed or failed above the collapsed table", () => { |
| 93 | const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" }; |
| 94 | const render = (currentScenarios, baselineScenarios) => |
| 95 | renderComment({ |
| 96 | current: result(), |
| 97 | baseline: result(), |
| 98 | currentScenarios, |
| 99 | baselineScenarios, |
| 100 | currentRun: run, |
| 101 | baselineRun: { ...run, matchesBase: true }, |
| 102 | }); |
| 103 | const open = (body) => body.slice(0, body.indexOf("<details>")); |
| 104 | |
| 105 | const unchanged = render(scenarios(), scenarios()); |
| 106 | assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/); |
| 107 | assert.match(open(unchanged), /All 4 scenario measurements match `main`/); |
| 108 | assert.doesNotMatch(open(unchanged), /long turn finished, cold open/); |
| 109 | |
| 110 | const grown = render(scenarios({ observed: 400_000 }), scenarios()); |
| 111 | assert.match(open(grown), /1 of 4 scenario measurements changed or failed/); |
| 112 | assert.match( |
| 113 | open(grown), |
| 114 | /\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/, |
| 115 | ); |
| 116 | |
| 117 | const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios()); |
| 118 | assert.match(over, /❌ One or more thread transfer ceilings were exceeded/); |
| 119 | assert.match( |
| 120 | over, |
| 121 | /Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/, |
| 122 | ); |
| 123 | |
| 124 | // A row `main` does not have is not reported as matching it. |
| 125 | const added = render(scenarios({ phase: "GH-26" }), scenarios()); |
| 126 | assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./); |
| 127 | assert.match(added, /\| Codex \| `GH-26` \|/); |
| 128 | |
| 129 | // A main run from before the scenario file has nothing to compare; a PR run from before it has no section. |
| 130 | assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/); |
| 131 | assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/); |
| 132 | }); |
| 133 | |
| 134 | test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => { |
| 135 | const outputs = {}; |
| 136 | const listWorkflowRunArtifacts = () => {}; |
| 137 | // The exact base run sits far past the newest page of main CI runs. |
| 138 | const mainRuns = Array.from({ length: 1_000 }, (_, index) => ({ |
| 139 | id: 1_000 - index, |
| 140 | head_sha: index === 900 ? "base-sha" : `main-sha-${index}`, |
| 141 | })); |
| 142 | const listWorkflowRuns = async ({ head_sha, per_page }) => ({ |
| 143 | data: { |
| 144 | workflow_runs: mainRuns |
| 145 | .filter((run) => head_sha === undefined || run.head_sha === head_sha) |
| 146 | .slice(0, per_page), |
| 147 | }, |
| 148 | }); |
| 149 | const listPullRequestsAssociatedWithCommit = () => {}; |
| 150 | const github = { |
| 151 | paginate: async (method, input) => { |
| 152 | if (method === listPullRequestsAssociatedWithCommit) { |
| 153 | return [ |
| 154 | { |
| 155 | number: 5350, |
| 156 | state: "open", |
| 157 | head: { sha: "head-sha", ref: "feature-branch", repo: null }, |
| 158 | }, |
| 159 | ]; |
| 160 | } |
| 161 | if (method === listWorkflowRunArtifacts) { |
| 162 | return [ |
| 163 | { |
| 164 | name: "thread-transfer-results", |
| 165 | expired: false, |
| 166 | runId: input.run_id, |
| 167 | }, |
| 168 | ]; |
| 169 | } |
| 170 | throw new Error("unexpected pagination call"); |
| 171 | }, |
| 172 | rest: { |
| 173 | actions: { listWorkflowRunArtifacts, listWorkflowRuns }, |
| 174 | pulls: { |
| 175 | get: async () => ({ |
| 176 | data: { |
| 177 | head: { sha: "head-sha" }, |
| 178 | base: { sha: "base-sha", ref: "main" }, |
| 179 | }, |
| 180 | }), |
| 181 | }, |
| 182 | repos: { listPullRequestsAssociatedWithCommit }, |
| 183 | }, |
| 184 | }; |
| 185 | await resolve({ |
| 186 | github, |
| 187 | context: { |
| 188 | repo: { owner: "pingdotgg", repo: "t3code" }, |
| 189 | payload: { |
| 190 | workflow_run: { |
| 191 | id: 2, |
| 192 | event: "pull_request", |
| 193 | workflow_id: 3, |
| 194 | head_sha: "head-sha", |
| 195 | head_branch: "feature-branch", |
| 196 | head_repository: { full_name: "pingdotgg/t3code" }, |
| 197 | conclusion: "success", |
| 198 | pull_requests: [], |
| 199 | }, |
| 200 | }, |
| 201 | }, |
| 202 | core: { |
| 203 | info: () => {}, |
| 204 | setOutput: (key, value) => { |
| 205 | outputs[key] = value; |
| 206 | }, |
| 207 | }, |
| 208 | }); |
| 209 | |
| 210 | assert.equal(outputs.publish, "true"); |
| 211 | assert.equal(outputs.pull_number, "5350"); |
| 212 | assert.equal(outputs.pr_artifact, "true"); |
| 213 | assert.equal(outputs.baseline_run_id, "100"); |
| 214 | assert.equal(outputs.baseline_matches_base, "true"); |
| 215 | }); |
| 216 | |
| 217 | test("does not guess when a fallback commit belongs to multiple PRs", async () => { |
| 218 | const outputs = {}; |
| 219 | const listPullRequestsAssociatedWithCommit = () => {}; |
| 220 | let fetchedPull = false; |
| 221 | await resolve({ |
| 222 | github: { |
| 223 | paginate: async (method) => { |
| 224 | assert.equal(method, listPullRequestsAssociatedWithCommit); |
| 225 | return [5350, 5351].map((number) => ({ |
| 226 | number, |
| 227 | state: "open", |
| 228 | head: { |
| 229 | sha: "head-sha", |
| 230 | ref: "feature-branch", |
| 231 | repo: { full_name: "pingdotgg/t3code" }, |
| 232 | }, |
| 233 | })); |
| 234 | }, |
| 235 | rest: { |
| 236 | actions: {}, |
| 237 | pulls: { |
| 238 | get: async () => { |
| 239 | fetchedPull = true; |
| 240 | }, |
| 241 | }, |
| 242 | repos: { listPullRequestsAssociatedWithCommit }, |
| 243 | }, |
| 244 | }, |
| 245 | context: { |
| 246 | repo: { owner: "pingdotgg", repo: "t3code" }, |
| 247 | payload: { |
| 248 | workflow_run: { |
| 249 | id: 2, |
| 250 | event: "pull_request", |
| 251 | workflow_id: 3, |
| 252 | head_sha: "head-sha", |
| 253 | head_branch: "feature-branch", |
| 254 | head_repository: { full_name: "pingdotgg/t3code" }, |
| 255 | conclusion: "success", |
| 256 | pull_requests: [], |
| 257 | }, |
| 258 | }, |
| 259 | }, |
| 260 | core: { |
| 261 | info: () => {}, |
| 262 | setOutput: (key, value) => { |
| 263 | outputs[key] = value; |
| 264 | }, |
| 265 | }, |
| 266 | }); |
| 267 | |
| 268 | assert.equal(outputs.publish, "false"); |
| 269 | assert.equal(fetchedPull, false); |
| 270 | }); |
| 271 | |
| 272 | test("skips a missing PR commit but preserves other lookup failures", async () => { |
| 273 | for (const [status, message, shouldSkip] of [ |
| 274 | [422, "No commit found for SHA: old-head", true], |
| 275 | [422, "Validation Failed", false], |
| 276 | [403, "API rate limit exceeded", false], |
| 277 | ]) { |
| 278 | const outputs = {}; |
| 279 | const error = Object.assign(new Error(message), { |
| 280 | status, |
| 281 | response: { data: { message } }, |
| 282 | }); |
| 283 | const run = resolve({ |
| 284 | github: { |
| 285 | paginate: async () => { |
| 286 | throw error; |
| 287 | }, |
| 288 | rest: { repos: { listPullRequestsAssociatedWithCommit: () => {} } }, |
| 289 | }, |
| 290 | context: { |
| 291 | repo: { owner: "pingdotgg", repo: "t3code" }, |
| 292 | payload: { |
| 293 | workflow_run: { event: "pull_request", head_sha: "old-head", pull_requests: [] }, |
| 294 | }, |
| 295 | }, |
| 296 | core: { |
| 297 | info: () => {}, |
| 298 | setOutput: (key, value) => { |
| 299 | outputs[key] = value; |
| 300 | }, |
| 301 | }, |
| 302 | }); |
| 303 | if (shouldSkip) { |
| 304 | await run; |
| 305 | assert.equal(outputs.publish, "false"); |
| 306 | } else { |
| 307 | await assert.rejects(run, (caught) => caught === error); |
| 308 | assert.deepEqual(outputs, {}); |
| 309 | } |
| 310 | } |
| 311 | }); |
| 312 | |
| 313 | test("does not report a passing run that skipped the server tests", async () => { |
| 314 | const outputs = {}; |
| 315 | const listWorkflowRunArtifacts = () => {}; |
| 316 | await resolve({ |
| 317 | github: { |
| 318 | paginate: async (method) => { |
| 319 | assert.equal(method, listWorkflowRunArtifacts); |
| 320 | return []; |
| 321 | }, |
| 322 | rest: { |
| 323 | actions: { listWorkflowRunArtifacts }, |
| 324 | pulls: { |
| 325 | get: async () => ({ |
| 326 | data: { head: { sha: "head-sha" }, base: { sha: "base-sha", ref: "main" } }, |
| 327 | }), |
| 328 | }, |
| 329 | }, |
| 330 | }, |
| 331 | context: { |
| 332 | repo: { owner: "pingdotgg", repo: "t3code" }, |
| 333 | payload: { |
| 334 | workflow_run: { |
| 335 | id: 2, |
| 336 | event: "pull_request", |
| 337 | workflow_id: 3, |
| 338 | head_sha: "head-sha", |
| 339 | conclusion: "success", |
| 340 | pull_requests: [{ number: 5350 }], |
| 341 | }, |
| 342 | }, |
| 343 | }, |
| 344 | core: { |
| 345 | info: () => {}, |
| 346 | setOutput: (key, value) => { |
| 347 | outputs[key] = value; |
| 348 | }, |
| 349 | }, |
| 350 | }); |
| 351 | |
| 352 | assert.equal(outputs.publish, "false"); |
| 353 | }); |
| 354 | |
| 355 | test("does not publish a stale result after the PR head advances", async () => { |
| 356 | let listedComments = false; |
| 357 | const info = []; |
| 358 | const published = await upsertCommentForCurrentHead( |
| 359 | { |
| 360 | paginate: async () => { |
| 361 | listedComments = true; |
| 362 | return []; |
| 363 | }, |
| 364 | rest: { |
| 365 | issues: { |
| 366 | listComments: () => {}, |
| 367 | createComment: () => { |
| 368 | throw new Error("must not create a stale comment"); |
| 369 | }, |
| 370 | updateComment: () => { |
| 371 | throw new Error("must not update a stale comment"); |
| 372 | }, |
| 373 | }, |
| 374 | pulls: { |
| 375 | get: async () => ({ data: { head: { sha: "new-head-sha" } } }), |
| 376 | }, |
| 377 | }, |
| 378 | }, |
| 379 | { repo: { owner: "pingdotgg", repo: "t3code" } }, |
| 380 | { info: (message) => info.push(message) }, |
| 381 | 5350, |
| 382 | "old-head-sha", |
| 383 | "stale body", |
| 384 | ); |
| 385 | |
| 386 | assert.equal(published, false); |
| 387 | assert.equal(listedComments, false); |
| 388 | assert.deepEqual(info, ["Skipping stale CI result old-head-sha; PR head is new-head-sha."]); |
| 389 | }); |
| 390 | |
| 391 | test("preserves a successful result when a same-SHA rerun has no artifact", async () => { |
| 392 | let updatedComment = false; |
| 393 | const sha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; |
| 394 | const published = await upsertCommentForCurrentHead( |
| 395 | { |
| 396 | paginate: async () => [ |
| 397 | { |
| 398 | id: 1, |
| 399 | user: { login: "github-actions[bot]" }, |
| 400 | body: `<!-- t3-thread-transfer-report -->\n<!-- t3-thread-transfer-result-sha:${sha} -->`, |
| 401 | }, |
| 402 | ], |
| 403 | rest: { |
| 404 | issues: { |
| 405 | listComments: () => {}, |
| 406 | createComment: () => { |
| 407 | updatedComment = true; |
| 408 | }, |
| 409 | updateComment: () => { |
| 410 | updatedComment = true; |
| 411 | }, |
| 412 | }, |
| 413 | pulls: { |
| 414 | get: async () => ({ data: { head: { sha } } }), |
| 415 | }, |
| 416 | }, |
| 417 | }, |
| 418 | { repo: { owner: "pingdotgg", repo: "t3code" } }, |
| 419 | { info: () => {} }, |
| 420 | 5350, |
| 421 | sha, |
| 422 | "missing artifact warning", |
| 423 | { preserveResultSha: sha }, |
| 424 | ); |
| 425 | |
| 426 | assert.equal(published, true); |
| 427 | assert.equal(updatedComment, false); |
| 428 | }); |