The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { MessageId, ProviderDriverKind, ThreadId } from "@t3tools/contracts"; |
| 2 | import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 3 | import * as DateTime from "effect/DateTime"; |
| 4 | import { assert, describe, expect, it, vi } from "vite-plus/test"; |
| 5 | import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts"; |
| 6 | |
| 7 | describe("V2 assistant streaming", () => { |
| 8 | it("keeps code fences intact", () => { |
| 9 | expect(splitBufferedAssistantText("Intro\n\n```ts\nx()\n\n")).toEqual({ |
| 10 | ready: "Intro\n\n", |
| 11 | rest: "```ts\nx()\n\n", |
| 12 | }); |
| 13 | expect(splitBufferedAssistantText("```ts\nx()\n```\nrest")).toEqual({ |
| 14 | ready: "```ts\nx()\n```\n", |
| 15 | rest: "rest", |
| 16 | }); |
| 17 | expect(splitBufferedAssistantText("```a``` is inline.\n\n```\ncode\n\nmore\n")).toEqual({ |
| 18 | ready: "```a``` is inline.\n\n", |
| 19 | rest: "```\ncode\n\nmore\n", |
| 20 | }); |
| 21 | }); |
| 22 | }); |
| 23 | |
| 24 | it("holds a streamed section heading until its content has a boundary", () => { |
| 25 | expect(splitBufferedAssistantText("Intro\n\n## Results\n\n")).toEqual({ |
| 26 | ready: "Intro\n\n", |
| 27 | rest: "## Results\n\n", |
| 28 | }); |
| 29 | expect(splitBufferedAssistantText("**Results**\n\nBody\n\nNext")).toEqual({ |
| 30 | ready: "**Results**\n\nBody\n\n", |
| 31 | rest: "Next", |
| 32 | }); |
| 33 | }); |
| 34 | |
| 35 | type Step = |
| 36 | // A snapshot at `at` ms, and the text it delivers at once (null when held back). |
| 37 | | { readonly at: number; readonly text: string; readonly shows: string | null } |
| 38 | | { readonly at: number; readonly final: string } |
| 39 | | { readonly id: string; readonly at: number; readonly text: string; readonly shows: string } |
| 40 | // Held snapshots due by `idle` ms, delivered whole. |
| 41 | | { readonly idle: number; readonly shows: ReadonlyArray<string> } |
| 42 | | { readonly dueAt: number | null }; |
| 43 | |
| 44 | const createdAt = DateTime.makeUnsafe(0); |
| 45 | const snapshot = (id: string, text: string, streaming: boolean) => |
| 46 | ({ |
| 47 | type: "message.updated", |
| 48 | driver: ProviderDriverKind.make("codex"), |
| 49 | message: { |
| 50 | createdBy: "agent", |
| 51 | creationSource: "provider", |
| 52 | id: MessageId.make(id), |
| 53 | threadId: ThreadId.make("thread"), |
| 54 | runId: null, |
| 55 | nodeId: null, |
| 56 | role: "assistant", |
| 57 | text, |
| 58 | attachments: [], |
| 59 | streaming, |
| 60 | createdAt, |
| 61 | updatedAt: createdAt, |
| 62 | }, |
| 63 | }) satisfies ProviderAdapter.ProviderAdapterV2Event; |
| 64 | const shown = (event: ProviderAdapter.ProviderAdapterV2Event | null) => |
| 65 | event?.type === "message.updated" ? event.message.text : null; |
| 66 | |
| 67 | it.each<{ readonly name: string; readonly mode?: "turn"; readonly steps: ReadonlyArray<Step> }>([ |
| 68 | { |
| 69 | name: "an unfinished code fence", |
| 70 | steps: [ |
| 71 | { at: 0, text: "A\n\n```ts\nx()", shows: "A\n\n" }, |
| 72 | { dueAt: 800 }, |
| 73 | { idle: 799, shows: [] }, |
| 74 | { idle: 800, shows: ["A\n\n```ts\nx()"] }, |
| 75 | { dueAt: null }, |
| 76 | { at: 1_300, text: "A\n\n```ts\nx()\n```\n\nB", shows: "A\n\n```ts\nx()\n```\n\n" }, |
| 77 | ], |
| 78 | }, |
| 79 | { |
| 80 | name: "a list's partial last item", |
| 81 | steps: [ |
| 82 | { at: 0, text: "Steps:\n\n- one\n- tw", shows: "Steps:\n\n- one\n" }, |
| 83 | { idle: 800, shows: ["Steps:\n\n- one\n- tw"] }, |
| 84 | { at: 1_300, text: "Steps:\n\n- one\n- two\n- th", shows: "Steps:\n\n- one\n- two\n" }, |
| 85 | ], |
| 86 | }, |
| 87 | { |
| 88 | name: "an incomplete table row", |
| 89 | steps: [ |
| 90 | { at: 0, text: "Intro.\n\n| a | b |\n|---|---|\n| 1 |", shows: "Intro.\n\n" }, |
| 91 | { idle: 800, shows: ["Intro.\n\n| a | b |\n|---|---|\n| 1 |"] }, |
| 92 | { |
| 93 | at: 1_300, |
| 94 | text: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter", |
| 95 | shows: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n", |
| 96 | }, |
| 97 | ], |
| 98 | }, |
| 99 | { |
| 100 | name: "more of the same paragraph after an idle delivery", |
| 101 | steps: [ |
| 102 | { at: 0, text: "A\n\nB", shows: "A\n\n" }, |
| 103 | { idle: 800, shows: ["A\n\nB"] }, |
| 104 | { at: 900, text: "A\n\nB more", shows: null }, |
| 105 | { idle: 1_699, shows: [] }, |
| 106 | { idle: 1_700, shows: ["A\n\nB more"] }, |
| 107 | { idle: 5_000, shows: [] }, |
| 108 | // Its only boundary is behind what the idle delivery showed. |
| 109 | { at: 5_100, text: "A\n\nB more and", shows: null }, |
| 110 | { at: 5_200, text: "A\n\nB more and.\n\nC", shows: "A\n\nB more and.\n\n" }, |
| 111 | { at: 5_300, final: "A\n\nB more and.\n\nC." }, |
| 112 | { idle: 10_000, shows: [] }, |
| 113 | ], |
| 114 | }, |
| 115 | { |
| 116 | name: "completion at the idle deadline", |
| 117 | steps: [ |
| 118 | { at: 0, text: "A\n\nB", shows: "A\n\n" }, |
| 119 | { at: 800, final: "A\n\nB." }, |
| 120 | { idle: 800, shows: [] }, |
| 121 | ], |
| 122 | }, |
| 123 | { |
| 124 | name: "completion right after an idle delivery at the same instant", |
| 125 | steps: [ |
| 126 | { at: 0, text: "A\n\nB", shows: "A\n\n" }, |
| 127 | { idle: 800, shows: ["A\n\nB"] }, |
| 128 | { at: 800, final: "A\n\nB." }, |
| 129 | { idle: 5_000, shows: [] }, |
| 130 | ], |
| 131 | }, |
| 132 | { |
| 133 | name: "several fast paragraphs inside the throttle", |
| 134 | steps: [ |
| 135 | { at: 0, text: "A\n\nB", shows: "A\n\n" }, |
| 136 | { at: 100, text: "A\n\nB\n\nC", shows: null }, |
| 137 | { at: 200, text: "A\n\nB\n\nC\n\nD", shows: null }, |
| 138 | { at: 300, text: "A\n\nB\n\nC\n\nD\n\nE", shows: null }, |
| 139 | { idle: 1_099, shows: [] }, |
| 140 | { idle: 1_100, shows: ["A\n\nB\n\nC\n\nD\n\nE"] }, |
| 141 | ], |
| 142 | }, |
| 143 | { |
| 144 | name: "two messages due at different times", |
| 145 | steps: [ |
| 146 | { id: "a", at: 0, text: "A\n\nx", shows: "A\n\n" }, |
| 147 | { id: "b", at: 300, text: "B\n\ny", shows: "B\n\n" }, |
| 148 | { dueAt: 800 }, |
| 149 | { idle: 800, shows: ["A\n\nx"] }, |
| 150 | { dueAt: 1_100 }, |
| 151 | { idle: 1_100, shows: ["B\n\ny"] }, |
| 152 | ], |
| 153 | }, |
| 154 | { |
| 155 | name: "empty text", |
| 156 | steps: [{ at: 0, text: "", shows: null }, { dueAt: null }], |
| 157 | }, |
| 158 | { |
| 159 | name: "whitespace-only text", |
| 160 | steps: [ |
| 161 | { at: 0, text: " \n", shows: null }, |
| 162 | { idle: 800, shows: [" \n"] }, |
| 163 | { idle: 5_000, shows: [] }, |
| 164 | ], |
| 165 | }, |
| 166 | { |
| 167 | // Rewrites replace delivered text: shorter at once, same length after the throttle. |
| 168 | name: "provider rewrites and truncation", |
| 169 | steps: [ |
| 170 | { at: 0, text: "Original long paragraph.\n\n", shows: "Original long paragraph.\n\n" }, |
| 171 | { at: 1_000, text: "New.\n\n", shows: "New.\n\n" }, |
| 172 | { at: 1_100, text: "Old.\n\n", shows: null }, |
| 173 | { idle: 1_900, shows: ["Old.\n\n"] }, |
| 174 | { at: 2_500, text: "Old", shows: null }, |
| 175 | { idle: 3_300, shows: ["Old"] }, |
| 176 | { at: 4_000, text: "", shows: null }, |
| 177 | { idle: 4_800, shows: [""] }, |
| 178 | ], |
| 179 | }, |
| 180 | { |
| 181 | name: "a rewrite that drops an open code fence", |
| 182 | steps: [ |
| 183 | { at: 0, text: "Intro\n\n```ts\nx()\n", shows: "Intro\n\n" }, |
| 184 | { at: 400, text: "Retry\n\n- a\n- b", shows: "Retry\n\n- a\n" }, |
| 185 | ], |
| 186 | }, |
| 187 | { |
| 188 | name: "turn mode", |
| 189 | mode: "turn", |
| 190 | steps: [{ at: 0, text: "A\n\nB", shows: null }, { dueAt: null }], |
| 191 | }, |
| 192 | ])("paragraph streaming holds and idle-delivers $name", ({ mode, steps }) => { |
| 193 | const filter = makeAssistantStreamingFilter(mode ?? "paragraph", 800); |
| 194 | for (const step of steps) { |
| 195 | if ("dueAt" in step) assert.equal(filter.nextIdleFlushAt(), step.dueAt); |
| 196 | else if ("idle" in step) |
| 197 | assert.deepEqual(filter.takeIdleFlushes(step.idle).map(shown), [...step.shows]); |
| 198 | else if ("final" in step) |
| 199 | assert.equal(shown(filter.filter(snapshot("m", step.final, false), step.at)), step.final); |
| 200 | else |
| 201 | assert.equal( |
| 202 | shown(filter.filter(snapshot("id" in step ? step.id : "m", step.text, true), step.at)), |
| 203 | step.shows, |
| 204 | ); |
| 205 | } |
| 206 | }); |
| 207 | |
| 208 | it("scans each streamed line once while a long code fence is still open", () => { |
| 209 | const filter = makeAssistantStreamingFilter("paragraph", 800); |
| 210 | const lineCount = 400; |
| 211 | let text = "Intro\n\n```ts\n"; |
| 212 | const indexOf = vi.spyOn(String.prototype, "indexOf"); |
| 213 | try { |
| 214 | for (let line = 0; line < lineCount; line++) { |
| 215 | text += `const value${line} = ${line};\n`; |
| 216 | filter.filter(snapshot("m", text, true), line * 400); |
| 217 | } |
| 218 | const newlineSearches = indexOf.mock.calls.filter(([search]) => search === "\n").length; |
| 219 | expect(newlineSearches).toBeLessThan(lineCount * 4); |
| 220 | } finally { |
| 221 | indexOf.mockRestore(); |
| 222 | } |
| 223 | text += "```\n"; |
| 224 | assert.equal(shown(filter.filter(snapshot("m", text, true), lineCount * 400)), text); |
| 225 | }); |