@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 346 lines 13.7 KB
Raw
1 import { OpenAiClient } from "@effect/ai-openai"
2 import * as OpenAiDecisionModel from "@effect/ai-openai/OpenAiDecisionModel"
3 import { assert, describe, it } from "@effect/vitest"
4 import { Effect, Layer, Redacted, Result, Schema } from "effect"
5 import { Decision, DecisionModel, Model } from "effect/ai"
6 import { HttpClient, type HttpClientError, type HttpClientRequest, HttpClientResponse } from "effect/http"
7
8 const Triage = Decision.make({
9 input: Schema.Struct({ message: Schema.String }),
10 decisions: {
11 team: Decision.classify({
12 instructions: "Which team should handle this request?",
13 criteria: { billing: "Payments", technical: "Outages" }
14 }),
15 severity: Decision.rate({
16 instructions: "How severe is the impact?",
17 criteria: ["Minor", "Major", "Critical"]
18 }),
19 urgent: Decision.probability({
20 instructions: "Does this need immediate attention?",
21 criteria: { false: "Can wait", true: "Act now" }
22 })
23 }
24 })
25
26 const Urgency = Decision.make({
27 input: Schema.String,
28 decisions: {
29 urgent: Decision.probability({ instructions: "Does this need immediate attention?" })
30 }
31 })
32
33 const Screenshots = Decision.make({
34 input: Schema.Struct({ message: Schema.String, shots: Schema.Array(Decision.Image) }),
35 decisions: {
36 urgent: Decision.probability({ instructions: "Does this need immediate attention?" })
37 }
38 })
39
40 const input = { message: "Checkout is down" }
41
42 const answers = [
43 {
44 name: "team",
45 type: "choice",
46 choice: "technical",
47 probabilities: [{ value: "billing", probability: 0.1 }, { value: "technical", probability: 0.9 }],
48 confidence: 0.9
49 },
50 {
51 name: "severity",
52 type: "score",
53 score: 1.75,
54 probabilities: [
55 { value: 0, label: "minor", probability: 0.05 },
56 { value: 1, label: "major", probability: 0.15 },
57 { value: 2, label: "critical", probability: 0.8 }
58 ],
59 confidence: 0.8
60 },
61 { name: "urgent", type: "predicate", probability: 0.99 }
62 ]
63
64 const response = (
65 answers: ReadonlyArray<unknown>,
66 fields: object = { usage: { input_tokens: 120, output_tokens: 0 } }
67 ) => ({ id: "dec_123", object: "decision", model: "gpt-6-luna", answers, ...fields })
68
69 const clientLayer = (
70 body: unknown,
71 onRequest: (request: HttpClientRequest.HttpClientRequest) => void = () => {}
72 ) =>
73 OpenAiClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe(
74 Layer.provide(Layer.succeed(
75 HttpClient.HttpClient,
76 HttpClient.makeWith(
77 Effect.map((request) => {
78 onRequest(request)
79 return HttpClientResponse.fromWeb(request, Response.json(body))
80 }),
81 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
82 )
83 ))
84 )
85
86 const layer = (body: unknown) =>
87 OpenAiDecisionModel.layer({ model: "gpt-6-luna" }).pipe(Layer.provide(clientLayer(body)))
88
89 const requestBody = (request: HttpClientRequest.HttpClientRequest) => {
90 assert.strictEqual(request.body._tag, "Uint8Array")
91 return request.body._tag === "Uint8Array" ? JSON.parse(new TextDecoder().decode(request.body.body)) : undefined
92 }
93
94 describe("OpenAiDecisionModel", () => {
95 it.effect("keeps explicit Rate labels separate from their string descriptions", () => {
96 const definition = Decision.make({
97 input: Schema.String,
98 decisions: {
99 severity: Decision.rate({
100 instructions: "Severity?",
101 criteria: ["Minor", { label: "Critical", description: "Checkout unavailable" }]
102 })
103 }
104 })
105 return Effect.gen(function*() {
106 const result = yield* DecisionModel.decide(definition, { input: "Checkout is down" })
107 assert.strictEqual(result.answers.severity.label, "Critical")
108 assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.1, Critical: 0.9 })
109 }).pipe(
110 Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })),
111 Effect.provide(clientLayer(
112 response([{
113 name: "severity",
114 type: "score",
115 score: 0.9,
116 confidence: 0.9,
117 probabilities: [{ value: 0, label: "ignored", probability: 0.1 }, {
118 value: 1,
119 label: "ignored",
120 probability: 0.9
121 }]
122 }]),
123 (request) => {
124 assert.deepStrictEqual(requestBody(request).questions, [{
125 type: "score",
126 name: "severity",
127 instructions: "Severity?",
128 levels: [{ label: "Minor", description: "Minor" }, {
129 label: "Critical",
130 description: "Checkout unavailable"
131 }]
132 }])
133 }
134 ))
135 )
136 })
137
138 it.effect("serializes structured instructions and descriptions to strings, including null choices", () => {
139 const definition = Decision.make({
140 input: Schema.String,
141 decisions: {
142 team: Decision.classify({
143 instructions: { question: "Team?" },
144 criteria: { billing: null, technical: { handles: "Outages" } }
145 }),
146 severity: Decision.rate({
147 instructions: ["Severity?"],
148 criteria: ["Minor", "Major", { label: "Critical", description: ["outage"] }]
149 }),
150 urgent: Decision.probability({
151 instructions: { question: "Urgent?" },
152 criteria: { false: ["Can wait"], true: { action: "Act now" } }
153 })
154 }
155 })
156 return DecisionModel.decide(definition, { input: "Checkout is down" }).pipe(
157 Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })),
158 Effect.provide(clientLayer(response(answers), (request) => {
159 assert.deepStrictEqual(requestBody(request).questions, [
160 {
161 type: "choice",
162 name: "team",
163 instructions: "{\"question\":\"Team?\"}",
164 choices: [
165 { value: "billing", description: "null" },
166 { value: "technical", description: "{\"handles\":\"Outages\"}" }
167 ]
168 },
169 {
170 type: "score",
171 name: "severity",
172 instructions: "[\"Severity?\"]",
173 levels: [
174 { label: "Minor", description: "Minor" },
175 { label: "Major", description: "Major" },
176 { label: "Critical", description: "[\"outage\"]" }
177 ]
178 },
179 {
180 type: "predicate",
181 name: "urgent",
182 instructions: "{\"question\":\"Urgent?\"}\n\nfalse: [\"Can wait\"]\ntrue: {\"action\":\"Act now\"}"
183 }
184 ])
185 }))
186 )
187 })
188
189 it.effect("sends every decision as a named question and keys the answers by name", () =>
190 Effect.gen(function*() {
191 const result = yield* DecisionModel.decide(Triage, { input })
192 assert.strictEqual(yield* Model.ProviderName, "openai")
193 assert.strictEqual(yield* Model.ModelName, "gpt-6-luna")
194 assert.strictEqual(result.answers.team.label, "technical")
195 assert.deepStrictEqual({ ...result.answers.team.probabilities }, { billing: 0.1, technical: 0.9 })
196 assert.strictEqual(result.answers.team.confidence, 0.9)
197 assert.strictEqual(result.answers.severity.label, "Critical")
198 assert.strictEqual(result.answers.severity.rating, 1.75)
199 assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.05, Major: 0.15, Critical: 0.8 })
200 assert.strictEqual(result.answers.severity.confidence, 0.8)
201 assert.strictEqual(result.answers.urgent.probability, 0.99)
202 assert.strictEqual(result.usage.inputTokens, 120)
203 assert.strictEqual(result.usage.outputTokens, 0)
204 }).pipe(
205 Effect.provide(OpenAiDecisionModel.model("gpt-6-luna")),
206 Effect.provide(clientLayer(response([...answers].reverse()), (request) => {
207 assert.strictEqual(request.method, "POST")
208 assert.strictEqual(request.url, "https://api.openai.com/v1/decisions")
209 assert.deepStrictEqual(requestBody(request), {
210 model: "gpt-6-luna",
211 input: JSON.stringify(input),
212 questions: [
213 {
214 type: "choice",
215 name: "team",
216 instructions: "Which team should handle this request?",
217 choices: [
218 { value: "billing", description: "Payments" },
219 { value: "technical", description: "Outages" }
220 ]
221 },
222 {
223 type: "score",
224 name: "severity",
225 instructions: "How severe is the impact?",
226 levels: [
227 { label: "Minor", description: "Minor" },
228 { label: "Major", description: "Major" },
229 { label: "Critical", description: "Critical" }
230 ]
231 },
232 {
233 type: "predicate",
234 name: "urgent",
235 instructions: "Does this need immediate attention?\n\nfalse: Can wait\ntrue: Act now"
236 }
237 ]
238 })
239 }))
240 ))
241
242 it.effect("sends string state unchanged without images and leaves omitted usage undefined", () =>
243 Effect.gen(function*() {
244 const result = yield* DecisionModel.decide(Urgency, { input: "Checkout is down" })
245 assert.isUndefined(result.usage.inputTokens)
246 assert.isUndefined(result.usage.outputTokens)
247 }).pipe(
248 Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })),
249 Effect.provide(clientLayer(response([answers[2]], {}), (request) => {
250 assert.strictEqual(requestBody(request).input, "Checkout is down")
251 }))
252 ))
253
254 it.effect("fails with InvalidOutputError when an answer is missing", () =>
255 Effect.gen(function*() {
256 const error = yield* Effect.flip(DecisionModel.decide(Triage, { input }))
257 assert.strictEqual(error.reason._tag, "InvalidOutputError")
258 }).pipe(Effect.provide(layer(response(answers.slice(0, 2))))))
259
260 it.effect("decide fails with ContentPolicyError when any answer is a refusal, even beside a missing answer", () =>
261 Effect.gen(function*() {
262 const error = yield* Effect.flip(DecisionModel.decide(Triage, { input }))
263 assert.strictEqual(error.reason._tag, "ContentPolicyError")
264 if (error.reason._tag === "ContentPolicyError") {
265 assert.include(error.reason.description, "severity")
266 assert.notInclude(error.reason.description, "team")
267 }
268 }).pipe(Effect.provide(layer(response([
269 { name: "severity", type: "refusal", refusal: "I can't help with that." },
270 answers[2]
271 ])))))
272
273 it.effect("decideSettled reports a refusal and a mismatched score per decision", () =>
274 Effect.gen(function*() {
275 const { outcomes } = yield* DecisionModel.decideSettled(Triage, { input })
276 assert.isTrue(Result.isFailure(outcomes.team) && outcomes.team.failure.reason === "Invalid")
277 assert.isTrue(Result.isFailure(outcomes.severity) && outcomes.severity.failure.reason === "Refused")
278 assert.deepStrictEqual(outcomes.urgent, Result.succeed({ probability: 0.99 }))
279 }).pipe(Effect.provide(layer(response([
280 { ...answers[1], name: "team" },
281 { name: "severity", type: "refusal", refusal: "I can't help with that." },
282 answers[2]
283 ])))))
284
285 it.effect("a refusal outranks a later answer with the same name", () =>
286 Effect.gen(function*() {
287 const error = yield* Effect.flip(DecisionModel.decide(Urgency, { input: "Checkout is down" }))
288 assert.strictEqual(error.reason._tag, "ContentPolicyError")
289 const { outcomes } = yield* DecisionModel.decideSettled(Urgency, { input: "Checkout is down" })
290 assert.isTrue(Result.isFailure(outcomes.urgent) && outcomes.urgent.failure.reason === "Refused")
291 }).pipe(Effect.provide(layer(response([
292 { name: "urgent", type: "refusal" },
293 { name: "urgent", type: "predicate", probability: 0.99 }
294 ])))))
295
296 it.effect("sends the state text, then each byte array, base64, or data URL image after its reference", () => {
297 const dataUrl = "data:image/webp;base64,UklGRg=="
298 const shots = [
299 { mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
300 { mediaType: "image/jpeg", data: "/9j/4AAQ" },
301 { mediaType: "image/webp", data: new URL(dataUrl) }
302 ]
303 return Effect.gen(function*() {
304 const result = yield* DecisionModel.decide(Screenshots, { input: { message: "Checkout is down", shots } })
305 assert.strictEqual(result.answers.urgent.probability, 0.99)
306 }).pipe(
307 Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })),
308 Effect.provide(clientLayer(response([answers[2]]), (request) => {
309 const body = requestBody(request)
310 assert.strictEqual(body.input.length, 1)
311 assert.strictEqual(body.input[0].role, "user")
312 const content = body.input[0].content
313 const state = JSON.parse(content[0].text)
314 const [png, jpeg, webp] = state.shots
315 assert.deepStrictEqual(content, [
316 { type: "input_text", text: JSON.stringify({ message: "Checkout is down", shots: [png, jpeg, webp] }) },
317 { type: "input_text", text: `Image ${png}:` },
318 { type: "input_image", image_url: "data:image/png;base64,iVBORw==" },
319 { type: "input_text", text: `Image ${jpeg}:` },
320 { type: "input_image", image_url: "data:image/jpeg;base64,/9j/4AAQ" },
321 { type: "input_text", text: `Image ${webp}:` },
322 { type: "input_image", image_url: dataUrl }
323 ])
324 }))
325 )
326 })
327
328 it.effect("rejects remote image URLs without sending a request", () => {
329 let requests = 0
330 return Effect.gen(function*() {
331 const error = yield* Effect.flip(DecisionModel.decide(Screenshots, {
332 input: {
333 message: "Checkout is down",
334 shots: [{ mediaType: "image/png", data: new URL("https://example.com/cat.png") }]
335 }
336 }))
337 assert.strictEqual(error.reason._tag, "InvalidUserInputError")
338 assert.strictEqual(requests, 0)
339 }).pipe(
340 Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })),
341 Effect.provide(clientLayer(response([answers[2]]), () => {
342 requests++
343 }))
344 )
345 })
346 })