The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import { OpenAiClient } from "@effect/ai-openai" |
| 2 | import * as OpenAiDecisionModel from "@effect/ai-openai/OpenAiDecisionModel" |
| 3 | import { assert, describe, it } from "@effect/vitest" |
| 4 | import { Effect, Layer, Redacted, Result, Schema } from "effect" |
| 5 | import { Decision, DecisionModel, Model } from "effect/ai" |
| 6 | import { HttpClient, type HttpClientError, type HttpClientRequest, HttpClientResponse } from "effect/http" |
| 7 | |
| 8 | const Triage = Decision.make({ |
| 9 | input: Schema.Struct({ message: Schema.String }), |
| 10 | decisions: { |
| 11 | team: Decision.classify({ |
| 12 | instructions: "Which team should handle this request?", |
| 13 | criteria: { billing: "Payments", technical: "Outages" } |
| 14 | }), |
| 15 | severity: Decision.rate({ |
| 16 | instructions: "How severe is the impact?", |
| 17 | criteria: ["Minor", "Major", "Critical"] |
| 18 | }), |
| 19 | urgent: Decision.probability({ |
| 20 | instructions: "Does this need immediate attention?", |
| 21 | criteria: { false: "Can wait", true: "Act now" } |
| 22 | }) |
| 23 | } |
| 24 | }) |
| 25 | |
| 26 | const Urgency = Decision.make({ |
| 27 | input: Schema.String, |
| 28 | decisions: { |
| 29 | urgent: Decision.probability({ instructions: "Does this need immediate attention?" }) |
| 30 | } |
| 31 | }) |
| 32 | |
| 33 | const Screenshots = Decision.make({ |
| 34 | input: Schema.Struct({ message: Schema.String, shots: Schema.Array(Decision.Image) }), |
| 35 | decisions: { |
| 36 | urgent: Decision.probability({ instructions: "Does this need immediate attention?" }) |
| 37 | } |
| 38 | }) |
| 39 | |
| 40 | const input = { message: "Checkout is down" } |
| 41 | |
| 42 | const answers = [ |
| 43 | { |
| 44 | name: "team", |
| 45 | type: "choice", |
| 46 | choice: "technical", |
| 47 | probabilities: [{ value: "billing", probability: 0.1 }, { value: "technical", probability: 0.9 }], |
| 48 | confidence: 0.9 |
| 49 | }, |
| 50 | { |
| 51 | name: "severity", |
| 52 | type: "score", |
| 53 | score: 1.75, |
| 54 | probabilities: [ |
| 55 | { value: 0, label: "minor", probability: 0.05 }, |
| 56 | { value: 1, label: "major", probability: 0.15 }, |
| 57 | { value: 2, label: "critical", probability: 0.8 } |
| 58 | ], |
| 59 | confidence: 0.8 |
| 60 | }, |
| 61 | { name: "urgent", type: "predicate", probability: 0.99 } |
| 62 | ] |
| 63 | |
| 64 | const response = ( |
| 65 | answers: ReadonlyArray<unknown>, |
| 66 | fields: object = { usage: { input_tokens: 120, output_tokens: 0 } } |
| 67 | ) => ({ id: "dec_123", object: "decision", model: "gpt-6-luna", answers, ...fields }) |
| 68 | |
| 69 | const clientLayer = ( |
| 70 | body: unknown, |
| 71 | onRequest: (request: HttpClientRequest.HttpClientRequest) => void = () => {} |
| 72 | ) => |
| 73 | OpenAiClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( |
| 74 | Layer.provide(Layer.succeed( |
| 75 | HttpClient.HttpClient, |
| 76 | HttpClient.makeWith( |
| 77 | Effect.map((request) => { |
| 78 | onRequest(request) |
| 79 | return HttpClientResponse.fromWeb(request, Response.json(body)) |
| 80 | }), |
| 81 | Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never> |
| 82 | ) |
| 83 | )) |
| 84 | ) |
| 85 | |
| 86 | const layer = (body: unknown) => |
| 87 | OpenAiDecisionModel.layer({ model: "gpt-6-luna" }).pipe(Layer.provide(clientLayer(body))) |
| 88 | |
| 89 | const requestBody = (request: HttpClientRequest.HttpClientRequest) => { |
| 90 | assert.strictEqual(request.body._tag, "Uint8Array") |
| 91 | return request.body._tag === "Uint8Array" ? JSON.parse(new TextDecoder().decode(request.body.body)) : undefined |
| 92 | } |
| 93 | |
| 94 | describe("OpenAiDecisionModel", () => { |
| 95 | it.effect("keeps explicit Rate labels separate from their string descriptions", () => { |
| 96 | const definition = Decision.make({ |
| 97 | input: Schema.String, |
| 98 | decisions: { |
| 99 | severity: Decision.rate({ |
| 100 | instructions: "Severity?", |
| 101 | criteria: ["Minor", { label: "Critical", description: "Checkout unavailable" }] |
| 102 | }) |
| 103 | } |
| 104 | }) |
| 105 | return Effect.gen(function*() { |
| 106 | const result = yield* DecisionModel.decide(definition, { input: "Checkout is down" }) |
| 107 | assert.strictEqual(result.answers.severity.label, "Critical") |
| 108 | assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.1, Critical: 0.9 }) |
| 109 | }).pipe( |
| 110 | Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })), |
| 111 | Effect.provide(clientLayer( |
| 112 | response([{ |
| 113 | name: "severity", |
| 114 | type: "score", |
| 115 | score: 0.9, |
| 116 | confidence: 0.9, |
| 117 | probabilities: [{ value: 0, label: "ignored", probability: 0.1 }, { |
| 118 | value: 1, |
| 119 | label: "ignored", |
| 120 | probability: 0.9 |
| 121 | }] |
| 122 | }]), |
| 123 | (request) => { |
| 124 | assert.deepStrictEqual(requestBody(request).questions, [{ |
| 125 | type: "score", |
| 126 | name: "severity", |
| 127 | instructions: "Severity?", |
| 128 | levels: [{ label: "Minor", description: "Minor" }, { |
| 129 | label: "Critical", |
| 130 | description: "Checkout unavailable" |
| 131 | }] |
| 132 | }]) |
| 133 | } |
| 134 | )) |
| 135 | ) |
| 136 | }) |
| 137 | |
| 138 | it.effect("serializes structured instructions and descriptions to strings, including null choices", () => { |
| 139 | const definition = Decision.make({ |
| 140 | input: Schema.String, |
| 141 | decisions: { |
| 142 | team: Decision.classify({ |
| 143 | instructions: { question: "Team?" }, |
| 144 | criteria: { billing: null, technical: { handles: "Outages" } } |
| 145 | }), |
| 146 | severity: Decision.rate({ |
| 147 | instructions: ["Severity?"], |
| 148 | criteria: ["Minor", "Major", { label: "Critical", description: ["outage"] }] |
| 149 | }), |
| 150 | urgent: Decision.probability({ |
| 151 | instructions: { question: "Urgent?" }, |
| 152 | criteria: { false: ["Can wait"], true: { action: "Act now" } } |
| 153 | }) |
| 154 | } |
| 155 | }) |
| 156 | return DecisionModel.decide(definition, { input: "Checkout is down" }).pipe( |
| 157 | Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })), |
| 158 | Effect.provide(clientLayer(response(answers), (request) => { |
| 159 | assert.deepStrictEqual(requestBody(request).questions, [ |
| 160 | { |
| 161 | type: "choice", |
| 162 | name: "team", |
| 163 | instructions: "{\"question\":\"Team?\"}", |
| 164 | choices: [ |
| 165 | { value: "billing", description: "null" }, |
| 166 | { value: "technical", description: "{\"handles\":\"Outages\"}" } |
| 167 | ] |
| 168 | }, |
| 169 | { |
| 170 | type: "score", |
| 171 | name: "severity", |
| 172 | instructions: "[\"Severity?\"]", |
| 173 | levels: [ |
| 174 | { label: "Minor", description: "Minor" }, |
| 175 | { label: "Major", description: "Major" }, |
| 176 | { label: "Critical", description: "[\"outage\"]" } |
| 177 | ] |
| 178 | }, |
| 179 | { |
| 180 | type: "predicate", |
| 181 | name: "urgent", |
| 182 | instructions: "{\"question\":\"Urgent?\"}\n\nfalse: [\"Can wait\"]\ntrue: {\"action\":\"Act now\"}" |
| 183 | } |
| 184 | ]) |
| 185 | })) |
| 186 | ) |
| 187 | }) |
| 188 | |
| 189 | it.effect("sends every decision as a named question and keys the answers by name", () => |
| 190 | Effect.gen(function*() { |
| 191 | const result = yield* DecisionModel.decide(Triage, { input }) |
| 192 | assert.strictEqual(yield* Model.ProviderName, "openai") |
| 193 | assert.strictEqual(yield* Model.ModelName, "gpt-6-luna") |
| 194 | assert.strictEqual(result.answers.team.label, "technical") |
| 195 | assert.deepStrictEqual({ ...result.answers.team.probabilities }, { billing: 0.1, technical: 0.9 }) |
| 196 | assert.strictEqual(result.answers.team.confidence, 0.9) |
| 197 | assert.strictEqual(result.answers.severity.label, "Critical") |
| 198 | assert.strictEqual(result.answers.severity.rating, 1.75) |
| 199 | assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.05, Major: 0.15, Critical: 0.8 }) |
| 200 | assert.strictEqual(result.answers.severity.confidence, 0.8) |
| 201 | assert.strictEqual(result.answers.urgent.probability, 0.99) |
| 202 | assert.strictEqual(result.usage.inputTokens, 120) |
| 203 | assert.strictEqual(result.usage.outputTokens, 0) |
| 204 | }).pipe( |
| 205 | Effect.provide(OpenAiDecisionModel.model("gpt-6-luna")), |
| 206 | Effect.provide(clientLayer(response([...answers].reverse()), (request) => { |
| 207 | assert.strictEqual(request.method, "POST") |
| 208 | assert.strictEqual(request.url, "https://api.openai.com/v1/decisions") |
| 209 | assert.deepStrictEqual(requestBody(request), { |
| 210 | model: "gpt-6-luna", |
| 211 | input: JSON.stringify(input), |
| 212 | questions: [ |
| 213 | { |
| 214 | type: "choice", |
| 215 | name: "team", |
| 216 | instructions: "Which team should handle this request?", |
| 217 | choices: [ |
| 218 | { value: "billing", description: "Payments" }, |
| 219 | { value: "technical", description: "Outages" } |
| 220 | ] |
| 221 | }, |
| 222 | { |
| 223 | type: "score", |
| 224 | name: "severity", |
| 225 | instructions: "How severe is the impact?", |
| 226 | levels: [ |
| 227 | { label: "Minor", description: "Minor" }, |
| 228 | { label: "Major", description: "Major" }, |
| 229 | { label: "Critical", description: "Critical" } |
| 230 | ] |
| 231 | }, |
| 232 | { |
| 233 | type: "predicate", |
| 234 | name: "urgent", |
| 235 | instructions: "Does this need immediate attention?\n\nfalse: Can wait\ntrue: Act now" |
| 236 | } |
| 237 | ] |
| 238 | }) |
| 239 | })) |
| 240 | )) |
| 241 | |
| 242 | it.effect("sends string state unchanged without images and leaves omitted usage undefined", () => |
| 243 | Effect.gen(function*() { |
| 244 | const result = yield* DecisionModel.decide(Urgency, { input: "Checkout is down" }) |
| 245 | assert.isUndefined(result.usage.inputTokens) |
| 246 | assert.isUndefined(result.usage.outputTokens) |
| 247 | }).pipe( |
| 248 | Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })), |
| 249 | Effect.provide(clientLayer(response([answers[2]], {}), (request) => { |
| 250 | assert.strictEqual(requestBody(request).input, "Checkout is down") |
| 251 | })) |
| 252 | )) |
| 253 | |
| 254 | it.effect("fails with InvalidOutputError when an answer is missing", () => |
| 255 | Effect.gen(function*() { |
| 256 | const error = yield* Effect.flip(DecisionModel.decide(Triage, { input })) |
| 257 | assert.strictEqual(error.reason._tag, "InvalidOutputError") |
| 258 | }).pipe(Effect.provide(layer(response(answers.slice(0, 2)))))) |
| 259 | |
| 260 | it.effect("decide fails with ContentPolicyError when any answer is a refusal, even beside a missing answer", () => |
| 261 | Effect.gen(function*() { |
| 262 | const error = yield* Effect.flip(DecisionModel.decide(Triage, { input })) |
| 263 | assert.strictEqual(error.reason._tag, "ContentPolicyError") |
| 264 | if (error.reason._tag === "ContentPolicyError") { |
| 265 | assert.include(error.reason.description, "severity") |
| 266 | assert.notInclude(error.reason.description, "team") |
| 267 | } |
| 268 | }).pipe(Effect.provide(layer(response([ |
| 269 | { name: "severity", type: "refusal", refusal: "I can't help with that." }, |
| 270 | answers[2] |
| 271 | ]))))) |
| 272 | |
| 273 | it.effect("decideSettled reports a refusal and a mismatched score per decision", () => |
| 274 | Effect.gen(function*() { |
| 275 | const { outcomes } = yield* DecisionModel.decideSettled(Triage, { input }) |
| 276 | assert.isTrue(Result.isFailure(outcomes.team) && outcomes.team.failure.reason === "Invalid") |
| 277 | assert.isTrue(Result.isFailure(outcomes.severity) && outcomes.severity.failure.reason === "Refused") |
| 278 | assert.deepStrictEqual(outcomes.urgent, Result.succeed({ probability: 0.99 })) |
| 279 | }).pipe(Effect.provide(layer(response([ |
| 280 | { ...answers[1], name: "team" }, |
| 281 | { name: "severity", type: "refusal", refusal: "I can't help with that." }, |
| 282 | answers[2] |
| 283 | ]))))) |
| 284 | |
| 285 | it.effect("a refusal outranks a later answer with the same name", () => |
| 286 | Effect.gen(function*() { |
| 287 | const error = yield* Effect.flip(DecisionModel.decide(Urgency, { input: "Checkout is down" })) |
| 288 | assert.strictEqual(error.reason._tag, "ContentPolicyError") |
| 289 | const { outcomes } = yield* DecisionModel.decideSettled(Urgency, { input: "Checkout is down" }) |
| 290 | assert.isTrue(Result.isFailure(outcomes.urgent) && outcomes.urgent.failure.reason === "Refused") |
| 291 | }).pipe(Effect.provide(layer(response([ |
| 292 | { name: "urgent", type: "refusal" }, |
| 293 | { name: "urgent", type: "predicate", probability: 0.99 } |
| 294 | ]))))) |
| 295 | |
| 296 | it.effect("sends the state text, then each byte array, base64, or data URL image after its reference", () => { |
| 297 | const dataUrl = "data:image/webp;base64,UklGRg==" |
| 298 | const shots = [ |
| 299 | { mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) }, |
| 300 | { mediaType: "image/jpeg", data: "/9j/4AAQ" }, |
| 301 | { mediaType: "image/webp", data: new URL(dataUrl) } |
| 302 | ] |
| 303 | return Effect.gen(function*() { |
| 304 | const result = yield* DecisionModel.decide(Screenshots, { input: { message: "Checkout is down", shots } }) |
| 305 | assert.strictEqual(result.answers.urgent.probability, 0.99) |
| 306 | }).pipe( |
| 307 | Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })), |
| 308 | Effect.provide(clientLayer(response([answers[2]]), (request) => { |
| 309 | const body = requestBody(request) |
| 310 | assert.strictEqual(body.input.length, 1) |
| 311 | assert.strictEqual(body.input[0].role, "user") |
| 312 | const content = body.input[0].content |
| 313 | const state = JSON.parse(content[0].text) |
| 314 | const [png, jpeg, webp] = state.shots |
| 315 | assert.deepStrictEqual(content, [ |
| 316 | { type: "input_text", text: JSON.stringify({ message: "Checkout is down", shots: [png, jpeg, webp] }) }, |
| 317 | { type: "input_text", text: `Image ${png}:` }, |
| 318 | { type: "input_image", image_url: "data:image/png;base64,iVBORw==" }, |
| 319 | { type: "input_text", text: `Image ${jpeg}:` }, |
| 320 | { type: "input_image", image_url: "data:image/jpeg;base64,/9j/4AAQ" }, |
| 321 | { type: "input_text", text: `Image ${webp}:` }, |
| 322 | { type: "input_image", image_url: dataUrl } |
| 323 | ]) |
| 324 | })) |
| 325 | ) |
| 326 | }) |
| 327 | |
| 328 | it.effect("rejects remote image URLs without sending a request", () => { |
| 329 | let requests = 0 |
| 330 | return Effect.gen(function*() { |
| 331 | const error = yield* Effect.flip(DecisionModel.decide(Screenshots, { |
| 332 | input: { |
| 333 | message: "Checkout is down", |
| 334 | shots: [{ mediaType: "image/png", data: new URL("https://example.com/cat.png") }] |
| 335 | } |
| 336 | })) |
| 337 | assert.strictEqual(error.reason._tag, "InvalidUserInputError") |
| 338 | assert.strictEqual(requests, 0) |
| 339 | }).pipe( |
| 340 | Effect.provide(OpenAiDecisionModel.layer({ model: "gpt-6-luna" })), |
| 341 | Effect.provide(clientLayer(response([answers[2]]), () => { |
| 342 | requests++ |
| 343 | })) |
| 344 | ) |
| 345 | }) |
| 346 | }) |