The best way to control Claude Code, Codex and more from any computer or phone

main
ts 937 lines 37.3 KB
Raw
1 import { OpenRouterClient, OpenRouterConfig, OpenRouterDecisionModel } from "@effect/ai-openrouter"
2 import { assert, describe, it } from "@effect/vitest"
3 import { Effect, Layer, Redacted, Result, Schema } from "effect"
4 import { Decision, DecisionModel, Model } from "effect/ai"
5 import { HttpClient, type HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"
6
7 const Ticket = Schema.Struct({
8 message: Schema.String,
9 orderId: Schema.String
10 })
11
12 const TicketTriage = Decision.make({
13 input: Ticket,
14 decisions: {
15 department: Decision.classify({
16 instructions: "Which team should handle this",
17 criteria: {
18 billing: "Payment or subscription issues",
19 technical: "Bugs or integration problems",
20 sales: "Pricing or account questions"
21 }
22 }),
23 frustration: Decision.rate({
24 instructions: "How frustrated the customer appears",
25 criteria: ["Calm, just stating facts", "Frustrated but civil", "Very angry, strong language"]
26 }),
27 urgent: Decision.probability({
28 instructions: "The message conveys urgency or time-sensitivity",
29 criteria: {
30 false: "No time pressure",
31 true: "Needs action now"
32 }
33 })
34 }
35 })
36
37 const ticket = {
38 message: "Hi, I've been trying to connect my Stripe account for 3 days and it keeps failing. Please help ASAP.",
39 orderId: "A-104"
40 }
41
42 const decisionsResponse = {
43 model: "test/decision-model",
44 answers: {
45 department: {
46 type: "choice",
47 choice: "billing",
48 probabilities: { billing: 0.84, technical: 0.159, sales: 0.001 },
49 confidence: 0.596
50 },
51 frustration: {
52 type: "score",
53 score: 1.6,
54 legend: {
55 "0": "Calm, just stating facts",
56 "1": "Frustrated but civil",
57 "2": "Very angry, strong language"
58 },
59 probabilities: { "0": 0.05, "1": 0.3, "2": 0.65 },
60 confidence: 0.78
61 },
62 urgent: {
63 type: "noul",
64 noul: 0.999
65 }
66 },
67 usage: { input_tokens: 312, output_tokens: 48 }
68 }
69
70 const Structured = Decision.make({
71 input: Schema.String,
72 decisions: {
73 severity: Decision.rate({
74 instructions: ["How severe?"],
75 criteria: ["Minor", { label: "Critical", description: { impact: "outage" } }]
76 })
77 }
78 })
79
80 describe("OpenRouterDecisionModel", () => {
81 it.effect("accepts rounded choice probabilities that total 0.99", () =>
82 Effect.gen(function*() {
83 const definition = Decision.make({
84 input: Schema.String,
85 decisions: {
86 neutral3: Decision.classify({
87 instructions: "Choose a sentiment",
88 criteria: { negative: "Negative", neutral: "Neutral", positive: "Positive" }
89 })
90 }
91 })
92 const { answers } = yield* DecisionModel.decide(definition, { input: "It is fine" }).pipe(
93 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
94 Effect.provide(makeClientLayer((request) =>
95 Effect.succeed(jsonResponse(request, {
96 model: "test/decision-model",
97 answers: {
98 neutral3: {
99 type: "choice",
100 choice: "neutral",
101 probabilities: { negative: 0.02, neutral: 0.93, positive: 0.04 },
102 confidence: 0.93
103 }
104 },
105 usage: { input_tokens: 10, output_tokens: 5 }
106 }))
107 ))
108 )
109
110 assert.strictEqual(answers.neutral3.label, "neutral")
111 assert.closeTo(Object.values(answers.neutral3.probabilities).reduce((sum, value) => sum + value, 0), 1, 1e-6)
112 }))
113
114 for (
115 const [name, probabilities, succeeds] of [
116 ["extra provider key", { a: 0.5, b: 0.5, extra: 0.01 }, true],
117 ["drift outside two-decimal rounding", { a: 0.49, b: 0.49 }, false],
118 ["out-of-range probability", { a: 1.01, b: -0.02 }, false],
119 ["all-zero distribution", { a: 0, b: 0 }, false],
120 ["eight rounded labels", { a: 0.12, b: 0.12, c: 0.12, d: 0.12, e: 0.12, f: 0.12, g: 0.12, h: 0.13 }, true]
121 ] as const
122 ) {
123 it.effect(name, () =>
124 Effect.gen(function*() {
125 const definition = Decision.make({
126 input: Schema.String,
127 decisions: {
128 choice: Decision.classify({
129 instructions: "Pick a label",
130 criteria: Object.fromEntries(
131 Object.keys(probabilities).filter((key) => key !== "extra").map((key) => [key, key])
132 )
133 })
134 }
135 })
136 const result = yield* Effect.result(
137 DecisionModel.decide(definition, { input: "test" }).pipe(
138 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
139 Effect.provide(makeClientLayer((request) =>
140 Effect.succeed(jsonResponse(request, {
141 model: "test/decision-model",
142 answers: { choice: { type: "choice", choice: "a", probabilities, confidence: 0.5 } },
143 usage: { input_tokens: 1, output_tokens: 1 }
144 }))
145 ))
146 )
147 )
148 if (succeeds) {
149 assert.isTrue(Result.isSuccess(result))
150 if (Result.isSuccess(result)) {
151 assert.closeTo(
152 Object.values(result.success.answers.choice.probabilities).reduce((sum, value) => sum + value, 0),
153 1,
154 1e-6
155 )
156 if (name === "extra provider key") {
157 assert.deepStrictEqual(result.success.answers.choice.probabilities, { a: 0.5, b: 0.5 })
158 }
159 }
160 } else {
161 assert.isTrue(Result.isFailure(result))
162 if (Result.isFailure(result)) assert.strictEqual(result.failure.reason._tag, "InvalidOutputError")
163 }
164 }))
165 }
166
167 it.effect("normalizes rounded score probabilities", () =>
168 Effect.gen(function*() {
169 const definition = Decision.make({
170 input: Schema.String,
171 decisions: { score: Decision.rate({ instructions: "Rate", criteria: ["low", "mid", "high"] }) }
172 })
173 const { answers } = yield* DecisionModel.decide(definition, { input: "test" }).pipe(
174 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
175 Effect.provide(makeClientLayer((request) =>
176 Effect.succeed(jsonResponse(request, {
177 model: "test/decision-model",
178 answers: {
179 score: { type: "score", score: 1, probabilities: { "0": 0.1, "1": 0.7, "2": 0.19 }, confidence: 0.7 }
180 },
181 usage: { input_tokens: 1, output_tokens: 1 }
182 }))
183 ))
184 )
185 assert.closeTo(Object.values(answers.score.probabilities).reduce((sum, value) => sum + value, 0), 1, 1e-6)
186 }))
187
188 it.effect("omits absent probability criteria from the encoded question", () =>
189 Effect.gen(function*() {
190 const definition = Decision.make({
191 input: Schema.String,
192 decisions: {
193 urgent: Decision.probability({ instructions: "Needs action now" })
194 }
195 })
196 const { answers } = yield* DecisionModel.decide(definition, { input: "Help ASAP" }).pipe(
197 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
198 Effect.provide(makeClientLayer((request) =>
199 Effect.gen(function*() {
200 const { questions } = yield* getRequestBody(request)
201 assert.deepStrictEqual(questions.urgent, {
202 type: "noul",
203 instructions: "Needs action now"
204 })
205
206 return jsonResponse(request, {
207 model: "test/decision-model",
208 answers: { urgent: { type: "noul", noul: 0.75 } },
209 usage: { input_tokens: 20, output_tokens: 4 }
210 })
211 })
212 ))
213 )
214
215 assert.deepStrictEqual(answers.urgent, { probability: 0.75 })
216 }))
217
218 for (const key of ["__proto__", "constructor", "toString"]) {
219 it.effect("round trips a classification label named " + key, () =>
220 Effect.gen(function*() {
221 const definition = Decision.make({
222 input: Schema.String,
223 decisions: {
224 category: Decision.classify({
225 instructions: "Choose a category",
226 criteria: { [key]: "Special category", ordinary: "Ordinary category" }
227 })
228 }
229 })
230 const { answers } = yield* DecisionModel.decide(definition, { input: "Help" }).pipe(
231 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
232 Effect.provide(makeClientLayer((request) =>
233 Effect.gen(function*() {
234 const { questions } = yield* getRequestBody(request)
235 const criteria = questions.category.criteria
236
237 assert.isTrue(Object.hasOwn(criteria, key))
238 assert.isTrue(Object.getOwnPropertyDescriptor(criteria, key)?.enumerable)
239 assert.deepStrictEqual(Object.keys(criteria), [key, "ordinary"])
240 assert.strictEqual(criteria[key], "Special category")
241 assert.strictEqual(Object.getPrototypeOf(criteria), Object.prototype)
242
243 return jsonResponse(request, {
244 model: "test/decision-model",
245 answers: {
246 category: {
247 type: "choice",
248 choice: key,
249 probabilities: { [key]: 0.75, ordinary: 0.25 },
250 confidence: 0.5
251 }
252 },
253 usage: { input_tokens: 10, output_tokens: 5 }
254 })
255 })
256 ))
257 )
258 const { label, probabilities } = answers.category
259
260 assert.isTrue(Object.hasOwn(probabilities, key))
261 assert.isTrue(Object.getOwnPropertyDescriptor(probabilities, key)?.enumerable)
262 assert.deepStrictEqual(Object.keys(probabilities), [key, "ordinary"])
263 assert.deepStrictEqual(probabilities, { [key]: 0.75, ordinary: 0.25 })
264 assert.strictEqual(label, key)
265 assert.strictEqual(Object.getPrototypeOf(probabilities), null)
266 }))
267
268 it.effect("maps a score distribution to a rate level named " + key, () =>
269 Effect.gen(function*() {
270 const definition = Decision.make({
271 input: Schema.String,
272 decisions: {
273 intensity: Decision.rate({ instructions: "How intense", criteria: ["low", key, "high"] })
274 }
275 })
276 const { answers } = yield* DecisionModel.decide(definition, { input: "Help" }).pipe(
277 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
278 Effect.provide(makeClientLayer((request) =>
279 Effect.gen(function*() {
280 const { questions } = yield* getRequestBody(request)
281 assert.deepStrictEqual(questions.intensity.criteria, ["low", key, "high"])
282
283 return jsonResponse(request, {
284 model: "test/decision-model",
285 answers: {
286 intensity: {
287 type: "score",
288 score: 1.1,
289 legend: { "0": "low", "1": key, "2": "high" },
290 probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 },
291 confidence: 0.5
292 }
293 },
294 usage: { input_tokens: 10, output_tokens: 5 }
295 })
296 })
297 ))
298 )
299 const { label, probabilities, rating } = answers.intensity
300
301 assert.isTrue(Object.hasOwn(probabilities, key))
302 assert.isTrue(Object.getOwnPropertyDescriptor(probabilities, key)?.enumerable)
303 assert.deepStrictEqual(Object.keys(probabilities), ["low", key, "high"])
304 assert.deepStrictEqual(probabilities, { low: 0.1, [key]: 0.7, high: 0.2 })
305 assert.strictEqual(label, key)
306 assert.strictEqual(rating, 1.1)
307 assert.strictEqual(Object.getPrototypeOf(probabilities), null)
308 }))
309
310 it.effect("sends an own enumerable question named " + key, () =>
311 Effect.gen(function*() {
312 const definition = Decision.make({
313 input: Schema.String,
314 decisions: {
315 [key]: Decision.probability({
316 instructions: "Needs action now",
317 criteria: { false: "No", true: "Yes" }
318 })
319 }
320 })
321 const { answers } = yield* DecisionModel.decide(definition, { input: "Help" }).pipe(
322 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
323 Effect.provide(makeClientLayer((request) =>
324 Effect.gen(function*() {
325 const { questions } = yield* getRequestBody(request)
326
327 assert.strictEqual(Object.getPrototypeOf(questions), Object.prototype)
328 assert.isTrue(Object.hasOwn(questions, key))
329 assert.isTrue(Object.getOwnPropertyDescriptor(questions, key)?.enumerable)
330 assert.deepStrictEqual(Object.keys(questions), [key])
331 assert.deepStrictEqual(questions[key], {
332 type: "noul",
333 instructions: "Needs action now",
334 criteria: { false: "No", true: "Yes" }
335 })
336
337 return jsonResponse(request, {
338 model: "test/decision-model",
339 answers: { [key]: { type: "noul", noul: 0.75 } },
340 usage: { input_tokens: 20, output_tokens: 4 }
341 })
342 })
343 ))
344 )
345
346 assert.strictEqual(Object.getPrototypeOf(answers), null)
347 assert.isTrue(Object.hasOwn(answers, key))
348 assert.isTrue(Object.getOwnPropertyDescriptor(answers, key)?.enumerable)
349 assert.deepStrictEqual(Object.keys(answers), [key])
350 assert.deepStrictEqual(answers[key], { probability: 0.75 })
351 }))
352 }
353
354 for (
355 const apiUrl of [
356 "https://proxy.test/openrouter/v1",
357 "https://proxy.test/openrouter/v1/",
358 "https://proxy.test/openrouter",
359 "https://proxy.test/openrouter/"
360 ]
361 ) {
362 it.effect("resolves the alpha endpoint for " + apiUrl, () =>
363 Effect.gen(function*() {
364 const requests: Array<HttpClientRequest.HttpClientRequest> = []
365 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
366 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
367 Effect.provide(makeClientLayer((request) => {
368 requests.push(request)
369 return Effect.succeed(jsonResponse(request, decisionsResponse))
370 }, apiUrl))
371 )
372 assert.strictEqual(requests.length, 1)
373 assert.strictEqual(requests[0].url, "https://proxy.test/openrouter/alpha/decisions")
374 }))
375 }
376
377 it.effect("applies a scoped client transform only to the enclosed decisions request", () =>
378 Effect.gen(function*() {
379 const requests: Array<HttpClientRequest.HttpClientRequest> = []
380 yield* Effect.gen(function*() {
381 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
382 OpenRouterConfig.withClientTransform(
383 HttpClient.mapRequest(HttpClientRequest.setHeader("x-decision-scope", "scoped"))
384 )
385 )
386 yield* DecisionModel.decide(TicketTriage, { input: ticket })
387 }).pipe(
388 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
389 Effect.provide(makeClientLayer((request) => {
390 requests.push(request)
391 return Effect.succeed(jsonResponse(request, decisionsResponse))
392 }))
393 )
394 assert.strictEqual(requests.length, 2)
395 assert.strictEqual(requests[0].headers["x-decision-scope"], "scoped")
396 assert.isUndefined(requests[1].headers["x-decision-scope"])
397 }))
398
399 it.effect("rejects numeric state before making an HTTP request", () =>
400 Effect.gen(function*() {
401 const definition = Decision.make({ input: Schema.Number, decisions: TicketTriage.decisions })
402 const error = yield* DecisionModel.decide(definition, { input: 42 }).pipe(
403 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
404 Effect.provide(makeClientLayer(() => Effect.die(new Error("Unexpected HTTP request")))),
405 Effect.flip
406 )
407 assert.strictEqual(error._tag, "AiError")
408 assert.strictEqual(error.reason._tag, "InvalidUserInputError")
409 }))
410
411 it.effect("maps a malformed successful response to a non-retryable client InvalidResponseError", () =>
412 Effect.gen(function*() {
413 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
414 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
415 Effect.provide(makeClientLayer((request) =>
416 Effect.succeed(jsonResponse(request, {
417 ...decisionsResponse,
418 usage: { input_tokens: "not a number", output_tokens: 48 }
419 }))
420 )),
421 Effect.flip
422 )
423 assert.strictEqual(error._tag, "AiError")
424 assert.strictEqual(error.module, "OpenRouterClient")
425 assert.strictEqual(error.method, "createDecisions")
426 assert.strictEqual(error.reason._tag, "InvalidResponseError")
427 }))
428
429 it.effect("make constructs a DecisionModel service", () =>
430 Effect.gen(function*() {
431 const service = yield* OpenRouterDecisionModel.make({ model: "test/decision-model" })
432 assert.isFunction(service.decide)
433 }).pipe(Effect.provide(makeClientLayer(() => Effect.die(new Error("Unexpected HTTP request"))))))
434
435 it.effect("forwards request config and scoped overrides", () =>
436 Effect.gen(function*() {
437 const requests: Array<HttpClientRequest.HttpClientRequest> = []
438 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
439 Effect.provideService(OpenRouterDecisionModel.Config, { user: "scoped-user", trace: { trace_id: "scoped" } }),
440 Effect.provide(OpenRouterDecisionModel.layer({
441 model: "test/decision-model",
442 config: {
443 provider: { order: ["test-provider"] },
444 session_id: "session-1",
445 user: "default-user",
446 trace: { trace_id: "default" }
447 }
448 })),
449 Effect.provide(makeClientLayer((request) => {
450 requests.push(request)
451 return Effect.succeed(jsonResponse(request, decisionsResponse))
452 }))
453 )
454 const body = yield* getRequestBody(requests[0])
455 assert.deepStrictEqual(body.provider, { order: ["test-provider"] })
456 assert.strictEqual(body.session_id, "session-1")
457 assert.strictEqual(body.user, "scoped-user")
458 assert.deepStrictEqual(body.trace, { trace_id: "scoped" })
459 }))
460
461 it.effect("accepts absent confidence, cost, id and provider without inventing values", () =>
462 Effect.gen(function*() {
463 const { confidence: _choiceConfidence, ...department } = decisionsResponse.answers.department
464 const { confidence: _scoreConfidence, legend: _legend, ...frustration } = decisionsResponse.answers.frustration
465 const { answers, id, metadata, usage } = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
466 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
467 Effect.provide(makeClientLayer((request) =>
468 Effect.succeed(jsonResponse(request, {
469 ...decisionsResponse,
470 answers: { ...decisionsResponse.answers, department, frustration }
471 }))
472 ))
473 )
474 assert.isUndefined(answers.department.confidence)
475 assert.isUndefined(answers.frustration.confidence)
476 assert.strictEqual(answers.frustration.label, "Very angry, strong language")
477 assert.strictEqual(answers.urgent.probability, 0.999)
478 assert.deepStrictEqual({ ...usage }, { inputTokens: 312, outputTokens: 48 })
479 assert.isUndefined(id)
480 assert.deepStrictEqual(metadata.openrouter, {})
481 }))
482
483 it.effect("returns the response id, resolved model, cost and serving provider", () =>
484 Effect.gen(function*() {
485 const result = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
486 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model-latest" })),
487 Effect.provide(makeClientLayer((request) =>
488 Effect.succeed(jsonResponse(request, {
489 ...decisionsResponse,
490 model: "test/decision-model-1.2",
491 id: "decision-1",
492 provider: "test-provider",
493 usage: { ...decisionsResponse.usage, cost: 0.001 }
494 }))
495 ))
496 )
497 assert.deepStrictEqual({ ...result.usage }, { inputTokens: 312, outputTokens: 48 })
498 assert.strictEqual(result.id, "decision-1")
499 assert.strictEqual(result.modelId, "test/decision-model-1.2")
500 assert.deepStrictEqual(result.metadata.openrouter, { cost: 0.001, provider: "test-provider" })
501 assert.isFalse("cost" in result.answers.department)
502 }))
503
504 it.effect("fails with InvalidOutputError when a choice answer omits probabilities", () =>
505 Effect.gen(function*() {
506 const { probabilities: _probabilities, ...department } = decisionsResponse.answers.department
507 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
508 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
509 Effect.provide(makeClientLayer((request) =>
510 Effect.succeed(jsonResponse(request, {
511 ...decisionsResponse,
512 answers: { ...decisionsResponse.answers, department }
513 }))
514 )),
515 Effect.flip
516 )
517 assert.strictEqual(error._tag, "AiError")
518 assert.strictEqual(error.reason._tag, "InvalidOutputError")
519 }))
520
521 it.effect("model provides DecisionModel with openrouter provider metadata", () =>
522 Effect.gen(function*() {
523 const provider = yield* Model.ProviderName
524 const modelName = yield* Model.ModelName
525 const model = yield* DecisionModel.DecisionModel
526
527 assert.strictEqual(provider, "openrouter")
528 assert.strictEqual(modelName, "test/decision-model")
529 assert.isDefined(model.decide)
530 }).pipe(
531 Effect.provide(OpenRouterDecisionModel.model("test/decision-model")),
532 Effect.provide(makeClientLayer(() => Effect.die(new Error("Unexpected HTTP request"))))
533 ))
534
535 it.effect("sends the encoded input as state and every decision as a question in one request", () =>
536 Effect.gen(function*() {
537 const requests: Array<HttpClientRequest.HttpClientRequest> = []
538 const clientLayer = makeClientLayer((request) => {
539 requests.push(request)
540 return Effect.succeed(jsonResponse(request, decisionsResponse))
541 })
542
543 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
544 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
545 Effect.provide(clientLayer)
546 )
547
548 assert.strictEqual(requests.length, 1)
549 assert.strictEqual(requests[0].method, "POST")
550 assert.strictEqual(requests[0].url, "https://openrouter.ai/api/alpha/decisions")
551 assert.strictEqual(requests[0].headers["authorization"], "Bearer or-test-key")
552
553 assert.strictEqual(requests[0].headers["http-referer"], "https://example.com")
554 assert.strictEqual(requests[0].headers["x-title"], "Decision contract tests")
555
556 const body = yield* getRequestBody(requests[0])
557 assert.deepStrictEqual(body, {
558 state: {
559 message: ticket.message,
560 orderId: "A-104"
561 },
562 model: "test/decision-model",
563 questions: {
564 department: {
565 type: "choice",
566 instructions: "Which team should handle this",
567 criteria: {
568 billing: "Payment or subscription issues",
569 technical: "Bugs or integration problems",
570 sales: "Pricing or account questions"
571 }
572 },
573 frustration: {
574 type: "score",
575 instructions: "How frustrated the customer appears",
576 criteria: ["Calm, just stating facts", "Frustrated but civil", "Very angry, strong language"]
577 },
578 urgent: {
579 type: "noul",
580 instructions: "The message conveys urgency or time-sensitivity",
581 criteria: {
582 false: "No time pressure",
583 true: "Needs action now"
584 }
585 }
586 }
587 })
588 }))
589
590 it.effect("maps choice, score, and noul answers onto classify, rate, and probability answers", () =>
591 Effect.gen(function*() {
592 const { answers, usage } = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
593 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
594 Effect.provide(makeClientLayer((request) => Effect.succeed(jsonResponse(request, decisionsResponse))))
595 )
596
597 assert.strictEqual(answers.department.label, "billing")
598 assert.deepStrictEqual(answers.department.probabilities, { billing: 0.84, technical: 0.159, sales: 0.001 })
599 assert.strictEqual(answers.department.confidence, 0.596)
600
601 assert.strictEqual(answers.frustration.rating, 1.6)
602 assert.strictEqual(answers.frustration.label, "Very angry, strong language")
603 assert.deepStrictEqual(answers.frustration.probabilities, {
604 "Calm, just stating facts": 0.05,
605 "Frustrated but civil": 0.3,
606 "Very angry, strong language": 0.65
607 })
608 assert.strictEqual(answers.frustration.confidence, 0.78)
609
610 assert.strictEqual(answers.urgent.probability, 0.999)
611 assert.isFalse("confidence" in answers.urgent)
612
613 assert.strictEqual(usage.inputTokens, 312)
614 assert.strictEqual(usage.outputTokens, 48)
615 }))
616
617 it.effect("encodes an explicitly undefined optional input field as JSON null", () =>
618 Effect.gen(function*() {
619 const requests: Array<HttpClientRequest.HttpClientRequest> = []
620 const definition = Decision.make({
621 input: Schema.Struct({ message: Schema.String, orderId: Schema.optional(Schema.String) }),
622 decisions: TicketTriage.decisions
623 })
624 const { answers } = yield* DecisionModel.decide(definition, {
625 input: { message: ticket.message, orderId: undefined }
626 }).pipe(
627 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
628 Effect.provide(makeClientLayer((request) => {
629 requests.push(request)
630 return Effect.succeed(jsonResponse(request, decisionsResponse))
631 }))
632 )
633
634 assert.strictEqual(requests.length, 1)
635 const body = yield* getRequestBody(requests[0])
636 assert.deepStrictEqual(body.state, { message: ticket.message, orderId: null })
637 assert.strictEqual(answers.urgent.probability, 0.999)
638 }))
639
640 it.effect("sends a pinned model id through unchanged", () =>
641 Effect.gen(function*() {
642 const requests: Array<HttpClientRequest.HttpClientRequest> = []
643
644 const modelName = yield* Effect.gen(function*() {
645 yield* DecisionModel.decide(TicketTriage, { input: ticket })
646 return yield* Model.ModelName
647 }).pipe(
648 Effect.provide(OpenRouterDecisionModel.model("test/decision-model-pinned")),
649 Effect.provide(makeClientLayer((request) => {
650 requests.push(request)
651 return Effect.succeed(jsonResponse(request, { ...decisionsResponse, model: "test/decision-model-pinned" }))
652 }))
653 )
654
655 const body = yield* getRequestBody(requests[0])
656 assert.strictEqual(modelName, "test/decision-model-pinned")
657 assert.strictEqual(body.model, "test/decision-model-pinned")
658 }))
659
660 it.effect("passes a string input through as a string state", () =>
661 Effect.gen(function*() {
662 const requests: Array<HttpClientRequest.HttpClientRequest> = []
663 const Sentiment = Decision.make({
664 input: Schema.String,
665 decisions: {
666 urgent: Decision.probability({
667 instructions: "The message conveys urgency",
668 criteria: { false: "No time pressure", true: "Needs action now" }
669 })
670 }
671 })
672
673 yield* DecisionModel.decide(Sentiment, { input: "My card was charged twice." }).pipe(
674 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
675 Effect.provide(makeClientLayer((request) => {
676 requests.push(request)
677 return Effect.succeed(jsonResponse(request, {
678 model: "test/decision-model",
679 answers: { urgent: { type: "noul", noul: 0.93 } },
680 usage: { input_tokens: 20, output_tokens: 4 }
681 }))
682 }))
683 )
684
685 const body = yield* getRequestBody(requests[0])
686 assert.strictEqual(body.state, "My card was charged twice.")
687 }))
688
689 it.effect("passes an array input through as an array state", () =>
690 Effect.gen(function*() {
691 const requests: Array<HttpClientRequest.HttpClientRequest> = []
692 const Conversation = Decision.make({
693 input: Schema.Array(Schema.String),
694 decisions: {
695 urgent: Decision.probability({
696 instructions: "The message conveys urgency",
697 criteria: { false: "No time pressure", true: "Needs action now" }
698 })
699 }
700 })
701
702 yield* DecisionModel.decide(Conversation, {
703 input: ["Hi", "My customer number is TS1337.", "My card was charged twice."]
704 }).pipe(
705 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
706 Effect.provide(makeClientLayer((request) => {
707 requests.push(request)
708 return Effect.succeed(jsonResponse(request, {
709 model: "test/decision-model",
710 answers: { urgent: { type: "noul", noul: 0.93 } },
711 usage: { input_tokens: 20, output_tokens: 4 }
712 }))
713 }))
714 )
715
716 const body = yield* getRequestBody(requests[0])
717 assert.deepStrictEqual(body.state, ["Hi", "My customer number is TS1337.", "My card was charged twice."])
718 }))
719
720 it.effect("fails with InvalidOutputError when an answer type does not match the decision", () =>
721 Effect.gen(function*() {
722 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
723 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
724 Effect.provide(makeClientLayer((request) =>
725 Effect.succeed(jsonResponse(request, {
726 ...decisionsResponse,
727 answers: {
728 ...decisionsResponse.answers,
729 urgent: {
730 type: "choice",
731 choice: "true",
732 probabilities: { true: 0.9, false: 0.1 },
733 confidence: 0.8
734 }
735 }
736 }))
737 )),
738 Effect.flip
739 )
740
741 assert.strictEqual(error._tag, "AiError")
742 assert.strictEqual(error.reason._tag, "InvalidOutputError")
743 }))
744
745 it.effect("fails with InvalidOutputError when an answer is missing", () =>
746 Effect.gen(function*() {
747 const { urgent: _urgent, ...answers } = decisionsResponse.answers
748 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
749 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
750 Effect.provide(
751 makeClientLayer((request) => Effect.succeed(jsonResponse(request, { ...decisionsResponse, answers })))
752 ),
753 Effect.flip
754 )
755
756 assert.strictEqual(error._tag, "AiError")
757 assert.strictEqual(error.reason._tag, "InvalidOutputError")
758 }))
759
760 it.effect("fails with InvalidOutputError when a score answer omits probabilities", () =>
761 Effect.gen(function*() {
762 const { probabilities: _probabilities, ...frustration } = decisionsResponse.answers.frustration
763 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
764 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
765 Effect.provide(makeClientLayer((request) =>
766 Effect.succeed(jsonResponse(request, {
767 ...decisionsResponse,
768 answers: { ...decisionsResponse.answers, frustration }
769 }))
770 )),
771 Effect.flip
772 )
773
774 assert.strictEqual(error._tag, "AiError")
775 assert.strictEqual(error.reason._tag, "InvalidOutputError")
776 }))
777
778 it.effect("decideSettled reports choice and score answers without probabilities as NoDistribution", () =>
779 Effect.gen(function*() {
780 const { probabilities: _choice, ...department } = decisionsResponse.answers.department
781 const { probabilities: _score, ...frustration } = decisionsResponse.answers.frustration
782 const { outcomes } = yield* DecisionModel.decideSettled(TicketTriage, { input: ticket }).pipe(
783 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
784 Effect.provide(makeClientLayer((request) =>
785 Effect.succeed(jsonResponse(request, {
786 ...decisionsResponse,
787 answers: { ...decisionsResponse.answers, department, frustration }
788 }))
789 ))
790 )
791
792 assert.isTrue(Result.isFailure(outcomes.department) && outcomes.department.failure.reason === "NoDistribution")
793 assert.isTrue(
794 Result.isFailure(outcomes.frustration) && outcomes.frustration.failure.reason === "NoDistribution"
795 )
796 assert.isTrue(Result.isSuccess(outcomes.urgent))
797 }))
798
799 it.effect("fails with InvalidOutputError when a score distribution skips a level index", () =>
800 Effect.gen(function*() {
801 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
802 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
803 Effect.provide(makeClientLayer((request) =>
804 Effect.succeed(jsonResponse(request, {
805 ...decisionsResponse,
806 answers: {
807 ...decisionsResponse.answers,
808 frustration: {
809 ...decisionsResponse.answers.frustration,
810 probabilities: { "0": 0.35, "2": 0.65 }
811 }
812 }
813 }))
814 )),
815 Effect.flip
816 )
817
818 assert.strictEqual(error._tag, "AiError")
819 assert.strictEqual(error.reason._tag, "InvalidOutputError")
820 }))
821
822 for (
823 const [status, reason] of [
824 [400, "InvalidRequestError"],
825 [401, "AuthenticationError"],
826 [402, "UnknownError"],
827 [403, "AuthenticationError"],
828 [404, "InvalidRequestError"],
829 [413, "UnknownError"],
830 [429, "RateLimitError"],
831 [500, "InternalProviderError"],
832 [502, "InternalProviderError"],
833 [503, "InternalProviderError"],
834 [524, "InternalProviderError"],
835 [529, "InternalProviderError"]
836 ] as const
837 ) {
838 it.effect("maps HTTP " + status + " using the existing OpenRouter error mapping", () =>
839 Effect.gen(function*() {
840 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
841 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
842 Effect.provide(makeClientLayer((request) =>
843 Effect.succeed(HttpClientResponse.fromWeb(
844 request,
845 new Response(
846 JSON.stringify({ error: { message: "Decision request failed", code: status } }),
847 { status, headers: { "content-type": "application/json", "retry-after": "3" } }
848 )
849 ))
850 )),
851 Effect.flip
852 )
853 assert.strictEqual(error._tag, "AiError")
854 assert.strictEqual(error.reason._tag, reason)
855 }))
856 }
857
858 it.effect("sends structured descriptions and maps score indices to labels", () =>
859 Effect.gen(function*() {
860 const result = yield* DecisionModel.decide(Structured, { input: "Checkout is down" })
861 assert.strictEqual(result.answers.severity.label, "Critical")
862 assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.1, Critical: 0.9 })
863 }).pipe(
864 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
865 Effect.provide(makeClientLayer((request) =>
866 Effect.gen(function*() {
867 const body = yield* getRequestBody(request)
868 assert.deepStrictEqual(body.questions, {
869 severity: { type: "score", instructions: ["How severe?"], criteria: ["Minor", { impact: "outage" }] }
870 })
871 return jsonResponse(request, {
872 model: "test/decision-model",
873 usage: { input_tokens: 120, output_tokens: 0 },
874 answers: {
875 severity: {
876 type: "score",
877 score: 0.9,
878 probabilities: { "0": 0.1, "1": 0.9 },
879 confidence: 0.9,
880 legend: { "0": "Minor", "1": { impact: "outage" } }
881 }
882 }
883 })
884 })
885 ))
886 ))
887 })
888 const makeClientLayer = (
889 handler: (
890 request: HttpClientRequest.HttpClientRequest
891 ) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>,
892 apiUrl?: string
893 ) =>
894 OpenRouterClient.layer({
895 apiUrl,
896 apiKey: Redacted.make("or-test-key"),
897 siteReferrer: "https://example.com",
898 siteTitle: "Decision contract tests"
899 }).pipe(
900 Layer.provide(Layer.succeed(HttpClient.HttpClient, makeHttpClient(handler)))
901 )
902
903 const makeHttpClient = (
904 handler: (
905 request: HttpClientRequest.HttpClientRequest
906 ) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>
907 ) =>
908 HttpClient.makeWith(
909 Effect.fnUntraced(function*(requestEffect) {
910 const request = yield* requestEffect
911 return yield* handler(request)
912 }),
913 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
914 )
915
916 const jsonResponse = (
917 request: HttpClientRequest.HttpClientRequest,
918 body: unknown
919 ): HttpClientResponse.HttpClientResponse =>
920 HttpClientResponse.fromWeb(
921 request,
922 new Response(JSON.stringify(body), {
923 status: 200,
924 headers: {
925 "content-type": "application/json"
926 }
927 })
928 )
929
930 const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) {
931 const body = request.body
932 if (body._tag === "Uint8Array") {
933 const text = new TextDecoder().decode(body.body)
934 return JSON.parse(text)
935 }
936 return yield* Effect.die(new Error("Expected Uint8Array body"))
937 })