@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 263 lines 10.1 KB
Raw
1 import * as CloudflareClient from "@effect/ai-cloudflare/CloudflareClient"
2 import * as CloudflareDecisionModel from "@effect/ai-cloudflare/CloudflareDecisionModel"
3 import { assert, describe, it } from "@effect/vitest"
4 import { Effect, Layer, Redacted, Schema } from "effect"
5 import { Decision, DecisionModel, Model } from "effect/ai"
6 import * as HttpClient from "effect/http/HttpClient"
7 import type * as HttpClientError from "effect/http/HttpClientError"
8 import type * as HttpClientRequest from "effect/http/HttpClientRequest"
9 import * as HttpClientResponse from "effect/http/HttpClientResponse"
10
11 const Triage = Decision.make({
12 input: Schema.Struct({ message: Schema.String }),
13 decisions: {
14 team: Decision.classify({
15 instructions: "Which team should handle this request?",
16 criteria: { billing: "Payments", technical: "Outages" }
17 }),
18 severity: Decision.rate({
19 instructions: "How severe is the impact?",
20 criteria: ["Minor", "Major", "Critical"]
21 }),
22 urgent: Decision.probability({
23 instructions: "Does this need immediate attention?",
24 criteria: { false: "Can wait", true: "Act now" }
25 })
26 }
27 })
28
29 const input = { message: "Checkout is down" }
30
31 const answers = {
32 team: {
33 type: "choice",
34 choice: "technical",
35 confidence: 0.9,
36 probabilities: { billing: 0.1, technical: 0.9 }
37 },
38 severity: {
39 type: "score",
40 score: 1.75,
41 confidence: 0.8,
42 probabilities: { "0": 0.05, "1": 0.15, "2": 0.8 },
43 legend: { "0": "Minor", "1": "Major", "2": "Critical" }
44 },
45 urgent: { type: "noul", noul: 0.99 }
46 }
47
48 const clientLayer = (
49 answers: unknown,
50 onRequest: (request: HttpClientRequest.HttpClientRequest) => void = () => {},
51 model = "clef"
52 ) =>
53 CloudflareClient.layer({ accountId: "test-account", apiKey: Redacted.make("test-token") }).pipe(
54 Layer.provide(Layer.succeed(
55 HttpClient.HttpClient,
56 HttpClient.makeWith(
57 Effect.map((request) => {
58 onRequest(request)
59 return HttpClientResponse.fromWeb(
60 request,
61 Response.json({
62 success: true,
63 result: { model, answers, usage: { input_tokens: 120, output_tokens: 0 } }
64 })
65 )
66 }),
67 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
68 )
69 ))
70 )
71
72 const layer = (answers: unknown) =>
73 CloudflareDecisionModel.layer({ model: "clef" }).pipe(Layer.provide(clientLayer(answers)))
74
75 const Structured = Decision.make({
76 input: Schema.String,
77 decisions: {
78 severity: Decision.rate({
79 instructions: ["How severe?"],
80 criteria: ["Minor", { label: "Critical", description: { impact: "outage" } }]
81 })
82 }
83 })
84
85 describe("CloudflareDecisionModel", () => {
86 it.effect("sends every decision as a question and maps the answers", () =>
87 Effect.gen(function*() {
88 const result = yield* DecisionModel.decide(Triage, { input })
89 assert.strictEqual(yield* Model.ProviderName, "cloudflare")
90 assert.strictEqual(yield* Model.ModelName, "clef")
91 assert.strictEqual(result.answers.team.label, "technical")
92 assert.deepStrictEqual({ ...result.answers.team.probabilities }, { billing: 0.1, technical: 0.9 })
93 assert.strictEqual(result.answers.team.confidence, 0.9)
94 assert.strictEqual(result.answers.severity.label, "Critical")
95 assert.strictEqual(result.answers.severity.rating, 1.75)
96 assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.05, Major: 0.15, Critical: 0.8 })
97 assert.strictEqual(result.answers.urgent.probability, 0.99)
98 assert.strictEqual(result.usage.inputTokens, 120)
99 assert.strictEqual(result.usage.outputTokens, 0)
100 }).pipe(
101 Effect.provide(CloudflareDecisionModel.model("clef")),
102 Effect.provide(clientLayer(answers, (request) => {
103 assert.strictEqual(request.body._tag, "Uint8Array")
104 if (request.body._tag === "Uint8Array") {
105 assert.deepStrictEqual(JSON.parse(new TextDecoder().decode(request.body.body)), {
106 model: "clef",
107 state: input,
108 questions: {
109 team: {
110 type: "choice",
111 instructions: Triage.decisions.team.instructions,
112 criteria: Triage.decisions.team.criteria
113 },
114 severity: {
115 type: "score",
116 instructions: Triage.decisions.severity.instructions,
117 criteria: ["Minor", "Major", "Critical"]
118 },
119 urgent: {
120 type: "noul",
121 instructions: Triage.decisions.urgent.instructions,
122 criteria: Triage.decisions.urgent.criteria
123 }
124 }
125 })
126 }
127 }))
128 ))
129
130 it.effect("sends an arbitrary model identifier through the decision request", () => {
131 const model = "clef-next"
132 return Effect.gen(function*() {
133 const result = yield* DecisionModel.decide(Triage, { input })
134 assert.strictEqual(yield* Model.ModelName, model)
135 assert.strictEqual(result.answers.urgent.probability, 0.99)
136 }).pipe(
137 Effect.provide(CloudflareDecisionModel.model(model)),
138 Effect.provide(clientLayer(answers, (request) => {
139 assert.strictEqual(
140 request.url,
141 `https://api.cloudflare.com/client/v4/accounts/test-account/ai/run/@cf/cloudflare/${model}`
142 )
143 assert.strictEqual(request.body._tag, "Uint8Array")
144 if (request.body._tag === "Uint8Array") {
145 assert.strictEqual(JSON.parse(new TextDecoder().decode(request.body.body)).model, model)
146 }
147 }, model))
148 )
149 })
150
151 it.effect("accepts probabilities with four-decimal rounding drift", () =>
152 Effect.gen(function*() {
153 const result = yield* DecisionModel.decide(Triage, { input })
154 assert.strictEqual(result.answers.severity.label, "Minor")
155 assert.closeTo(result.answers.severity.probabilities.Minor, 1 / 3, 1e-10)
156 }).pipe(Effect.provide(layer({
157 ...answers,
158 severity: {
159 ...answers.severity,
160 score: 1,
161 confidence: 0.3333,
162 probabilities: { "0": 0.3333, "1": 0.3333, "2": 0.3333 }
163 }
164 }))))
165
166 for (
167 const [name, invalid] of [
168 ["a missing answer", { team: answers.team, severity: answers.severity }],
169 ["a skipped rating level", {
170 ...answers,
171 severity: { ...answers.severity, probabilities: { "0": 0.2, "2": 0.8 } }
172 }],
173 ["drift beyond four decimals", {
174 ...answers,
175 team: { ...answers.team, probabilities: { billing: 0.1, technical: 0.89 } }
176 }]
177 ] as const
178 ) {
179 it.effect(`fails with InvalidOutputError for ${name}`, () =>
180 Effect.gen(function*() {
181 const error = yield* Effect.flip(DecisionModel.decide(Triage, { input }))
182 assert.strictEqual(error.reason._tag, "InvalidOutputError")
183 }).pipe(Effect.provide(layer(invalid))))
184 }
185
186 it.effect("sends byte arrays, base64 strings, and data URLs as images listed by reference in the state", () => {
187 const dataUrl = "data:image/webp;base64,UklGRg=="
188 const Screenshots = Decision.make({
189 input: Schema.Struct({ message: Schema.String, shots: Schema.Array(Decision.Image) }),
190 decisions: Triage.decisions
191 })
192 const shots = [
193 { mediaType: "image/png", data: new Uint8Array([137, 80, 78, 71]) },
194 { mediaType: "image/jpeg", data: "/9j/4AAQ" },
195 { mediaType: "image/webp", data: new URL(dataUrl) }
196 ]
197 return Effect.gen(function*() {
198 const result = yield* DecisionModel.decide(Screenshots, { input: { ...input, shots } })
199 assert.strictEqual(result.answers.urgent.probability, 0.99)
200 }).pipe(
201 Effect.provide(CloudflareDecisionModel.layer({ model: "clef" })),
202 Effect.provide(clientLayer(answers, (request) => {
203 assert.strictEqual(request.body._tag, "Uint8Array")
204 if (request.body._tag === "Uint8Array") {
205 const body = JSON.parse(new TextDecoder().decode(request.body.body))
206 const [png, jpeg, webp] = JSON.parse(body.state.slice(body.state.indexOf("{"))).shots
207 assert.strictEqual(
208 body.state,
209 `Image 1: ${png}\nImage 2: ${jpeg}\nImage 3: ${webp}\n\n` +
210 JSON.stringify({ ...input, shots: [png, jpeg, webp] })
211 )
212 assert.deepStrictEqual(body.images, [
213 { content_type: "image/png", base64: "iVBORw==" },
214 { content_type: "image/jpeg", base64: "/9j/4AAQ" },
215 dataUrl
216 ])
217 }
218 }))
219 )
220 })
221
222 it.effect("rejects remote image URLs without sending a request", () => {
223 let requests = 0
224 const Photo = Decision.make({ input: Schema.Struct({ photo: Decision.Image }), decisions: Triage.decisions })
225 return Effect.gen(function*() {
226 const error = yield* Effect.flip(DecisionModel.decide(Photo, {
227 input: { photo: { mediaType: "image/png", data: new URL("https://example.com/cat.png") } }
228 }))
229 assert.strictEqual(error.reason._tag, "InvalidUserInputError")
230 assert.strictEqual(requests, 0)
231 }).pipe(
232 Effect.provide(CloudflareDecisionModel.layer({ model: "clef" })),
233 Effect.provide(clientLayer(answers, () => {
234 requests++
235 }))
236 )
237 })
238
239 it.effect("sends structured descriptions and maps score indices to labels", () =>
240 Effect.gen(function*() {
241 const result = yield* DecisionModel.decide(Structured, { input: "Checkout is down" })
242 assert.strictEqual(result.answers.severity.label, "Critical")
243 assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.1, Critical: 0.9 })
244 }).pipe(
245 Effect.provide(CloudflareDecisionModel.layer({ model: "clef" })),
246 Effect.provide(clientLayer({
247 severity: {
248 type: "score",
249 score: 0.9,
250 probabilities: { "0": 0.1, "1": 0.9 },
251 confidence: 0.9,
252 legend: { "0": "Minor", "1": { impact: "outage" } }
253 }
254 }, (request) => {
255 assert.strictEqual(request.body._tag, "Uint8Array")
256 if (request.body._tag === "Uint8Array") {
257 assert.deepStrictEqual(JSON.parse(new TextDecoder().decode(request.body.body)).questions, {
258 severity: { type: "score", instructions: ["How severe?"], criteria: ["Minor", { impact: "outage" }] }
259 })
260 }
261 }))
262 ))
263 })