The best way to control Claude Code, Codex and more from any computer or phone

main
ts 138 lines 4.83 KB
Raw
1 import * as GCP from "@alchemy.run/gcp";
2 import { GcpEnvironment } from "@alchemy.run/gcp/Environment";
3 import { expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
6 import * as Effect from "effect/Effect";
7 import { MinimumLogLevel } from "effect/References";
8 import * as Schedule from "effect/Schedule";
9
10 const { test } = Test.make({ providers: GCP.providers() });
11
12 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
13
14 // Evaluators need an existing evaluation metric; a missing one fails with
15 // BadRequest "Request contains an invalid argument." Set
16 // GCP_TEST_AIPLATFORM_EVALUATION_METRIC to a metric resource name to run
17 // the lifecycle.
18 const evaluationMetric = process.env.GCP_TEST_AIPLATFORM_EVALUATION_METRIC;
19 const runLifecycle = !process.env.FAST && !!evaluationMetric;
20
21 const waitUntilGone = (name: string) =>
22 aiplatform.getProjectsLocationsOnlineEvaluators({ name }).pipe(
23 Effect.as("found" as const),
24 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
25 Effect.repeat({
26 schedule: Schedule.spaced("2 seconds"),
27 until: (status) => status === "gone",
28 times: 10,
29 }),
30 );
31
32 test.provider(
33 "getProjectsLocationsOnlineEvaluators on a missing evaluator fails with a typed tag",
34 (stack) =>
35 Effect.gen(function* () {
36 const { project } = yield* GcpEnvironment.current;
37 const parent = `projects/${project}/locations/us-central1`;
38 yield* stack.destroy();
39
40 const error = yield* Effect.flip(
41 aiplatform.getProjectsLocationsOnlineEvaluators({
42 name: `${parent}/onlineEvaluators/1234567890123456789`,
43 }),
44 );
45 expect(error._tag).toEqual("NotFound");
46 const page = yield* aiplatform.listProjectsLocationsOnlineEvaluators({
47 parent,
48 pageSize: 10,
49 });
50 expect((page.onlineEvaluators ?? []).map((item) => item.name)).not.toContain(
51 `${parent}/onlineEvaluators/1234567890123456789`,
52 );
53
54 yield* stack.destroy();
55 }).pipe(logLevel),
56 {
57 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
58 timeout: 90_000,
59 },
60 );
61
62 test.provider.skipIf(!runLifecycle)(
63 "create, update, and delete an online evaluator",
64 (stack) =>
65 Effect.gen(function* () {
66 yield* stack.destroy();
67
68 const created = yield* stack.deploy(
69 Effect.gen(function* () {
70 const engine = yield* GCP.AIPlatform.ReasoningEngine("Agent", {
71 location: "us-central1",
72 displayName: "alchemy-eval-engine",
73 description: "test",
74 labels: { env: "test" },
75 spec: { agentFramework: "custom" },
76 });
77 const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
78 location: "us-central1",
79 displayName: "alchemy-online-eval",
80 agentResource: engine.name,
81 metricSources: [
82 {
83 metricResourceName: evaluationMetric,
84 },
85 ],
86 config: { randomSampling: { percentage: 10 } },
87 });
88 return { engine, evaluator };
89 }),
90 );
91
92 expect(created.evaluator.name).toContain("/onlineEvaluators/");
93 expect(created.evaluator.location).toEqual("us-central1");
94 expect(created.evaluator.displayName).toEqual("alchemy-online-eval");
95
96 const fetched = yield* aiplatform.getProjectsLocationsOnlineEvaluators({
97 name: created.evaluator.name,
98 });
99 expect(fetched.name).toEqual(created.evaluator.name);
100
101 const updated = yield* stack.deploy(
102 Effect.gen(function* () {
103 const engine = yield* GCP.AIPlatform.ReasoningEngine("Agent", {
104 reasoningEngineId: created.engine.reasoningEngineId,
105 location: "us-central1",
106 displayName: "alchemy-eval-engine",
107 description: "test",
108 labels: { env: "test" },
109 spec: { agentFramework: "custom" },
110 });
111 const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
112 location: "us-central1",
113 displayName: "alchemy-online-eval-v2",
114 agentResource: engine.name,
115 metricSources: [
116 {
117 metricResourceName: evaluationMetric,
118 },
119 ],
120 config: { randomSampling: { percentage: 20 } },
121 });
122 return { engine, evaluator };
123 }),
124 );
125
126 expect(updated.evaluator.name).toEqual(created.evaluator.name);
127 expect(updated.evaluator.displayName).toEqual("alchemy-online-eval-v2");
128
129 yield* stack.destroy();
130
131 const gone = yield* waitUntilGone(created.evaluator.name);
132 expect(gone).toEqual("gone");
133 }).pipe(logLevel),
134 {
135 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
136 timeout: 180_000,
137 },
138 );