The best way to control Claude Code, Codex and more from any computer or phone

main
ts 127 lines 4.43 KB
Raw
1 import * as GCP from "@alchemy.run/gcp";
2 import { GcpEnvironment } from "@alchemy.run/gcp/Environment";
3 import { expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
6 import * as Effect from "effect/Effect";
7 import { MinimumLogLevel } from "effect/References";
8 import * as Schedule from "effect/Schedule";
9
10 const { test } = Test.make({ providers: GCP.providers() });
11
12 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
13
14 const waitUntilGone = (name: string) =>
15 aiplatform.getProjectsLocationsEvaluationSets({ name }).pipe(
16 Effect.as("found" as const),
17 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
18 Effect.repeat({
19 schedule: Schedule.spaced("2 seconds"),
20 until: (status) => status === "gone",
21 times: 10,
22 }),
23 );
24
25 test.provider(
26 "getProjectsLocationsEvaluationSets on a missing set fails with a typed tag",
27 (stack) =>
28 Effect.gen(function* () {
29 const { project } = yield* GcpEnvironment.current;
30 const parent = `projects/${project}/locations/us-central1`;
31 yield* stack.destroy();
32
33 const error = yield* Effect.flip(
34 aiplatform.getProjectsLocationsEvaluationSets({
35 name: `${parent}/evaluationSets/1234567890123456789`,
36 }),
37 );
38 expect(error._tag).toEqual("NotFound");
39 const page = yield* aiplatform.listProjectsLocationsEvaluationSets({
40 parent,
41 pageSize: 10,
42 });
43 expect((page.evaluationSets ?? []).map((item) => item.name)).not.toContain(
44 `${parent}/evaluationSets/1234567890123456789`,
45 );
46
47 yield* stack.destroy();
48 }).pipe(logLevel),
49 {
50 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
51 timeout: 90_000,
52 },
53 );
54
55 test.provider(
56 "create, update, and delete a vertex evaluation set",
57 (stack) =>
58 Effect.gen(function* () {
59 yield* stack.destroy();
60
61 const created = yield* stack.deploy(
62 Effect.gen(function* () {
63 const item = yield* GCP.AIPlatform.EvaluationItem("Prompt", {
64 location: "us-central1",
65 displayName: "alchemy-eval-set-item",
66 evaluationItemType: "REQUEST",
67 evaluationRequest: { prompt: { text: "What is 2+2?" } },
68 labels: { env: "test" },
69 });
70 const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
71 location: "us-central1",
72 displayName: "alchemy-eval-set",
73 evaluationItems: [item.name],
74 });
75 return { item, set };
76 }),
77 );
78
79 expect(created.set.name).toContain("/evaluationSets/");
80 expect(created.set.location).toEqual("us-central1");
81 expect(created.set.evaluationItems).toContain(created.item.name);
82
83 const fetched = yield* aiplatform.getProjectsLocationsEvaluationSets({
84 name: created.set.name,
85 });
86 expect(fetched.name).toEqual(created.set.name);
87
88 const updated = yield* stack.deploy(
89 Effect.gen(function* () {
90 const item = yield* GCP.AIPlatform.EvaluationItem("Prompt", {
91 evaluationItemId: created.item.evaluationItemId,
92 location: "us-central1",
93 displayName: "alchemy-eval-set-item",
94 evaluationItemType: "REQUEST",
95 evaluationRequest: { prompt: { text: "What is 2+2?" } },
96 labels: { env: "test" },
97 });
98 const second = yield* GCP.AIPlatform.EvaluationItem("Followup", {
99 location: "us-central1",
100 displayName: "alchemy-eval-set-item-2",
101 evaluationItemType: "REQUEST",
102 evaluationRequest: { prompt: { text: "What is 3+3?" } },
103 labels: { env: "test" },
104 });
105 const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
106 evaluationSetId: created.set.evaluationSetId,
107 location: "us-central1",
108 displayName: "alchemy-eval-set",
109 evaluationItems: [item.name, second.name],
110 });
111 return { item, second, set };
112 }),
113 );
114
115 expect(updated.set.name).toEqual(created.set.name);
116 expect(updated.set.evaluationItems).toContain(updated.second.name);
117
118 yield* stack.destroy();
119
120 const gone = yield* waitUntilGone(created.set.name);
121 expect(gone).toEqual("gone");
122 }).pipe(logLevel),
123 {
124 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
125 timeout: 180_000,
126 },
127 );