The best way to control Claude Code, Codex and more from any computer or phone

main
ts 405 lines 13.8 KB
Raw
1 import * as datalabeling from "@distilled.cloud/gcp/datalabeling_v1beta1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import { GcpEnvironment } from "../Environment.ts";
9 import { createInternalLabels } from "../Labels.ts";
10 import type { Providers } from "../Providers.ts";
11 import {
12 encodeOwnership,
13 findOwned,
14 hasOwnershipMarker,
15 ignoreGone,
16 listEvaluationJobs,
17 MAX_EVALUATION_DESCRIPTION_LENGTH,
18 noRetryLayer,
19 ownedByAlchemy,
20 parseOwnership,
21 parseResourceName,
22 projectParent,
23 replaceOnIdentity,
24 retryDelete,
25 retryTransient,
26 sameJson,
27 sameText,
28 updateMaskOf,
29 waitUntilGone,
30 } from "./internal.ts";
31
32 export type EvaluationJobConfig = datalabeling.GoogleCloudDatalabelingV1beta1EvaluationJobConfig;
33
34 export type EvaluationJobProps = {
35 /**
36 * Evaluation job id (the last segment of
37 * `projects/{project}/evaluationJobs/{evaluation_job}`).
38 * Server-assigned on create. Immutable — changing it replaces the
39 * job.
40 */
41 evaluationJobId?: string;
42 /**
43 * Description of the job. Maximum 25,000 characters. Evaluation jobs
44 * have no labels field, so Alchemy stamps ownership into this field.
45 * Immutable — changing it replaces the job.
46 */
47 description?: string;
48 /**
49 * Annotation spec set resource name describing the labels the model
50 * outputs. Immutable — changing it replaces the job.
51 */
52 annotationSpecSet: string;
53 /**
54 * AI Platform Prediction model version sampled for evaluation, as
55 * `projects/{project}/models/{model}/versions/{version}`. Immutable —
56 * changing it replaces the job.
57 */
58 modelVersion: string;
59 /**
60 * Run interval, in crontab or English-like format. Must be at least
61 * one day. Immutable — changing it replaces the job.
62 */
63 schedule: string;
64 /**
65 * When true, Data Labeling assigns human labelers for missing ground
66 * truth. When false, ground truth is read from the job's BigQuery
67 * table.
68 * @default false
69 */
70 labelMissingGroundTruth?: boolean;
71 /**
72 * Configuration for sampling, import keys, human annotation, and
73 * metrics. Only `humanAnnotationConfig.instruction`, `exampleCount`,
74 * and `exampleSamplePercentage` update in place; any other change
75 * replaces the job.
76 */
77 evaluationJobConfig: EvaluationJobConfig;
78 /**
79 * When true, pause the job after ensure. Pausing a paused job is a
80 * no-op; resuming a running or scheduled job is a no-op.
81 * @default false
82 */
83 paused?: boolean;
84 };
85
86 export type EvaluationJob = Resource<
87 "GCP.DataLabeling.EvaluationJob",
88 EvaluationJobProps,
89 {
90 /** Full resource name `projects/{project}/evaluationJobs/{job}`. */
91 name: string;
92 /** Evaluation job id (last path segment). */
93 evaluationJobId: string;
94 /** Project id. */
95 project: string;
96 /** User description with the Alchemy ownership prefix stripped. */
97 description: string | undefined;
98 /** Annotation spec set resource name. */
99 annotationSpecSet: string | undefined;
100 /** Model version sampled for evaluation. */
101 modelVersion: string | undefined;
102 /** Run interval. */
103 schedule: string | undefined;
104 /** Whether the service provides ground-truth labels. */
105 labelMissingGroundTruth: boolean;
106 /** Job configuration. */
107 evaluationJobConfig: EvaluationJobConfig | undefined;
108 /** Server-reported job state. */
109 state: string | undefined;
110 /** Whether the job is paused. */
111 paused: boolean;
112 /** Failed attempts, if any. */
113 attempts: datalabeling.GoogleCloudDatalabelingV1beta1AttemptList | undefined;
114 /** RFC3339 creation timestamp. */
115 createTime: string | undefined;
116 },
117 never,
118 Providers
119 >;
120
121 /**
122 * A Data Labeling evaluation job that periodically samples predictions
123 * from an AI Platform model version and scores them.
124 *
125 * Patch can only update `evaluationJobConfig.humanAnnotationConfig.instruction`,
126 * `evaluationJobConfig.exampleCount`, and
127 * `evaluationJobConfig.exampleSamplePercentage`. Pause and resume are
128 * separate RPCs. Every other field is identity — changing it replaces
129 * the job. There is no labels API, so Alchemy stamps ownership into
130 * `description`.
131 *
132 * ### Creating an Evaluation Job
133 * **Example:** Daily image classification eval
134 * ```typescript
135 * const evaluationJobConfig = {
136 * exampleCount: 100,
137 * exampleSamplePercentage: 0.1,
138 * evaluationConfig: {},
139 * inputConfig: {
140 * dataType: "IMAGE",
141 * annotationType: "IMAGE_CLASSIFICATION_ANNOTATION",
142 * classificationMetadata: { isMultiLabel: false },
143 * bigquerySource: {
144 * inputUri: "bq://my-project.eval.predictions",
145 * },
146 * },
147 * bigqueryImportKeys: {
148 * data_json_key: "data",
149 * label_json_key: "label",
150 * label_score_json_key: "score",
151 * },
152 * imageClassificationConfig: {
153 * annotationSpecSet: specs.name,
154 * },
155 * };
156 * const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
157 * annotationSpecSet: specs.name,
158 * modelVersion:
159 * "projects/my-project/models/classifier/versions/v1",
160 * schedule: "every 24 hours",
161 * evaluationJobConfig,
162 * });
163 * ```
164 *
165 * ### Pausing an Evaluation Job
166 * **Example:** Pause sampling
167 * ```typescript
168 * // Same logical id, changed props: the engine updates it in place.
169 * const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
170 * annotationSpecSet: specs.name,
171 * modelVersion:
172 * "projects/my-project/models/classifier/versions/v1",
173 * schedule: "every 24 hours",
174 * evaluationJobConfig,
175 * paused: true,
176 * });
177 * ```
178 *
179 * @resource
180 * @category DataLabeling
181 */
182 export const EvaluationJob = Resource<EvaluationJob>("GCP.DataLabeling.EvaluationJob");
183
184 export class EvaluationJobNotResolved extends Data.TaggedError(
185 "GCP.DataLabeling.EvaluationJobNotResolved",
186 )<{
187 resourceName: string;
188 }> {}
189
190 const resourceName = (project: string, evaluationJobId: string) =>
191 `${projectParent(project)}/evaluationJobs/${evaluationJobId}`;
192
193 const isPaused = (state: string | undefined) => state === "PAUSED";
194
195 const configIdentity = (config: EvaluationJobConfig | undefined) => {
196 if (config === undefined) return undefined;
197 const {
198 exampleCount: _exampleCount,
199 exampleSamplePercentage: _exampleSamplePercentage,
200 humanAnnotationConfig,
201 ...rest
202 } = config;
203 const { instruction: _instruction, ...humanRest } = humanAnnotationConfig ?? {};
204 return {
205 ...rest,
206 humanAnnotationConfig: Object.keys(humanRest).length > 0 ? humanRest : undefined,
207 };
208 };
209
210 const toAttrs = (
211 job: datalabeling.GoogleCloudDatalabelingV1beta1EvaluationJob,
212 project: string,
213 ) => {
214 const name = job.name ?? "";
215 const parsed = parseResourceName(name, "evaluationJobs");
216 return {
217 name,
218 evaluationJobId: parsed.id,
219 project: parsed.project || project,
220 description: parseOwnership(job.description).text,
221 annotationSpecSet: job.annotationSpecSet,
222 modelVersion: job.modelVersion,
223 schedule: job.schedule,
224 labelMissingGroundTruth: job.labelMissingGroundTruth === true,
225 evaluationJobConfig: job.evaluationJobConfig,
226 state: job.state,
227 paused: isPaused(job.state),
228 attempts: job.attempts,
229 createTime: job.createTime,
230 };
231 };
232
233 const getByName = (name: string) =>
234 name.length === 0
235 ? Effect.undefined
236 : datalabeling.getProjectsEvaluationJobs({ name }).pipe(
237 Effect.provide(noRetryLayer),
238 Effect.catchTags({
239 NotFound: () => Effect.undefined,
240 BadGateway: () => Effect.undefined,
241 }),
242 );
243
244 const findByOwnership = (id: string, project: string) =>
245 Effect.gen(function* () {
246 const rows = yield* listEvaluationJobs(projectParent(project));
247 return yield* findOwned(id, rows, (row) => row.description);
248 });
249
250 export const EvaluationJobProvider = () =>
251 Provider.succeed(EvaluationJob, {
252 stables: ["name", "evaluationJobId", "project", "createTime"],
253
254 diff: Effect.fn(function* ({ news, olds, output }) {
255 if (!isResolved(news)) return undefined;
256 const extra =
257 (output?.annotationSpecSet !== undefined &&
258 !sameText(news.annotationSpecSet, output.annotationSpecSet)) ||
259 (output?.modelVersion !== undefined && !sameText(news.modelVersion, output.modelVersion)) ||
260 (output?.schedule !== undefined && !sameText(news.schedule, output.schedule)) ||
261 (output !== undefined &&
262 (news.labelMissingGroundTruth === true) !== output.labelMissingGroundTruth) ||
263 (olds !== undefined && !sameText(news.description, output?.description)) ||
264 (output !== undefined &&
265 !sameJson(
266 configIdentity(news.evaluationJobConfig),
267 configIdentity(output.evaluationJobConfig),
268 ));
269 return replaceOnIdentity({
270 previousId: olds?.evaluationJobId ?? output?.evaluationJobId,
271 nextId: news.evaluationJobId,
272 extra,
273 });
274 }),
275
276 read: Effect.fn(function* ({ id, olds, output }) {
277 const env = yield* GcpEnvironment.current;
278 const evaluationJobId =
279 olds?.evaluationJobId ??
280 output?.evaluationJobId ??
281 (output?.name ? parseResourceName(output.name, "evaluationJobs").id : "");
282 const name =
283 output?.name ??
284 (evaluationJobId.length > 0 ? resourceName(env.project, evaluationJobId) : "");
285 const existing = (yield* getByName(name)) ?? (yield* findByOwnership(id, env.project));
286 if (existing === undefined) return undefined;
287 const attrs = toAttrs(existing, env.project);
288 return (yield* ownedByAlchemy(id, existing.description)) ? attrs : Unowned(attrs);
289 }),
290
291 list: () =>
292 Effect.gen(function* () {
293 const env = yield* GcpEnvironment.current;
294 const rows = yield* listEvaluationJobs(projectParent(env.project));
295 return rows
296 .filter((row) => hasOwnershipMarker(row.description))
297 .map((row) => toAttrs(row, env.project));
298 }),
299
300 reconcile: Effect.fn(function* ({ id, news, output }) {
301 const env = yield* GcpEnvironment.current;
302 const evaluationJobId = news.evaluationJobId ?? output?.evaluationJobId;
303 const name =
304 output?.name ??
305 (evaluationJobId !== undefined ? resourceName(env.project, evaluationJobId) : "");
306 const ownership = yield* createInternalLabels(id);
307 const description = encodeOwnership(
308 ownership,
309 news.description,
310 MAX_EVALUATION_DESCRIPTION_LENGTH,
311 );
312 const labelMissingGroundTruth = news.labelMissingGroundTruth === true;
313 const desiredPaused = news.paused === true;
314
315 let current = (yield* getByName(name)) ?? (yield* findByOwnership(id, env.project));
316
317 if (current === undefined) {
318 const created = yield* retryTransient(
319 datalabeling.createProjectsEvaluationJobs({
320 parent: projectParent(env.project),
321 body: {
322 job: {
323 description,
324 annotationSpecSet: news.annotationSpecSet,
325 modelVersion: news.modelVersion,
326 schedule: news.schedule,
327 labelMissingGroundTruth,
328 evaluationJobConfig: news.evaluationJobConfig,
329 },
330 },
331 }),
332 ).pipe(Effect.catchTag("Conflict", () => findByOwnership(id, env.project)));
333 current = created ?? undefined;
334 }
335
336 if (current === undefined) {
337 return yield* new EvaluationJobNotResolved({
338 resourceName: name || projectParent(env.project),
339 });
340 }
341
342 const currentName = current.name ?? name;
343 const observedConfig = current.evaluationJobConfig;
344 const desiredConfig = news.evaluationJobConfig;
345 const instructionChanged = !sameText(
346 observedConfig?.humanAnnotationConfig?.instruction,
347 desiredConfig.humanAnnotationConfig?.instruction,
348 );
349 const exampleCountChanged =
350 (observedConfig?.exampleCount ?? undefined) !== (desiredConfig.exampleCount ?? undefined);
351 const sampleChanged =
352 (observedConfig?.exampleSamplePercentage ?? undefined) !==
353 (desiredConfig.exampleSamplePercentage ?? undefined);
354
355 if (instructionChanged || exampleCountChanged || sampleChanged) {
356 current = yield* retryTransient(
357 datalabeling.patchProjectsEvaluationJobs({
358 name: currentName,
359 updateMask: updateMaskOf(
360 instructionChanged
361 ? "evaluationJobConfig.humanAnnotationConfig.instruction"
362 : undefined,
363 exampleCountChanged ? "evaluationJobConfig.exampleCount" : undefined,
364 sampleChanged ? "evaluationJobConfig.exampleSamplePercentage" : undefined,
365 ),
366 body: {
367 evaluationJobConfig: {
368 humanAnnotationConfig: desiredConfig.humanAnnotationConfig,
369 exampleCount: desiredConfig.exampleCount,
370 exampleSamplePercentage: desiredConfig.exampleSamplePercentage,
371 },
372 },
373 }),
374 );
375 }
376
377 if (desiredPaused && !isPaused(current.state)) {
378 yield* retryTransient(
379 datalabeling.pauseProjectsEvaluationJobs({
380 name: currentName,
381 body: {},
382 }),
383 );
384 current = (yield* getByName(currentName)) ?? current;
385 } else if (!desiredPaused && isPaused(current.state)) {
386 yield* retryTransient(
387 datalabeling.resumeProjectsEvaluationJobs({
388 name: currentName,
389 body: {},
390 }),
391 );
392 current = (yield* getByName(currentName)) ?? current;
393 }
394
395 return toAttrs(current, env.project);
396 }),
397
398 delete: Effect.fn(function* ({ output }) {
399 if (!output.name) return;
400 yield* datalabeling
401 .deleteProjectsEvaluationJobs({ name: output.name })
402 .pipe(retryDelete, ignoreGone);
403 yield* waitUntilGone(getByName(output.name));
404 }),
405 });