@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 343 lines 11.6 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { GcpEnvironment } from "../Environment.ts";
13 import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
14 import type { Providers } from "../Providers.ts";
15 import {
16 hasAlchemyLabelKeys,
17 isJobTerminal,
18 normalizeLocation,
19 parentOf,
20 parseResourceName,
21 rfc1035,
22 userLabels,
23 waitForOperation,
24 } from "./internal.ts";
25 import { listLocations } from "./names.ts";
26 import type { EncryptionSpec } from "./shared.ts";
27
28 export type ActiveLearningConfig = aiplatform.GoogleCloudAiplatformV1ActiveLearningConfig;
29
30 export type DataLabelingJobProps = {
31 /**
32 * Region. Immutable — changing it replaces the job.
33 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
34 */
35 location?: string;
36 /**
37 * User-facing display name. Defaults to a generated id. Immutable after
38 * create — DataLabelingJob has no update API.
39 */
40 displayName?: string;
41 /**
42 * Dataset resource names to label. Currently a single dataset.
43 * Format `projects/{project}/locations/{location}/datasets/{dataset}`.
44 */
45 datasets: string[];
46 /**
47 * Number of labelers per DataItem.
48 */
49 labelerCount: number;
50 /**
51 * GCS URI of the instruction PDF shared with labelers.
52 */
53 instructionUri: string;
54 /**
55 * GCS URI of the YAML schema describing the job config. Schema files
56 * live under
57 * `https://storage.googleapis.com/google-cloud-aiplatform/schema/datalabelingjob/inputs/`.
58 */
59 inputsSchemaUri: string;
60 /**
61 * Input config parameters for the job (schema-specific).
62 */
63 inputs: unknown;
64 /**
65 * Specialist pool resource names associated with this job.
66 */
67 specialistPools?: string[];
68 /**
69 * Labels applied to annotations created by this job.
70 */
71 annotationLabels?: Record<string, string>;
72 /**
73 * Active learning pipeline configuration.
74 */
75 activeLearningConfig?: ActiveLearningConfig;
76 /**
77 * User labels. Alchemy ownership labels are merged in automatically.
78 */
79 labels?: Record<string, string>;
80 /**
81 * Customer-managed encryption.
82 */
83 encryptionSpec?: EncryptionSpec;
84 };
85
86 export type DataLabelingJob = Resource<
87 "GCP.AIPlatform.DataLabelingJob",
88 DataLabelingJobProps,
89 {
90 /** Full resource name `.../dataLabelingJobs/{job}`. */
91 name: string;
92 /** Job id (last path segment). */
93 dataLabelingJobId: string;
94 /** Project id. */
95 project: string;
96 /** Location id. */
97 location: string;
98 /** Display name. */
99 displayName: string | undefined;
100 /** Dataset resource names. */
101 datasets: string[];
102 /** Labelers per DataItem. */
103 labelerCount: number | undefined;
104 /** Instruction PDF URI. */
105 instructionUri: string | undefined;
106 /** Inputs schema URI. */
107 inputsSchemaUri: string | undefined;
108 /** Detailed job state. */
109 state: string | undefined;
110 /** Labeling progress in `[0, 100]`. */
111 labelingProgress: number | undefined;
112 /** User labels (Alchemy ownership labels stripped). */
113 labels: Record<string, string>;
114 /** Customer-managed KMS key, if any. */
115 kmsKeyName: string | undefined;
116 /** RFC3339 creation timestamp. */
117 createTime: string | undefined;
118 /** RFC3339 last-update timestamp. */
119 updateTime: string | undefined;
120 /** Failure/cancel status, if any. */
121 error: { code?: number; message?: string } | undefined;
122 },
123 never,
124 Providers
125 >;
126
127 /**
128 * A Vertex AI DataLabelingJob — human labeling of Dataset DataItems.
129 *
130 * Creating a job starts labeling immediately. There is no update API, so
131 * reconcile is observe-ensure. Delete cancels a running job, then deletes
132 * it. Alchemy ownership labels are merged into `labels` so `list` / nuke
133 * can find the job.
134 *
135 * ### Creating a Data Labeling Job
136 * **Example:** Single-label image classification
137 * ```typescript
138 * const job = yield* GCP.AIPlatform.DataLabelingJob("Label", {
139 * datasets: [dataset.name],
140 * displayName: "label-images",
141 * labelerCount: 1,
142 * instructionUri: "gs://bucket/instructions.pdf",
143 * inputsSchemaUri:
144 * "gs://google-cloud-aiplatform/schema/datalabelingjob/inputs/image_classification_1.0.0.yaml",
145 * inputs: { annotationSpecs: ["cat", "dog"] },
146 * });
147 * ```
148 *
149 * @resource
150 * @category AIPlatform
151 */
152 export const DataLabelingJob = Resource<DataLabelingJob>("GCP.AIPlatform.DataLabelingJob");
153
154 export class DataLabelingJobNotResolved extends Data.TaggedError(
155 "GCP.AIPlatform.DataLabelingJobNotResolved",
156 )<{
157 resourceName: string;
158 }> {}
159
160 const toId = (id: string, existing?: string) =>
161 Effect.gen(function* () {
162 if (existing !== undefined) return existing;
163 return rfc1035(
164 yield* createPhysicalName({
165 id,
166 maxLength: 63,
167 lowercase: true,
168 }),
169 );
170 });
171
172 const toAttrs = (job: aiplatform.GoogleCloudAiplatformV1DataLabelingJob, project: string) => {
173 const name = job.name ?? "";
174 const parsed = parseResourceName(name, "dataLabelingJobs");
175 return {
176 name,
177 dataLabelingJobId: parsed.id,
178 project: parsed.project || project,
179 location: parsed.location,
180 displayName: job.displayName,
181 datasets: job.datasets ?? [],
182 labelerCount: job.labelerCount,
183 instructionUri: job.instructionUri,
184 inputsSchemaUri: job.inputsSchemaUri,
185 state: job.state,
186 labelingProgress: job.labelingProgress,
187 labels: userLabels(job.labels),
188 kmsKeyName: job.encryptionSpec?.kmsKeyName,
189 createTime: job.createTime,
190 updateTime: job.updateTime,
191 error: job.error ? { code: job.error.code, message: job.error.message } : undefined,
192 };
193 };
194
195 const getByName = (name: string) =>
196 aiplatform
197 .getProjectsLocationsDataLabelingJobs({ name })
198 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
199
200 const listJobs = (project: string, region: string) => {
201 const collect = (parent: string) =>
202 aiplatform.listProjectsLocationsDataLabelingJobs.pages({ parent, pageSize: 100 }).pipe(
203 Stream.flatMap((page) => Stream.fromIterable(page.dataLabelingJobs ?? [])),
204 Stream.filter((job) => hasAlchemyLabelKeys(job.labels)),
205 Stream.runCollect,
206 Effect.map((chunk) => Array.from(chunk)),
207 );
208 const fallback = Effect.forEach(listLocations(region), (location) =>
209 collect(`projects/${project}/locations/${location}`),
210 ).pipe(Effect.map((pages) => pages.flat()));
211 // Vertex AI has no `locations/-` wildcard; scan known locations.
212 return fallback.pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
213 };
214
215 const findOwned = (id: string, project: string, region: string, hinted?: string) =>
216 Effect.gen(function* () {
217 if (hinted !== undefined && hinted.length > 0) {
218 const existing = yield* getByName(hinted);
219 if (existing !== undefined) return existing;
220 }
221 const jobs = yield* listJobs(project, region);
222 for (const job of jobs) {
223 if (yield* hasAlchemyLabels(id, tagRecord(job.labels))) return job;
224 }
225 return undefined as aiplatform.GoogleCloudAiplatformV1DataLabelingJob | undefined;
226 });
227
228 const cancelAndDelete = (name: string) =>
229 Effect.gen(function* () {
230 const existing = yield* getByName(name);
231 if (existing === undefined) return;
232 if (!isJobTerminal(existing.state)) {
233 yield* aiplatform
234 .cancelProjectsLocationsDataLabelingJobs({ name })
235 .pipe(Effect.catchTag("NotFound", () => Effect.void));
236 yield* getByName(name).pipe(
237 Effect.filterOrFail(
238 (job) => job === undefined || isJobTerminal(job.state),
239 () => new DataLabelingJobNotResolved({ resourceName: name }),
240 ),
241 Effect.retry({
242 while: (error) => error._tag === "GCP.AIPlatform.DataLabelingJobNotResolved",
243 times: 8,
244 schedule: Schedule.spaced("4 seconds"),
245 }),
246 Effect.catchTag("GCP.AIPlatform.DataLabelingJobNotResolved", () => Effect.void),
247 );
248 }
249 const operation = yield* aiplatform.deleteProjectsLocationsDataLabelingJobs({ name }).pipe(
250 Effect.retry({
251 while: (error) => error._tag === "Conflict",
252 times: 8,
253 schedule: Schedule.spaced("3 seconds"),
254 }),
255 Effect.catchTags({
256 NotFound: () => Effect.undefined,
257 Conflict: () => Effect.undefined,
258 }),
259 );
260 if (operation !== undefined) {
261 yield* waitForOperation(operation, { notFoundOk: true });
262 }
263 });
264
265 export const DataLabelingJobProvider = () =>
266 Provider.succeed(DataLabelingJob, {
267 stables: ["name", "dataLabelingJobId", "project", "location", "createTime"],
268
269 diff: Effect.fn(function* ({ news, olds, output }) {
270 const env = yield* GcpEnvironment.current;
271 if (!isResolved(news)) return undefined;
272 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
273 const nextLocation = normalizeLocation(
274 news.location ?? olds?.location ?? output?.location,
275 env.region,
276 );
277 if (previousLocation !== nextLocation) {
278 return { action: "replace" as const, deleteFirst: false };
279 }
280 return undefined;
281 }),
282
283 read: Effect.fn(function* ({ id, output }) {
284 const env = yield* GcpEnvironment.current;
285 const existing = yield* findOwned(id, env.project, env.region, output?.name);
286 if (existing === undefined) return undefined;
287 const attrs = toAttrs(existing, env.project);
288 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
289 }),
290
291 list: () =>
292 Effect.gen(function* () {
293 const env = yield* GcpEnvironment.current;
294 const jobs = yield* listJobs(env.project, env.region);
295 return jobs.map((job) => toAttrs(job, env.project));
296 }),
297
298 reconcile: Effect.fn(function* ({ id, news, output }) {
299 const env = yield* GcpEnvironment.current;
300 const location = normalizeLocation(news.location ?? output?.location, env.region);
301 const displayName = news.displayName ?? (yield* toId(id, output?.dataLabelingJobId));
302 const desiredLabels = {
303 ...toLabels(news.labels),
304 ...(yield* createInternalLabels(id)),
305 };
306
307 let current = yield* findOwned(id, env.project, env.region, output?.name);
308
309 if (current === undefined) {
310 const created = yield* aiplatform
311 .createProjectsLocationsDataLabelingJobs({
312 parent: parentOf(env.project, location),
313 body: {
314 displayName,
315 datasets: news.datasets,
316 labelerCount: news.labelerCount,
317 instructionUri: news.instructionUri,
318 inputsSchemaUri: news.inputsSchemaUri,
319 inputs: news.inputs,
320 specialistPools: news.specialistPools,
321 annotationLabels: news.annotationLabels,
322 activeLearningConfig: news.activeLearningConfig,
323 labels: desiredLabels,
324 encryptionSpec: news.encryptionSpec,
325 },
326 })
327 .pipe(Effect.catchTag("Conflict", () => findOwned(id, env.project, env.region)));
328 current = created ?? undefined;
329 }
330
331 if (current === undefined) {
332 return yield* new DataLabelingJobNotResolved({
333 resourceName: output?.name ?? parentOf(env.project, location),
334 });
335 }
336
337 return toAttrs(current, env.project);
338 }),
339
340 delete: Effect.fn(function* ({ output }) {
341 yield* cancelAndDelete(output.name);
342 }),
343 });