@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 603 lines 19.6 KB
Raw
1 import * as kafka from "@distilled.cloud/gcp/managedkafka_v1";
2 import { createPhysicalName } from "alchemy/PhysicalName";
3 import { tagRecord } from "alchemy/Tags";
4 import * as Data from "effect/Data";
5 import * as Effect from "effect/Effect";
6 import * as Schedule from "effect/Schedule";
7 import * as Stream from "effect/Stream";
8 import { isTransientGcpError } from "../Errors.ts";
9 import { alchemyLabelKeys, createInternalLabels, stripInternalLabels } from "../Labels.ts";
10 import { waitForOperation as waitForGcpOperation } from "../Operation.ts";
11
12 export const MAX_NAME_LENGTH = 63;
13 /** The API rejects longer cluster and Connect cluster ids. */
14 export const MAX_CLUSTER_ID_LENGTH = 53;
15 export const DEFAULT_VCPU_COUNT = 3;
16 export const DEFAULT_MEMORY_BYTES = 3_221_225_472;
17 export const OWNERSHIP_SUBJECT = "alchemy_ownership";
18
19 export class ResourceNotResolved extends Data.TaggedError("GCP.ManagedKafka.ResourceNotResolved")<{
20 resourceName: string;
21 }> {}
22
23 export class ResourceStillExists extends Data.TaggedError("GCP.ManagedKafka.ResourceStillExists")<{
24 resourceName: string;
25 }> {}
26
27 export class ResourceNotReady extends Data.TaggedError("GCP.ManagedKafka.ResourceNotReady")<{
28 resourceName: string;
29 state: string;
30 }> {}
31
32 export class ResourceFailed extends Data.TaggedError("GCP.ManagedKafka.ResourceFailed")<{
33 resourceName: string;
34 state: string;
35 }> {}
36
37 export const lastSegment = (value: string) => {
38 const trimmed = value.replace(/\/+$/, "");
39 const parts = trimmed.split("/");
40 return parts[parts.length - 1] || trimmed;
41 };
42
43 export const rfc1035 = (name: string, fallback = "kafka", maxLength = MAX_NAME_LENGTH): string => {
44 let next = name
45 .toLowerCase()
46 .replace(/[^a-z0-9-]/g, "-")
47 .replace(/-+/g, "-")
48 .replace(/^-+|-+$/g, "");
49 if (!/^[a-z]/.test(next)) next = `k${next}`;
50 next = next.slice(0, maxLength).replace(/-+$/g, "");
51 if (next.length === 0) return fallback;
52 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, maxLength - 1)}0`;
53 return next.slice(0, maxLength);
54 };
55
56 export const schemaRegistryIdOf = (name: string, fallback = "schema"): string => {
57 let next = name
58 .toLowerCase()
59 .replace(/[^a-z0-9_]/g, "_")
60 .replace(/_+/g, "_")
61 .replace(/^_|_$/g, "");
62 if (!/^[a-z]/.test(next)) next = `s${next}`;
63 next = next.slice(0, MAX_NAME_LENGTH).replace(/_+$/g, "");
64 return next.length > 0 ? next : fallback;
65 };
66
67 export const normalizeLocation = (location: string | undefined, defaultLocation: string) =>
68 lastSegment(location ?? defaultLocation).toLowerCase();
69
70 export const locationParent = (project: string, location: string) =>
71 `projects/${project}/locations/${location}`;
72
73 export const defaultSubnet = (project: string, location: string) =>
74 `projects/${project}/regions/${location}/subnetworks/default`;
75
76 export const toPhysicalId = (
77 id: string,
78 explicit: string | undefined,
79 existing: string | undefined,
80 fallback = "kafka",
81 maxLength = MAX_NAME_LENGTH,
82 ) =>
83 Effect.gen(function* () {
84 if (explicit !== undefined) return rfc1035(explicit, fallback, maxLength);
85 if (existing !== undefined) return existing;
86 return rfc1035(
87 yield* createPhysicalName({
88 id,
89 maxLength,
90 lowercase: true,
91 }),
92 fallback,
93 maxLength,
94 );
95 });
96
97 export const toSchemaRegistryId = (
98 id: string,
99 explicit: string | undefined,
100 existing: string | undefined,
101 ) =>
102 Effect.gen(function* () {
103 if (explicit !== undefined) return schemaRegistryIdOf(explicit);
104 if (existing !== undefined) return existing;
105 return schemaRegistryIdOf(
106 yield* createPhysicalName({
107 id,
108 maxLength: MAX_NAME_LENGTH,
109 lowercase: true,
110 delimiter: "_",
111 }),
112 );
113 });
114
115 export const toSubjectId = (
116 id: string,
117 explicit: string | undefined,
118 existing: string | undefined,
119 ) =>
120 Effect.gen(function* () {
121 if (explicit !== undefined) return explicit;
122 if (existing !== undefined) return existing;
123 return rfc1035(
124 yield* createPhysicalName({
125 id,
126 maxLength: MAX_NAME_LENGTH,
127 lowercase: true,
128 }),
129 "subject",
130 );
131 });
132
133 export const parseName = (name: string, collection: string) => {
134 const parts = name.split("/").filter((part) => part.length > 0);
135 const collectionAt = parts.lastIndexOf(collection);
136 const locationsAt = parts.lastIndexOf("locations");
137 const projectsAt = parts.lastIndexOf("projects");
138 return {
139 project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
140 // API resource names always carry a `locations/{location}` segment.
141 location: locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : "",
142 id:
143 collectionAt >= 0 && parts[collectionAt + 1]
144 ? parts.slice(collectionAt + 1).join("/")
145 : lastSegment(name),
146 parent:
147 collectionAt > 0
148 ? parts.slice(0, collectionAt).join("/")
149 : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
150 };
151 };
152
153 export const expandParent = (
154 value: string,
155 project: string,
156 location: string,
157 collection: string,
158 ) => {
159 if (value.includes("/")) return value.replace(/\/+$/, "");
160 return `projects/${project}/locations/${location}/${collection}/${value}`;
161 };
162
163 export const userLabels = (
164 labels: Record<string, string | undefined> | null | undefined,
165 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
166
167 export const hasAlchemyLabelMap = (labels: Record<string, string | undefined> | null | undefined) =>
168 Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));
169
170 export const stringMapOf = (
171 value: Record<string, string | undefined> | null | undefined,
172 ): Record<string, string> =>
173 Object.fromEntries(
174 Object.entries(value ?? {}).filter(
175 (entry): entry is [string, string] => entry[1] !== undefined,
176 ),
177 );
178
179 export const sameText = (left: string | undefined, right: string | undefined) =>
180 (left ?? "") === (right ?? "");
181
182 export const fingerprint = (value: unknown): string => JSON.stringify(value ?? null);
183
184 export const fieldMask = (fields: Array<string | false | undefined>) =>
185 fields.filter((field): field is string => typeof field === "string").join(",");
186
187 export const asCount = (value: number | string | undefined, fallback: number) => {
188 if (value === undefined) return fallback;
189 const parsed = typeof value === "number" ? value : Number(value);
190 return Number.isFinite(parsed) ? parsed : fallback;
191 };
192
193 export const asCountString = (value: number | string | undefined, fallback: number) =>
194 String(asCount(value, fallback));
195
196 export const replaceOnIdentity = (input: {
197 previousId?: string;
198 nextId?: string;
199 previousLocation?: string;
200 nextLocation?: string;
201 previousParent?: string;
202 nextParent?: string;
203 extra?: boolean;
204 }) => {
205 const parentChanged =
206 (input.previousParent ?? "") !== "" &&
207 (input.nextParent ?? "") !== "" &&
208 (input.previousParent ?? "") !== (input.nextParent ?? "");
209 const locationChanged =
210 (input.previousLocation ?? "") !== "" &&
211 (input.nextLocation ?? "") !== "" &&
212 (input.previousLocation ?? "") !== (input.nextLocation ?? "");
213 const idChanged =
214 input.previousId !== undefined &&
215 input.nextId !== undefined &&
216 input.previousId !== input.nextId;
217 if (!(input.extra === true || parentChanged || locationChanged || idChanged)) {
218 return undefined;
219 }
220 const samePhysical =
221 !parentChanged &&
222 !locationChanged &&
223 input.previousId !== undefined &&
224 input.nextId === input.previousId;
225 return {
226 action: "replace" as const,
227 deleteFirst: samePhysical,
228 };
229 };
230
231 const READY_STATES = new Set(["ACTIVE", "RUNNING"]);
232 const FAILED_STATES = new Set(["FAILED", "ERROR"]);
233
234 export const isReadyState = (state: string | undefined) =>
235 READY_STATES.has((state ?? "").toUpperCase());
236
237 export const isFailedState = (state: string | undefined) =>
238 FAILED_STATES.has((state ?? "").toUpperCase());
239
240 /**
241 * Wait for a Managed Kafka operation. Cluster creates take 20–30 minutes;
242 * Connect clusters are similar. An operation that finished with
243 * `ALREADY_EXISTS` (6) is a lost create race; `notFoundOk` also accepts
244 * `NOT_FOUND` (5).
245 */
246 export const waitForOperation = (operation: kafka.Operation, options?: { notFoundOk?: boolean }) =>
247 waitForGcpOperation(
248 operation,
249 (name) =>
250 kafka
251 .getProjectsLocationsOperations({ name })
252 .pipe(
253 Effect.catchTag("NotFound", (error) =>
254 options?.notFoundOk === true
255 ? Effect.succeed<kafka.Operation>({ name, done: true })
256 : Effect.fail(error),
257 ),
258 ),
259 { budget: "45 minutes" },
260 ).pipe(
261 Effect.catchIf(
262 (error) =>
263 error._tag === "GCP.OperationFailed" &&
264 (error.code === 6 || (options?.notFoundOk === true && error.code === 5)),
265 () => Effect.void,
266 ),
267 Effect.asVoid,
268 );
269
270 export const waitUntilExists = <A, E extends { readonly _tag: string }, R>(
271 get: Effect.Effect<A | undefined, E, R>,
272 name: string,
273 ) =>
274 get.pipe(
275 Effect.filterOrFail(
276 (value): value is A => value !== undefined,
277 () => new ResourceNotResolved({ resourceName: name }),
278 ),
279 Effect.retry({
280 while: (error) => error._tag === "GCP.ManagedKafka.ResourceNotResolved",
281 times: 8,
282 schedule: Schedule.spaced("1 second"),
283 }),
284 );
285
286 export const waitUntilGone = <A, E extends { readonly _tag: string }, R>(
287 get: Effect.Effect<A | undefined, E, R>,
288 name: string,
289 ) =>
290 get.pipe(
291 Effect.flatMap((value) =>
292 value === undefined
293 ? Effect.void
294 : Effect.fail(new ResourceStillExists({ resourceName: name })),
295 ),
296 Effect.retry({
297 while: (error) => error._tag === "GCP.ManagedKafka.ResourceStillExists",
298 times: 10,
299 schedule: Schedule.spaced("2 seconds"),
300 }),
301 );
302
303 export const waitUntilReady = <A, E extends { readonly _tag: string }, R>(
304 get: Effect.Effect<A | undefined, E, R>,
305 name: string,
306 stateOf: (value: A) => string | undefined,
307 ) =>
308 get.pipe(
309 Effect.filterOrFail(
310 (value): value is A => value !== undefined,
311 () => new ResourceNotResolved({ resourceName: name }),
312 ),
313 Effect.filterOrFail(
314 (value) => !isFailedState(stateOf(value) ?? ""),
315 (value) => new ResourceFailed({ resourceName: name, state: stateOf(value) ?? "" }),
316 ),
317 Effect.filterOrFail(
318 (value) => {
319 const state = stateOf(value) ?? "";
320 return isReadyState(state) || state.length === 0;
321 },
322 (value) => new ResourceNotReady({ resourceName: name, state: stateOf(value) ?? "" }),
323 ),
324 Effect.retry({
325 while: (error) =>
326 error._tag === "GCP.ManagedKafka.ResourceNotReady" ||
327 error._tag === "GCP.ManagedKafka.ResourceNotResolved",
328 times: 10,
329 schedule: Schedule.spaced("8 seconds"),
330 }),
331 );
332
333 export const collectPages = <Page, A, E, R>(
334 pages: Stream.Stream<Page, E, R>,
335 items: (page: Page) => readonly A[] | undefined,
336 ) =>
337 pages.pipe(
338 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
339 Stream.runCollect,
340 Effect.map((chunk) => Array.from(chunk)),
341 );
342
343 export const listAtLocation = <A, E, R>(
344 project: string,
345 _region: string,
346 list: (parent: string) => Effect.Effect<A[], E, R>,
347 ) => list(`projects/${project}/locations/-`);
348
349 export const listClusters = (project: string, region: string) =>
350 listAtLocation(project, region, (parent) =>
351 collectPages(
352 kafka.listProjectsLocationsClusters.pages({ parent, pageSize: 1000 }),
353 (page) => page.clusters,
354 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([] as kafka.Cluster[]))),
355 );
356
357 export const listAlchemyClusters = (project: string, region: string) =>
358 listClusters(project, region).pipe(
359 Effect.map((clusters: kafka.Cluster[]) =>
360 clusters.filter((cluster) => hasAlchemyLabelMap(cluster.labels)),
361 ),
362 );
363
364 export const listConnectClusters = (project: string, region: string) =>
365 listAtLocation(project, region, (parent) =>
366 collectPages(
367 kafka.listProjectsLocationsConnectClusters.pages({
368 parent,
369 pageSize: 1000,
370 }),
371 (page) => page.connectClusters,
372 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([] as kafka.ConnectCluster[]))),
373 );
374
375 export const listAlchemyConnectClusters = (project: string, region: string) =>
376 listConnectClusters(project, region).pipe(
377 Effect.map((clusters: kafka.ConnectCluster[]) =>
378 clusters.filter((cluster) => hasAlchemyLabelMap(cluster.labels)),
379 ),
380 );
381
382 export const listSchemaRegistries = (project: string, location: string) => {
383 // The `locations/-` wildcard hangs for Schema Registry; list the region.
384 const parents = [`projects/${project}/locations/${location}`];
385 return Effect.forEach(
386 parents,
387 (parent) =>
388 kafka
389 .listProjectsLocationsSchemaRegistries({
390 parent,
391 view: "SCHEMA_REGISTRY_VIEW_FULL",
392 })
393 .pipe(
394 Effect.map((page) => page.schemaRegistries ?? []),
395 Effect.catchTag(["NotFound", "SchemaRegistryRequiresCluster"], () =>
396 Effect.succeed([] as kafka.SchemaRegistry[]),
397 ),
398 ),
399 { concurrency: 1 },
400 ).pipe(
401 Effect.map((groups) => {
402 const seen = new Set<string>();
403 const out: kafka.SchemaRegistry[] = [];
404 for (const registry of groups.flat()) {
405 const name = registry.name ?? "";
406 if (name.length === 0 || seen.has(name)) continue;
407 seen.add(name);
408 out.push(registry);
409 }
410 return out;
411 }),
412 );
413 };
414
415 export const encodeOwnership = (labels: Record<string, string>) =>
416 `[alchemy ${alchemyLabelKeys.stack}=${labels[alchemyLabelKeys.stack]} ${alchemyLabelKeys.stage}=${labels[alchemyLabelKeys.stage]} ${alchemyLabelKeys.id}=${labels[alchemyLabelKeys.id]}]`;
417
418 export const parseOwnership = (text: string | undefined) => {
419 if (!text?.includes("[alchemy ")) return {} as Record<string, string>;
420 const start = text.indexOf("[alchemy ");
421 const end = text.indexOf("]", start);
422 if (end < 0) return {} as Record<string, string>;
423 const labels: Record<string, string> = {};
424 for (const part of text.slice(start + "[alchemy ".length, end).split(/\s+/)) {
425 const eq = part.indexOf("=");
426 if (eq > 0) labels[part.slice(0, eq)] = part.slice(eq + 1);
427 }
428 return labels;
429 };
430
431 export const hasOwnershipMarker = (text: string | undefined) =>
432 Object.keys(parseOwnership(text)).some((key) => key.startsWith("alchemy-"));
433
434 export const ownershipSchema = (labels: Record<string, string>) =>
435 JSON.stringify({
436 type: "record",
437 name: "AlchemyOwnership",
438 namespace: "alchemy",
439 doc: encodeOwnership(labels),
440 fields: [{ name: "ok", type: "string" }],
441 });
442
443 export const parseHttpJson = (body: kafka.HttpBody | undefined) =>
444 Effect.sync(() => {
445 const data = body?.data;
446 if (data === undefined || data.length === 0) return undefined;
447 try {
448 return JSON.parse(data) as unknown;
449 } catch {
450 /* base64 */
451 }
452 try {
453 return JSON.parse(Buffer.from(data, "base64").toString("utf8")) as unknown;
454 } catch {
455 return undefined;
456 }
457 });
458
459 export const subjectParent = (schemaRegistry: string, subject: string, context?: string) =>
460 context !== undefined && context.length > 0
461 ? `${schemaRegistry}/contexts/${context}/subjects/${subject}`
462 : `${schemaRegistry}/subjects/${subject}`;
463
464 export const versionName = (parent: string, version: number | string) =>
465 `${parent}/versions/${version}`;
466
467 export const retryTransient = <A, E extends { readonly _tag: string }, R>(
468 effect: Effect.Effect<A, E, R>,
469 ) =>
470 effect.pipe(
471 Effect.retry({
472 while: (error) => isTransientGcpError(error) || error._tag === "NotFound",
473 times: 8,
474 schedule: Schedule.exponential("250 millis"),
475 }),
476 );
477
478 /**
479 * Cluster and Connect cluster creates are limited to one request per minute
480 * per region, so a burst of creates is throttled with `TooManyRequests`
481 * until the minute window rolls over. Retry for up to ~4 minutes.
482 */
483 export const retryCreateQuota = <A, E extends { readonly _tag: string }, R>(
484 effect: Effect.Effect<A, E, R>,
485 ) =>
486 effect.pipe(
487 Effect.retry({
488 while: (error) => error._tag === "TooManyRequests",
489 times: 12,
490 schedule: Schedule.spaced("20 seconds"),
491 }),
492 );
493
494 export const getCluster = (name: string) =>
495 name.length === 0
496 ? Effect.undefined
497 : kafka
498 .getProjectsLocationsClusters({ name })
499 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
500
501 export const getConnectCluster = (name: string) =>
502 name.length === 0
503 ? Effect.undefined
504 : kafka
505 .getProjectsLocationsConnectClusters({ name })
506 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
507
508 export const getAcl = (name: string) =>
509 name.length === 0
510 ? Effect.undefined
511 : kafka
512 .getProjectsLocationsClustersAcls({ name })
513 .pipe(Effect.catchTag(["NotFound", "AclClusterNotFound"], () => Effect.undefined));
514
515 export const getTopic = (name: string) =>
516 name.length === 0
517 ? Effect.undefined
518 : kafka
519 .getProjectsLocationsClustersTopics({ name })
520 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
521
522 export const getConnector = (name: string) =>
523 name.length === 0
524 ? Effect.undefined
525 : kafka
526 .getProjectsLocationsConnectClustersConnectors({ name })
527 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
528
529 export const getSchemaRegistry = (name: string) =>
530 name.length === 0
531 ? Effect.undefined
532 : kafka
533 .getProjectsLocationsSchemaRegistries({ name })
534 .pipe(
535 Effect.catchTag(["NotFound", "SchemaRegistryRequiresCluster"], () => Effect.undefined),
536 );
537
538 export const getSchemaVersion = (name: string, deleted = false) =>
539 name.length === 0
540 ? Effect.undefined
541 : kafka
542 .getProjectsLocationsSchemaRegistriesSubjectsVersions({
543 name,
544 deleted,
545 })
546 .pipe(
547 Effect.catchTag(
548 ["NotFound", "SchemaRegistryRequiresCluster", "SchemaRegistryPathNotFound"],
549 () => Effect.undefined,
550 ),
551 );
552
553 export const getContextSchemaVersion = (name: string, deleted = false) =>
554 name.length === 0
555 ? Effect.undefined
556 : kafka
557 .getProjectsLocationsSchemaRegistriesContextsSubjectsVersions({
558 name,
559 deleted,
560 })
561 .pipe(
562 Effect.catchTag(
563 ["NotFound", "SchemaRegistryRequiresCluster", "SchemaRegistryPathNotFound"],
564 () => Effect.undefined,
565 ),
566 );
567
568 export const stampSchemaRegistryOwnership = (
569 schemaRegistry: string,
570 labels: Record<string, string>,
571 ) =>
572 Effect.gen(function* () {
573 const parent = subjectParent(schemaRegistry, OWNERSHIP_SUBJECT);
574 const schema = ownershipSchema(labels);
575 const latest = yield* getSchemaVersion(versionName(parent, "latest"));
576 if (hasOwnershipMarker(latest?.schema) && latest?.schema === schema) {
577 return;
578 }
579 yield* kafka
580 .createProjectsLocationsSchemaRegistriesSubjectsVersions({
581 parent,
582 body: {
583 schemaType: "AVRO",
584 schema,
585 },
586 })
587 .pipe(Effect.catchTag(["Conflict", "BadRequest"], () => Effect.void));
588 });
589
590 export const schemaRegistryOwnership = (schemaRegistry: string) =>
591 getSchemaVersion(versionName(subjectParent(schemaRegistry, OWNERSHIP_SUBJECT), "latest")).pipe(
592 Effect.map((version) => parseOwnership(version?.schema)),
593 );
594
595 export const hasSchemaRegistryOwnership = Effect.fn(function* (id: string, schemaRegistry: string) {
596 const expected = yield* createInternalLabels(id);
597 const observed = yield* schemaRegistryOwnership(schemaRegistry);
598 return (
599 observed[alchemyLabelKeys.stack] === expected[alchemyLabelKeys.stack] &&
600 observed[alchemyLabelKeys.stage] === expected[alchemyLabelKeys.stage] &&
601 observed[alchemyLabelKeys.id] === expected[alchemyLabelKeys.id]
602 );
603 });