The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as bigquery from "@distilled.cloud/gcp/bigquery_v2"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import { tagRecord } from "alchemy/Tags"; |
| 8 | import * as Data from "effect/Data"; |
| 9 | import * as Effect from "effect/Effect"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Stream from "effect/Stream"; |
| 12 | import { GcpEnvironment } from "../Environment.ts"; |
| 13 | import { |
| 14 | createInternalLabels, |
| 15 | hasAlchemyLabels, |
| 16 | stripInternalLabels, |
| 17 | toLabels, |
| 18 | } from "../Labels.ts"; |
| 19 | import type { Providers } from "../Providers.ts"; |
| 20 | |
| 21 | const MAX_JOB_ID_LENGTH = 1024; |
| 22 | |
| 23 | export type JobQuery = bigquery.JobConfigurationQuery; |
| 24 | export type JobLoad = bigquery.JobConfigurationLoad; |
| 25 | export type JobCopy = bigquery.JobConfigurationTableCopy; |
| 26 | export type JobExtract = bigquery.JobConfigurationExtract; |
| 27 | export type JobError = bigquery.ErrorProto; |
| 28 | |
| 29 | export type JobProps = { |
| 30 | /** |
| 31 | * Job id (the `{jobId}` segment of `projects/{project}/jobs/{jobId}`). |
| 32 | * If omitted, a unique name is generated from the stack, stage, and |
| 33 | * logical id. Letters, numbers, underscores, and dashes; max 1,024 |
| 34 | * characters. Immutable — changing it replaces the job. |
| 35 | */ |
| 36 | jobId?: string; |
| 37 | /** |
| 38 | * Geographic location (`US`, `EU`, `US-CENTRAL1`, `us-central1`, …). |
| 39 | * Immutable — changing it replaces the job. |
| 40 | * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`) |
| 41 | */ |
| 42 | location?: string; |
| 43 | /** |
| 44 | * User labels. Alchemy ownership labels are merged in automatically. |
| 45 | * Immutable after create — changing labels replaces the job. |
| 46 | */ |
| 47 | labels?: Record<string, string>; |
| 48 | /** |
| 49 | * Query job configuration. Mutually exclusive with `load`, `copy`, and |
| 50 | * `extract`. GoogleSQL is the default (`useLegacySql: false`). |
| 51 | */ |
| 52 | query?: JobQuery; |
| 53 | /** |
| 54 | * Load job configuration. Mutually exclusive with `query`, `copy`, and |
| 55 | * `extract`. |
| 56 | */ |
| 57 | load?: JobLoad; |
| 58 | /** |
| 59 | * Copy job configuration. Mutually exclusive with `query`, `load`, and |
| 60 | * `extract`. |
| 61 | */ |
| 62 | copy?: JobCopy; |
| 63 | /** |
| 64 | * Extract job configuration. Mutually exclusive with `query`, `load`, |
| 65 | * and `copy`. |
| 66 | */ |
| 67 | extract?: JobExtract; |
| 68 | /** |
| 69 | * Job timeout in milliseconds relative to creation time. |
| 70 | */ |
| 71 | jobTimeoutMs?: string; |
| 72 | /** |
| 73 | * Reservation the job should run on, as |
| 74 | * `projects/{project}/locations/{location}/reservations/{reservation}`. |
| 75 | */ |
| 76 | reservation?: string; |
| 77 | }; |
| 78 | |
| 79 | export type Job = Resource< |
| 80 | "GCP.BigQuery.Job", |
| 81 | JobProps, |
| 82 | { |
| 83 | /** Resource path `projects/{project}/jobs/{jobId}`. */ |
| 84 | name: string; |
| 85 | /** Opaque id `project:jobId`. */ |
| 86 | id: string; |
| 87 | /** Job id. */ |
| 88 | jobId: string; |
| 89 | /** Project id. */ |
| 90 | project: string; |
| 91 | /** Geographic location. */ |
| 92 | location: string; |
| 93 | /** User labels (Alchemy ownership labels stripped). */ |
| 94 | labels: Record<string, string>; |
| 95 | /** Job kind (`QUERY`, `LOAD`, `EXTRACT`, `COPY`, …). */ |
| 96 | jobType: string | undefined; |
| 97 | /** Running state (`PENDING`, `RUNNING`, `DONE`). */ |
| 98 | state: string | undefined; |
| 99 | /** Final error when the job completed unsuccessfully. */ |
| 100 | errorResult: JobError | undefined; |
| 101 | /** Query text when this is a query job. */ |
| 102 | query: string | undefined; |
| 103 | /** Self-link URL. */ |
| 104 | selfLink: string | undefined; |
| 105 | /** Creator email. */ |
| 106 | userEmail: string | undefined; |
| 107 | /** Creation time in milliseconds since epoch. */ |
| 108 | creationTime: string | undefined; |
| 109 | /** Start time in milliseconds since epoch. */ |
| 110 | startTime: string | undefined; |
| 111 | /** End time in milliseconds since epoch. */ |
| 112 | endTime: string | undefined; |
| 113 | }, |
| 114 | never, |
| 115 | Providers |
| 116 | >; |
| 117 | |
| 118 | /** |
| 119 | * A Google BigQuery job — a one-shot query, load, copy, or extract. |
| 120 | * |
| 121 | * Jobs are immutable after insert. Changing `jobId`, `location`, labels, |
| 122 | * or the job configuration replaces the job. BigQuery job ids cannot be |
| 123 | * reused even after metadata delete, so replacement always creates a new |
| 124 | * id then deletes the previous generation. Destroy cancels a running job, |
| 125 | * then deletes its metadata. |
| 126 | * |
| 127 | * ### Creating a Job |
| 128 | * **Example:** GoogleSQL query |
| 129 | * ```typescript |
| 130 | * const job = yield* GCP.BigQuery.Job("Count", { |
| 131 | * query: { query: "SELECT 1 AS n", useLegacySql: false }, |
| 132 | * labels: { env: "prod" }, |
| 133 | * }); |
| 134 | * ``` |
| 135 | * |
| 136 | * **Example:** Query into a destination table |
| 137 | * ```typescript |
| 138 | * const dataset = yield* GCP.BigQuery.Dataset("Analytics", { |
| 139 | * forceDestroy: true, |
| 140 | * }); |
| 141 | * const table = yield* GCP.BigQuery.Table("Results", { |
| 142 | * datasetId: dataset.datasetId, |
| 143 | * schema: [{ name: "n", type: "INTEGER" }], |
| 144 | * }); |
| 145 | * const load = yield* GCP.BigQuery.Job("Seed", { |
| 146 | * query: { |
| 147 | * query: "SELECT 1 AS n", |
| 148 | * useLegacySql: false, |
| 149 | * destinationTable: { |
| 150 | * projectId: table.project, |
| 151 | * datasetId: table.datasetId, |
| 152 | * tableId: table.tableId, |
| 153 | * }, |
| 154 | * writeDisposition: "WRITE_TRUNCATE", |
| 155 | * }, |
| 156 | * }); |
| 157 | * ``` |
| 158 | * |
| 159 | * @resource |
| 160 | * @category BigQuery |
| 161 | */ |
| 162 | export const Job = Resource<Job>("GCP.BigQuery.Job"); |
| 163 | |
| 164 | export class JobNotResolved extends Data.TaggedError("GCP.BigQuery.JobNotResolved")<{ |
| 165 | resourceName: string; |
| 166 | }> {} |
| 167 | |
| 168 | export class JobNotDone extends Data.TaggedError("GCP.BigQuery.JobNotDone")<{ |
| 169 | jobId: string; |
| 170 | state: string | undefined; |
| 171 | }> {} |
| 172 | |
| 173 | const userLabels = ( |
| 174 | labels: Record<string, string | undefined> | null | undefined, |
| 175 | ): Record<string, string> => stripInternalLabels(tagRecord(labels)); |
| 176 | |
| 177 | const resourceName = (project: string, jobId: string) => `projects/${project}/jobs/${jobId}`; |
| 178 | |
| 179 | const normalizeLocation = (location: string | undefined, fallback: string) => |
| 180 | (location ?? fallback).toUpperCase(); |
| 181 | |
| 182 | const toId = (id: string, jobId: string | undefined, existing?: string) => |
| 183 | Effect.gen(function* () { |
| 184 | if (jobId !== undefined) return jobId; |
| 185 | if (existing !== undefined) return existing; |
| 186 | return yield* createPhysicalName({ |
| 187 | id, |
| 188 | maxLength: MAX_JOB_ID_LENGTH, |
| 189 | lowercase: true, |
| 190 | }); |
| 191 | }); |
| 192 | |
| 193 | const jobIdOf = (job: bigquery.Job | bigquery.JobListJobsItem, fallback: string) => |
| 194 | job.jobReference?.jobId ?? fallback; |
| 195 | |
| 196 | const projectOf = (job: bigquery.Job | bigquery.JobListJobsItem, fallback: string) => |
| 197 | job.jobReference?.projectId ?? fallback; |
| 198 | |
| 199 | const locationOf = (job: bigquery.Job | bigquery.JobListJobsItem, fallback: string) => |
| 200 | job.jobReference?.location ?? fallback; |
| 201 | |
| 202 | const stateOf = (job: bigquery.Job | bigquery.JobListJobsItem) => { |
| 203 | const full = job as bigquery.Job; |
| 204 | return full.status?.state ?? (job as bigquery.JobListJobsItem).state; |
| 205 | }; |
| 206 | |
| 207 | const toAttrs = ( |
| 208 | job: bigquery.Job | bigquery.JobListJobsItem, |
| 209 | fallbackProject: string, |
| 210 | region: string, |
| 211 | ) => { |
| 212 | const jobId = jobIdOf(job, ""); |
| 213 | const project = projectOf(job, fallbackProject); |
| 214 | const full = job as bigquery.Job; |
| 215 | return { |
| 216 | name: resourceName(project, jobId), |
| 217 | id: full.id ?? `${project}:${jobId}`, |
| 218 | jobId, |
| 219 | project, |
| 220 | location: locationOf(job, region), |
| 221 | labels: userLabels(job.configuration?.labels), |
| 222 | jobType: job.configuration?.jobType, |
| 223 | state: stateOf(job), |
| 224 | errorResult: full.status?.errorResult ?? (job as bigquery.JobListJobsItem).errorResult, |
| 225 | query: job.configuration?.query?.query, |
| 226 | selfLink: full.selfLink, |
| 227 | userEmail: full.user_email ?? (job as bigquery.JobListJobsItem).user_email, |
| 228 | creationTime: full.statistics?.creationTime ?? job.statistics?.creationTime, |
| 229 | startTime: full.statistics?.startTime ?? job.statistics?.startTime, |
| 230 | endTime: full.statistics?.endTime ?? job.statistics?.endTime, |
| 231 | }; |
| 232 | }; |
| 233 | |
| 234 | const getByRef = (projectId: string, jobId: string, location: string) => |
| 235 | bigquery |
| 236 | .getJobs({ projectId, jobId, location }) |
| 237 | .pipe(Effect.catchTag("NotFound", () => Effect.undefined)); |
| 238 | |
| 239 | const waitUntilDone = (projectId: string, jobId: string, location: string) => |
| 240 | getByRef(projectId, jobId, location).pipe( |
| 241 | Effect.flatMap((job) => { |
| 242 | if (job === undefined) return Effect.undefined; |
| 243 | const state = job.status?.state ?? ""; |
| 244 | if (state === "DONE") return Effect.succeed(job); |
| 245 | return Effect.fail(new JobNotDone({ jobId, state })); |
| 246 | }), |
| 247 | Effect.retry({ |
| 248 | while: (error) => error._tag === "GCP.BigQuery.JobNotDone", |
| 249 | times: 10, |
| 250 | schedule: Schedule.spaced("2 seconds"), |
| 251 | }), |
| 252 | ); |
| 253 | |
| 254 | const queryBody = (query: JobQuery): JobQuery => ({ |
| 255 | ...query, |
| 256 | useLegacySql: query.useLegacySql ?? false, |
| 257 | }); |
| 258 | |
| 259 | const toJobBody = ( |
| 260 | projectId: string, |
| 261 | jobId: string, |
| 262 | location: string, |
| 263 | news: JobProps, |
| 264 | labels: Record<string, string>, |
| 265 | ): bigquery.Job => ({ |
| 266 | jobReference: { projectId, jobId, location }, |
| 267 | configuration: { |
| 268 | labels, |
| 269 | query: news.query !== undefined ? queryBody(news.query) : undefined, |
| 270 | load: news.load, |
| 271 | copy: news.copy, |
| 272 | extract: news.extract, |
| 273 | jobTimeoutMs: news.jobTimeoutMs, |
| 274 | reservation: news.reservation, |
| 275 | }, |
| 276 | }); |
| 277 | |
| 278 | const stable = (value: unknown): unknown => { |
| 279 | if (Array.isArray(value)) return value.map(stable); |
| 280 | if (value !== null && typeof value === "object") { |
| 281 | return Object.fromEntries( |
| 282 | Object.entries(value as Record<string, unknown>) |
| 283 | .filter(([, nested]) => nested !== undefined) |
| 284 | .sort(([left], [right]) => left.localeCompare(right)) |
| 285 | .map(([key, nested]) => [key, stable(nested)]), |
| 286 | ); |
| 287 | } |
| 288 | return value; |
| 289 | }; |
| 290 | |
| 291 | const jsonEqual = (left: unknown, right: unknown) => |
| 292 | JSON.stringify(stable(left ?? null)) === JSON.stringify(stable(right ?? null)); |
| 293 | |
| 294 | const pickDefined = (news: Record<string, unknown>, observed: Record<string, unknown>) => { |
| 295 | const next: Record<string, unknown> = {}; |
| 296 | const previous: Record<string, unknown> = {}; |
| 297 | for (const [key, value] of Object.entries(news)) { |
| 298 | if (value !== undefined) { |
| 299 | next[key] = value; |
| 300 | previous[key] = observed[key]; |
| 301 | } |
| 302 | } |
| 303 | return { news: next, observed: previous }; |
| 304 | }; |
| 305 | |
| 306 | const nestedChanged = ( |
| 307 | news: Record<string, unknown> | undefined, |
| 308 | olds: Record<string, unknown> | undefined, |
| 309 | hadOlds: boolean, |
| 310 | ) => { |
| 311 | if (news === undefined) return false; |
| 312 | if (olds === undefined) return hadOlds; |
| 313 | const picked = pickDefined(news, olds); |
| 314 | return !jsonEqual(picked.news, picked.observed); |
| 315 | }; |
| 316 | |
| 317 | export const JobProvider = () => |
| 318 | Provider.succeed(Job, { |
| 319 | stables: ["name", "id", "jobId", "project", "location", "creationTime"], |
| 320 | |
| 321 | diff: Effect.fn(function* ({ news, olds, output }) { |
| 322 | if (!isResolved(news)) return undefined; |
| 323 | const env = yield* GcpEnvironment.current; |
| 324 | const previousId = olds?.jobId ?? output?.jobId; |
| 325 | const jobIdChanged = |
| 326 | news.jobId !== undefined && previousId !== undefined && news.jobId !== previousId; |
| 327 | |
| 328 | const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region); |
| 329 | const nextLocation = normalizeLocation( |
| 330 | news.location ?? olds?.location ?? output?.location, |
| 331 | env.region, |
| 332 | ); |
| 333 | const locationChanged = output !== undefined && previousLocation !== nextLocation; |
| 334 | |
| 335 | const labelsChanged = |
| 336 | news.labels !== undefined && |
| 337 | output !== undefined && |
| 338 | !jsonEqual(toLabels(news.labels), output.labels); |
| 339 | |
| 340 | const timeoutChanged = |
| 341 | news.jobTimeoutMs !== undefined && (olds?.jobTimeoutMs ?? "") !== news.jobTimeoutMs; |
| 342 | const reservationChanged = |
| 343 | news.reservation !== undefined && (olds?.reservation ?? "") !== news.reservation; |
| 344 | |
| 345 | const replaced = |
| 346 | jobIdChanged || |
| 347 | locationChanged || |
| 348 | labelsChanged || |
| 349 | timeoutChanged || |
| 350 | reservationChanged || |
| 351 | nestedChanged( |
| 352 | news.query as Record<string, unknown> | undefined, |
| 353 | olds?.query as Record<string, unknown> | undefined, |
| 354 | olds !== undefined, |
| 355 | ) || |
| 356 | nestedChanged( |
| 357 | news.load as Record<string, unknown> | undefined, |
| 358 | olds?.load as Record<string, unknown> | undefined, |
| 359 | olds !== undefined, |
| 360 | ) || |
| 361 | nestedChanged( |
| 362 | news.copy as Record<string, unknown> | undefined, |
| 363 | olds?.copy as Record<string, unknown> | undefined, |
| 364 | olds !== undefined, |
| 365 | ) || |
| 366 | nestedChanged( |
| 367 | news.extract as Record<string, unknown> | undefined, |
| 368 | olds?.extract as Record<string, unknown> | undefined, |
| 369 | olds !== undefined, |
| 370 | ); |
| 371 | |
| 372 | if (!replaced) return undefined; |
| 373 | // Job ids cannot be reused after delete (the API returns NotFound on |
| 374 | // insert of a retired id), so never delete-first. |
| 375 | return { action: "replace" as const, deleteFirst: false }; |
| 376 | }), |
| 377 | |
| 378 | read: Effect.fn(function* ({ id, olds, output }) { |
| 379 | const env = yield* GcpEnvironment.current; |
| 380 | const jobId = yield* toId(id, olds?.jobId, output?.jobId); |
| 381 | const location = olds?.location ?? output?.location ?? env.region; |
| 382 | const project = output?.project ?? env.project; |
| 383 | const existing = yield* getByRef(project, jobId, location); |
| 384 | if (existing === undefined) return undefined; |
| 385 | const attrs = toAttrs(existing, project, env.region); |
| 386 | return (yield* hasAlchemyLabels(id, tagRecord(existing.configuration?.labels))) |
| 387 | ? attrs |
| 388 | : Unowned(attrs); |
| 389 | }), |
| 390 | |
| 391 | list: () => |
| 392 | Effect.gen(function* () { |
| 393 | const env = yield* GcpEnvironment.current; |
| 394 | return yield* bigquery.listJobs |
| 395 | .pages({ |
| 396 | projectId: env.project, |
| 397 | maxResults: 1000, |
| 398 | projection: "full", |
| 399 | allUsers: true, |
| 400 | }) |
| 401 | .pipe( |
| 402 | Stream.flatMap((page) => Stream.fromIterable(page.jobs ?? [])), |
| 403 | Stream.filter((job) => |
| 404 | Object.keys(job.configuration?.labels ?? {}).some((key) => |
| 405 | key.startsWith("alchemy-"), |
| 406 | ), |
| 407 | ), |
| 408 | Stream.map((job) => toAttrs(job, env.project, env.region)), |
| 409 | Stream.runCollect, |
| 410 | Effect.map((chunk) => Array.from(chunk)), |
| 411 | Effect.catchTag("Forbidden", () => |
| 412 | bigquery.listJobs |
| 413 | .pages({ |
| 414 | projectId: env.project, |
| 415 | maxResults: 1000, |
| 416 | projection: "full", |
| 417 | }) |
| 418 | .pipe( |
| 419 | Stream.flatMap((page) => Stream.fromIterable(page.jobs ?? [])), |
| 420 | Stream.filter((job) => |
| 421 | Object.keys(job.configuration?.labels ?? {}).some((key) => |
| 422 | key.startsWith("alchemy-"), |
| 423 | ), |
| 424 | ), |
| 425 | Stream.map((job) => toAttrs(job, env.project, env.region)), |
| 426 | Stream.runCollect, |
| 427 | Effect.map((chunk) => Array.from(chunk)), |
| 428 | ), |
| 429 | ), |
| 430 | Effect.catchTag("NotFound", () => Effect.succeed([] as ReturnType<typeof toAttrs>[])), |
| 431 | ); |
| 432 | }), |
| 433 | |
| 434 | reconcile: Effect.fn(function* ({ id, news, output }) { |
| 435 | const env = yield* GcpEnvironment.current; |
| 436 | const jobId = yield* toId(id, news.jobId, output?.jobId); |
| 437 | const location = news.location ?? output?.location ?? env.region; |
| 438 | const project = output?.project ?? env.project; |
| 439 | const name = resourceName(project, jobId); |
| 440 | const desiredLabels = { |
| 441 | ...toLabels(news.labels), |
| 442 | ...(yield* createInternalLabels(id)), |
| 443 | }; |
| 444 | |
| 445 | let current = yield* getByRef(project, jobId, location); |
| 446 | |
| 447 | if (current === undefined) { |
| 448 | const created = yield* bigquery |
| 449 | .insertJobs({ |
| 450 | projectId: project, |
| 451 | body: toJobBody(project, jobId, location, news, desiredLabels), |
| 452 | }) |
| 453 | .pipe(Effect.catchTag("Conflict", () => getByRef(project, jobId, location))); |
| 454 | current = created ?? undefined; |
| 455 | } |
| 456 | |
| 457 | if (current === undefined) { |
| 458 | return yield* new JobNotResolved({ resourceName: name }); |
| 459 | } |
| 460 | |
| 461 | const state = current.status?.state ?? ""; |
| 462 | if (state !== "DONE") { |
| 463 | current = (yield* waitUntilDone(project, jobId, location)) ?? current; |
| 464 | } |
| 465 | |
| 466 | return toAttrs(current, project, env.region); |
| 467 | }), |
| 468 | |
| 469 | delete: Effect.fn(function* ({ output }) { |
| 470 | const state = output.state ?? ""; |
| 471 | if (state === "PENDING" || state === "RUNNING") { |
| 472 | yield* bigquery |
| 473 | .cancelJobs({ |
| 474 | projectId: output.project, |
| 475 | jobId: output.jobId, |
| 476 | location: output.location, |
| 477 | }) |
| 478 | .pipe( |
| 479 | Effect.catchTags({ |
| 480 | NotFound: () => Effect.void, |
| 481 | BadRequest: () => Effect.void, |
| 482 | }), |
| 483 | ); |
| 484 | yield* waitUntilDone(output.project, output.jobId, output.location).pipe( |
| 485 | Effect.catchTag("GCP.BigQuery.JobNotDone", () => Effect.void), |
| 486 | ); |
| 487 | } |
| 488 | yield* bigquery |
| 489 | .deleteJobs({ |
| 490 | projectId: output.project, |
| 491 | jobId: output.jobId, |
| 492 | location: output.location, |
| 493 | }) |
| 494 | .pipe( |
| 495 | Effect.retry({ |
| 496 | while: (error) => error._tag === "Conflict" || error._tag === "BadRequest", |
| 497 | times: 8, |
| 498 | schedule: Schedule.spaced("2 seconds"), |
| 499 | }), |
| 500 | Effect.catchTag("NotFound", () => Effect.void), |
| 501 | ); |
| 502 | }), |
| 503 | }); |