The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as bqdt from "@distilled.cloud/gcp/bigquerydatatransfer_v1"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import * as Provider from "alchemy/Provider"; |
| 5 | import { Resource } from "alchemy/Resource"; |
| 6 | import * as Data from "effect/Data"; |
| 7 | import * as Duration from "effect/Duration"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Schedule from "effect/Schedule"; |
| 10 | import * as Stream from "effect/Stream"; |
| 11 | import { GcpEnvironment } from "../Environment.ts"; |
| 12 | import { alchemyLabelKeys, createInternalLabels, hasAlchemyLabels } from "../Labels.ts"; |
| 13 | import type { Providers } from "../Providers.ts"; |
| 14 | |
| 15 | const backoff = Schedule.min([ |
| 16 | Schedule.exponential(Duration.millis(300), 1.5), |
| 17 | Schedule.spaced(Duration.seconds(2)), |
| 18 | ]); |
| 19 | |
| 20 | export type TransferParams = Record<string, unknown>; |
| 21 | |
| 22 | export type EmailPreferences = { |
| 23 | /** |
| 24 | * Send email to the config owner when a transfer run fails. |
| 25 | */ |
| 26 | enableFailureEmail?: boolean; |
| 27 | }; |
| 28 | |
| 29 | export type ScheduleOptions = { |
| 30 | /** |
| 31 | * Disable automatic scheduling. Runs can still be started with |
| 32 | * StartManualTransferRuns. When true, `schedule` is ignored. |
| 33 | */ |
| 34 | disableAutoScheduling?: boolean; |
| 35 | /** |
| 36 | * RFC3339 time to start scheduling runs. The first run is at or after |
| 37 | * this time according to `schedule`. |
| 38 | */ |
| 39 | startTime?: string; |
| 40 | /** |
| 41 | * RFC3339 time to stop scheduling runs. A run is not scheduled at or |
| 42 | * after this time. |
| 43 | */ |
| 44 | endTime?: string; |
| 45 | }; |
| 46 | |
| 47 | export type EventDrivenSchedule = { |
| 48 | /** |
| 49 | * Pub/Sub subscription that triggers runs. Only `google_cloud_storage` |
| 50 | * supports this. Format: `projects/{project}/subscriptions/{subscription}`. |
| 51 | */ |
| 52 | pubsubSubscription?: string; |
| 53 | }; |
| 54 | |
| 55 | export type TimeBasedSchedule = { |
| 56 | /** RFC3339 time to start scheduling runs. */ |
| 57 | startTime?: string; |
| 58 | /** RFC3339 time to stop scheduling runs. */ |
| 59 | endTime?: string; |
| 60 | /** |
| 61 | * Recurrence in UTC (e.g. `"every 24 hours"`, `"first sunday of quarter |
| 62 | * 00:00"`). Empty uses the data source default. |
| 63 | */ |
| 64 | schedule?: string; |
| 65 | }; |
| 66 | |
| 67 | export type ScheduleOptionsV2 = { |
| 68 | /** |
| 69 | * Event-driven schedule. Cannot be combined with `timeBasedSchedule` or |
| 70 | * `manualSchedule`. |
| 71 | */ |
| 72 | eventDrivenSchedule?: EventDrivenSchedule; |
| 73 | /** |
| 74 | * Time-based schedule. Cannot be combined with `eventDrivenSchedule` or |
| 75 | * `manualSchedule`. Replaces top-level `schedule` / `scheduleOptions`. |
| 76 | */ |
| 77 | timeBasedSchedule?: TimeBasedSchedule; |
| 78 | /** |
| 79 | * Manual schedule (equivalent to `disableAutoScheduling: true`). Empty |
| 80 | * object selects this mode. |
| 81 | */ |
| 82 | manualSchedule?: Record<string, never>; |
| 83 | }; |
| 84 | |
| 85 | export type EncryptionConfiguration = { |
| 86 | /** |
| 87 | * Cloud KMS key used to encrypt transferred BigQuery data, as |
| 88 | * `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`. |
| 89 | */ |
| 90 | kmsKeyName?: string; |
| 91 | }; |
| 92 | |
| 93 | export type DataplexConfiguration = { |
| 94 | /** |
| 95 | * Dataplex Universal Catalog entry group, as |
| 96 | * `projects/{project}/locations/{region}/entryGroups/{entryGroup}`. |
| 97 | */ |
| 98 | entryGroup?: string; |
| 99 | }; |
| 100 | |
| 101 | export type MetadataDestination = { |
| 102 | /** Dataplex Universal Catalog destination. */ |
| 103 | dataplexConfiguration?: DataplexConfiguration; |
| 104 | }; |
| 105 | |
| 106 | export type ManagedTableType = |
| 107 | | "MANAGED_TABLE_TYPE_UNSPECIFIED" |
| 108 | | "NATIVE" |
| 109 | | "BIGLAKE" |
| 110 | | (string & {}); |
| 111 | |
| 112 | export type TransferConfigProps = { |
| 113 | /** |
| 114 | * Data Transfer location (`us-central1`, `us-east1`, `us`, …). Must |
| 115 | * match the destination dataset location. Immutable — changing it |
| 116 | * replaces the config. `US-CENTRAL1` is accepted and normalized to |
| 117 | * `us-central1`. |
| 118 | * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`) |
| 119 | */ |
| 120 | location?: string; |
| 121 | /** |
| 122 | * Data source id (e.g. `"scheduled_query"`, `"google_cloud_storage"`). |
| 123 | * Immutable — changing it replaces the config. The full list is |
| 124 | * available from `projects.locations.dataSources.list`. |
| 125 | */ |
| 126 | dataSourceId: string; |
| 127 | /** |
| 128 | * User-specified display name. Transfer configs have no labels field, |
| 129 | * so Alchemy ownership (`alchemy-stack` / `alchemy-stage` / |
| 130 | * `alchemy-id`) is stored in a `[alchemy …]` prefix for `read` / |
| 131 | * `list` / nuke. |
| 132 | */ |
| 133 | displayName?: string; |
| 134 | /** |
| 135 | * Destination BigQuery dataset id. Required for most data sources |
| 136 | * (including scheduled queries and Cloud Storage). Must live in the |
| 137 | * same location as this config. |
| 138 | */ |
| 139 | destinationDatasetId?: string; |
| 140 | /** |
| 141 | * Parameters specific to `dataSourceId`. Values are typically strings. |
| 142 | * Scheduled query: `query`, `destination_table_name_template`, |
| 143 | * `write_disposition`. Cloud Storage: `data_path_template`, |
| 144 | * `destination_table_name_template`, `file_format`. |
| 145 | */ |
| 146 | params: TransferParams; |
| 147 | /** |
| 148 | * Recurrence in UTC (e.g. `"every 24 hours"`). Empty uses the data |
| 149 | * source default. Ignored when `scheduleOptions.disableAutoScheduling` |
| 150 | * is true. Do not set together with `scheduleOptionsV2`. |
| 151 | */ |
| 152 | schedule?: string; |
| 153 | /** |
| 154 | * Time-window and auto-schedule options. Do not set together with |
| 155 | * `scheduleOptionsV2`. |
| 156 | */ |
| 157 | scheduleOptions?: ScheduleOptions; |
| 158 | /** |
| 159 | * V2 schedule (time-based, event-driven, or manual). Replaces |
| 160 | * `schedule` and `scheduleOptions` — the two families cannot be |
| 161 | * combined. |
| 162 | */ |
| 163 | scheduleOptionsV2?: ScheduleOptionsV2; |
| 164 | /** |
| 165 | * Days of data to refresh on each run. `0` uses the data source |
| 166 | * default. Only valid when the data source supports it. |
| 167 | */ |
| 168 | dataRefreshWindowDays?: number; |
| 169 | /** |
| 170 | * When true, the config exists but no runs are scheduled. |
| 171 | * @default false |
| 172 | */ |
| 173 | disabled?: boolean; |
| 174 | /** |
| 175 | * Pub/Sub topic notified when a run finishes, as |
| 176 | * `projects/{project}/topics/{topic}`. |
| 177 | */ |
| 178 | notificationPubsubTopic?: string; |
| 179 | /** |
| 180 | * Email notification preferences for the config owner. |
| 181 | */ |
| 182 | emailPreferences?: EmailPreferences; |
| 183 | /** |
| 184 | * Optional CMEK for transferred BigQuery data. |
| 185 | */ |
| 186 | encryptionConfiguration?: EncryptionConfiguration; |
| 187 | /** |
| 188 | * Classification of the destination table (`NATIVE`, `BIGLAKE`). |
| 189 | */ |
| 190 | managedTableType?: ManagedTableType; |
| 191 | /** |
| 192 | * Metadata destination (Dataplex) when the transfer writes catalog |
| 193 | * metadata rather than a dataset. |
| 194 | */ |
| 195 | metadataDestination?: MetadataDestination; |
| 196 | /** |
| 197 | * Service account the transfer runs as. The caller needs |
| 198 | * `iam.serviceAccounts.actAs`. Not all data sources support this; |
| 199 | * omitted uses the caller's credentials. Not returned by get — passed |
| 200 | * on create and whenever it changes. |
| 201 | */ |
| 202 | serviceAccountName?: string; |
| 203 | /** |
| 204 | * OAuth version info for data sources that need user credentials |
| 205 | * (e.g. `youtube_channel`). Do not set with `serviceAccountName`. |
| 206 | */ |
| 207 | versionInfo?: string; |
| 208 | /** |
| 209 | * Deprecated OAuth authorization code. Prefer `versionInfo`. |
| 210 | */ |
| 211 | authorizationCode?: string; |
| 212 | }; |
| 213 | |
| 214 | export type TransferConfig = Resource< |
| 215 | "GCP.BigQueryDataTransfer.TransferConfig", |
| 216 | TransferConfigProps, |
| 217 | { |
| 218 | /** Full resource name `projects/{project}/locations/{location}/transferConfigs/{id}`. */ |
| 219 | name: string; |
| 220 | /** Server-assigned config id (last path segment, usually a UUID). */ |
| 221 | transferConfigId: string; |
| 222 | /** Project id. */ |
| 223 | project: string; |
| 224 | /** Location id (`us-central1`, …). */ |
| 225 | location: string; |
| 226 | /** User display name with the Alchemy ownership prefix stripped. */ |
| 227 | displayName: string | undefined; |
| 228 | /** Data source id. */ |
| 229 | dataSourceId: string | undefined; |
| 230 | /** Destination dataset id. */ |
| 231 | destinationDatasetId: string | undefined; |
| 232 | /** Data-source parameters currently stored on the config. */ |
| 233 | params: TransferParams; |
| 234 | /** Recurrence string. */ |
| 235 | schedule: string | undefined; |
| 236 | /** Time-window / auto-schedule options. */ |
| 237 | scheduleOptions: ScheduleOptions | undefined; |
| 238 | /** V2 schedule options. */ |
| 239 | scheduleOptionsV2: ScheduleOptionsV2 | undefined; |
| 240 | /** Refresh window in days. */ |
| 241 | dataRefreshWindowDays: number | undefined; |
| 242 | /** Whether the config is disabled. */ |
| 243 | disabled: boolean; |
| 244 | /** Run-completion Pub/Sub topic. */ |
| 245 | notificationPubsubTopic: string | undefined; |
| 246 | /** Email notification preferences. */ |
| 247 | emailPreferences: EmailPreferences | undefined; |
| 248 | /** CMEK configuration. */ |
| 249 | encryptionConfiguration: EncryptionConfiguration | undefined; |
| 250 | /** Destination table classification. */ |
| 251 | managedTableType: string | undefined; |
| 252 | /** Metadata destination. */ |
| 253 | metadataDestination: MetadataDestination | undefined; |
| 254 | /** Region of the destination dataset. */ |
| 255 | datasetRegion: string | undefined; |
| 256 | /** Next scheduled run (RFC3339). */ |
| 257 | nextRunTime: string | undefined; |
| 258 | /** State of the most recently updated run. */ |
| 259 | state: string | undefined; |
| 260 | /** Last modification time (RFC3339). */ |
| 261 | updateTime: string | undefined; |
| 262 | /** Deprecated user id. */ |
| 263 | userId: string | undefined; |
| 264 | /** Owner whose credentials the transfer uses. */ |
| 265 | ownerInfo: bqdt.UserInfo | undefined; |
| 266 | /** Latest config-level error, if any. */ |
| 267 | error: bqdt.Status | undefined; |
| 268 | }, |
| 269 | never, |
| 270 | Providers |
| 271 | >; |
| 272 | |
| 273 | /** |
| 274 | * A BigQuery Data Transfer Service transfer configuration. |
| 275 | * |
| 276 | * Transfer configs have no labels field — Alchemy stamps ownership into |
| 277 | * the display name (`[alchemy alchemy-stack=… alchemy-stage=… |
| 278 | * alchemy-id=…]`) so `read`, `list`, and `pnpm nuke:gcp` can find them. |
| 279 | * The physical id is a server-assigned UUID. `dataSourceId` and |
| 280 | * `location` are immutable; changing either replaces the config. |
| 281 | * |
| 282 | * ### Creating a Transfer Config |
| 283 | * **Example:** Scheduled query with auto-scheduling disabled |
| 284 | * ```typescript |
| 285 | * const dataset = yield* GCP.BigQuery.Dataset("Analytics", { |
| 286 | * location: "US-CENTRAL1", |
| 287 | * forceDestroy: true, |
| 288 | * }); |
| 289 | * const nightly = yield* GCP.BigQueryDataTransfer.TransferConfig("Nightly", { |
| 290 | * dataSourceId: "scheduled_query", |
| 291 | * destinationDatasetId: dataset.datasetId, |
| 292 | * displayName: "nightly", |
| 293 | * schedule: "every 24 hours", |
| 294 | * scheduleOptions: { disableAutoScheduling: true }, |
| 295 | * params: { |
| 296 | * query: "SELECT 1 AS n", |
| 297 | * destination_table_name_template: "nightly", |
| 298 | * write_disposition: "WRITE_TRUNCATE", |
| 299 | * }, |
| 300 | * }); |
| 301 | * ``` |
| 302 | * |
| 303 | * **Example:** Cloud Storage load |
| 304 | * ```typescript |
| 305 | * const load = yield* GCP.BigQueryDataTransfer.TransferConfig("GcsLoad", { |
| 306 | * dataSourceId: "google_cloud_storage", |
| 307 | * destinationDatasetId: dataset.datasetId, |
| 308 | * schedule: "every 24 hours", |
| 309 | * scheduleOptions: { disableAutoScheduling: true }, |
| 310 | * params: { |
| 311 | * data_path_template: "gs://my-bucket/*.csv", |
| 312 | * destination_table_name_template: "events", |
| 313 | * file_format: "CSV", |
| 314 | * skip_leading_rows: "1", |
| 315 | * max_bad_records: "0", |
| 316 | * }, |
| 317 | * }); |
| 318 | * ``` |
| 319 | * |
| 320 | * @resource |
| 321 | * @category BigQueryDataTransfer |
| 322 | */ |
| 323 | export const TransferConfig = Resource<TransferConfig>("GCP.BigQueryDataTransfer.TransferConfig"); |
| 324 | |
| 325 | export class TransferConfigNotResolved extends Data.TaggedError( |
| 326 | "GCP.BigQueryDataTransfer.TransferConfigNotResolved", |
| 327 | )<{ |
| 328 | resourceName: string; |
| 329 | }> {} |
| 330 | |
| 331 | const lastSegment = (value: string) => { |
| 332 | const trimmed = value.replace(/\/+$/, ""); |
| 333 | const parts = trimmed.split("/"); |
| 334 | return parts[parts.length - 1] || trimmed; |
| 335 | }; |
| 336 | |
| 337 | const normalizeLocation = (location: string) => lastSegment(location).toLowerCase(); |
| 338 | |
| 339 | const parentOf = (project: string, location: string) => `projects/${project}/locations/${location}`; |
| 340 | |
| 341 | const parseName = (name: string) => { |
| 342 | const parts = name.split("/").filter((part) => part.length > 0); |
| 343 | const configsAt = parts.lastIndexOf("transferConfigs"); |
| 344 | const locationsAt = parts.lastIndexOf("locations"); |
| 345 | const projectsAt = parts.lastIndexOf("projects"); |
| 346 | return { |
| 347 | project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "", |
| 348 | location: locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : "", |
| 349 | transferConfigId: |
| 350 | configsAt >= 0 && parts[configsAt + 1] ? parts[configsAt + 1]! : lastSegment(name), |
| 351 | }; |
| 352 | }; |
| 353 | |
| 354 | const encodeDisplayName = ( |
| 355 | labels: Record<string, string>, |
| 356 | displayName: string | undefined, |
| 357 | ): string => { |
| 358 | const marker = `[alchemy ${alchemyLabelKeys.stack}=${labels[alchemyLabelKeys.stack]} ${alchemyLabelKeys.stage}=${labels[alchemyLabelKeys.stage]} ${alchemyLabelKeys.id}=${labels[alchemyLabelKeys.id]}]`; |
| 359 | const trimmed = displayName?.replace(/[\r\n]+/g, " ").trim(); |
| 360 | const combined = trimmed && trimmed.length > 0 ? `${marker} ${trimmed}` : marker; |
| 361 | return combined.slice(0, 1024); |
| 362 | }; |
| 363 | |
| 364 | const parseDisplayName = ( |
| 365 | displayName: string | undefined, |
| 366 | ): { |
| 367 | labels: Record<string, string>; |
| 368 | displayName: string | undefined; |
| 369 | } => { |
| 370 | if (!displayName?.startsWith("[alchemy ")) { |
| 371 | return { labels: {}, displayName }; |
| 372 | } |
| 373 | const end = displayName.indexOf("]"); |
| 374 | if (end < 0) return { labels: {}, displayName }; |
| 375 | const labels: Record<string, string> = {}; |
| 376 | for (const part of displayName.slice("[alchemy ".length, end).split(/\s+/)) { |
| 377 | const eq = part.indexOf("="); |
| 378 | if (eq > 0) { |
| 379 | labels[part.slice(0, eq)] = part.slice(eq + 1); |
| 380 | } |
| 381 | } |
| 382 | const rest = displayName.slice(end + 1).replace(/^[\s\n]+/, ""); |
| 383 | return { labels, displayName: rest.length > 0 ? rest : undefined }; |
| 384 | }; |
| 385 | |
| 386 | const hasOwnershipMarker = (displayName: string | undefined): boolean => |
| 387 | Object.keys(parseDisplayName(displayName).labels).some((key) => key.startsWith("alchemy-")); |
| 388 | |
| 389 | const compact = <T extends Record<string, unknown>>(value: T): T => |
| 390 | Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined)) as T; |
| 391 | |
| 392 | const jsonEqual = (left: unknown, right: unknown) => |
| 393 | JSON.stringify(left ?? null) === JSON.stringify(right ?? null); |
| 394 | |
| 395 | const canonParams = (params: TransferParams | undefined): TransferParams => { |
| 396 | const out: TransferParams = {}; |
| 397 | for (const key of Object.keys(params ?? {}).sort()) { |
| 398 | const value = params![key]; |
| 399 | if (value === undefined) continue; |
| 400 | out[key] = value; |
| 401 | } |
| 402 | return out; |
| 403 | }; |
| 404 | |
| 405 | const toScheduleOptions = ( |
| 406 | options: bqdt.ScheduleOptions | undefined, |
| 407 | ): ScheduleOptions | undefined => { |
| 408 | if (options === undefined) return undefined; |
| 409 | if ( |
| 410 | options.disableAutoScheduling === undefined && |
| 411 | options.startTime === undefined && |
| 412 | options.endTime === undefined |
| 413 | ) { |
| 414 | return undefined; |
| 415 | } |
| 416 | return compact({ |
| 417 | disableAutoScheduling: options.disableAutoScheduling, |
| 418 | startTime: options.startTime, |
| 419 | endTime: options.endTime, |
| 420 | }); |
| 421 | }; |
| 422 | |
| 423 | const toScheduleOptionsV2 = ( |
| 424 | options: bqdt.ScheduleOptionsV2 | undefined, |
| 425 | ): ScheduleOptionsV2 | undefined => { |
| 426 | if (options === undefined) return undefined; |
| 427 | const timeBased = |
| 428 | options.timeBasedSchedule === undefined |
| 429 | ? undefined |
| 430 | : compact({ |
| 431 | startTime: options.timeBasedSchedule.startTime, |
| 432 | endTime: options.timeBasedSchedule.endTime, |
| 433 | schedule: options.timeBasedSchedule.schedule, |
| 434 | }); |
| 435 | const eventDriven = |
| 436 | options.eventDrivenSchedule === undefined |
| 437 | ? undefined |
| 438 | : compact({ |
| 439 | pubsubSubscription: options.eventDrivenSchedule.pubsubSubscription, |
| 440 | }); |
| 441 | const manual = options.manualSchedule === undefined ? undefined : ({} as Record<string, never>); |
| 442 | if (timeBased === undefined && eventDriven === undefined && manual === undefined) { |
| 443 | return undefined; |
| 444 | } |
| 445 | return compact({ |
| 446 | timeBasedSchedule: timeBased && Object.keys(timeBased).length > 0 ? timeBased : undefined, |
| 447 | eventDrivenSchedule: |
| 448 | eventDriven && Object.keys(eventDriven).length > 0 ? eventDriven : undefined, |
| 449 | manualSchedule: manual, |
| 450 | }); |
| 451 | }; |
| 452 | |
| 453 | const toEmailPreferences = ( |
| 454 | preferences: bqdt.EmailPreferences | undefined, |
| 455 | ): EmailPreferences | undefined => { |
| 456 | if (preferences === undefined) return undefined; |
| 457 | if (preferences.enableFailureEmail === undefined) return undefined; |
| 458 | return { enableFailureEmail: preferences.enableFailureEmail }; |
| 459 | }; |
| 460 | |
| 461 | const toEncryption = ( |
| 462 | encryption: bqdt.EncryptionConfiguration | undefined, |
| 463 | ): EncryptionConfiguration | undefined => { |
| 464 | if (encryption?.kmsKeyName === undefined) return undefined; |
| 465 | return { kmsKeyName: encryption.kmsKeyName }; |
| 466 | }; |
| 467 | |
| 468 | const toMetadataDestination = ( |
| 469 | destination: bqdt.MetadataDestination | undefined, |
| 470 | ): MetadataDestination | undefined => { |
| 471 | const entryGroup = destination?.dataplexConfiguration?.entryGroup; |
| 472 | if (entryGroup === undefined) return undefined; |
| 473 | return { dataplexConfiguration: { entryGroup } }; |
| 474 | }; |
| 475 | |
| 476 | const toAttrs = (config: bqdt.TransferConfig, project: string, location: string) => { |
| 477 | const name = config.name ?? ""; |
| 478 | const parsed = parseName(name); |
| 479 | const { displayName } = parseDisplayName(config.displayName); |
| 480 | return { |
| 481 | name, |
| 482 | transferConfigId: parsed.transferConfigId, |
| 483 | project: parsed.project || project, |
| 484 | location: parsed.location || location, |
| 485 | displayName, |
| 486 | dataSourceId: config.dataSourceId, |
| 487 | destinationDatasetId: config.destinationDatasetId, |
| 488 | params: config.params ?? {}, |
| 489 | schedule: config.schedule, |
| 490 | scheduleOptions: toScheduleOptions(config.scheduleOptions), |
| 491 | scheduleOptionsV2: toScheduleOptionsV2(config.scheduleOptionsV2), |
| 492 | dataRefreshWindowDays: config.dataRefreshWindowDays, |
| 493 | disabled: config.disabled === true, |
| 494 | notificationPubsubTopic: config.notificationPubsubTopic, |
| 495 | emailPreferences: toEmailPreferences(config.emailPreferences), |
| 496 | encryptionConfiguration: toEncryption(config.encryptionConfiguration), |
| 497 | managedTableType: config.managedTableType, |
| 498 | metadataDestination: toMetadataDestination(config.metadataDestination), |
| 499 | datasetRegion: config.datasetRegion, |
| 500 | nextRunTime: config.nextRunTime, |
| 501 | state: config.state, |
| 502 | updateTime: config.updateTime, |
| 503 | userId: config.userId, |
| 504 | ownerInfo: config.ownerInfo, |
| 505 | error: config.error, |
| 506 | }; |
| 507 | }; |
| 508 | |
| 509 | const getByName = (name: string) => |
| 510 | bqdt |
| 511 | .getProjectsLocationsTransferConfigs({ name }) |
| 512 | .pipe(Effect.catchTag("NotFound", () => Effect.undefined)); |
| 513 | |
| 514 | const listAt = (parent: string) => |
| 515 | bqdt.listProjectsLocationsTransferConfigs |
| 516 | .pages({ |
| 517 | parent, |
| 518 | pageSize: 1000, |
| 519 | }) |
| 520 | .pipe( |
| 521 | Stream.flatMap((page) => Stream.fromIterable(page.transferConfigs ?? [])), |
| 522 | Stream.runCollect, |
| 523 | Effect.map((chunk) => Array.from(chunk)), |
| 524 | Effect.catchTag("NotFound", () => Effect.succeed([] as bqdt.TransferConfig[])), |
| 525 | ); |
| 526 | |
| 527 | const findOwned = Effect.fn(function* ( |
| 528 | parent: string, |
| 529 | id: string, |
| 530 | dataSourceId: string | undefined, |
| 531 | ) { |
| 532 | const configs = yield* listAt(parent); |
| 533 | for (const config of configs) { |
| 534 | const { labels } = parseDisplayName(config.displayName); |
| 535 | if (!(yield* hasAlchemyLabels(id, labels))) continue; |
| 536 | if (dataSourceId !== undefined && (config.dataSourceId ?? "") !== dataSourceId) { |
| 537 | continue; |
| 538 | } |
| 539 | if (config.name === undefined) continue; |
| 540 | // List can lag a delete; confirm the config still exists. |
| 541 | return yield* getByName(config.name); |
| 542 | } |
| 543 | return undefined; |
| 544 | }); |
| 545 | |
| 546 | const toBody = (news: TransferConfigProps, displayName: string): bqdt.TransferConfig => |
| 547 | compact({ |
| 548 | displayName, |
| 549 | dataSourceId: news.dataSourceId, |
| 550 | destinationDatasetId: news.destinationDatasetId, |
| 551 | params: news.params, |
| 552 | schedule: news.schedule, |
| 553 | scheduleOptions: news.scheduleOptions, |
| 554 | scheduleOptionsV2: news.scheduleOptionsV2, |
| 555 | dataRefreshWindowDays: news.dataRefreshWindowDays, |
| 556 | disabled: news.disabled === true ? true : news.disabled, |
| 557 | notificationPubsubTopic: news.notificationPubsubTopic, |
| 558 | emailPreferences: news.emailPreferences, |
| 559 | encryptionConfiguration: news.encryptionConfiguration, |
| 560 | managedTableType: news.managedTableType, |
| 561 | metadataDestination: news.metadataDestination, |
| 562 | }) as bqdt.TransferConfig; |
| 563 | |
| 564 | const retryTransient = <A, E extends { _tag: string }, R>(effect: Effect.Effect<A, E, R>) => |
| 565 | effect.pipe( |
| 566 | Effect.retry({ |
| 567 | while: (error) => error._tag === "NotFound" || error._tag === "Conflict", |
| 568 | times: 8, |
| 569 | schedule: backoff, |
| 570 | }), |
| 571 | ); |
| 572 | |
| 573 | const listLocationParents = (project: string, region: string) => |
| 574 | bqdt.listProjectsLocations |
| 575 | .pages({ |
| 576 | name: `projects/${project}`, |
| 577 | pageSize: 100, |
| 578 | }) |
| 579 | .pipe( |
| 580 | Stream.flatMap((page) => Stream.fromIterable(page.locations ?? [])), |
| 581 | Stream.map((location) => location.name), |
| 582 | Stream.filter((name): name is string => !!name && name.length > 0), |
| 583 | Stream.runCollect, |
| 584 | Effect.map((chunk) => Array.from(chunk)), |
| 585 | Effect.catchTag("NotFound", () => Effect.succeed([parentOf(project, region)])), |
| 586 | ); |
| 587 | |
| 588 | export const TransferConfigProvider = () => |
| 589 | Provider.succeed(TransferConfig, { |
| 590 | stables: ["name", "transferConfigId", "project", "location", "dataSourceId"], |
| 591 | |
| 592 | diff: Effect.fn(function* ({ news, olds, output }) { |
| 593 | if (!isResolved(news)) return; |
| 594 | const previousLocation = olds?.location ?? output?.location; |
| 595 | const nextLocation = typeof news.location === "string" ? news.location : undefined; |
| 596 | if ( |
| 597 | previousLocation !== undefined && |
| 598 | nextLocation !== undefined && |
| 599 | normalizeLocation(previousLocation) !== normalizeLocation(nextLocation) |
| 600 | ) { |
| 601 | return { action: "replace" as const, deleteFirst: false }; |
| 602 | } |
| 603 | |
| 604 | const previousSource = olds?.dataSourceId ?? output?.dataSourceId; |
| 605 | const nextSource = typeof news.dataSourceId === "string" ? news.dataSourceId : undefined; |
| 606 | if ( |
| 607 | previousSource !== undefined && |
| 608 | nextSource !== undefined && |
| 609 | nextSource !== previousSource |
| 610 | ) { |
| 611 | return { action: "replace" as const, deleteFirst: false }; |
| 612 | } |
| 613 | return undefined; |
| 614 | }), |
| 615 | |
| 616 | read: Effect.fn(function* ({ id, olds, output }) { |
| 617 | const env = yield* GcpEnvironment.current; |
| 618 | const location = normalizeLocation(olds?.location ?? output?.location ?? env.region); |
| 619 | const parent = parentOf(env.project, location); |
| 620 | const existing = |
| 621 | output?.name !== undefined |
| 622 | ? yield* getByName(output.name) |
| 623 | : yield* findOwned(parent, id, olds?.dataSourceId); |
| 624 | if (existing === undefined) return undefined; |
| 625 | const attrs = toAttrs(existing, env.project, location); |
| 626 | return (yield* hasAlchemyLabels(id, parseDisplayName(existing.displayName).labels)) |
| 627 | ? attrs |
| 628 | : Unowned(attrs); |
| 629 | }), |
| 630 | |
| 631 | list: () => |
| 632 | Effect.gen(function* () { |
| 633 | const env = yield* GcpEnvironment.current; |
| 634 | const parents = yield* listLocationParents(env.project, env.region); |
| 635 | const unique = |
| 636 | parents.length > 0 ? [...new Set(parents)] : [parentOf(env.project, env.region)]; |
| 637 | const pages = yield* Effect.forEach(unique, (parent) => listAt(parent), { concurrency: 4 }); |
| 638 | return pages |
| 639 | .flat() |
| 640 | .filter((config) => hasOwnershipMarker(config.displayName)) |
| 641 | .map((config) => toAttrs(config, env.project, env.region)); |
| 642 | }), |
| 643 | |
| 644 | reconcile: Effect.fn(function* ({ id, news, output, olds }) { |
| 645 | const env = yield* GcpEnvironment.current; |
| 646 | const location = normalizeLocation(news.location ?? output?.location ?? env.region); |
| 647 | const parent = parentOf(env.project, location); |
| 648 | const ownership = yield* createInternalLabels(id); |
| 649 | const desiredDisplayName = encodeDisplayName(ownership, news.displayName); |
| 650 | const desiredDisabled = news.disabled === true; |
| 651 | const desiredBody = toBody({ ...news, disabled: desiredDisabled }, desiredDisplayName); |
| 652 | |
| 653 | let current = output?.name !== undefined ? yield* getByName(output.name) : undefined; |
| 654 | if (current === undefined) { |
| 655 | current = yield* findOwned(parent, id, news.dataSourceId); |
| 656 | } |
| 657 | |
| 658 | if (current === undefined) { |
| 659 | const created = yield* bqdt |
| 660 | .createProjectsLocationsTransferConfigs({ |
| 661 | parent, |
| 662 | serviceAccountName: news.serviceAccountName, |
| 663 | versionInfo: news.versionInfo, |
| 664 | authorizationCode: news.authorizationCode, |
| 665 | body: desiredBody, |
| 666 | }) |
| 667 | .pipe(Effect.catchTag("Conflict", () => findOwned(parent, id, news.dataSourceId))); |
| 668 | current = created ?? undefined; |
| 669 | } |
| 670 | |
| 671 | if (current === undefined) { |
| 672 | return yield* new TransferConfigNotResolved({ |
| 673 | resourceName: `${parent}/transferConfigs`, |
| 674 | }); |
| 675 | } |
| 676 | |
| 677 | const name = current.name ?? ""; |
| 678 | const mask: string[] = []; |
| 679 | if ((current.displayName ?? "") !== desiredDisplayName) { |
| 680 | mask.push("displayName"); |
| 681 | } |
| 682 | if ((current.destinationDatasetId ?? "") !== (news.destinationDatasetId ?? "")) { |
| 683 | mask.push("destinationDatasetId"); |
| 684 | } |
| 685 | if (!jsonEqual(canonParams(current.params), canonParams(news.params))) { |
| 686 | mask.push("params"); |
| 687 | } |
| 688 | if (news.schedule !== undefined && (current.schedule ?? "") !== news.schedule) { |
| 689 | mask.push("schedule"); |
| 690 | } |
| 691 | if ( |
| 692 | news.scheduleOptions !== undefined && |
| 693 | !jsonEqual( |
| 694 | toScheduleOptions(current.scheduleOptions) ?? null, |
| 695 | compact({ ...news.scheduleOptions }), |
| 696 | ) |
| 697 | ) { |
| 698 | mask.push("scheduleOptions"); |
| 699 | } |
| 700 | if ( |
| 701 | news.scheduleOptionsV2 !== undefined && |
| 702 | !jsonEqual(toScheduleOptionsV2(current.scheduleOptionsV2) ?? null, news.scheduleOptionsV2) |
| 703 | ) { |
| 704 | mask.push("scheduleOptionsV2"); |
| 705 | } |
| 706 | if ( |
| 707 | news.dataRefreshWindowDays !== undefined && |
| 708 | (current.dataRefreshWindowDays ?? 0) !== news.dataRefreshWindowDays |
| 709 | ) { |
| 710 | mask.push("dataRefreshWindowDays"); |
| 711 | } |
| 712 | if ((current.disabled === true) !== desiredDisabled) { |
| 713 | mask.push("disabled"); |
| 714 | } |
| 715 | if ( |
| 716 | news.notificationPubsubTopic !== undefined && |
| 717 | (current.notificationPubsubTopic ?? "") !== news.notificationPubsubTopic |
| 718 | ) { |
| 719 | mask.push("notificationPubsubTopic"); |
| 720 | } |
| 721 | if ( |
| 722 | news.emailPreferences !== undefined && |
| 723 | !jsonEqual( |
| 724 | toEmailPreferences(current.emailPreferences) ?? null, |
| 725 | compact({ ...news.emailPreferences }), |
| 726 | ) |
| 727 | ) { |
| 728 | mask.push("emailPreferences"); |
| 729 | } |
| 730 | if ( |
| 731 | news.encryptionConfiguration !== undefined && |
| 732 | (current.encryptionConfiguration?.kmsKeyName ?? "") !== |
| 733 | (news.encryptionConfiguration.kmsKeyName ?? "") |
| 734 | ) { |
| 735 | mask.push("encryptionConfiguration"); |
| 736 | } |
| 737 | if ( |
| 738 | news.managedTableType !== undefined && |
| 739 | (current.managedTableType ?? "") !== news.managedTableType |
| 740 | ) { |
| 741 | mask.push("managedTableType"); |
| 742 | } |
| 743 | if ( |
| 744 | news.metadataDestination !== undefined && |
| 745 | !jsonEqual( |
| 746 | toMetadataDestination(current.metadataDestination) ?? null, |
| 747 | news.metadataDestination, |
| 748 | ) |
| 749 | ) { |
| 750 | mask.push("metadataDestination"); |
| 751 | } |
| 752 | |
| 753 | const serviceAccountChanged = |
| 754 | (olds?.serviceAccountName ?? "") !== (news.serviceAccountName ?? ""); |
| 755 | |
| 756 | if (mask.length > 0 || serviceAccountChanged) { |
| 757 | current = yield* retryTransient( |
| 758 | bqdt.patchProjectsLocationsTransferConfigs({ |
| 759 | name, |
| 760 | updateMask: mask.length > 0 ? mask.join(",") : "displayName", |
| 761 | serviceAccountName: news.serviceAccountName, |
| 762 | versionInfo: news.versionInfo, |
| 763 | authorizationCode: news.authorizationCode, |
| 764 | body: { ...desiredBody, name }, |
| 765 | }), |
| 766 | ); |
| 767 | current = (yield* getByName(name)) ?? current; |
| 768 | } |
| 769 | |
| 770 | return toAttrs(current, env.project, location); |
| 771 | }), |
| 772 | |
| 773 | delete: Effect.fn(function* ({ output }) { |
| 774 | yield* bqdt.deleteProjectsLocationsTransferConfigs({ name: output.name }).pipe( |
| 775 | Effect.retry({ |
| 776 | while: (error) => error._tag === "Conflict", |
| 777 | times: 8, |
| 778 | schedule: Schedule.spaced("2 seconds"), |
| 779 | }), |
| 780 | Effect.catchTag("NotFound", () => Effect.void), |
| 781 | ); |
| 782 | }), |
| 783 | }); |