The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as composer from "@distilled.cloud/gcp/composer_v1"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import { tagRecord } from "alchemy/Tags"; |
| 8 | import * as Data from "effect/Data"; |
| 9 | import * as Effect from "effect/Effect"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Stream from "effect/Stream"; |
| 12 | import { GcpEnvironment } from "../Environment.ts"; |
| 13 | import { |
| 14 | createInternalLabels, |
| 15 | diffLabels, |
| 16 | hasAlchemyLabels, |
| 17 | stripInternalLabels, |
| 18 | toLabels, |
| 19 | } from "../Labels.ts"; |
| 20 | import { waitForOperation as waitForGcpOperation } from "../Operation.ts"; |
| 21 | import type { Providers } from "../Providers.ts"; |
| 22 | |
| 23 | // Composer rejects 64-character ids (even on GET) despite documenting |
| 24 | // "a letter followed by up to 63" characters. |
| 25 | const MAX_NAME_LENGTH = 63; |
| 26 | |
| 27 | /** |
| 28 | * Composer rejects `locations/-` (`Unexpected location: -`). Nuke walks |
| 29 | * known Composer regions instead. |
| 30 | */ |
| 31 | const LIST_LOCATIONS = [ |
| 32 | "africa-south1", |
| 33 | "asia-east1", |
| 34 | "asia-east2", |
| 35 | "asia-northeast1", |
| 36 | "asia-northeast2", |
| 37 | "asia-northeast3", |
| 38 | "asia-south1", |
| 39 | "asia-south2", |
| 40 | "asia-southeast1", |
| 41 | "asia-southeast2", |
| 42 | "australia-southeast1", |
| 43 | "australia-southeast2", |
| 44 | "europe-central2", |
| 45 | "europe-north1", |
| 46 | "europe-southwest1", |
| 47 | "europe-west1", |
| 48 | "europe-west2", |
| 49 | "europe-west3", |
| 50 | "europe-west4", |
| 51 | "europe-west6", |
| 52 | "europe-west8", |
| 53 | "europe-west9", |
| 54 | "europe-west10", |
| 55 | "europe-west12", |
| 56 | "me-central1", |
| 57 | "me-central2", |
| 58 | "me-west1", |
| 59 | "northamerica-northeast1", |
| 60 | "northamerica-northeast2", |
| 61 | "southamerica-east1", |
| 62 | "southamerica-west1", |
| 63 | "us-central1", |
| 64 | "us-east1", |
| 65 | "us-east4", |
| 66 | "us-east5", |
| 67 | "us-south1", |
| 68 | "us-west1", |
| 69 | "us-west2", |
| 70 | "us-west3", |
| 71 | "us-west4", |
| 72 | ] as const; |
| 73 | |
| 74 | export type EnvironmentConfig = composer.EnvironmentConfig; |
| 75 | export type SoftwareConfig = composer.SoftwareConfig; |
| 76 | export type NodeConfig = composer.NodeConfig; |
| 77 | export type WorkloadsConfig = composer.WorkloadsConfig; |
| 78 | export type PrivateEnvironmentConfig = composer.PrivateEnvironmentConfig; |
| 79 | export type StorageConfig = composer.StorageConfig; |
| 80 | export type MaintenanceWindow = composer.MaintenanceWindow; |
| 81 | export type WebServerNetworkAccessControl = composer.WebServerNetworkAccessControl; |
| 82 | export type MasterAuthorizedNetworksConfig = composer.MasterAuthorizedNetworksConfig; |
| 83 | export type RecoveryConfig = composer.RecoveryConfig; |
| 84 | export type DataRetentionConfig = composer.DataRetentionConfig; |
| 85 | export type DatabaseConfig = composer.DatabaseConfig; |
| 86 | export type WebServerConfig = composer.WebServerConfig; |
| 87 | export type EncryptionConfig = composer.EncryptionConfig; |
| 88 | |
| 89 | export type EnvironmentSize = composer.EnvironmentConfigEnvironmentSizeEnum | (string & {}); |
| 90 | export type ResilienceMode = composer.EnvironmentConfigResilienceModeEnum | (string & {}); |
| 91 | export type EnvironmentState = composer.EnvironmentStateEnum | (string & {}); |
| 92 | |
| 93 | export type EnvironmentProps = { |
| 94 | /** |
| 95 | * Environment id (the `{environment}` segment of |
| 96 | * `projects/{project}/locations/{location}/environments/{environment}`). |
| 97 | * If omitted, a unique RFC1035 name is generated from the stack, stage, |
| 98 | * and logical id. Must start with a lowercase letter, be 1-63 characters, |
| 99 | * and match `[a-z]([-a-z0-9]*[a-z0-9])?`. Immutable — changing it |
| 100 | * replaces the environment. |
| 101 | */ |
| 102 | environmentId?: string; |
| 103 | /** |
| 104 | * Region (`us-central1`, `us-east1`, …). Immutable — changing it |
| 105 | * replaces the environment. `US-CENTRAL1` is accepted and normalized to |
| 106 | * `us-central1`. |
| 107 | * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`) |
| 108 | */ |
| 109 | location?: string; |
| 110 | /** |
| 111 | * User labels. Alchemy ownership labels are merged in automatically. |
| 112 | */ |
| 113 | labels?: Record<string, string>; |
| 114 | /** |
| 115 | * Configuration for software, nodes, workloads, size, networking, and |
| 116 | * related settings. Node network/subnetwork, service account, encryption, |
| 117 | * private-IP, python version, and storage bucket are immutable. |
| 118 | * |
| 119 | * New environments require an explicit `nodeConfig.serviceAccount` — |
| 120 | * Composer rejects creates without one (`Composer environment service |
| 121 | * account is required to be explicitly specified`). The account needs |
| 122 | * `roles/composer.worker` (or broader) on the project. Changing it |
| 123 | * replaces the environment. |
| 124 | */ |
| 125 | config?: EnvironmentConfig; |
| 126 | /** |
| 127 | * Cloud Storage bucket used by the environment (`bucket` has no `gs://` |
| 128 | * prefix). Immutable — changing it replaces the environment. |
| 129 | */ |
| 130 | storageConfig?: StorageConfig; |
| 131 | }; |
| 132 | |
| 133 | export type Environment = Resource< |
| 134 | "GCP.Composer.Environment", |
| 135 | EnvironmentProps, |
| 136 | { |
| 137 | /** Full resource name `projects/{project}/locations/{location}/environments/{environment}`. */ |
| 138 | name: string; |
| 139 | /** Environment id (last path segment). */ |
| 140 | environmentId: string; |
| 141 | /** Project id. */ |
| 142 | project: string; |
| 143 | /** Region id (`us-central1`, …). */ |
| 144 | location: string; |
| 145 | /** Server-generated UUID. */ |
| 146 | uuid: string | undefined; |
| 147 | /** Server-reported state (`RUNNING`, `CREATING`, …). */ |
| 148 | state: string | undefined; |
| 149 | /** User labels (Alchemy ownership labels stripped). */ |
| 150 | labels: Record<string, string>; |
| 151 | /** Configuration currently applied, including output-only URIs. */ |
| 152 | config: EnvironmentConfig | undefined; |
| 153 | /** Storage configuration currently applied. */ |
| 154 | storageConfig: StorageConfig | undefined; |
| 155 | /** Apache Airflow Web UI URI. */ |
| 156 | airflowUri: string | undefined; |
| 157 | /** Airflow Web UI URI for workforce identity federation. */ |
| 158 | airflowByoidUri: string | undefined; |
| 159 | /** GKE cluster that runs this environment. */ |
| 160 | gkeCluster: string | undefined; |
| 161 | /** Cloud Storage prefix of the DAGs for this environment. */ |
| 162 | dagGcsPrefix: string | undefined; |
| 163 | /** RFC3339 creation timestamp. */ |
| 164 | createTime: string | undefined; |
| 165 | /** RFC3339 last-update timestamp. */ |
| 166 | updateTime: string | undefined; |
| 167 | }, |
| 168 | never, |
| 169 | Providers |
| 170 | >; |
| 171 | |
| 172 | /** |
| 173 | * A Cloud Composer environment (Managed Apache Airflow). |
| 174 | * |
| 175 | * Changing `environmentId`, `location`, node network/subnetwork, node |
| 176 | * service account, encryption, private-IP settings, python version, or the |
| 177 | * storage bucket replaces the environment. Labels, PyPI packages, Airflow config overrides, env vars, |
| 178 | * image version, workloads, and environment size update in place — one |
| 179 | * update type per Composer patch, applied sequentially. |
| 180 | * |
| 181 | * Provisioning typically takes 20–45 minutes. Polls the LRO via |
| 182 | * `getProjectsLocationsOperations` (Composer has no wait long-poll). |
| 183 | * |
| 184 | * New environments must name their service account explicitly in |
| 185 | * `config.nodeConfig.serviceAccount`; Composer no longer falls back to the |
| 186 | * default Compute Engine account. The account needs `roles/composer.worker` |
| 187 | * (or broader) on the project. Changing it replaces the environment. |
| 188 | * |
| 189 | * ### Creating an Environment |
| 190 | * **Example:** Generated name, Composer 3 small |
| 191 | * ```typescript |
| 192 | * const airflow = yield* GCP.Composer.Environment("Airflow", { |
| 193 | * config: { |
| 194 | * environmentSize: "ENVIRONMENT_SIZE_SMALL", |
| 195 | * nodeConfig: { |
| 196 | * serviceAccount: "composer-env@my-project.iam.gserviceaccount.com", |
| 197 | * }, |
| 198 | * softwareConfig: { imageVersion: "composer-3-airflow-2" }, |
| 199 | * }, |
| 200 | * }); |
| 201 | * ``` |
| 202 | * |
| 203 | * **Example:** Explicit id, labels, and PyPI packages |
| 204 | * ```typescript |
| 205 | * const airflow = yield* GCP.Composer.Environment("Airflow", { |
| 206 | * environmentId: "app-airflow", |
| 207 | * location: "us-central1", |
| 208 | * labels: { env: "prod" }, |
| 209 | * config: { |
| 210 | * environmentSize: "ENVIRONMENT_SIZE_SMALL", |
| 211 | * nodeConfig: { |
| 212 | * serviceAccount: "composer-env@my-project.iam.gserviceaccount.com", |
| 213 | * }, |
| 214 | * softwareConfig: { |
| 215 | * imageVersion: "composer-3-airflow-2", |
| 216 | * pypiPackages: { numpy: "==2.1.0" }, |
| 217 | * }, |
| 218 | * }, |
| 219 | * }); |
| 220 | * ``` |
| 221 | * |
| 222 | * ### Updating software |
| 223 | * **Example:** Override Airflow config and env vars |
| 224 | * ```typescript |
| 225 | * const airflow = yield* GCP.Composer.Environment("Airflow", { |
| 226 | * environmentId: "app-airflow", |
| 227 | * config: { |
| 228 | * softwareConfig: { |
| 229 | * airflowConfigOverrides: { "core-dags_are_paused_at_creation": "True" }, |
| 230 | * envVariables: { EXAMPLE_VAR: "test" }, |
| 231 | * }, |
| 232 | * }, |
| 233 | * }); |
| 234 | * ``` |
| 235 | * |
| 236 | * @resource |
| 237 | * @category Composer |
| 238 | */ |
| 239 | export const Environment = Resource<Environment>("GCP.Composer.Environment"); |
| 240 | |
| 241 | export class EnvironmentNotResolved extends Data.TaggedError( |
| 242 | "GCP.Composer.EnvironmentNotResolved", |
| 243 | )<{ |
| 244 | resourceName: string; |
| 245 | }> {} |
| 246 | |
| 247 | export class EnvironmentNotReady extends Data.TaggedError("GCP.Composer.EnvironmentNotReady")<{ |
| 248 | resourceName: string; |
| 249 | state: string; |
| 250 | }> {} |
| 251 | |
| 252 | export class EnvironmentFailed extends Data.TaggedError("GCP.Composer.EnvironmentFailed")<{ |
| 253 | resourceName: string; |
| 254 | state: string | undefined; |
| 255 | }> {} |
| 256 | |
| 257 | export class EnvironmentStillExists extends Data.TaggedError( |
| 258 | "GCP.Composer.EnvironmentStillExists", |
| 259 | )<{ |
| 260 | resourceName: string; |
| 261 | }> {} |
| 262 | |
| 263 | const lastSegment = (value: string) => { |
| 264 | const trimmed = value.replace(/\/+$/, ""); |
| 265 | const parts = trimmed.split("/"); |
| 266 | return parts[parts.length - 1] || trimmed; |
| 267 | }; |
| 268 | |
| 269 | const normalizeLocation = (location: string | undefined, defaultLocation: string) => |
| 270 | lastSegment(location ?? defaultLocation).toLowerCase(); |
| 271 | |
| 272 | const rfc1035 = (name: string): string => { |
| 273 | let next = name |
| 274 | .toLowerCase() |
| 275 | .replace(/[^a-z0-9-]/g, "-") |
| 276 | .replace(/-+/g, "-") |
| 277 | .replace(/^-+|-+$/g, ""); |
| 278 | if (!/^[a-z]/.test(next)) next = `c${next}`; |
| 279 | next = next.slice(0, MAX_NAME_LENGTH).replace(/-+$/g, ""); |
| 280 | if (next.length === 0) return "environment"; |
| 281 | if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, MAX_NAME_LENGTH - 1)}0`; |
| 282 | return next.slice(0, MAX_NAME_LENGTH); |
| 283 | }; |
| 284 | |
| 285 | const resourceName = (project: string, location: string, environmentId: string) => |
| 286 | `projects/${project}/locations/${location}/environments/${environmentId}`; |
| 287 | |
| 288 | const parseName = (name: string, defaultLocation: string) => { |
| 289 | const parts = name.split("/").filter((part) => part.length > 0); |
| 290 | const environmentsAt = parts.lastIndexOf("environments"); |
| 291 | const locationsAt = parts.lastIndexOf("locations"); |
| 292 | const projectsAt = parts.lastIndexOf("projects"); |
| 293 | return { |
| 294 | project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "", |
| 295 | location: |
| 296 | locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : defaultLocation, |
| 297 | environmentId: |
| 298 | environmentsAt >= 0 && parts[environmentsAt + 1] |
| 299 | ? parts[environmentsAt + 1]! |
| 300 | : lastSegment(name), |
| 301 | }; |
| 302 | }; |
| 303 | |
| 304 | const userLabels = ( |
| 305 | labels: Record<string, string | undefined> | null | undefined, |
| 306 | ): Record<string, string> => stripInternalLabels(tagRecord(labels)); |
| 307 | |
| 308 | const toId = (id: string, environmentId: string | undefined, existing?: string) => |
| 309 | Effect.gen(function* () { |
| 310 | return ( |
| 311 | environmentId ?? |
| 312 | existing ?? |
| 313 | rfc1035( |
| 314 | yield* createPhysicalName({ |
| 315 | id, |
| 316 | maxLength: MAX_NAME_LENGTH, |
| 317 | lowercase: true, |
| 318 | }), |
| 319 | ) |
| 320 | ); |
| 321 | }); |
| 322 | |
| 323 | const comparable = (value: unknown): unknown => { |
| 324 | if (value === undefined) return undefined; |
| 325 | if (value === null) return null; |
| 326 | if (Array.isArray(value)) return value.map(comparable); |
| 327 | if (typeof value === "object") { |
| 328 | return Object.fromEntries( |
| 329 | Object.entries(value as Record<string, unknown>) |
| 330 | .filter(([, item]) => item !== undefined) |
| 331 | .sort(([a], [b]) => a.localeCompare(b)) |
| 332 | .map(([key, item]) => [key, comparable(item)]), |
| 333 | ); |
| 334 | } |
| 335 | return value; |
| 336 | }; |
| 337 | |
| 338 | const jsonKey = (value: unknown) => JSON.stringify(comparable(value) ?? null); |
| 339 | |
| 340 | const mapOf = ( |
| 341 | map: Record<string, string | undefined> | null | undefined, |
| 342 | ): Record<string, string> => |
| 343 | Object.fromEntries( |
| 344 | Object.entries(map ?? {}).filter((entry): entry is [string, string] => entry[1] !== undefined), |
| 345 | ); |
| 346 | |
| 347 | const mapKey = (map: Record<string, string | undefined> | null | undefined) => jsonKey(mapOf(map)); |
| 348 | |
| 349 | const sizeOf = (size: string | undefined) => { |
| 350 | const value = (size ?? "").toUpperCase(); |
| 351 | return value === "ENVIRONMENT_SIZE_UNSPECIFIED" ? "" : value; |
| 352 | }; |
| 353 | |
| 354 | const resilienceOf = (mode: string | undefined) => { |
| 355 | const value = (mode ?? "").toUpperCase(); |
| 356 | return value === "RESILIENCE_MODE_UNSPECIFIED" ? "" : value; |
| 357 | }; |
| 358 | |
| 359 | const pluginsOf = (mode: string | undefined) => { |
| 360 | const value = (mode ?? "").toUpperCase(); |
| 361 | return value === "WEB_SERVER_PLUGINS_MODE_UNSPECIFIED" ? "" : value; |
| 362 | }; |
| 363 | |
| 364 | const imageVersionMatches = (desired: string | undefined, observed: string | undefined) => { |
| 365 | if (desired === undefined || desired.length === 0) return true; |
| 366 | if (observed === undefined) return false; |
| 367 | if (desired === observed) return true; |
| 368 | if (!observed.startsWith(desired)) return false; |
| 369 | const next = observed[desired.length]; |
| 370 | return next === undefined || next === "-" || next === "."; |
| 371 | }; |
| 372 | |
| 373 | const networkId = (value: string | undefined) => |
| 374 | value === undefined || value.length === 0 ? undefined : lastSegment(value); |
| 375 | |
| 376 | const fieldChanged = (next: unknown, prev: unknown) => |
| 377 | next !== undefined && prev !== undefined && jsonKey(next) !== jsonKey(prev); |
| 378 | |
| 379 | const networkChanged = (next: string | undefined, prev: string | undefined) => |
| 380 | next !== undefined && prev !== undefined && networkId(next) !== networkId(prev); |
| 381 | |
| 382 | const listChanged = ( |
| 383 | next: ReadonlyArray<string> | undefined, |
| 384 | prev: ReadonlyArray<string> | undefined, |
| 385 | ) => |
| 386 | next !== undefined && |
| 387 | prev !== undefined && |
| 388 | jsonKey([...next].map((item) => item.toLowerCase()).sort()) !== |
| 389 | jsonKey([...prev].map((item) => item.toLowerCase()).sort()); |
| 390 | |
| 391 | const immutableChanged = ( |
| 392 | news: EnvironmentProps, |
| 393 | olds: Partial<EnvironmentProps> | undefined, |
| 394 | output: Environment["Attributes"] | undefined, |
| 395 | ) => { |
| 396 | const prevConfig = olds?.config ?? output?.config; |
| 397 | const nextConfig = news.config; |
| 398 | const prevStorage = olds?.storageConfig ?? output?.storageConfig; |
| 399 | const nextStorage = news.storageConfig; |
| 400 | const prevNode = prevConfig?.nodeConfig; |
| 401 | const nextNode = nextConfig?.nodeConfig; |
| 402 | const prevPrivate = prevConfig?.privateEnvironmentConfig; |
| 403 | const nextPrivate = nextConfig?.privateEnvironmentConfig; |
| 404 | |
| 405 | return ( |
| 406 | (nextConfig !== undefined && |
| 407 | prevConfig !== undefined && |
| 408 | (networkChanged(nextNode?.network, prevNode?.network) || |
| 409 | networkChanged(nextNode?.subnetwork, prevNode?.subnetwork) || |
| 410 | fieldChanged(nextNode?.machineType, prevNode?.machineType) || |
| 411 | fieldChanged(nextNode?.diskSizeGb, prevNode?.diskSizeGb) || |
| 412 | networkChanged(nextNode?.location, prevNode?.location) || |
| 413 | networkChanged(nextNode?.serviceAccount, prevNode?.serviceAccount) || |
| 414 | listChanged(nextNode?.oauthScopes, prevNode?.oauthScopes) || |
| 415 | listChanged(nextNode?.tags, prevNode?.tags) || |
| 416 | fieldChanged(nextNode?.ipAllocationPolicy, prevNode?.ipAllocationPolicy) || |
| 417 | fieldChanged(nextNode?.enableIpMasqAgent, prevNode?.enableIpMasqAgent) || |
| 418 | fieldChanged( |
| 419 | nextNode?.composerInternalIpv4CidrBlock, |
| 420 | prevNode?.composerInternalIpv4CidrBlock, |
| 421 | ) || |
| 422 | networkChanged(nextNode?.composerNetworkAttachment, prevNode?.composerNetworkAttachment) || |
| 423 | fieldChanged( |
| 424 | nextConfig.encryptionConfig?.kmsKeyName, |
| 425 | prevConfig.encryptionConfig?.kmsKeyName, |
| 426 | ) || |
| 427 | fieldChanged( |
| 428 | nextConfig.softwareConfig?.pythonVersion, |
| 429 | prevConfig.softwareConfig?.pythonVersion, |
| 430 | ) || |
| 431 | fieldChanged(nextConfig.databaseConfig?.zone, prevConfig.databaseConfig?.zone) || |
| 432 | fieldChanged( |
| 433 | nextPrivate?.enablePrivateEnvironment, |
| 434 | prevPrivate?.enablePrivateEnvironment, |
| 435 | ) || |
| 436 | fieldChanged(nextPrivate?.networkingType, prevPrivate?.networkingType) || |
| 437 | networkChanged( |
| 438 | nextPrivate?.cloudComposerConnectionSubnetwork, |
| 439 | prevPrivate?.cloudComposerConnectionSubnetwork, |
| 440 | ) || |
| 441 | fieldChanged( |
| 442 | nextPrivate?.cloudComposerNetworkIpv4CidrBlock, |
| 443 | prevPrivate?.cloudComposerNetworkIpv4CidrBlock, |
| 444 | ) || |
| 445 | fieldChanged(nextPrivate?.cloudSqlIpv4CidrBlock, prevPrivate?.cloudSqlIpv4CidrBlock) || |
| 446 | fieldChanged(nextPrivate?.webServerIpv4CidrBlock, prevPrivate?.webServerIpv4CidrBlock) || |
| 447 | fieldChanged( |
| 448 | nextPrivate?.enablePrivatelyUsedPublicIps, |
| 449 | prevPrivate?.enablePrivatelyUsedPublicIps, |
| 450 | ) || |
| 451 | fieldChanged(nextPrivate?.privateClusterConfig, prevPrivate?.privateClusterConfig) || |
| 452 | fieldChanged(nextPrivate?.networkingConfig, prevPrivate?.networkingConfig) || |
| 453 | fieldChanged( |
| 454 | nextPrivate?.enablePrivateBuildsOnly, |
| 455 | prevPrivate?.enablePrivateBuildsOnly, |
| 456 | ))) || |
| 457 | (nextStorage?.bucket !== undefined && |
| 458 | prevStorage?.bucket !== undefined && |
| 459 | nextStorage.bucket !== prevStorage.bucket) |
| 460 | ); |
| 461 | }; |
| 462 | |
| 463 | const toAttrs = (environment: composer.Environment, project: string, region: string) => { |
| 464 | const name = environment.name ?? ""; |
| 465 | const parsed = parseName(name, region); |
| 466 | return { |
| 467 | name, |
| 468 | environmentId: parsed.environmentId, |
| 469 | project: parsed.project || project, |
| 470 | location: parsed.location, |
| 471 | uuid: environment.uuid, |
| 472 | state: environment.state, |
| 473 | labels: userLabels(environment.labels), |
| 474 | config: environment.config, |
| 475 | storageConfig: environment.storageConfig, |
| 476 | airflowUri: environment.config?.airflowUri, |
| 477 | airflowByoidUri: environment.config?.airflowByoidUri, |
| 478 | gkeCluster: environment.config?.gkeCluster, |
| 479 | dagGcsPrefix: environment.config?.dagGcsPrefix, |
| 480 | createTime: environment.createTime, |
| 481 | updateTime: environment.updateTime, |
| 482 | }; |
| 483 | }; |
| 484 | |
| 485 | const isPlaceholder = (environment: composer.Environment) => { |
| 486 | const name = environment.name ?? ""; |
| 487 | return name.endsWith("/environments/-") || name.endsWith("/environments/"); |
| 488 | }; |
| 489 | |
| 490 | const getByName = (name: string) => |
| 491 | composer |
| 492 | .getProjectsLocationsEnvironments({ name }) |
| 493 | .pipe(Effect.catchTag("NotFound", () => Effect.undefined)); |
| 494 | |
| 495 | // Environment creates run 20-30 minutes, deletes 10-20. |
| 496 | const waitForOperation = (operation: composer.Operation) => |
| 497 | waitForGcpOperation(operation, (name) => composer.getProjectsLocationsOperations({ name }), { |
| 498 | budget: "45 minutes", |
| 499 | interval: "15 seconds", |
| 500 | }); |
| 501 | |
| 502 | const waitUntilRunning = (name: string) => |
| 503 | getByName(name).pipe( |
| 504 | Effect.filterOrFail( |
| 505 | (environment): environment is composer.Environment => environment !== undefined, |
| 506 | () => new EnvironmentNotResolved({ resourceName: name }), |
| 507 | ), |
| 508 | Effect.filterOrFail( |
| 509 | (environment) => environment.state !== "ERROR", |
| 510 | (environment) => |
| 511 | new EnvironmentFailed({ |
| 512 | resourceName: name, |
| 513 | state: environment.state, |
| 514 | }), |
| 515 | ), |
| 516 | Effect.filterOrFail( |
| 517 | (environment) => environment.state === "RUNNING", |
| 518 | (environment) => |
| 519 | new EnvironmentNotReady({ |
| 520 | resourceName: name, |
| 521 | state: environment.state ?? "STATE_UNSPECIFIED", |
| 522 | }), |
| 523 | ), |
| 524 | Effect.retry({ |
| 525 | while: (error) => |
| 526 | error._tag === "GCP.Composer.EnvironmentNotResolved" || |
| 527 | error._tag === "GCP.Composer.EnvironmentNotReady", |
| 528 | times: 240, |
| 529 | schedule: Schedule.spaced("15 seconds"), |
| 530 | }), |
| 531 | ); |
| 532 | |
| 533 | const waitUntilGone = (name: string) => |
| 534 | getByName(name).pipe( |
| 535 | Effect.flatMap((environment) => |
| 536 | environment === undefined |
| 537 | ? Effect.void |
| 538 | : Effect.fail(new EnvironmentStillExists({ resourceName: name })), |
| 539 | ), |
| 540 | Effect.retry({ |
| 541 | while: (error) => error._tag === "GCP.Composer.EnvironmentStillExists", |
| 542 | times: 120, |
| 543 | schedule: Schedule.spaced("15 seconds"), |
| 544 | }), |
| 545 | ); |
| 546 | |
| 547 | const listOwnedAt = (project: string, location: string) => |
| 548 | composer.listProjectsLocationsEnvironments |
| 549 | .pages({ |
| 550 | parent: `projects/${project}/locations/${location}`, |
| 551 | pageSize: 1000, |
| 552 | }) |
| 553 | .pipe( |
| 554 | Stream.flatMap((page) => Stream.fromIterable(page.environments ?? [])), |
| 555 | Stream.filter( |
| 556 | (environment) => |
| 557 | !isPlaceholder(environment) && |
| 558 | Object.keys(environment.labels ?? {}).some((key) => key.startsWith("alchemy-")), |
| 559 | ), |
| 560 | Stream.map((environment) => toAttrs(environment, project, location)), |
| 561 | Stream.runCollect, |
| 562 | Effect.map((chunk) => Array.from(chunk)), |
| 563 | // A location Composer does not serve has no environments. |
| 564 | Effect.catchTag("NotFound", () => Effect.succeed([])), |
| 565 | ); |
| 566 | |
| 567 | const toCreateConfig = ( |
| 568 | config: composer.EnvironmentConfig | undefined, |
| 569 | ): composer.EnvironmentConfig | undefined => { |
| 570 | if (config === undefined) return undefined; |
| 571 | const { |
| 572 | airflowByoidUri: _airflowByoidUri, |
| 573 | airflowUri: _airflowUri, |
| 574 | gkeCluster: _gkeCluster, |
| 575 | dagGcsPrefix: _dagGcsPrefix, |
| 576 | ...rest |
| 577 | } = config; |
| 578 | return rest; |
| 579 | }; |
| 580 | |
| 581 | type EnvironmentPatch = { |
| 582 | mask: string; |
| 583 | body: composer.Environment; |
| 584 | }; |
| 585 | |
| 586 | const desiredPatches = ( |
| 587 | news: EnvironmentProps, |
| 588 | current: composer.Environment, |
| 589 | desiredLabels: Record<string, string>, |
| 590 | labelsChanged: boolean, |
| 591 | ): EnvironmentPatch[] => { |
| 592 | const patches: EnvironmentPatch[] = []; |
| 593 | if (labelsChanged) { |
| 594 | patches.push({ mask: "labels", body: { labels: desiredLabels } }); |
| 595 | } |
| 596 | |
| 597 | const nextSoftware = news.config?.softwareConfig; |
| 598 | const currentSoftware = current.config?.softwareConfig; |
| 599 | |
| 600 | if ( |
| 601 | nextSoftware?.pypiPackages !== undefined && |
| 602 | mapKey(nextSoftware.pypiPackages) !== mapKey(currentSoftware?.pypiPackages) |
| 603 | ) { |
| 604 | patches.push({ |
| 605 | mask: "config.softwareConfig.pypiPackages", |
| 606 | body: { |
| 607 | config: { softwareConfig: { pypiPackages: nextSoftware.pypiPackages } }, |
| 608 | }, |
| 609 | }); |
| 610 | } |
| 611 | if ( |
| 612 | nextSoftware?.airflowConfigOverrides !== undefined && |
| 613 | mapKey(nextSoftware.airflowConfigOverrides) !== mapKey(currentSoftware?.airflowConfigOverrides) |
| 614 | ) { |
| 615 | patches.push({ |
| 616 | mask: "config.softwareConfig.airflowConfigOverrides", |
| 617 | body: { |
| 618 | config: { |
| 619 | softwareConfig: { |
| 620 | airflowConfigOverrides: nextSoftware.airflowConfigOverrides, |
| 621 | }, |
| 622 | }, |
| 623 | }, |
| 624 | }); |
| 625 | } |
| 626 | if ( |
| 627 | nextSoftware?.envVariables !== undefined && |
| 628 | mapKey(nextSoftware.envVariables) !== mapKey(currentSoftware?.envVariables) |
| 629 | ) { |
| 630 | patches.push({ |
| 631 | mask: "config.softwareConfig.envVariables", |
| 632 | body: { |
| 633 | config: { softwareConfig: { envVariables: nextSoftware.envVariables } }, |
| 634 | }, |
| 635 | }); |
| 636 | } |
| 637 | if ( |
| 638 | nextSoftware?.imageVersion !== undefined && |
| 639 | !imageVersionMatches(nextSoftware.imageVersion, currentSoftware?.imageVersion) |
| 640 | ) { |
| 641 | patches.push({ |
| 642 | mask: "config.softwareConfig.imageVersion", |
| 643 | body: { |
| 644 | config: { softwareConfig: { imageVersion: nextSoftware.imageVersion } }, |
| 645 | }, |
| 646 | }); |
| 647 | } |
| 648 | if ( |
| 649 | nextSoftware?.schedulerCount !== undefined && |
| 650 | (currentSoftware?.schedulerCount ?? 0) !== nextSoftware.schedulerCount |
| 651 | ) { |
| 652 | patches.push({ |
| 653 | mask: "config.softwareConfig.schedulerCount", |
| 654 | body: { |
| 655 | config: { |
| 656 | softwareConfig: { schedulerCount: nextSoftware.schedulerCount }, |
| 657 | }, |
| 658 | }, |
| 659 | }); |
| 660 | } |
| 661 | if ( |
| 662 | nextSoftware?.cloudDataLineageIntegration !== undefined && |
| 663 | jsonKey(nextSoftware.cloudDataLineageIntegration) !== |
| 664 | jsonKey(currentSoftware?.cloudDataLineageIntegration) |
| 665 | ) { |
| 666 | patches.push({ |
| 667 | mask: "config.softwareConfig.cloudDataLineageIntegration", |
| 668 | body: { |
| 669 | config: { |
| 670 | softwareConfig: { |
| 671 | cloudDataLineageIntegration: nextSoftware.cloudDataLineageIntegration, |
| 672 | }, |
| 673 | }, |
| 674 | }, |
| 675 | }); |
| 676 | } |
| 677 | if ( |
| 678 | nextSoftware?.webServerPluginsMode !== undefined && |
| 679 | pluginsOf(nextSoftware.webServerPluginsMode) !== |
| 680 | pluginsOf(currentSoftware?.webServerPluginsMode) |
| 681 | ) { |
| 682 | patches.push({ |
| 683 | mask: "config.softwareConfig.webServerPluginsMode", |
| 684 | body: { |
| 685 | config: { |
| 686 | softwareConfig: { |
| 687 | webServerPluginsMode: nextSoftware.webServerPluginsMode, |
| 688 | }, |
| 689 | }, |
| 690 | }, |
| 691 | }); |
| 692 | } |
| 693 | |
| 694 | const nextConfig = news.config; |
| 695 | if ( |
| 696 | nextConfig?.nodeCount !== undefined && |
| 697 | (current.config?.nodeCount ?? 0) !== nextConfig.nodeCount |
| 698 | ) { |
| 699 | patches.push({ |
| 700 | mask: "config.nodeCount", |
| 701 | body: { config: { nodeCount: nextConfig.nodeCount } }, |
| 702 | }); |
| 703 | } |
| 704 | if ( |
| 705 | nextConfig?.webServerNetworkAccessControl !== undefined && |
| 706 | jsonKey(nextConfig.webServerNetworkAccessControl) !== |
| 707 | jsonKey(current.config?.webServerNetworkAccessControl) |
| 708 | ) { |
| 709 | patches.push({ |
| 710 | mask: "config.webServerNetworkAccessControl", |
| 711 | body: { |
| 712 | config: { |
| 713 | webServerNetworkAccessControl: nextConfig.webServerNetworkAccessControl, |
| 714 | }, |
| 715 | }, |
| 716 | }); |
| 717 | } |
| 718 | if ( |
| 719 | nextConfig?.databaseConfig?.machineType !== undefined && |
| 720 | (current.config?.databaseConfig?.machineType ?? "") !== nextConfig.databaseConfig.machineType |
| 721 | ) { |
| 722 | patches.push({ |
| 723 | mask: "config.databaseConfig.machineType", |
| 724 | body: { |
| 725 | config: { |
| 726 | databaseConfig: { |
| 727 | machineType: nextConfig.databaseConfig.machineType, |
| 728 | }, |
| 729 | }, |
| 730 | }, |
| 731 | }); |
| 732 | } |
| 733 | if ( |
| 734 | nextConfig?.webServerConfig?.machineType !== undefined && |
| 735 | (current.config?.webServerConfig?.machineType ?? "") !== nextConfig.webServerConfig.machineType |
| 736 | ) { |
| 737 | patches.push({ |
| 738 | mask: "config.webServerConfig.machineType", |
| 739 | body: { |
| 740 | config: { |
| 741 | webServerConfig: { |
| 742 | machineType: nextConfig.webServerConfig.machineType, |
| 743 | }, |
| 744 | }, |
| 745 | }, |
| 746 | }); |
| 747 | } |
| 748 | if ( |
| 749 | nextConfig?.maintenanceWindow !== undefined && |
| 750 | jsonKey(nextConfig.maintenanceWindow) !== jsonKey(current.config?.maintenanceWindow) |
| 751 | ) { |
| 752 | patches.push({ |
| 753 | mask: "config.maintenanceWindow", |
| 754 | body: { config: { maintenanceWindow: nextConfig.maintenanceWindow } }, |
| 755 | }); |
| 756 | } |
| 757 | if ( |
| 758 | nextConfig?.workloadsConfig !== undefined && |
| 759 | jsonKey(nextConfig.workloadsConfig) !== jsonKey(current.config?.workloadsConfig) |
| 760 | ) { |
| 761 | patches.push({ |
| 762 | mask: "config.workloadsConfig", |
| 763 | body: { config: { workloadsConfig: nextConfig.workloadsConfig } }, |
| 764 | }); |
| 765 | } |
| 766 | if ( |
| 767 | nextConfig?.environmentSize !== undefined && |
| 768 | sizeOf(nextConfig.environmentSize) !== sizeOf(current.config?.environmentSize) |
| 769 | ) { |
| 770 | patches.push({ |
| 771 | mask: "config.environmentSize", |
| 772 | body: { config: { environmentSize: nextConfig.environmentSize } }, |
| 773 | }); |
| 774 | } |
| 775 | if ( |
| 776 | nextConfig?.resilienceMode !== undefined && |
| 777 | resilienceOf(nextConfig.resilienceMode) !== resilienceOf(current.config?.resilienceMode) |
| 778 | ) { |
| 779 | patches.push({ |
| 780 | mask: "config.resilienceMode", |
| 781 | body: { config: { resilienceMode: nextConfig.resilienceMode } }, |
| 782 | }); |
| 783 | } |
| 784 | if ( |
| 785 | nextConfig?.masterAuthorizedNetworksConfig !== undefined && |
| 786 | jsonKey(nextConfig.masterAuthorizedNetworksConfig) !== |
| 787 | jsonKey(current.config?.masterAuthorizedNetworksConfig) |
| 788 | ) { |
| 789 | patches.push({ |
| 790 | mask: "config.masterAuthorizedNetworksConfig", |
| 791 | body: { |
| 792 | config: { |
| 793 | masterAuthorizedNetworksConfig: nextConfig.masterAuthorizedNetworksConfig, |
| 794 | }, |
| 795 | }, |
| 796 | }); |
| 797 | } |
| 798 | if ( |
| 799 | nextConfig?.recoveryConfig !== undefined && |
| 800 | jsonKey(nextConfig.recoveryConfig) !== jsonKey(current.config?.recoveryConfig) |
| 801 | ) { |
| 802 | patches.push({ |
| 803 | mask: "config.recoveryConfig", |
| 804 | body: { config: { recoveryConfig: nextConfig.recoveryConfig } }, |
| 805 | }); |
| 806 | } |
| 807 | if ( |
| 808 | nextConfig?.dataRetentionConfig !== undefined && |
| 809 | jsonKey(nextConfig.dataRetentionConfig) !== jsonKey(current.config?.dataRetentionConfig) |
| 810 | ) { |
| 811 | patches.push({ |
| 812 | mask: "config.dataRetentionConfig", |
| 813 | body: { |
| 814 | config: { dataRetentionConfig: nextConfig.dataRetentionConfig }, |
| 815 | }, |
| 816 | }); |
| 817 | } |
| 818 | |
| 819 | return patches; |
| 820 | }; |
| 821 | |
| 822 | export const EnvironmentProvider = () => |
| 823 | Provider.succeed(Environment, { |
| 824 | stables: ["name", "environmentId", "project", "location", "uuid", "createTime"], |
| 825 | |
| 826 | diff: Effect.fn(function* ({ news, olds, output }) { |
| 827 | if (!isResolved(news)) return undefined; |
| 828 | const env = yield* GcpEnvironment.current; |
| 829 | |
| 830 | const previousId = olds?.environmentId ?? output?.environmentId; |
| 831 | const nextId = news.environmentId ?? previousId; |
| 832 | const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region); |
| 833 | const nextLocation = normalizeLocation(news.location ?? output?.location, env.region); |
| 834 | const replace = |
| 835 | (previousId !== undefined && nextId !== undefined && nextId !== previousId) || |
| 836 | previousLocation !== nextLocation || |
| 837 | immutableChanged(news, olds, output); |
| 838 | |
| 839 | if (!replace) return undefined; |
| 840 | return { |
| 841 | action: "replace" as const, |
| 842 | deleteFirst: |
| 843 | previousLocation === nextLocation && previousId !== undefined && nextId === previousId, |
| 844 | }; |
| 845 | }), |
| 846 | |
| 847 | read: Effect.fn(function* ({ id, olds, output }) { |
| 848 | const env = yield* GcpEnvironment.current; |
| 849 | const environmentId = yield* toId(id, olds?.environmentId, output?.environmentId); |
| 850 | const location = normalizeLocation(olds?.location ?? output?.location, env.region); |
| 851 | const name = output?.name ?? resourceName(env.project, location, environmentId); |
| 852 | const existing = yield* getByName(name); |
| 853 | if (existing === undefined) return undefined; |
| 854 | const attrs = toAttrs(existing, env.project, env.region); |
| 855 | return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs); |
| 856 | }), |
| 857 | |
| 858 | list: () => |
| 859 | Effect.gen(function* () { |
| 860 | const env = yield* GcpEnvironment.current; |
| 861 | const pages = yield* Effect.forEach( |
| 862 | Array.from(new Set<string>([...LIST_LOCATIONS, env.region])), |
| 863 | (location) => listOwnedAt(env.project, location), |
| 864 | { concurrency: 8 }, |
| 865 | ); |
| 866 | return pages.flat(); |
| 867 | }), |
| 868 | |
| 869 | reconcile: Effect.fn(function* ({ id, news, output }) { |
| 870 | const env = yield* GcpEnvironment.current; |
| 871 | const environmentId = yield* toId(id, news.environmentId, output?.environmentId); |
| 872 | const location = normalizeLocation(news.location ?? output?.location, env.region); |
| 873 | const name = resourceName(env.project, location, environmentId); |
| 874 | const desiredLabels = { |
| 875 | ...toLabels(news.labels), |
| 876 | ...(yield* createInternalLabels(id)), |
| 877 | }; |
| 878 | |
| 879 | let current = yield* getByName(output?.name ?? name); |
| 880 | |
| 881 | if (current === undefined) { |
| 882 | const created = yield* composer |
| 883 | .createProjectsLocationsEnvironments({ |
| 884 | parent: `projects/${env.project}/locations/${location}`, |
| 885 | body: { |
| 886 | name, |
| 887 | labels: desiredLabels, |
| 888 | config: toCreateConfig(news.config), |
| 889 | storageConfig: news.storageConfig, |
| 890 | }, |
| 891 | }) |
| 892 | .pipe(Effect.catchTag("Conflict", () => Effect.undefined)); |
| 893 | if (created !== undefined) { |
| 894 | // ALREADY_EXISTS (6): a concurrent create won the race. |
| 895 | yield* waitForOperation(created).pipe( |
| 896 | Effect.catchTag("GCP.OperationFailed", (error) => |
| 897 | error.code === 6 ? Effect.void : Effect.fail(error), |
| 898 | ), |
| 899 | ); |
| 900 | } |
| 901 | current = yield* waitUntilRunning(name); |
| 902 | } |
| 903 | |
| 904 | if (current === undefined) { |
| 905 | return yield* new EnvironmentNotResolved({ resourceName: name }); |
| 906 | } |
| 907 | |
| 908 | if (current.state === "ERROR") { |
| 909 | return yield* new EnvironmentFailed({ |
| 910 | resourceName: name, |
| 911 | state: current.state, |
| 912 | }); |
| 913 | } |
| 914 | |
| 915 | if (current.state !== "RUNNING") { |
| 916 | current = yield* waitUntilRunning(name); |
| 917 | } |
| 918 | |
| 919 | const observedLabels = tagRecord(current.labels); |
| 920 | const { upsert, removed } = diffLabels(observedLabels, desiredLabels); |
| 921 | const labelsChanged = upsert.length > 0 || removed.length > 0; |
| 922 | const patches = desiredPatches(news, current, desiredLabels, labelsChanged); |
| 923 | |
| 924 | for (const patch of patches) { |
| 925 | const operation = yield* composer |
| 926 | .patchProjectsLocationsEnvironments({ |
| 927 | name: current.name ?? name, |
| 928 | updateMask: patch.mask, |
| 929 | body: patch.body, |
| 930 | }) |
| 931 | .pipe( |
| 932 | Effect.retry({ |
| 933 | while: (error) => error._tag === "Conflict", |
| 934 | times: 8, |
| 935 | schedule: Schedule.spaced("5 seconds"), |
| 936 | }), |
| 937 | ); |
| 938 | yield* waitForOperation(operation); |
| 939 | current = yield* waitUntilRunning(current.name ?? name); |
| 940 | } |
| 941 | |
| 942 | return toAttrs(current, env.project, env.region); |
| 943 | }), |
| 944 | |
| 945 | delete: Effect.fn(function* ({ output }) { |
| 946 | const operation = yield* composer |
| 947 | .deleteProjectsLocationsEnvironments({ name: output.name }) |
| 948 | .pipe( |
| 949 | Effect.catchTag("NotFound", () => Effect.undefined), |
| 950 | Effect.retry({ |
| 951 | while: (error) => error._tag === "Conflict", |
| 952 | times: 8, |
| 953 | schedule: Schedule.spaced("5 seconds"), |
| 954 | }), |
| 955 | ); |
| 956 | if (operation !== undefined) { |
| 957 | // NOT_FOUND (5): the environment was already gone. |
| 958 | yield* waitForOperation(operation).pipe( |
| 959 | Effect.catchTag("GCP.OperationFailed", (error) => |
| 960 | error.code === 5 ? Effect.void : Effect.fail(error), |
| 961 | ), |
| 962 | ); |
| 963 | } |
| 964 | yield* waitUntilGone(output.name); |
| 965 | }), |
| 966 | }); |