The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as r2 from "@distilled.cloud/cloudflare/r2"; |
| 2 | import * as Retry from "@distilled.cloud/cloudflare/Retry"; |
| 3 | import { deepEqual, isResolved } from "alchemy/Diff"; |
| 4 | import * as Provider from "alchemy/Provider"; |
| 5 | import { Resource } from "alchemy/Resource"; |
| 6 | import * as Data from "effect/Data"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as Redacted from "effect/Redacted"; |
| 9 | import * as Schedule from "effect/Schedule"; |
| 10 | import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; |
| 11 | import type { Providers } from "../Providers.ts"; |
| 12 | |
| 13 | export class SuperSlurperJobUnexpectedResponse extends Data.TaggedError( |
| 14 | "SuperSlurperJobUnexpectedResponse", |
| 15 | )<{ |
| 16 | readonly message: string; |
| 17 | readonly cause?: unknown; |
| 18 | }> {} |
| 19 | |
| 20 | /** S3-compatible credentials with access to a migration bucket. */ |
| 21 | export interface SuperSlurperCredentials { |
| 22 | /** Access key ID. Changing credentials replaces the migration. */ |
| 23 | accessKeyId: Redacted.Redacted<string>; |
| 24 | /** Secret access key. Changing credentials replaces the migration. */ |
| 25 | secretAccessKey: Redacted.Redacted<string>; |
| 26 | } |
| 27 | |
| 28 | /** Object selection shared by all source vendors. */ |
| 29 | export interface SuperSlurperSourceSelection { |
| 30 | /** Name of the source bucket. Immutable. */ |
| 31 | bucket: string; |
| 32 | /** Object keys to migrate. Omit to migrate all matching objects. Immutable. */ |
| 33 | keys?: string[]; |
| 34 | /** Restrict migration to this object key prefix. Immutable. */ |
| 35 | pathPrefix?: string; |
| 36 | } |
| 37 | |
| 38 | /** AWS S3 or another HTTPS S3-compatible source. */ |
| 39 | export interface SuperSlurperS3Source extends SuperSlurperSourceSelection { |
| 40 | /** Select the S3 source API. */ |
| 41 | vendor: "s3"; |
| 42 | /** Source credentials with read access. Immutable. */ |
| 43 | secret: SuperSlurperCredentials; |
| 44 | /** Custom S3-compatible HTTPS endpoint. Immutable. */ |
| 45 | endpoint?: string; |
| 46 | /** Source bucket region. Immutable. */ |
| 47 | region?: string; |
| 48 | } |
| 49 | |
| 50 | /** Google Cloud Storage source. */ |
| 51 | export interface SuperSlurperGcsSource extends SuperSlurperSourceSelection { |
| 52 | /** Select Google Cloud Storage. */ |
| 53 | vendor: "gcs"; |
| 54 | /** Service account credentials with source read access. Immutable. */ |
| 55 | secret: { |
| 56 | /** Service account email. */ |
| 57 | clientEmail: Redacted.Redacted<string>; |
| 58 | /** Service account private key in PEM format. */ |
| 59 | privateKey: Redacted.Redacted<string>; |
| 60 | }; |
| 61 | } |
| 62 | |
| 63 | /** Cloudflare R2 source. */ |
| 64 | export interface SuperSlurperR2Source extends SuperSlurperSourceSelection { |
| 65 | /** Select Cloudflare R2. */ |
| 66 | vendor: "r2"; |
| 67 | /** S3-compatible R2 credentials with source read access. Immutable. */ |
| 68 | secret: SuperSlurperCredentials; |
| 69 | /** Source jurisdiction. Immutable. @default "default" */ |
| 70 | jurisdiction?: "default" | "eu" | "us" | "fedramp"; |
| 71 | } |
| 72 | |
| 73 | /** Supported source bucket configurations. */ |
| 74 | export type SuperSlurperSource = |
| 75 | | SuperSlurperS3Source |
| 76 | | SuperSlurperGcsSource |
| 77 | | SuperSlurperR2Source; |
| 78 | |
| 79 | /** R2 destination for migrated objects. */ |
| 80 | export interface SuperSlurperTarget { |
| 81 | /** Name of the destination R2 bucket. Immutable. */ |
| 82 | bucket: string; |
| 83 | /** The destination vendor must be R2. */ |
| 84 | vendor: "r2"; |
| 85 | /** S3-compatible R2 credentials with destination write access. Immutable. */ |
| 86 | secret: SuperSlurperCredentials; |
| 87 | /** Destination jurisdiction. Immutable. @default "default" */ |
| 88 | jurisdiction?: "default" | "eu" | "us" | "fedramp"; |
| 89 | } |
| 90 | |
| 91 | export interface SuperSlurperJobProps { |
| 92 | /** Source bucket and credentials. Any change replaces the job. */ |
| 93 | source: SuperSlurperSource; |
| 94 | /** Destination R2 bucket and credentials. Any change replaces the job. */ |
| 95 | target: SuperSlurperTarget; |
| 96 | /** Overwrite existing destination objects. Immutable. @default false */ |
| 97 | overwrite?: boolean; |
| 98 | /** |
| 99 | * Pause an active job, or resume it when false. Terminal jobs are never |
| 100 | * restarted. A new job starts running before the pause request is applied, |
| 101 | * so this is not a guarantee that no objects are transferred. |
| 102 | * @default false |
| 103 | */ |
| 104 | paused?: boolean; |
| 105 | } |
| 106 | |
| 107 | export interface SuperSlurperJobAttributes { |
| 108 | /** Cloudflare account that owns the migration. */ |
| 109 | accountId: string; |
| 110 | /** Server-assigned migration ID. Stable until replacement. */ |
| 111 | jobId: string; |
| 112 | /** Last observed job status; not a completion guarantee. */ |
| 113 | status: r2.GetSuperSlurperJobResponse["status"]; |
| 114 | /** Creation timestamp reported by Cloudflare. */ |
| 115 | createdAt: string | undefined; |
| 116 | /** Completion timestamp, if the job has finished. */ |
| 117 | finishedAt: string | undefined; |
| 118 | } |
| 119 | |
| 120 | export type SuperSlurperJob = Resource< |
| 121 | "Cloudflare.R2.SuperSlurperJob", |
| 122 | SuperSlurperJobProps, |
| 123 | SuperSlurperJobAttributes, |
| 124 | never, |
| 125 | Providers |
| 126 | >; |
| 127 | |
| 128 | /** |
| 129 | * Starts a bulk migration from S3, GCS, or R2 into an R2 bucket. |
| 130 | * |
| 131 | * Deployment returns the job identity without waiting for the migration. |
| 132 | * Completed and aborted jobs remain resources: redeploying does not run them |
| 133 | * again. Changing source, target, credentials, or overwrite replaces the job. |
| 134 | * Destroy only cancels this resource's active job; it never deletes copied |
| 135 | * objects or Cloudflare's job history. Transfers already in flight may finish. |
| 136 | * |
| 137 | * Cloudflare exposes no job name, ownership tags, or idempotency key. The |
| 138 | * persisted job ID is the only ownership record; similar jobs are never |
| 139 | * adopted. If a create response or the subsequent state write is lost, the |
| 140 | * job cannot be recovered automatically and retrying may create a second job. |
| 141 | * Inspect the account's migration history and cancel that orphan explicitly. |
| 142 | * Account-wide adoption and cancellation are intentionally not supported. |
| 143 | * If reading a cached ID fails with Cloudflare's ambiguous server error, |
| 144 | * paginated job history is checked for that exact ID before declaring it absent. |
| 145 | * |
| 146 | * ### Migrating from S3 |
| 147 | * **Example:** Copy an S3 prefix into R2 |
| 148 | * ```typescript |
| 149 | * const target = yield* Cloudflare.R2.Bucket("Archive"); |
| 150 | * const job = yield* Cloudflare.R2.SuperSlurperJob("Migration", { |
| 151 | * source: { |
| 152 | * vendor: "s3", |
| 153 | * bucket: "legacy-archive", |
| 154 | * region: "us-east-1", |
| 155 | * pathPrefix: "photos/", |
| 156 | * secret: { |
| 157 | * accessKeyId: yield* Config.Redacted("AWS_ACCESS_KEY_ID"), |
| 158 | * secretAccessKey: yield* Config.Redacted("AWS_SECRET_ACCESS_KEY"), |
| 159 | * }, |
| 160 | * }, |
| 161 | * target: { |
| 162 | * vendor: "r2", |
| 163 | * bucket: target.bucketName, |
| 164 | * secret: { |
| 165 | * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"), |
| 166 | * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"), |
| 167 | * }, |
| 168 | * }, |
| 169 | * }); |
| 170 | * ``` |
| 171 | * |
| 172 | * ### Migrating from GCS or R2 |
| 173 | * **Example:** Google Cloud Storage source |
| 174 | * ```typescript |
| 175 | * const source: Cloudflare.R2.SuperSlurperSource = { |
| 176 | * vendor: "gcs", |
| 177 | * bucket: "legacy-media", |
| 178 | * secret: { |
| 179 | * clientEmail: yield* Config.Redacted("GCS_CLIENT_EMAIL"), |
| 180 | * privateKey: yield* Config.Redacted("GCS_PRIVATE_KEY"), |
| 181 | * }, |
| 182 | * }; |
| 183 | * ``` |
| 184 | * |
| 185 | * **Example:** R2 source and paused desired state |
| 186 | * ```typescript |
| 187 | * yield* Cloudflare.R2.SuperSlurperJob("Migration", { |
| 188 | * source: { |
| 189 | * vendor: "r2", |
| 190 | * bucket: "old-media", |
| 191 | * secret: { |
| 192 | * accessKeyId: yield* Config.Redacted("SOURCE_R2_ACCESS_KEY_ID"), |
| 193 | * secretAccessKey: yield* Config.Redacted("SOURCE_R2_SECRET_ACCESS_KEY"), |
| 194 | * }, |
| 195 | * }, |
| 196 | * target: { |
| 197 | * vendor: "r2", |
| 198 | * bucket: "new-media", |
| 199 | * secret: { |
| 200 | * accessKeyId: yield* Config.Redacted("TARGET_R2_ACCESS_KEY_ID"), |
| 201 | * secretAccessKey: yield* Config.Redacted("TARGET_R2_SECRET_ACCESS_KEY"), |
| 202 | * }, |
| 203 | * }, |
| 204 | * paused: true, |
| 205 | * }); |
| 206 | * ``` |
| 207 | * |
| 208 | * ### Referencing a Migration |
| 209 | * **Example:** Read a persisted job identity without creating a new job |
| 210 | * ```typescript |
| 211 | * const job = yield* Cloudflare.R2.SuperSlurperJob.ref("Migration"); |
| 212 | * return { accountId: job.accountId, jobId: job.jobId }; |
| 213 | * ``` |
| 214 | * |
| 215 | * @resource |
| 216 | * @product R2 |
| 217 | * @category Storage & Databases |
| 218 | */ |
| 219 | export const SuperSlurperJob = Resource<SuperSlurperJob>("Cloudflare.R2.SuperSlurperJob"); |
| 220 | |
| 221 | export const SuperSlurperJobProvider = () => |
| 222 | Provider.succeed(SuperSlurperJob, { |
| 223 | stables: ["accountId", "jobId"], |
| 224 | nuke: { skip: true }, |
| 225 | diff: Effect.fn(function* ({ olds, news, output }) { |
| 226 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 227 | if (output && output.accountId !== accountId) { |
| 228 | return { action: "replace" } as const; |
| 229 | } |
| 230 | if ( |
| 231 | !("source" in news) || |
| 232 | !isResolved(news.source) || |
| 233 | !isResolved(news.target) || |
| 234 | !isResolved(news.overwrite) |
| 235 | ) { |
| 236 | return { action: "replace" } as const; |
| 237 | } |
| 238 | if ( |
| 239 | !deepEqual(olds.source, news.source) || |
| 240 | !deepEqual(olds.target, news.target) || |
| 241 | (olds.overwrite ?? false) !== (news.overwrite ?? false) |
| 242 | ) { |
| 243 | return { action: "replace" } as const; |
| 244 | } |
| 245 | }), |
| 246 | read: Effect.fn(function* ({ output }) { |
| 247 | if (!output) return undefined; |
| 248 | const observed = yield* observe(output); |
| 249 | return observed ? attributes(output, observed) : undefined; |
| 250 | }), |
| 251 | reconcile: Effect.fn(function* ({ news, output }) { |
| 252 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 253 | let identity = output; |
| 254 | let observed = identity ? yield* observe(identity) : undefined; |
| 255 | if (!observed) { |
| 256 | const created = yield* r2 |
| 257 | .createSuperSlurperJob({ |
| 258 | accountId, |
| 259 | source: sourceRequest(news.source), |
| 260 | target: { |
| 261 | ...news.target, |
| 262 | secret: credentialsRequest(news.target.secret), |
| 263 | }, |
| 264 | overwrite: news.overwrite ?? false, |
| 265 | }) |
| 266 | .pipe( |
| 267 | Retry.none, |
| 268 | // Preconnectivity rejects before creating a job; fresh tokens may not have propagated. |
| 269 | Effect.retry({ |
| 270 | while: (error) => error._tag === "SuperSlurperPreconnectivityFailed", |
| 271 | schedule: Schedule.spaced("3 seconds"), |
| 272 | times: 8, |
| 273 | }), |
| 274 | ); |
| 275 | if (!created.id) { |
| 276 | return yield* new SuperSlurperJobUnexpectedResponse({ |
| 277 | message: "Super Slurper did not return a job ID", |
| 278 | }); |
| 279 | } |
| 280 | identity = attributes({ accountId, jobId: created.id }, { status: "running" }); |
| 281 | observed = { id: created.id, status: "running" }; |
| 282 | } |
| 283 | const current = identity!; |
| 284 | if (news.paused && observed.status === "running") { |
| 285 | yield* mutate(current, "pause"); |
| 286 | observed = (yield* observe(current)) ?? observed; |
| 287 | } else if (!news.paused && observed.status === "paused") { |
| 288 | yield* mutate(current, "resume"); |
| 289 | observed = (yield* observe(current)) ?? observed; |
| 290 | } |
| 291 | return attributes(current, observed); |
| 292 | }), |
| 293 | delete: Effect.fn(function* ({ output }) { |
| 294 | const observed = yield* observe(output); |
| 295 | if (observed?.status === "running" || observed?.status === "paused") { |
| 296 | yield* mutate(output, "abort"); |
| 297 | } |
| 298 | }), |
| 299 | }); |
| 300 | |
| 301 | const observe = (identity: { accountId: string; jobId: string }) => |
| 302 | r2 |
| 303 | .getSuperSlurperJob({ |
| 304 | accountId: identity.accountId, |
| 305 | jobId: identity.jobId, |
| 306 | }) |
| 307 | .pipe( |
| 308 | Retry.none, |
| 309 | Effect.map((job) => (job.id ? job : undefined)), |
| 310 | Effect.catchTag("SuperSlurperJobOperationFailed", () => findInHistory(identity)), |
| 311 | ); |
| 312 | |
| 313 | const findInHistory = Effect.fn(function* (identity: { accountId: string; jobId: string }) { |
| 314 | const limit = 50; |
| 315 | const seen = new Set<string>(); |
| 316 | for (let offset = 0; ;) { |
| 317 | // The SDK's single-page paginator does not follow Slurper's offset parameter. |
| 318 | const page = yield* r2 |
| 319 | .listSuperSlurperJobs({ |
| 320 | accountId: identity.accountId, |
| 321 | limit, |
| 322 | offset, |
| 323 | }) |
| 324 | .pipe(Retry.none); |
| 325 | const job = page.result.find((item) => item.id === identity.jobId); |
| 326 | if (job) return job; |
| 327 | if (page.result.length === 0) return undefined; |
| 328 | const ids = page.result.flatMap((item) => (item.id ? [item.id] : [])); |
| 329 | if (ids.every((id) => seen.has(id))) { |
| 330 | return yield* new SuperSlurperJobUnexpectedResponse({ |
| 331 | message: "Super Slurper job history pagination did not advance", |
| 332 | }); |
| 333 | } |
| 334 | for (const id of ids) seen.add(id); |
| 335 | offset += page.result.length; |
| 336 | } |
| 337 | }); |
| 338 | |
| 339 | const mutate = ( |
| 340 | identity: { accountId: string; jobId: string }, |
| 341 | action: "pause" | "resume" | "abort", |
| 342 | ) => { |
| 343 | const operation = |
| 344 | action === "pause" |
| 345 | ? r2.pauseSuperSlurperJob |
| 346 | : action === "resume" |
| 347 | ? r2.resumeSuperSlurperJob |
| 348 | : r2.abortSuperSlurperJob; |
| 349 | return operation({ |
| 350 | accountId: identity.accountId, |
| 351 | jobId: identity.jobId, |
| 352 | }).pipe( |
| 353 | Retry.none, |
| 354 | Effect.asVoid, |
| 355 | Effect.catchTag("SuperSlurperJobOperationFailed", (error) => |
| 356 | Effect.gen(function* () { |
| 357 | const current = yield* observe(identity); |
| 358 | if ( |
| 359 | !current || |
| 360 | current.status === "completed" || |
| 361 | current.status === "aborted" || |
| 362 | (action === "pause" && current.status === "paused") || |
| 363 | (action === "resume" && current.status === "running") |
| 364 | ) { |
| 365 | return; |
| 366 | } |
| 367 | return yield* error; |
| 368 | }), |
| 369 | ), |
| 370 | ); |
| 371 | }; |
| 372 | |
| 373 | const attributes = ( |
| 374 | identity: { accountId: string; jobId: string }, |
| 375 | job: r2.GetSuperSlurperJobResponse, |
| 376 | ): SuperSlurperJobAttributes => ({ |
| 377 | accountId: identity.accountId, |
| 378 | jobId: identity.jobId, |
| 379 | status: job.status, |
| 380 | createdAt: job.createdAt ?? undefined, |
| 381 | finishedAt: job.finishedAt ?? undefined, |
| 382 | }); |
| 383 | |
| 384 | const credentialsRequest = (secret: SuperSlurperCredentials) => ({ |
| 385 | accessKeyId: Redacted.value(secret.accessKeyId), |
| 386 | secretAccessKey: Redacted.value(secret.secretAccessKey), |
| 387 | }); |
| 388 | |
| 389 | const sourceRequest = (source: SuperSlurperSource): r2.SuperSlurperJobsCreateRequestSource => |
| 390 | source.vendor === "gcs" |
| 391 | ? { |
| 392 | ...source, |
| 393 | secret: { |
| 394 | clientEmail: Redacted.value(source.secret.clientEmail), |
| 395 | privateKey: Redacted.value(source.secret.privateKey), |
| 396 | }, |
| 397 | } |
| 398 | : { ...source, secret: credentialsRequest(source.secret) }; |