@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 398 lines 13.5 KB
Raw
1 import * as r2 from "@distilled.cloud/cloudflare/r2";
2 import * as Retry from "@distilled.cloud/cloudflare/Retry";
3 import { deepEqual, isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Redacted from "effect/Redacted";
9 import * as Schedule from "effect/Schedule";
10 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
11 import type { Providers } from "../Providers.ts";
12
13 export class SuperSlurperJobUnexpectedResponse extends Data.TaggedError(
14 "SuperSlurperJobUnexpectedResponse",
15 )<{
16 readonly message: string;
17 readonly cause?: unknown;
18 }> {}
19
20 /** S3-compatible credentials with access to a migration bucket. */
21 export interface SuperSlurperCredentials {
22 /** Access key ID. Changing credentials replaces the migration. */
23 accessKeyId: Redacted.Redacted<string>;
24 /** Secret access key. Changing credentials replaces the migration. */
25 secretAccessKey: Redacted.Redacted<string>;
26 }
27
28 /** Object selection shared by all source vendors. */
29 export interface SuperSlurperSourceSelection {
30 /** Name of the source bucket. Immutable. */
31 bucket: string;
32 /** Object keys to migrate. Omit to migrate all matching objects. Immutable. */
33 keys?: string[];
34 /** Restrict migration to this object key prefix. Immutable. */
35 pathPrefix?: string;
36 }
37
38 /** AWS S3 or another HTTPS S3-compatible source. */
39 export interface SuperSlurperS3Source extends SuperSlurperSourceSelection {
40 /** Select the S3 source API. */
41 vendor: "s3";
42 /** Source credentials with read access. Immutable. */
43 secret: SuperSlurperCredentials;
44 /** Custom S3-compatible HTTPS endpoint. Immutable. */
45 endpoint?: string;
46 /** Source bucket region. Immutable. */
47 region?: string;
48 }
49
50 /** Google Cloud Storage source. */
51 export interface SuperSlurperGcsSource extends SuperSlurperSourceSelection {
52 /** Select Google Cloud Storage. */
53 vendor: "gcs";
54 /** Service account credentials with source read access. Immutable. */
55 secret: {
56 /** Service account email. */
57 clientEmail: Redacted.Redacted<string>;
58 /** Service account private key in PEM format. */
59 privateKey: Redacted.Redacted<string>;
60 };
61 }
62
63 /** Cloudflare R2 source. */
64 export interface SuperSlurperR2Source extends SuperSlurperSourceSelection {
65 /** Select Cloudflare R2. */
66 vendor: "r2";
67 /** S3-compatible R2 credentials with source read access. Immutable. */
68 secret: SuperSlurperCredentials;
69 /** Source jurisdiction. Immutable. @default "default" */
70 jurisdiction?: "default" | "eu" | "us" | "fedramp";
71 }
72
73 /** Supported source bucket configurations. */
74 export type SuperSlurperSource =
75 | SuperSlurperS3Source
76 | SuperSlurperGcsSource
77 | SuperSlurperR2Source;
78
79 /** R2 destination for migrated objects. */
80 export interface SuperSlurperTarget {
81 /** Name of the destination R2 bucket. Immutable. */
82 bucket: string;
83 /** The destination vendor must be R2. */
84 vendor: "r2";
85 /** S3-compatible R2 credentials with destination write access. Immutable. */
86 secret: SuperSlurperCredentials;
87 /** Destination jurisdiction. Immutable. @default "default" */
88 jurisdiction?: "default" | "eu" | "us" | "fedramp";
89 }
90
91 export interface SuperSlurperJobProps {
92 /** Source bucket and credentials. Any change replaces the job. */
93 source: SuperSlurperSource;
94 /** Destination R2 bucket and credentials. Any change replaces the job. */
95 target: SuperSlurperTarget;
96 /** Overwrite existing destination objects. Immutable. @default false */
97 overwrite?: boolean;
98 /**
99 * Pause an active job, or resume it when false. Terminal jobs are never
100 * restarted. A new job starts running before the pause request is applied,
101 * so this is not a guarantee that no objects are transferred.
102 * @default false
103 */
104 paused?: boolean;
105 }
106
107 export interface SuperSlurperJobAttributes {
108 /** Cloudflare account that owns the migration. */
109 accountId: string;
110 /** Server-assigned migration ID. Stable until replacement. */
111 jobId: string;
112 /** Last observed job status; not a completion guarantee. */
113 status: r2.GetSuperSlurperJobResponse["status"];
114 /** Creation timestamp reported by Cloudflare. */
115 createdAt: string | undefined;
116 /** Completion timestamp, if the job has finished. */
117 finishedAt: string | undefined;
118 }
119
120 export type SuperSlurperJob = Resource<
121 "Cloudflare.R2.SuperSlurperJob",
122 SuperSlurperJobProps,
123 SuperSlurperJobAttributes,
124 never,
125 Providers
126 >;
127
128 /**
129 * Starts a bulk migration from S3, GCS, or R2 into an R2 bucket.
130 *
131 * Deployment returns the job identity without waiting for the migration.
132 * Completed and aborted jobs remain resources: redeploying does not run them
133 * again. Changing source, target, credentials, or overwrite replaces the job.
134 * Destroy only cancels this resource's active job; it never deletes copied
135 * objects or Cloudflare's job history. Transfers already in flight may finish.
136 *
137 * Cloudflare exposes no job name, ownership tags, or idempotency key. The
138 * persisted job ID is the only ownership record; similar jobs are never
139 * adopted. If a create response or the subsequent state write is lost, the
140 * job cannot be recovered automatically and retrying may create a second job.
141 * Inspect the account's migration history and cancel that orphan explicitly.
142 * Account-wide adoption and cancellation are intentionally not supported.
143 * If reading a cached ID fails with Cloudflare's ambiguous server error,
144 * paginated job history is checked for that exact ID before declaring it absent.
145 *
146 * ### Migrating from S3
147 * **Example:** Copy an S3 prefix into R2
148 * ```typescript
149 * const target = yield* Cloudflare.R2.Bucket("Archive");
150 * const job = yield* Cloudflare.R2.SuperSlurperJob("Migration", {
151 * source: {
152 * vendor: "s3",
153 * bucket: "legacy-archive",
154 * region: "us-east-1",
155 * pathPrefix: "photos/",
156 * secret: {
157 * accessKeyId: yield* Config.Redacted("AWS_ACCESS_KEY_ID"),
158 * secretAccessKey: yield* Config.Redacted("AWS_SECRET_ACCESS_KEY"),
159 * },
160 * },
161 * target: {
162 * vendor: "r2",
163 * bucket: target.bucketName,
164 * secret: {
165 * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"),
166 * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"),
167 * },
168 * },
169 * });
170 * ```
171 *
172 * ### Migrating from GCS or R2
173 * **Example:** Google Cloud Storage source
174 * ```typescript
175 * const source: Cloudflare.R2.SuperSlurperSource = {
176 * vendor: "gcs",
177 * bucket: "legacy-media",
178 * secret: {
179 * clientEmail: yield* Config.Redacted("GCS_CLIENT_EMAIL"),
180 * privateKey: yield* Config.Redacted("GCS_PRIVATE_KEY"),
181 * },
182 * };
183 * ```
184 *
185 * **Example:** R2 source and paused desired state
186 * ```typescript
187 * yield* Cloudflare.R2.SuperSlurperJob("Migration", {
188 * source: {
189 * vendor: "r2",
190 * bucket: "old-media",
191 * secret: {
192 * accessKeyId: yield* Config.Redacted("SOURCE_R2_ACCESS_KEY_ID"),
193 * secretAccessKey: yield* Config.Redacted("SOURCE_R2_SECRET_ACCESS_KEY"),
194 * },
195 * },
196 * target: {
197 * vendor: "r2",
198 * bucket: "new-media",
199 * secret: {
200 * accessKeyId: yield* Config.Redacted("TARGET_R2_ACCESS_KEY_ID"),
201 * secretAccessKey: yield* Config.Redacted("TARGET_R2_SECRET_ACCESS_KEY"),
202 * },
203 * },
204 * paused: true,
205 * });
206 * ```
207 *
208 * ### Referencing a Migration
209 * **Example:** Read a persisted job identity without creating a new job
210 * ```typescript
211 * const job = yield* Cloudflare.R2.SuperSlurperJob.ref("Migration");
212 * return { accountId: job.accountId, jobId: job.jobId };
213 * ```
214 *
215 * @resource
216 * @product R2
217 * @category Storage & Databases
218 */
219 export const SuperSlurperJob = Resource<SuperSlurperJob>("Cloudflare.R2.SuperSlurperJob");
220
221 export const SuperSlurperJobProvider = () =>
222 Provider.succeed(SuperSlurperJob, {
223 stables: ["accountId", "jobId"],
224 nuke: { skip: true },
225 diff: Effect.fn(function* ({ olds, news, output }) {
226 const { accountId } = yield* yield* CloudflareEnvironment;
227 if (output && output.accountId !== accountId) {
228 return { action: "replace" } as const;
229 }
230 if (
231 !("source" in news) ||
232 !isResolved(news.source) ||
233 !isResolved(news.target) ||
234 !isResolved(news.overwrite)
235 ) {
236 return { action: "replace" } as const;
237 }
238 if (
239 !deepEqual(olds.source, news.source) ||
240 !deepEqual(olds.target, news.target) ||
241 (olds.overwrite ?? false) !== (news.overwrite ?? false)
242 ) {
243 return { action: "replace" } as const;
244 }
245 }),
246 read: Effect.fn(function* ({ output }) {
247 if (!output) return undefined;
248 const observed = yield* observe(output);
249 return observed ? attributes(output, observed) : undefined;
250 }),
251 reconcile: Effect.fn(function* ({ news, output }) {
252 const { accountId } = yield* yield* CloudflareEnvironment;
253 let identity = output;
254 let observed = identity ? yield* observe(identity) : undefined;
255 if (!observed) {
256 const created = yield* r2
257 .createSuperSlurperJob({
258 accountId,
259 source: sourceRequest(news.source),
260 target: {
261 ...news.target,
262 secret: credentialsRequest(news.target.secret),
263 },
264 overwrite: news.overwrite ?? false,
265 })
266 .pipe(
267 Retry.none,
268 // Preconnectivity rejects before creating a job; fresh tokens may not have propagated.
269 Effect.retry({
270 while: (error) => error._tag === "SuperSlurperPreconnectivityFailed",
271 schedule: Schedule.spaced("3 seconds"),
272 times: 8,
273 }),
274 );
275 if (!created.id) {
276 return yield* new SuperSlurperJobUnexpectedResponse({
277 message: "Super Slurper did not return a job ID",
278 });
279 }
280 identity = attributes({ accountId, jobId: created.id }, { status: "running" });
281 observed = { id: created.id, status: "running" };
282 }
283 const current = identity!;
284 if (news.paused && observed.status === "running") {
285 yield* mutate(current, "pause");
286 observed = (yield* observe(current)) ?? observed;
287 } else if (!news.paused && observed.status === "paused") {
288 yield* mutate(current, "resume");
289 observed = (yield* observe(current)) ?? observed;
290 }
291 return attributes(current, observed);
292 }),
293 delete: Effect.fn(function* ({ output }) {
294 const observed = yield* observe(output);
295 if (observed?.status === "running" || observed?.status === "paused") {
296 yield* mutate(output, "abort");
297 }
298 }),
299 });
300
301 const observe = (identity: { accountId: string; jobId: string }) =>
302 r2
303 .getSuperSlurperJob({
304 accountId: identity.accountId,
305 jobId: identity.jobId,
306 })
307 .pipe(
308 Retry.none,
309 Effect.map((job) => (job.id ? job : undefined)),
310 Effect.catchTag("SuperSlurperJobOperationFailed", () => findInHistory(identity)),
311 );
312
313 const findInHistory = Effect.fn(function* (identity: { accountId: string; jobId: string }) {
314 const limit = 50;
315 const seen = new Set<string>();
316 for (let offset = 0; ;) {
317 // The SDK's single-page paginator does not follow Slurper's offset parameter.
318 const page = yield* r2
319 .listSuperSlurperJobs({
320 accountId: identity.accountId,
321 limit,
322 offset,
323 })
324 .pipe(Retry.none);
325 const job = page.result.find((item) => item.id === identity.jobId);
326 if (job) return job;
327 if (page.result.length === 0) return undefined;
328 const ids = page.result.flatMap((item) => (item.id ? [item.id] : []));
329 if (ids.every((id) => seen.has(id))) {
330 return yield* new SuperSlurperJobUnexpectedResponse({
331 message: "Super Slurper job history pagination did not advance",
332 });
333 }
334 for (const id of ids) seen.add(id);
335 offset += page.result.length;
336 }
337 });
338
339 const mutate = (
340 identity: { accountId: string; jobId: string },
341 action: "pause" | "resume" | "abort",
342 ) => {
343 const operation =
344 action === "pause"
345 ? r2.pauseSuperSlurperJob
346 : action === "resume"
347 ? r2.resumeSuperSlurperJob
348 : r2.abortSuperSlurperJob;
349 return operation({
350 accountId: identity.accountId,
351 jobId: identity.jobId,
352 }).pipe(
353 Retry.none,
354 Effect.asVoid,
355 Effect.catchTag("SuperSlurperJobOperationFailed", (error) =>
356 Effect.gen(function* () {
357 const current = yield* observe(identity);
358 if (
359 !current ||
360 current.status === "completed" ||
361 current.status === "aborted" ||
362 (action === "pause" && current.status === "paused") ||
363 (action === "resume" && current.status === "running")
364 ) {
365 return;
366 }
367 return yield* error;
368 }),
369 ),
370 );
371 };
372
373 const attributes = (
374 identity: { accountId: string; jobId: string },
375 job: r2.GetSuperSlurperJobResponse,
376 ): SuperSlurperJobAttributes => ({
377 accountId: identity.accountId,
378 jobId: identity.jobId,
379 status: job.status,
380 createdAt: job.createdAt ?? undefined,
381 finishedAt: job.finishedAt ?? undefined,
382 });
383
384 const credentialsRequest = (secret: SuperSlurperCredentials) => ({
385 accessKeyId: Redacted.value(secret.accessKeyId),
386 secretAccessKey: Redacted.value(secret.secretAccessKey),
387 });
388
389 const sourceRequest = (source: SuperSlurperSource): r2.SuperSlurperJobsCreateRequestSource =>
390 source.vendor === "gcs"
391 ? {
392 ...source,
393 secret: {
394 clientEmail: Redacted.value(source.secret.clientEmail),
395 privateKey: Redacted.value(source.secret.privateKey),
396 },
397 }
398 : { ...source, secret: credentialsRequest(source.secret) };