The best way to control Claude Code, Codex and more from any computer or phone

main
ts 248 lines 9.07 KB
Raw
1 import { AbortMultipartUpload } from "@alchemy.run/aws/S3";
2 import { AbortMultipartUploadHttp } from "@alchemy.run/aws/S3";
3 import { CompleteMultipartUpload } from "@alchemy.run/aws/S3";
4 import { CompleteMultipartUploadHttp } from "@alchemy.run/aws/S3";
5 import { CreateMultipartUpload } from "@alchemy.run/aws/S3";
6 import { CreateMultipartUploadHttp } from "@alchemy.run/aws/S3";
7 import { DeleteObjects } from "@alchemy.run/aws/S3";
8 import { DeleteObjectsHttp } from "@alchemy.run/aws/S3";
9 import { GetObject } from "@alchemy.run/aws/S3";
10 import { GetObjectHttp } from "@alchemy.run/aws/S3";
11 import { HeadObject } from "@alchemy.run/aws/S3";
12 import { HeadObjectHttp } from "@alchemy.run/aws/S3";
13 import { ListObjectsV2 } from "@alchemy.run/aws/S3";
14 import { ListObjectsV2Http } from "@alchemy.run/aws/S3";
15 import { PutObject } from "@alchemy.run/aws/S3";
16 import { PutObjectHttp } from "@alchemy.run/aws/S3";
17 import { UploadPart } from "@alchemy.run/aws/S3";
18 import { UploadPartHttp } from "@alchemy.run/aws/S3";
19 /**
20 * `Git.BlobStoreS3` — AWS S3 implementation of {@link BlobStore}.
21 *
22 * Runs the git service's bulk bytes (packs, clone bundles, oversize
23 * objects, push spill) on an S3 bucket while compute stays on Cloudflare.
24 * It is built over the S3 bindings (`AWS.S3.GetObject`, `PutObject`, …),
25 * so one provision serves the Worker and the Repo DO alike, exactly like
26 * {@link BlobStoreR2}: the bucket is an `AWS.S3.Bucket` resource, the
27 * Worker gets a least-privilege IAM identity minted for it, and every
28 * request is signed with credentials assumed at runtime, in the bucket's
29 * region.
30 *
31 * Latency note: every blob operation crosses the internet to S3 (no
32 * Cloudflare-internal path), so serving-path reads are slower than R2.
33 * The window-coalescing in `ObjectStore`/`PackSource` keeps request
34 * counts low, but R2 remains the recommended default for
35 * Cloudflare-hosted assemblies.
36 *
37 * ### Providing the store
38 * **Example:** Packs on S3, compute on Cloudflare
39 * ```typescript
40 * const GitObjects = AWS.S3.Bucket("GitObjects");
41 *
42 * const GitLive = Git.ApiLive.pipe(
43 * Layer.provide(Git.ApiHandlersLive),
44 * Layer.provide(Git.ReposDurableObject),
45 * Layer.provide(Git.RegistryDurableObject),
46 * Layer.provide(Git.BlobStoreS3(GitObjects)),
47 * );
48 * ```
49 */
50 import * as Effect from "effect/Effect";
51 import * as Layer from "effect/Layer";
52 import * as Option from "effect/Option";
53 import * as Stream from "effect/Stream";
54 import {
55 BlobStore,
56 BlobStoreError,
57 type BlobBody,
58 type BlobMeta,
59 orderedParts,
60 type BlobMultipart,
61 type BlobStoreShape,
62 } from "./BlobStore.ts";
63
64 const s3Error = (what: string) => (error: { readonly _tag?: string; readonly message?: string }) =>
65 new BlobStoreError({
66 reason: `${what}: ${error._tag ?? "S3Error"}${error.message ? `: ${error.message}` : ""}`,
67 });
68
69 const collectBytes = (
70 stream: Stream.Stream<Uint8Array, Error>,
71 what: string,
72 ): Effect.Effect<Uint8Array, BlobStoreError> =>
73 Stream.runCollect(stream).pipe(
74 Effect.mapError((error) => s3Error(what)({ message: String(error) })),
75 Effect.map((chunks) => {
76 const total = chunks.reduce((n, c) => n + c.length, 0);
77 const out = new Uint8Array(total);
78 let at = 0;
79 for (const chunk of chunks) {
80 out.set(chunk, at);
81 at += chunk.length;
82 }
83 return out;
84 }),
85 );
86
87 /**
88 * S3-backed {@link BlobStore} using the given bucket. Accepts an
89 * `AWS.S3.Bucket` resource or its declaration Effect, like {@link BlobStoreR2}.
90 * See the module doc for identity and latency semantics.
91 *
92 * @layer
93 * @provides Git.BlobStore
94 */
95 export const BlobStoreS3 = (bucket: Parameters<typeof GetObject>[0]): Layer.Layer<BlobStore> =>
96 Layer.effect(
97 BlobStore,
98 Effect.gen(function* () {
99 const getObject = yield* GetObject(bucket);
100 const putObject = yield* PutObject(bucket);
101 const headObject = yield* HeadObject(bucket);
102 const deleteObjects = yield* DeleteObjects(bucket);
103 const listObjectsV2 = yield* ListObjectsV2(bucket);
104 const createMultipart = yield* CreateMultipartUpload(bucket);
105 const uploadPart = yield* UploadPart(bucket);
106 const completeMultipart = yield* CompleteMultipartUpload(bucket);
107 const abortMultipart = yield* AbortMultipartUpload(bucket);
108
109 return {
110 get: (key, range) =>
111 getObject({
112 Key: key,
113 Range:
114 range === undefined
115 ? undefined
116 : `bytes=${range.offset}-${range.offset + range.length - 1}`,
117 }).pipe(
118 Effect.map((output): BlobBody | null => {
119 const body = output.Body ?? Stream.empty;
120 return {
121 size: output.ContentLength ?? 0,
122 bytes: collectBytes(body, `get ${key}`),
123 stream: body.pipe(
124 Stream.mapError((error) => s3Error(`stream ${key}`)({ message: String(error) })),
125 ),
126 };
127 }),
128 Effect.catchTag("NoSuchKey", () => Effect.succeed(null)),
129 Effect.mapError(s3Error(`get ${key}`)),
130 ),
131
132 put: (key, body, opts) =>
133 putObject({
134 Key: key,
135 Body: body,
136 ContentLength: body instanceof Uint8Array ? body.length : opts?.contentLength,
137 }).pipe(Effect.mapError(s3Error(`put ${key}`)), Effect.asVoid),
138
139 head: (key) =>
140 headObject({ Key: key }).pipe(
141 Effect.map((output): BlobMeta => ({
142 key,
143 size: output.ContentLength ?? 0,
144 })),
145 Effect.catchTag("NotFound", () => Effect.succeed(null)),
146 Effect.mapError(s3Error(`head ${key}`)),
147 ),
148
149 multipart: (key) =>
150 createMultipart({ Key: key }).pipe(
151 Effect.mapError(s3Error(`multipart ${key}`)),
152 Effect.map((created): BlobMultipart => {
153 const UploadId = created.UploadId ?? "";
154 return {
155 uploadId: UploadId,
156 uploadPart: (partNumber, part) =>
157 uploadPart({
158 Key: key,
159 UploadId,
160 PartNumber: partNumber,
161 Body: part,
162 ContentLength: part.length,
163 }).pipe(
164 Effect.mapError(s3Error(`part ${partNumber} of ${key}`)),
165 Effect.map((uploaded) => ({
166 partNumber,
167 etag: uploaded.ETag ?? "",
168 })),
169 ),
170 complete: (parts) =>
171 completeMultipart({
172 Key: key,
173 UploadId,
174 MultipartUpload: {
175 Parts: orderedParts(parts).map((p) => ({
176 PartNumber: p.partNumber,
177 ETag: p.etag,
178 })),
179 },
180 }).pipe(Effect.mapError(s3Error(`complete ${key}`)), Effect.asVoid),
181 abort: abortMultipart({ Key: key, UploadId }).pipe(
182 Effect.mapError(s3Error(`abort ${key}`)),
183 Effect.asVoid,
184 ),
185 };
186 }),
187 ),
188 uploadPart: (key, uploadId, partNumber, part) =>
189 uploadPart({
190 Key: key,
191 UploadId: uploadId,
192 PartNumber: partNumber,
193 Body: part,
194 ContentLength: part.length,
195 }).pipe(
196 Effect.mapError(s3Error(`part ${partNumber} of ${key}`)),
197 Effect.map((uploaded) => ({
198 partNumber,
199 etag: uploaded.ETag ?? "",
200 })),
201 ),
202 delete: (keys) => {
203 const list = typeof keys === "string" ? [keys] : [...keys];
204 if (list.length === 0) return Effect.void;
205 return deleteObjects({
206 Delete: {
207 Objects: list.map((Key) => ({ Key })),
208 Quiet: true,
209 },
210 }).pipe(Effect.mapError(s3Error("delete")), Effect.asVoid);
211 },
212
213 list: (prefix) =>
214 Stream.paginate(undefined as string | undefined, (ContinuationToken) =>
215 listObjectsV2({
216 Prefix: prefix,
217 ContinuationToken,
218 MaxKeys: 1000,
219 }).pipe(
220 Effect.mapError(s3Error(`list ${prefix}`)),
221 Effect.map((page) => {
222 const metas = (page.Contents ?? []).flatMap((object): Array<BlobMeta> =>
223 object.Key === undefined ? [] : [{ key: object.Key, size: object.Size ?? 0 }],
224 );
225 return [
226 metas,
227 page.IsTruncated && page.NextContinuationToken
228 ? Option.some(page.NextContinuationToken)
229 : Option.none<string>(),
230 ] as const;
231 }),
232 ),
233 ),
234 } satisfies BlobStoreShape;
235 }),
236 ).pipe(
237 Layer.provide([
238 GetObjectHttp,
239 PutObjectHttp,
240 HeadObjectHttp,
241 DeleteObjectsHttp,
242 ListObjectsV2Http,
243 CreateMultipartUploadHttp,
244 UploadPartHttp,
245 CompleteMultipartUploadHttp,
246 AbortMultipartUploadHttp,
247 ]),
248 ) as never;