The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { AbortMultipartUpload } from "@alchemy.run/aws/S3"; |
| 2 | import { AbortMultipartUploadHttp } from "@alchemy.run/aws/S3"; |
| 3 | import { CompleteMultipartUpload } from "@alchemy.run/aws/S3"; |
| 4 | import { CompleteMultipartUploadHttp } from "@alchemy.run/aws/S3"; |
| 5 | import { CreateMultipartUpload } from "@alchemy.run/aws/S3"; |
| 6 | import { CreateMultipartUploadHttp } from "@alchemy.run/aws/S3"; |
| 7 | import { DeleteObjects } from "@alchemy.run/aws/S3"; |
| 8 | import { DeleteObjectsHttp } from "@alchemy.run/aws/S3"; |
| 9 | import { GetObject } from "@alchemy.run/aws/S3"; |
| 10 | import { GetObjectHttp } from "@alchemy.run/aws/S3"; |
| 11 | import { HeadObject } from "@alchemy.run/aws/S3"; |
| 12 | import { HeadObjectHttp } from "@alchemy.run/aws/S3"; |
| 13 | import { ListObjectsV2 } from "@alchemy.run/aws/S3"; |
| 14 | import { ListObjectsV2Http } from "@alchemy.run/aws/S3"; |
| 15 | import { PutObject } from "@alchemy.run/aws/S3"; |
| 16 | import { PutObjectHttp } from "@alchemy.run/aws/S3"; |
| 17 | import { UploadPart } from "@alchemy.run/aws/S3"; |
| 18 | import { UploadPartHttp } from "@alchemy.run/aws/S3"; |
| 19 | /** |
| 20 | * `Git.BlobStoreS3` — AWS S3 implementation of {@link BlobStore}. |
| 21 | * |
| 22 | * Runs the git service's bulk bytes (packs, clone bundles, oversize |
| 23 | * objects, push spill) on an S3 bucket while compute stays on Cloudflare. |
| 24 | * It is built over the S3 bindings (`AWS.S3.GetObject`, `PutObject`, …), |
| 25 | * so one provision serves the Worker and the Repo DO alike, exactly like |
| 26 | * {@link BlobStoreR2}: the bucket is an `AWS.S3.Bucket` resource, the |
| 27 | * Worker gets a least-privilege IAM identity minted for it, and every |
| 28 | * request is signed with credentials assumed at runtime, in the bucket's |
| 29 | * region. |
| 30 | * |
| 31 | * Latency note: every blob operation crosses the internet to S3 (no |
| 32 | * Cloudflare-internal path), so serving-path reads are slower than R2. |
| 33 | * The window-coalescing in `ObjectStore`/`PackSource` keeps request |
| 34 | * counts low, but R2 remains the recommended default for |
| 35 | * Cloudflare-hosted assemblies. |
| 36 | * |
| 37 | * ### Providing the store |
| 38 | * **Example:** Packs on S3, compute on Cloudflare |
| 39 | * ```typescript |
| 40 | * const GitObjects = AWS.S3.Bucket("GitObjects"); |
| 41 | * |
| 42 | * const GitLive = Git.ApiLive.pipe( |
| 43 | * Layer.provide(Git.ApiHandlersLive), |
| 44 | * Layer.provide(Git.ReposDurableObject), |
| 45 | * Layer.provide(Git.RegistryDurableObject), |
| 46 | * Layer.provide(Git.BlobStoreS3(GitObjects)), |
| 47 | * ); |
| 48 | * ``` |
| 49 | */ |
| 50 | import * as Effect from "effect/Effect"; |
| 51 | import * as Layer from "effect/Layer"; |
| 52 | import * as Option from "effect/Option"; |
| 53 | import * as Stream from "effect/Stream"; |
| 54 | import { |
| 55 | BlobStore, |
| 56 | BlobStoreError, |
| 57 | type BlobBody, |
| 58 | type BlobMeta, |
| 59 | orderedParts, |
| 60 | type BlobMultipart, |
| 61 | type BlobStoreShape, |
| 62 | } from "./BlobStore.ts"; |
| 63 | |
| 64 | const s3Error = (what: string) => (error: { readonly _tag?: string; readonly message?: string }) => |
| 65 | new BlobStoreError({ |
| 66 | reason: `${what}: ${error._tag ?? "S3Error"}${error.message ? `: ${error.message}` : ""}`, |
| 67 | }); |
| 68 | |
| 69 | const collectBytes = ( |
| 70 | stream: Stream.Stream<Uint8Array, Error>, |
| 71 | what: string, |
| 72 | ): Effect.Effect<Uint8Array, BlobStoreError> => |
| 73 | Stream.runCollect(stream).pipe( |
| 74 | Effect.mapError((error) => s3Error(what)({ message: String(error) })), |
| 75 | Effect.map((chunks) => { |
| 76 | const total = chunks.reduce((n, c) => n + c.length, 0); |
| 77 | const out = new Uint8Array(total); |
| 78 | let at = 0; |
| 79 | for (const chunk of chunks) { |
| 80 | out.set(chunk, at); |
| 81 | at += chunk.length; |
| 82 | } |
| 83 | return out; |
| 84 | }), |
| 85 | ); |
| 86 | |
| 87 | /** |
| 88 | * S3-backed {@link BlobStore} using the given bucket. Accepts an |
| 89 | * `AWS.S3.Bucket` resource or its declaration Effect, like {@link BlobStoreR2}. |
| 90 | * See the module doc for identity and latency semantics. |
| 91 | * |
| 92 | * @layer |
| 93 | * @provides Git.BlobStore |
| 94 | */ |
| 95 | export const BlobStoreS3 = (bucket: Parameters<typeof GetObject>[0]): Layer.Layer<BlobStore> => |
| 96 | Layer.effect( |
| 97 | BlobStore, |
| 98 | Effect.gen(function* () { |
| 99 | const getObject = yield* GetObject(bucket); |
| 100 | const putObject = yield* PutObject(bucket); |
| 101 | const headObject = yield* HeadObject(bucket); |
| 102 | const deleteObjects = yield* DeleteObjects(bucket); |
| 103 | const listObjectsV2 = yield* ListObjectsV2(bucket); |
| 104 | const createMultipart = yield* CreateMultipartUpload(bucket); |
| 105 | const uploadPart = yield* UploadPart(bucket); |
| 106 | const completeMultipart = yield* CompleteMultipartUpload(bucket); |
| 107 | const abortMultipart = yield* AbortMultipartUpload(bucket); |
| 108 | |
| 109 | return { |
| 110 | get: (key, range) => |
| 111 | getObject({ |
| 112 | Key: key, |
| 113 | Range: |
| 114 | range === undefined |
| 115 | ? undefined |
| 116 | : `bytes=${range.offset}-${range.offset + range.length - 1}`, |
| 117 | }).pipe( |
| 118 | Effect.map((output): BlobBody | null => { |
| 119 | const body = output.Body ?? Stream.empty; |
| 120 | return { |
| 121 | size: output.ContentLength ?? 0, |
| 122 | bytes: collectBytes(body, `get ${key}`), |
| 123 | stream: body.pipe( |
| 124 | Stream.mapError((error) => s3Error(`stream ${key}`)({ message: String(error) })), |
| 125 | ), |
| 126 | }; |
| 127 | }), |
| 128 | Effect.catchTag("NoSuchKey", () => Effect.succeed(null)), |
| 129 | Effect.mapError(s3Error(`get ${key}`)), |
| 130 | ), |
| 131 | |
| 132 | put: (key, body, opts) => |
| 133 | putObject({ |
| 134 | Key: key, |
| 135 | Body: body, |
| 136 | ContentLength: body instanceof Uint8Array ? body.length : opts?.contentLength, |
| 137 | }).pipe(Effect.mapError(s3Error(`put ${key}`)), Effect.asVoid), |
| 138 | |
| 139 | head: (key) => |
| 140 | headObject({ Key: key }).pipe( |
| 141 | Effect.map((output): BlobMeta => ({ |
| 142 | key, |
| 143 | size: output.ContentLength ?? 0, |
| 144 | })), |
| 145 | Effect.catchTag("NotFound", () => Effect.succeed(null)), |
| 146 | Effect.mapError(s3Error(`head ${key}`)), |
| 147 | ), |
| 148 | |
| 149 | multipart: (key) => |
| 150 | createMultipart({ Key: key }).pipe( |
| 151 | Effect.mapError(s3Error(`multipart ${key}`)), |
| 152 | Effect.map((created): BlobMultipart => { |
| 153 | const UploadId = created.UploadId ?? ""; |
| 154 | return { |
| 155 | uploadId: UploadId, |
| 156 | uploadPart: (partNumber, part) => |
| 157 | uploadPart({ |
| 158 | Key: key, |
| 159 | UploadId, |
| 160 | PartNumber: partNumber, |
| 161 | Body: part, |
| 162 | ContentLength: part.length, |
| 163 | }).pipe( |
| 164 | Effect.mapError(s3Error(`part ${partNumber} of ${key}`)), |
| 165 | Effect.map((uploaded) => ({ |
| 166 | partNumber, |
| 167 | etag: uploaded.ETag ?? "", |
| 168 | })), |
| 169 | ), |
| 170 | complete: (parts) => |
| 171 | completeMultipart({ |
| 172 | Key: key, |
| 173 | UploadId, |
| 174 | MultipartUpload: { |
| 175 | Parts: orderedParts(parts).map((p) => ({ |
| 176 | PartNumber: p.partNumber, |
| 177 | ETag: p.etag, |
| 178 | })), |
| 179 | }, |
| 180 | }).pipe(Effect.mapError(s3Error(`complete ${key}`)), Effect.asVoid), |
| 181 | abort: abortMultipart({ Key: key, UploadId }).pipe( |
| 182 | Effect.mapError(s3Error(`abort ${key}`)), |
| 183 | Effect.asVoid, |
| 184 | ), |
| 185 | }; |
| 186 | }), |
| 187 | ), |
| 188 | uploadPart: (key, uploadId, partNumber, part) => |
| 189 | uploadPart({ |
| 190 | Key: key, |
| 191 | UploadId: uploadId, |
| 192 | PartNumber: partNumber, |
| 193 | Body: part, |
| 194 | ContentLength: part.length, |
| 195 | }).pipe( |
| 196 | Effect.mapError(s3Error(`part ${partNumber} of ${key}`)), |
| 197 | Effect.map((uploaded) => ({ |
| 198 | partNumber, |
| 199 | etag: uploaded.ETag ?? "", |
| 200 | })), |
| 201 | ), |
| 202 | delete: (keys) => { |
| 203 | const list = typeof keys === "string" ? [keys] : [...keys]; |
| 204 | if (list.length === 0) return Effect.void; |
| 205 | return deleteObjects({ |
| 206 | Delete: { |
| 207 | Objects: list.map((Key) => ({ Key })), |
| 208 | Quiet: true, |
| 209 | }, |
| 210 | }).pipe(Effect.mapError(s3Error("delete")), Effect.asVoid); |
| 211 | }, |
| 212 | |
| 213 | list: (prefix) => |
| 214 | Stream.paginate(undefined as string | undefined, (ContinuationToken) => |
| 215 | listObjectsV2({ |
| 216 | Prefix: prefix, |
| 217 | ContinuationToken, |
| 218 | MaxKeys: 1000, |
| 219 | }).pipe( |
| 220 | Effect.mapError(s3Error(`list ${prefix}`)), |
| 221 | Effect.map((page) => { |
| 222 | const metas = (page.Contents ?? []).flatMap((object): Array<BlobMeta> => |
| 223 | object.Key === undefined ? [] : [{ key: object.Key, size: object.Size ?? 0 }], |
| 224 | ); |
| 225 | return [ |
| 226 | metas, |
| 227 | page.IsTruncated && page.NextContinuationToken |
| 228 | ? Option.some(page.NextContinuationToken) |
| 229 | : Option.none<string>(), |
| 230 | ] as const; |
| 231 | }), |
| 232 | ), |
| 233 | ), |
| 234 | } satisfies BlobStoreShape; |
| 235 | }), |
| 236 | ).pipe( |
| 237 | Layer.provide([ |
| 238 | GetObjectHttp, |
| 239 | PutObjectHttp, |
| 240 | HeadObjectHttp, |
| 241 | DeleteObjectsHttp, |
| 242 | ListObjectsV2Http, |
| 243 | CreateMultipartUploadHttp, |
| 244 | UploadPartHttp, |
| 245 | CompleteMultipartUploadHttp, |
| 246 | AbortMultipartUploadHttp, |
| 247 | ]), |
| 248 | ) as never; |