The best way to control Claude Code, Codex and more from any computer or phone

main
ts 169 lines 5.87 KB
Raw
1 /**
2 * Bytes on S3, compute on Cloudflare: the Worker and the Repo DO reach the
3 * bucket through the S3 bindings with an identity Alchemy minted. A push
4 * that spills, compaction into an S3 pack, and a clone back under
5 * `fsck --strict`. Every byte of the clone came back through S3.
6 */
7 import * as AWS from "@alchemy.run/aws";
8 import * as Cloudflare from "@alchemy.run/cloudflare";
9 import { GitApi } from "@alchemy.run/git/Api";
10 import { expect } from "@alchemy.run/test";
11 import * as Test from "@alchemy.run/test/Alchemy";
12 import * as Data from "effect/Data";
13 import * as Effect from "effect/Effect";
14 import * as FileSystem from "effect/FileSystem";
15 import * as HttpApiClient from "effect/http-api/HttpApiClient";
16 import * as HttpClient from "effect/http/HttpClient";
17 import * as HttpClientRequest from "effect/http/HttpClientRequest";
18 import * as Layer from "effect/Layer";
19 import * as ChildProcess from "effect/process/ChildProcess";
20 import { MinimumLogLevel } from "effect/References";
21 import * as Schedule from "effect/Schedule";
22 import * as Stream from "effect/Stream";
23 import { makeS3TestStack, TEST_SECRET } from "./fixtures/s3-stack.ts";
24
25 const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
26 providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()),
27 });
28
29 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
30
31 const Stack = makeS3TestStack("GitBlobStoreS3Stack");
32
33 const makeClient = (url: string, token: string) =>
34 HttpApiClient.make(GitApi, {
35 baseUrl: url,
36 transformClient: HttpClient.mapRequest((request) =>
37 request.pipe(HttpClientRequest.bearerToken(token)),
38 ),
39 });
40
41 const edgeRetry = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
42 effect.pipe(
43 Effect.timeout("10 seconds"),
44 Effect.retry({
45 while: (error: unknown) =>
46 (error as { _tag?: string })._tag === "TimeoutError" ||
47 (error as { _tag?: string })._tag === "HttpClientError",
48 schedule: Schedule.spaced("1500 millis"),
49 times: 20,
50 }),
51 );
52
53 class ShellError extends Data.TaggedError("ShellError")<{
54 readonly script: string;
55 readonly exitCode: number;
56 readonly stdout: string;
57 readonly stderr: string;
58 }> {
59 override get message(): string {
60 return `shell script exited ${this.exitCode}\n--- script ---\n${this.script}\n--- stdout ---\n${this.stdout}\n--- stderr ---\n${this.stderr}`;
61 }
62 }
63
64 const sh = Effect.fn(function* (cwd: string, script: string) {
65 const handle = yield* ChildProcess.make("sh", ["-ec", script], {
66 cwd,
67 env: {
68 GIT_TERMINAL_PROMPT: "0",
69 GIT_CONFIG_NOSYSTEM: "1",
70 GIT_CONFIG_GLOBAL: "/dev/null",
71 GIT_AUTHOR_NAME: "S3 Bot",
72 GIT_AUTHOR_EMAIL: "s3@example.com",
73 GIT_COMMITTER_NAME: "S3 Bot",
74 GIT_COMMITTER_EMAIL: "s3@example.com",
75 GIT_AUTHOR_DATE: "2026-01-01T00:00:00Z",
76 GIT_COMMITTER_DATE: "2026-01-01T00:00:00Z",
77 },
78 extendEnv: true,
79 });
80 const [exitCode, stdout, stderr] = yield* Effect.all(
81 [
82 handle.exitCode,
83 Stream.mkString(Stream.decodeText(handle.stdout)),
84 Stream.mkString(Stream.decodeText(handle.stderr)),
85 ],
86 { concurrency: 3 },
87 );
88 if (exitCode !== 0) {
89 return yield* new ShellError({ script, exitCode, stdout, stderr });
90 }
91 return { stdout: stdout.trim(), stderr };
92 }, Effect.timeout("180 seconds"));
93
94 const stack = beforeAll(
95 deploy(Stack).pipe(
96 Effect.tap(({ url }) =>
97 Effect.gen(function* () {
98 yield* Effect.logInfo(`git-service (S3 bytes) deployed at ${url}`);
99 const admin = yield* makeClient(url, TEST_SECRET);
100 yield* admin.repos.list({ query: {} }).pipe(
101 edgeRetry,
102 Effect.retry({
103 schedule: Schedule.spaced("1500 millis"),
104 times: 40,
105 }),
106 );
107 }),
108 ),
109 logLevel,
110 ),
111 );
112 afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack).pipe(logLevel));
113
114 test(
115 "a spilling push lands in S3, compacts into an S3 pack, and clones back under fsck --strict",
116 Effect.gen(function* () {
117 const { url } = yield* stack;
118 const admin = yield* makeClient(url, TEST_SECRET);
119 const owner = "s3";
120 const name = "bytes";
121 yield* admin.repos.delete({ params: { owner, repo: name } }).pipe(
122 Effect.catchTag("RepoNotFound", () => Effect.void),
123 edgeRetry,
124 );
125 yield* admin.repos.create({ payload: { owner, name } }).pipe(edgeRetry);
126 const parsed = new URL(url);
127 const remote = `${parsed.protocol}//x:${TEST_SECRET}@${parsed.host}/${owner}/${name}.git`;
128 const fs = yield* FileSystem.FileSystem;
129 const dir = yield* fs.makeTempDirectory({ prefix: "git-s3-" });
130 // ~14 MiB of incompressible blobs: a body that spills to the store.
131 yield* sh(
132 dir,
133 `git init -q -b main src && cd src
134 for i in $(seq 1 14); do head -c 1000000 /dev/urandom > bin-$i.dat; done
135 git add -A && git commit -qm base
136 git push -q ${remote} main`,
137 );
138 // Compact now; the pack is written to the store.
139 yield* admin.repos.compact({ params: { owner, repo: name } }).pipe(edgeRetry);
140 const repo = yield* admin.repos.get({ params: { owner, repo: name } }).pipe(
141 edgeRetry,
142 Effect.repeat({
143 schedule: Schedule.spaced("3 seconds"),
144 until: (r) => r.objects.loose === 0,
145 times: 30,
146 }),
147 );
148 expect(repo.objects.loose).toBe(0);
149 expect(repo.objects.packed).toBeGreaterThan(0);
150 const back = yield* sh(
151 dir,
152 `git clone -q ${remote} back && cd back && git fsck --strict && git rev-parse HEAD`,
153 );
154 const head = yield* sh(dir, "cd src && git rev-parse HEAD");
155 expect(back.stdout.trim().split("\n").pop()).toBe(head.stdout.trim());
156 yield* Effect.logInfo(`objects: ${JSON.stringify(repo.objects)}`);
157 }),
158 {
159 tags: [
160 "provider:aws",
161 "provider:aws:s3",
162 "provider:cloudflare",
163 "provider:cloudflare:r2",
164 "provider:cloudflare:worker",
165 "live",
166 ],
167 timeout: 600_000,
168 },
169 );