The best way to control Claude Code, Codex and more from any computer or phone

main
ts 396 lines 15.5 KB
Raw
1 import type { Progress } from "@alchemy.run/domain/Backfill";
2 import * as workflows from "@distilled.cloud/cloudflare/workflows";
3 import { isResolved } from "alchemy/Diff";
4 import * as ProviderLayer from "alchemy/Local/ProviderLayer";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import * as Clock from "effect/Clock";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
12
13 export interface BackfillProps {
14 /** The Domain's Worker: a run starts once the Worker carrying its Workflow is deployed. */
15 scriptName: string;
16 /** The Workflow, hosted by the Domain's Worker, that starts the runs and waits for them. */
17 workflowName: string;
18 /**
19 * Every view and projection in the model, by target id (`view:Name`, with
20 * `#<generation>` once rebuilt). Those new to the Domain are backfilled.
21 */
22 targets: string[];
23 /**
24 * What the backfill needs to know of each consumer (the sources it reads, and
25 * whether a key reads its one source's log itself), by target id. The
26 * deploy passes them to each run, so a run doesn't depend on what an
27 * object's code knows.
28 */
29 plans?: Record<string, { sources?: string[]; singleSource: boolean }>;
30 /**
31 * How long a deploy waits for the backfill, in seconds. Past it the deploy
32 * fails, while the backfill keeps running on Cloudflare; the next deploy
33 * waits on the same runs.
34 * @default 1800
35 */
36 timeoutSeconds?: number;
37 }
38
39 export interface BackfillAttrs {
40 accountId: string;
41 workflowName: string;
42 /** The Workflow instance the last deploy waited through. */
43 instanceId: string;
44 targets: string[];
45 /** `complete` once every target's run is complete. */
46 status: string;
47 }
48
49 const BackfillTypeId = "Cloudflare.Domain.Backfill";
50
51 /**
52 * The backfill of a Domain's views and projections, run by reconciliation.
53 * The Durable Object platform declares it with the model's consumers: you
54 * don't declare it yourself. A deploy that adds a view or projection, or
55 * bumps a version, waits here until every new consumer has received the
56 * history recorded before it, and reports how far each one has got.
57 *
58 * ### What a deploy waits for <!-- api-prose -->
59 *
60 * Each consumer generation (a view, or a view after a version bump) has one
61 * run, held by one Durable Object of a class only that generation has. The
62 * first deploy to reach it starts it; every later start finds it started, so
63 * there is never more than one, however many deploys ask. The run lives on
64 * Cloudflare and moves on its own: this resource starts it, waits for it,
65 * and reports its progress, but holds none of its state.
66 *
67 * So a deploy is safe to repeat and to interrupt. Run it again after a
68 * failure, a timeout or a Ctrl-C, and it waits on the same runs, from where
69 * they are; once they are complete it returns at once. A deploy that changes
70 * no consumer plans no change here.
71 *
72 * ### Progress <!-- api-prose -->
73 *
74 * While it waits, the deploy notes each run's phase and how far through the
75 * directory it is:
76 *
77 * ```sh
78 * Backfilling view:Deposits reading history 41% (1,690/4,096 subtrees, 410,211 instances) (48s) ...
79 * ```
80 *
81 * A consumer keyed by its source reads its history in one pass; any other
82 * reads it, then applies it once every source's history is in. `refused`
83 * counts calls Cloudflare turned down for the account's load: the run backs
84 * off and sends them again.
85 *
86 * ### Rebuilding and cancelling <!-- api-prose -->
87 *
88 * A run reads the sources its consumer read when it started. A deploy whose
89 * consumer reads other sources at the same version (a view that now reads a
90 * journal as well as accounts) fails, saying to bump the version: waiting on
91 * the run would report a history that misses the new source.
92 *
93 * To run a backfill again, bump the consumer's version: the new generation is
94 * a new class with its own run, and the same Worker upload deletes the old
95 * generation's class, data and run. To stop a backfill, remove the consumer
96 * from the model and deploy: its classes, and the run with them, are deleted.
97 * Either is one Worker upload, so a consumer never has two runs at once.
98 *
99 * @resource
100 * @product Durable Objects
101 */
102 export interface Backfill extends Resource<typeof BackfillTypeId, BackfillProps, BackfillAttrs> {}
103
104 export const Backfill = Resource<Backfill>(BackfillTypeId);
105
106 export class BackfillStillRunning extends Data.TaggedError("BackfillStillRunning")<{
107 readonly message: string;
108 readonly instanceId: string;
109 }> {}
110
111 export class BackfillFailed extends Data.TaggedError("BackfillFailed")<{
112 readonly message: string;
113 readonly instanceId: string;
114 readonly status: string;
115 }> {}
116
117 /** The Workflow instance for one deploy's wait: named after the targets, and new each time. */
118 const instanceIdOf = (targets: ReadonlyArray<string>, resource: string, attempt: string) => {
119 let hash = 0x811c9dc5;
120 for (const char of [...targets].sort().join("|")) {
121 hash ^= char.charCodeAt(0);
122 hash = Math.imul(hash, 0x01000193) >>> 0;
123 }
124 return `backfill-${resource.slice(0, 12)}-${hash.toString(16).padStart(8, "0")}-${attempt}`;
125 };
126
127 const POLL = "1 second";
128 /** How often the deploy reads the runs' progress from the instance's steps. */
129 const PROGRESS_EVERY = 3;
130 /** How many times a wait whose instance can't carry it on is started again, and how soon. */
131 const MAX_RESTARTS = 30;
132 const RESTART_DELAY = "2 seconds";
133
134 /**
135 * An instance started on an edge that doesn't have the deployed Worker yet (or
136 * its entrypoint, or the consumer's class) stays on the code it started with:
137 * a new instance runs the deployed code.
138 */
139 const startedTooSoon = (message: string) =>
140 /entrypoint name .* was not found|^Worker not found|isn't in the code this runs yet/.test(
141 message,
142 );
143
144 /**
145 * Workflows answers instance calls for a Workflow uploaded moments before with
146 * an internal error (code 10001) for a while: call again.
147 */
148 const retryInternal = {
149 while: (error: { readonly _tag: string }) => error._tag === "WorkflowInternalError",
150 schedule: Schedule.spaced("2 seconds"),
151 times: 30,
152 };
153
154 /** Each run's progress from the latest check step's output (an array of {@link Progress}). */
155 export const progressOf = (steps: ReadonlyArray<object>) => {
156 for (let i = steps.length - 1; i >= 0; i--) {
157 const step = steps[i]!;
158 if (!("output" in step) || typeof step.output !== "string") continue;
159 if (!("name" in step) || typeof step.name !== "string" || !step.name.startsWith("check-"))
160 continue;
161 const found = findProgress(parseJson(step.output));
162 if (found !== undefined) return found;
163 }
164 return undefined;
165 };
166
167 const parseJson = (text: string): unknown => {
168 try {
169 return JSON.parse(text);
170 } catch {
171 return undefined;
172 }
173 };
174
175 /** The first array of runs' progress in a step's output, however the step encoded it. */
176 const findProgress = (value: unknown): ReadonlyArray<Progress> | undefined => {
177 if (Array.isArray(value)) {
178 if (value.length > 0 && value.every((item) => isProgress(item))) return value;
179 for (const item of value) {
180 const found = findProgress(item);
181 if (found !== undefined) return found;
182 }
183 return undefined;
184 }
185 if (value !== null && typeof value === "object")
186 for (const item of Object.values(value)) {
187 const found = findProgress(item);
188 if (found !== undefined) return found;
189 }
190 if (typeof value === "string" && (value.startsWith("[") || value.startsWith("{")))
191 return findProgress(parseJson(value));
192 return undefined;
193 };
194
195 const isProgress = (value: unknown): value is Progress =>
196 value !== null &&
197 typeof value === "object" &&
198 "target" in value &&
199 "phase" in value &&
200 "subtrees" in value;
201
202 const count = (n: number) => n.toLocaleString("en-US");
203
204 /** One run's progress, as the deploy notes it. */
205 export const describeProgress = (run: Progress) => {
206 if (run.phase === "complete") return `${run.target} complete`;
207 if (run.phase === "unknown") return `${run.target} starting`;
208 const what = run.phase === "discover" ? "reading history" : "applying history";
209 if (run.subtrees === 0) return `${run.target} ${what}`;
210 const percent = Math.floor((100 * run.done) / run.subtrees);
211 return `${run.target} ${what} ${percent}% (${count(run.done)}/${count(run.subtrees)} subtrees, ${count(run.instances)} instances${run.refused > 0 ? `, ${count(run.refused)} refused` : ""})`;
212 };
213
214 const BackfillLive = () =>
215 Provider.succeed(Backfill, {
216 stables: ["accountId", "workflowName"],
217 diff: Effect.fn(function* ({ olds, news }) {
218 if (!isResolved(news)) return { action: "update" } as const;
219 const changed =
220 olds.workflowName !== news.workflowName ||
221 [...olds.targets].sort().join("|") !== [...(news.targets ?? [])].sort().join("|");
222 if (changed) return { action: "update" } as const;
223 }),
224 // The runs are the state, and they live on Cloudflare: what was recorded stands
225 // until a deploy changes the targets (an instance Workflows has since dropped
226 // changes nothing).
227 read: Effect.fn(function* ({ output }) {
228 return output;
229 }),
230 reconcile: Effect.fn(function* ({ instanceId: resource, news, output, session }) {
231 const { accountId } = yield* yield* CloudflareEnvironment;
232 const label = news.targets.join(", ");
233 const started = yield* Clock.currentTimeMillis;
234 const timeout = (news.timeoutSeconds ?? 1800) * 1000;
235 const instance = (id: string) =>
236 workflows
237 .getInstance({ accountId, workflowName: news.workflowName, instanceId: id })
238 .pipe(Effect.retry(retryInternal));
239
240 // The instance a deploy that stopped waiting left behind: stop it, so it doesn't keep
241 // checking runs no one waits on. Best effort: it ends by itself once they complete.
242 if (output?.instanceId !== undefined && output.status !== "complete")
243 yield* workflows
244 .patchInstanceStatus({
245 accountId,
246 workflowName: news.workflowName,
247 instanceId: output.instanceId,
248 status: "terminate",
249 })
250 .pipe(
251 Effect.asVoid,
252 Effect.catchTag(
253 ["InstanceNotFound", "WorkflowNotFound", "InstanceCannotTerminate"],
254 () => Effect.void,
255 ),
256 );
257
258 /**
259 * Start an instance for this wait. Each deploy (and each start again) gets a new one:
260 * it only starts the runs and reports on them, and starting a run that is already
261 * started finds it, so a new instance never starts a backfill over.
262 */
263 const start = (attempt: number) =>
264 Effect.gen(function* () {
265 const id = instanceIdOf(news.targets, resource, `${started.toString(36)}${attempt}`);
266 yield* workflows
267 .createInstance({
268 accountId,
269 workflowName: news.workflowName,
270 instanceId: id,
271 params: { targets: news.targets, plans: news.plans ?? {} },
272 })
273 .pipe(
274 Effect.asVoid,
275 Effect.catchTag("InstanceAlreadyExists", () => Effect.void),
276 // The Workflow is uploaded with the Worker: it can take a moment to appear.
277 Effect.retry({
278 while: (error) =>
279 error._tag === "WorkflowNotFound" || error._tag === "WorkflowInternalError",
280 schedule: Schedule.spaced("2 seconds"),
281 times: 30,
282 }),
283 );
284 return id;
285 });
286
287 let attempt = 0;
288 let id = yield* start(attempt);
289 let progress: ReadonlyArray<Progress> | undefined;
290 const attrs = () => ({ accountId, workflowName: news.workflowName, targets: news.targets });
291 const where = () =>
292 progress === undefined ? label : progress.map((run) => describeProgress(run)).join("; ");
293
294 for (let poll = 0; ; poll++) {
295 const elapsed = (yield* Clock.currentTimeMillis) - started;
296 const status = poll % PROGRESS_EVERY === 0 ? yield* instance(id) : undefined;
297 const current =
298 status ??
299 (yield* workflows
300 .getInstance({
301 accountId,
302 workflowName: news.workflowName,
303 instanceId: id,
304 simple: true,
305 })
306 .pipe(Effect.retry(retryInternal)));
307 if (status !== undefined) progress = progressOf(status.steps ?? []) ?? progress;
308
309 if (current.status === "complete") {
310 yield* session.note(`Backfilled ${label}`, { kind: "status" });
311 return { ...attrs(), instanceId: id, status: "complete" };
312 }
313 if (current.status === "errored" || current.status === "terminated") {
314 const failed = status ?? (yield* instance(id));
315 const message = failed.error?.message ?? "";
316 // The runs are unaffected: a new instance waits on them, from where they are.
317 if (
318 (current.status === "terminated" || startedTooSoon(message)) &&
319 attempt < MAX_RESTARTS
320 ) {
321 attempt += 1;
322 yield* Effect.sleep(RESTART_DELAY);
323 id = yield* start(attempt);
324 continue;
325 }
326 // A consumer that reads other sources than its run: the reason is the whole message.
327 const changed = /BackfillPlanChanged: (.*)$/s.exec(message)?.[1];
328 const reason = message
329 ? `: ${failed.error?.name ?? "Error"}: ${message.replace(/\.$/, "")}`
330 : "";
331 return yield* new BackfillFailed({
332 message:
333 changed ??
334 `The backfill of ${label} ended ${current.status}${reason}.${progress === undefined ? "" : ` Where it got to: ${where()}.`}`,
335 instanceId: id,
336 status: current.status,
337 });
338 }
339 if (elapsed > timeout) {
340 return yield* new BackfillStillRunning({
341 message: `The backfill of ${label} is still running after ${Math.round(elapsed / 1000)}s: ${where()}. It keeps running on Cloudflare; the next deploy waits on it again.`,
342 instanceId: id,
343 });
344 }
345 yield* session.note(`Backfilling ${where()} (${Math.round(elapsed / 1000)}s) ...`, {
346 kind: "status",
347 });
348 yield* Effect.sleep(POLL);
349 }
350 }),
351 // The runs belong to the consumers' classes, which the Worker deletes with them: here,
352 // only stop the instance a deploy may have left waiting.
353 delete: Effect.fn(function* ({ output }) {
354 yield* workflows
355 .patchInstanceStatus({
356 accountId: output.accountId,
357 workflowName: output.workflowName,
358 instanceId: output.instanceId,
359 status: "terminate",
360 })
361 .pipe(
362 Effect.asVoid,
363 Effect.catchTag(
364 ["InstanceNotFound", "WorkflowNotFound", "InstanceCannotTerminate"],
365 () => Effect.void,
366 ),
367 );
368 }),
369 });
370
371 /** In `alchemy dev` the Domain backfills itself on its first call: nothing to wait for. */
372 const BackfillLocal = () =>
373 Provider.succeed(Backfill, {
374 diff: Effect.fn(function* () {
375 return undefined;
376 }),
377 read: Effect.fn(function* ({ output }) {
378 return output;
379 }),
380 reconcile: Effect.fn(function* ({ instanceId, news }) {
381 return {
382 accountId: "dev",
383 workflowName: news.workflowName,
384 instanceId: `dev:${instanceIdOf(news.targets, instanceId, "local")}`,
385 targets: news.targets,
386 status: "complete",
387 };
388 }),
389 delete: Effect.fn(function* () {}),
390 });
391
392 export const BackfillProvider = () =>
393 ProviderLayer.dual(Backfill, {
394 local: () => BackfillLocal(),
395 live: () => BackfillLive(),
396 });