The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import type { Progress } from "@alchemy.run/domain/Backfill"; |
| 2 | import * as workflows from "@distilled.cloud/cloudflare/workflows"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import * as ProviderLayer from "alchemy/Local/ProviderLayer"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import * as Clock from "effect/Clock"; |
| 8 | import * as Data from "effect/Data"; |
| 9 | import * as Effect from "effect/Effect"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; |
| 12 | |
| 13 | export interface BackfillProps { |
| 14 | /** The Domain's Worker: a run starts once the Worker carrying its Workflow is deployed. */ |
| 15 | scriptName: string; |
| 16 | /** The Workflow, hosted by the Domain's Worker, that starts the runs and waits for them. */ |
| 17 | workflowName: string; |
| 18 | /** |
| 19 | * Every view and projection in the model, by target id (`view:Name`, with |
| 20 | * `#<generation>` once rebuilt). Those new to the Domain are backfilled. |
| 21 | */ |
| 22 | targets: string[]; |
| 23 | /** |
| 24 | * What the backfill needs to know of each consumer (the sources it reads, and |
| 25 | * whether a key reads its one source's log itself), by target id. The |
| 26 | * deploy passes them to each run, so a run doesn't depend on what an |
| 27 | * object's code knows. |
| 28 | */ |
| 29 | plans?: Record<string, { sources?: string[]; singleSource: boolean }>; |
| 30 | /** |
| 31 | * How long a deploy waits for the backfill, in seconds. Past it the deploy |
| 32 | * fails, while the backfill keeps running on Cloudflare; the next deploy |
| 33 | * waits on the same runs. |
| 34 | * @default 1800 |
| 35 | */ |
| 36 | timeoutSeconds?: number; |
| 37 | } |
| 38 | |
| 39 | export interface BackfillAttrs { |
| 40 | accountId: string; |
| 41 | workflowName: string; |
| 42 | /** The Workflow instance the last deploy waited through. */ |
| 43 | instanceId: string; |
| 44 | targets: string[]; |
| 45 | /** `complete` once every target's run is complete. */ |
| 46 | status: string; |
| 47 | } |
| 48 | |
| 49 | const BackfillTypeId = "Cloudflare.Domain.Backfill"; |
| 50 | |
| 51 | /** |
| 52 | * The backfill of a Domain's views and projections, run by reconciliation. |
| 53 | * The Durable Object platform declares it with the model's consumers: you |
| 54 | * don't declare it yourself. A deploy that adds a view or projection, or |
| 55 | * bumps a version, waits here until every new consumer has received the |
| 56 | * history recorded before it, and reports how far each one has got. |
| 57 | * |
| 58 | * ### What a deploy waits for <!-- api-prose --> |
| 59 | * |
| 60 | * Each consumer generation (a view, or a view after a version bump) has one |
| 61 | * run, held by one Durable Object of a class only that generation has. The |
| 62 | * first deploy to reach it starts it; every later start finds it started, so |
| 63 | * there is never more than one, however many deploys ask. The run lives on |
| 64 | * Cloudflare and moves on its own: this resource starts it, waits for it, |
| 65 | * and reports its progress, but holds none of its state. |
| 66 | * |
| 67 | * So a deploy is safe to repeat and to interrupt. Run it again after a |
| 68 | * failure, a timeout or a Ctrl-C, and it waits on the same runs, from where |
| 69 | * they are; once they are complete it returns at once. A deploy that changes |
| 70 | * no consumer plans no change here. |
| 71 | * |
| 72 | * ### Progress <!-- api-prose --> |
| 73 | * |
| 74 | * While it waits, the deploy notes each run's phase and how far through the |
| 75 | * directory it is: |
| 76 | * |
| 77 | * ```sh |
| 78 | * Backfilling view:Deposits reading history 41% (1,690/4,096 subtrees, 410,211 instances) (48s) ... |
| 79 | * ``` |
| 80 | * |
| 81 | * A consumer keyed by its source reads its history in one pass; any other |
| 82 | * reads it, then applies it once every source's history is in. `refused` |
| 83 | * counts calls Cloudflare turned down for the account's load: the run backs |
| 84 | * off and sends them again. |
| 85 | * |
| 86 | * ### Rebuilding and cancelling <!-- api-prose --> |
| 87 | * |
| 88 | * A run reads the sources its consumer read when it started. A deploy whose |
| 89 | * consumer reads other sources at the same version (a view that now reads a |
| 90 | * journal as well as accounts) fails, saying to bump the version: waiting on |
| 91 | * the run would report a history that misses the new source. |
| 92 | * |
| 93 | * To run a backfill again, bump the consumer's version: the new generation is |
| 94 | * a new class with its own run, and the same Worker upload deletes the old |
| 95 | * generation's class, data and run. To stop a backfill, remove the consumer |
| 96 | * from the model and deploy: its classes, and the run with them, are deleted. |
| 97 | * Either is one Worker upload, so a consumer never has two runs at once. |
| 98 | * |
| 99 | * @resource |
| 100 | * @product Durable Objects |
| 101 | */ |
| 102 | export interface Backfill extends Resource<typeof BackfillTypeId, BackfillProps, BackfillAttrs> {} |
| 103 | |
| 104 | export const Backfill = Resource<Backfill>(BackfillTypeId); |
| 105 | |
| 106 | export class BackfillStillRunning extends Data.TaggedError("BackfillStillRunning")<{ |
| 107 | readonly message: string; |
| 108 | readonly instanceId: string; |
| 109 | }> {} |
| 110 | |
| 111 | export class BackfillFailed extends Data.TaggedError("BackfillFailed")<{ |
| 112 | readonly message: string; |
| 113 | readonly instanceId: string; |
| 114 | readonly status: string; |
| 115 | }> {} |
| 116 | |
| 117 | /** The Workflow instance for one deploy's wait: named after the targets, and new each time. */ |
| 118 | const instanceIdOf = (targets: ReadonlyArray<string>, resource: string, attempt: string) => { |
| 119 | let hash = 0x811c9dc5; |
| 120 | for (const char of [...targets].sort().join("|")) { |
| 121 | hash ^= char.charCodeAt(0); |
| 122 | hash = Math.imul(hash, 0x01000193) >>> 0; |
| 123 | } |
| 124 | return `backfill-${resource.slice(0, 12)}-${hash.toString(16).padStart(8, "0")}-${attempt}`; |
| 125 | }; |
| 126 | |
| 127 | const POLL = "1 second"; |
| 128 | /** How often the deploy reads the runs' progress from the instance's steps. */ |
| 129 | const PROGRESS_EVERY = 3; |
| 130 | /** How many times a wait whose instance can't carry it on is started again, and how soon. */ |
| 131 | const MAX_RESTARTS = 30; |
| 132 | const RESTART_DELAY = "2 seconds"; |
| 133 | |
| 134 | /** |
| 135 | * An instance started on an edge that doesn't have the deployed Worker yet (or |
| 136 | * its entrypoint, or the consumer's class) stays on the code it started with: |
| 137 | * a new instance runs the deployed code. |
| 138 | */ |
| 139 | const startedTooSoon = (message: string) => |
| 140 | /entrypoint name .* was not found|^Worker not found|isn't in the code this runs yet/.test( |
| 141 | message, |
| 142 | ); |
| 143 | |
| 144 | /** |
| 145 | * Workflows answers instance calls for a Workflow uploaded moments before with |
| 146 | * an internal error (code 10001) for a while: call again. |
| 147 | */ |
| 148 | const retryInternal = { |
| 149 | while: (error: { readonly _tag: string }) => error._tag === "WorkflowInternalError", |
| 150 | schedule: Schedule.spaced("2 seconds"), |
| 151 | times: 30, |
| 152 | }; |
| 153 | |
| 154 | /** Each run's progress from the latest check step's output (an array of {@link Progress}). */ |
| 155 | export const progressOf = (steps: ReadonlyArray<object>) => { |
| 156 | for (let i = steps.length - 1; i >= 0; i--) { |
| 157 | const step = steps[i]!; |
| 158 | if (!("output" in step) || typeof step.output !== "string") continue; |
| 159 | if (!("name" in step) || typeof step.name !== "string" || !step.name.startsWith("check-")) |
| 160 | continue; |
| 161 | const found = findProgress(parseJson(step.output)); |
| 162 | if (found !== undefined) return found; |
| 163 | } |
| 164 | return undefined; |
| 165 | }; |
| 166 | |
| 167 | const parseJson = (text: string): unknown => { |
| 168 | try { |
| 169 | return JSON.parse(text); |
| 170 | } catch { |
| 171 | return undefined; |
| 172 | } |
| 173 | }; |
| 174 | |
| 175 | /** The first array of runs' progress in a step's output, however the step encoded it. */ |
| 176 | const findProgress = (value: unknown): ReadonlyArray<Progress> | undefined => { |
| 177 | if (Array.isArray(value)) { |
| 178 | if (value.length > 0 && value.every((item) => isProgress(item))) return value; |
| 179 | for (const item of value) { |
| 180 | const found = findProgress(item); |
| 181 | if (found !== undefined) return found; |
| 182 | } |
| 183 | return undefined; |
| 184 | } |
| 185 | if (value !== null && typeof value === "object") |
| 186 | for (const item of Object.values(value)) { |
| 187 | const found = findProgress(item); |
| 188 | if (found !== undefined) return found; |
| 189 | } |
| 190 | if (typeof value === "string" && (value.startsWith("[") || value.startsWith("{"))) |
| 191 | return findProgress(parseJson(value)); |
| 192 | return undefined; |
| 193 | }; |
| 194 | |
| 195 | const isProgress = (value: unknown): value is Progress => |
| 196 | value !== null && |
| 197 | typeof value === "object" && |
| 198 | "target" in value && |
| 199 | "phase" in value && |
| 200 | "subtrees" in value; |
| 201 | |
| 202 | const count = (n: number) => n.toLocaleString("en-US"); |
| 203 | |
| 204 | /** One run's progress, as the deploy notes it. */ |
| 205 | export const describeProgress = (run: Progress) => { |
| 206 | if (run.phase === "complete") return `${run.target} complete`; |
| 207 | if (run.phase === "unknown") return `${run.target} starting`; |
| 208 | const what = run.phase === "discover" ? "reading history" : "applying history"; |
| 209 | if (run.subtrees === 0) return `${run.target} ${what}`; |
| 210 | const percent = Math.floor((100 * run.done) / run.subtrees); |
| 211 | return `${run.target} ${what} ${percent}% (${count(run.done)}/${count(run.subtrees)} subtrees, ${count(run.instances)} instances${run.refused > 0 ? `, ${count(run.refused)} refused` : ""})`; |
| 212 | }; |
| 213 | |
| 214 | const BackfillLive = () => |
| 215 | Provider.succeed(Backfill, { |
| 216 | stables: ["accountId", "workflowName"], |
| 217 | diff: Effect.fn(function* ({ olds, news }) { |
| 218 | if (!isResolved(news)) return { action: "update" } as const; |
| 219 | const changed = |
| 220 | olds.workflowName !== news.workflowName || |
| 221 | [...olds.targets].sort().join("|") !== [...(news.targets ?? [])].sort().join("|"); |
| 222 | if (changed) return { action: "update" } as const; |
| 223 | }), |
| 224 | // The runs are the state, and they live on Cloudflare: what was recorded stands |
| 225 | // until a deploy changes the targets (an instance Workflows has since dropped |
| 226 | // changes nothing). |
| 227 | read: Effect.fn(function* ({ output }) { |
| 228 | return output; |
| 229 | }), |
| 230 | reconcile: Effect.fn(function* ({ instanceId: resource, news, output, session }) { |
| 231 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 232 | const label = news.targets.join(", "); |
| 233 | const started = yield* Clock.currentTimeMillis; |
| 234 | const timeout = (news.timeoutSeconds ?? 1800) * 1000; |
| 235 | const instance = (id: string) => |
| 236 | workflows |
| 237 | .getInstance({ accountId, workflowName: news.workflowName, instanceId: id }) |
| 238 | .pipe(Effect.retry(retryInternal)); |
| 239 | |
| 240 | // The instance a deploy that stopped waiting left behind: stop it, so it doesn't keep |
| 241 | // checking runs no one waits on. Best effort: it ends by itself once they complete. |
| 242 | if (output?.instanceId !== undefined && output.status !== "complete") |
| 243 | yield* workflows |
| 244 | .patchInstanceStatus({ |
| 245 | accountId, |
| 246 | workflowName: news.workflowName, |
| 247 | instanceId: output.instanceId, |
| 248 | status: "terminate", |
| 249 | }) |
| 250 | .pipe( |
| 251 | Effect.asVoid, |
| 252 | Effect.catchTag( |
| 253 | ["InstanceNotFound", "WorkflowNotFound", "InstanceCannotTerminate"], |
| 254 | () => Effect.void, |
| 255 | ), |
| 256 | ); |
| 257 | |
| 258 | /** |
| 259 | * Start an instance for this wait. Each deploy (and each start again) gets a new one: |
| 260 | * it only starts the runs and reports on them, and starting a run that is already |
| 261 | * started finds it, so a new instance never starts a backfill over. |
| 262 | */ |
| 263 | const start = (attempt: number) => |
| 264 | Effect.gen(function* () { |
| 265 | const id = instanceIdOf(news.targets, resource, `${started.toString(36)}${attempt}`); |
| 266 | yield* workflows |
| 267 | .createInstance({ |
| 268 | accountId, |
| 269 | workflowName: news.workflowName, |
| 270 | instanceId: id, |
| 271 | params: { targets: news.targets, plans: news.plans ?? {} }, |
| 272 | }) |
| 273 | .pipe( |
| 274 | Effect.asVoid, |
| 275 | Effect.catchTag("InstanceAlreadyExists", () => Effect.void), |
| 276 | // The Workflow is uploaded with the Worker: it can take a moment to appear. |
| 277 | Effect.retry({ |
| 278 | while: (error) => |
| 279 | error._tag === "WorkflowNotFound" || error._tag === "WorkflowInternalError", |
| 280 | schedule: Schedule.spaced("2 seconds"), |
| 281 | times: 30, |
| 282 | }), |
| 283 | ); |
| 284 | return id; |
| 285 | }); |
| 286 | |
| 287 | let attempt = 0; |
| 288 | let id = yield* start(attempt); |
| 289 | let progress: ReadonlyArray<Progress> | undefined; |
| 290 | const attrs = () => ({ accountId, workflowName: news.workflowName, targets: news.targets }); |
| 291 | const where = () => |
| 292 | progress === undefined ? label : progress.map((run) => describeProgress(run)).join("; "); |
| 293 | |
| 294 | for (let poll = 0; ; poll++) { |
| 295 | const elapsed = (yield* Clock.currentTimeMillis) - started; |
| 296 | const status = poll % PROGRESS_EVERY === 0 ? yield* instance(id) : undefined; |
| 297 | const current = |
| 298 | status ?? |
| 299 | (yield* workflows |
| 300 | .getInstance({ |
| 301 | accountId, |
| 302 | workflowName: news.workflowName, |
| 303 | instanceId: id, |
| 304 | simple: true, |
| 305 | }) |
| 306 | .pipe(Effect.retry(retryInternal))); |
| 307 | if (status !== undefined) progress = progressOf(status.steps ?? []) ?? progress; |
| 308 | |
| 309 | if (current.status === "complete") { |
| 310 | yield* session.note(`Backfilled ${label}`, { kind: "status" }); |
| 311 | return { ...attrs(), instanceId: id, status: "complete" }; |
| 312 | } |
| 313 | if (current.status === "errored" || current.status === "terminated") { |
| 314 | const failed = status ?? (yield* instance(id)); |
| 315 | const message = failed.error?.message ?? ""; |
| 316 | // The runs are unaffected: a new instance waits on them, from where they are. |
| 317 | if ( |
| 318 | (current.status === "terminated" || startedTooSoon(message)) && |
| 319 | attempt < MAX_RESTARTS |
| 320 | ) { |
| 321 | attempt += 1; |
| 322 | yield* Effect.sleep(RESTART_DELAY); |
| 323 | id = yield* start(attempt); |
| 324 | continue; |
| 325 | } |
| 326 | // A consumer that reads other sources than its run: the reason is the whole message. |
| 327 | const changed = /BackfillPlanChanged: (.*)$/s.exec(message)?.[1]; |
| 328 | const reason = message |
| 329 | ? `: ${failed.error?.name ?? "Error"}: ${message.replace(/\.$/, "")}` |
| 330 | : ""; |
| 331 | return yield* new BackfillFailed({ |
| 332 | message: |
| 333 | changed ?? |
| 334 | `The backfill of ${label} ended ${current.status}${reason}.${progress === undefined ? "" : ` Where it got to: ${where()}.`}`, |
| 335 | instanceId: id, |
| 336 | status: current.status, |
| 337 | }); |
| 338 | } |
| 339 | if (elapsed > timeout) { |
| 340 | return yield* new BackfillStillRunning({ |
| 341 | message: `The backfill of ${label} is still running after ${Math.round(elapsed / 1000)}s: ${where()}. It keeps running on Cloudflare; the next deploy waits on it again.`, |
| 342 | instanceId: id, |
| 343 | }); |
| 344 | } |
| 345 | yield* session.note(`Backfilling ${where()} (${Math.round(elapsed / 1000)}s) ...`, { |
| 346 | kind: "status", |
| 347 | }); |
| 348 | yield* Effect.sleep(POLL); |
| 349 | } |
| 350 | }), |
| 351 | // The runs belong to the consumers' classes, which the Worker deletes with them: here, |
| 352 | // only stop the instance a deploy may have left waiting. |
| 353 | delete: Effect.fn(function* ({ output }) { |
| 354 | yield* workflows |
| 355 | .patchInstanceStatus({ |
| 356 | accountId: output.accountId, |
| 357 | workflowName: output.workflowName, |
| 358 | instanceId: output.instanceId, |
| 359 | status: "terminate", |
| 360 | }) |
| 361 | .pipe( |
| 362 | Effect.asVoid, |
| 363 | Effect.catchTag( |
| 364 | ["InstanceNotFound", "WorkflowNotFound", "InstanceCannotTerminate"], |
| 365 | () => Effect.void, |
| 366 | ), |
| 367 | ); |
| 368 | }), |
| 369 | }); |
| 370 | |
| 371 | /** In `alchemy dev` the Domain backfills itself on its first call: nothing to wait for. */ |
| 372 | const BackfillLocal = () => |
| 373 | Provider.succeed(Backfill, { |
| 374 | diff: Effect.fn(function* () { |
| 375 | return undefined; |
| 376 | }), |
| 377 | read: Effect.fn(function* ({ output }) { |
| 378 | return output; |
| 379 | }), |
| 380 | reconcile: Effect.fn(function* ({ instanceId, news }) { |
| 381 | return { |
| 382 | accountId: "dev", |
| 383 | workflowName: news.workflowName, |
| 384 | instanceId: `dev:${instanceIdOf(news.targets, instanceId, "local")}`, |
| 385 | targets: news.targets, |
| 386 | status: "complete", |
| 387 | }; |
| 388 | }), |
| 389 | delete: Effect.fn(function* () {}), |
| 390 | }); |
| 391 | |
| 392 | export const BackfillProvider = () => |
| 393 | ProviderLayer.dual(Backfill, { |
| 394 | local: () => BackfillLocal(), |
| 395 | live: () => BackfillLive(), |
| 396 | }); |