@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 62 lines 2.44 KB
Raw
1 import * as GCP from "@alchemy.run/gcp";
2 import * as Effect from "effect/Effect";
3 import * as Option from "effect/Option";
4 import { EventsTable, Inbox, type EventRow } from "./resources.ts";
5
6 /** One pull returns at most this many messages. */
7 const BATCH = 100;
8
9 /**
10 * The batch half of the pipeline: drain Pub/Sub into BigQuery.
11 *
12 * A Cloud Run Job is the right host for this. It runs to completion and
13 * exits, so nothing is billed between drains, and Cloud Scheduler or an
14 * operator can start it whenever a batch is due.
15 *
16 * The order matters: insert first, ack second. A crash between them
17 * redelivers the batch and duplicates rows, which a `SELECT DISTINCT id`
18 * removes; acking first would lose events outright.
19 */
20 export default class Drain extends GCP.Run.Job<Drain>()(
21 "Drain",
22 {
23 main: import.meta.url,
24 },
25 Effect.gen(function* () {
26 const inbox = yield* Inbox;
27 const table = yield* EventsTable;
28
29 // pubsub.subscriber on the subscription, and bigquery.dataEditor on
30 // the table — nothing on the project.
31 const subscription = yield* GCP.PubSub.ReadSubscription(inbox);
32 const warehouse = yield* GCP.BigQuery.WriteTable(table);
33
34 return {
35 // A pull may return fewer messages than are waiting, so drain
36 // batch by batch until one comes back empty.
37 run: Effect.gen(function* () {
38 // A pull waits for messages; an empty subscription answers
39 // nothing, so a quiet 10 seconds means the backlog is drained.
40 const received = yield* subscription
41 .pull({ maxMessages: BATCH })
42 .pipe(Effect.timeoutOption("10 seconds"));
43 const messages = Option.getOrElse(received, () => []);
44 if (messages.length === 0) {
45 yield* Effect.log("drain: nothing left");
46 return 0;
47 }
48
49 const rows = messages.map((message) => JSON.parse(message.text) as EventRow);
50
51 // insertIds make the streaming insert idempotent inside
52 // BigQuery's dedup window, so a redelivered batch collapses.
53 yield* warehouse.insert(rows, { insertIds: rows.map((row) => row.id) });
54
55 yield* subscription.acknowledge(messages.map((message) => message.ackId));
56
57 yield* Effect.log(`drain: wrote ${rows.length} row(s)`);
58 return messages.length;
59 }).pipe(Effect.repeat({ until: (count) => count === 0 }), Effect.asVoid, Effect.orDie),
60 };
61 }).pipe(Effect.provide([GCP.PubSub.ReadSubscriptionHttp, GCP.BigQuery.WriteTableHttp])),
62 ) {}