The best way to control Claude Code, Codex and more from any computer or phone

main
ts 231 lines 7.82 KB
Raw
1 import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
2 import * as Effect from "effect/Effect";
3 import * as GCPBindings from "../GCPBindings.ts";
4 import { type BindingIam, grantFor } from "../HttpBinding.ts";
5 import { type ListTableResult, QueryNotComplete, type ReadTableClient } from "./ReadTable.ts";
6 import { decodeRows, encodeRow, queryParameter } from "./Rows.ts";
7 import type { Table } from "./Table.ts";
8 import { InsertRowsFailed, type WriteTableClient } from "./WriteTable.ts";
9
10 // bigquery.jobs.create is only grantable on the project; data access is
11 // granted on the table's own IAM policy.
12 export const readTableIam: ReadonlyArray<BindingIam> = [
13 { role: "roles/bigquery.dataViewer", on: "bigquery.table" },
14 { role: "roles/bigquery.jobUser" },
15 ];
16 export const writeTableIam: ReadonlyArray<BindingIam> = [
17 { role: "roles/bigquery.dataEditor", on: "bigquery.table" },
18 ];
19 export const readWriteTableIam: ReadonlyArray<BindingIam> = [
20 { role: "roles/bigquery.dataEditor", on: "bigquery.table" },
21 { role: "roles/bigquery.jobUser" },
22 ];
23
24 export interface TableRef {
25 project: string;
26 datasetId: string;
27 tableId: string;
28 location: string | undefined;
29 }
30
31 // `getQueryResults` long-polls up to `timeoutMs` per call, so this bounds
32 // the total wait for a query job at ~2 minutes.
33 const QUERY_WAIT_MS = 10_000;
34 const QUERY_POLLS = 12;
35
36 interface QueryPage {
37 jobComplete?: boolean;
38 schema?: bigquery.TableSchema;
39 rows?: bigquery.TableRow[];
40 pageToken?: string;
41 }
42
43 /** Fetch every result page after `pageToken`. */
44 const remainingRows = <E>(
45 fetchPage: (pageToken: string) => Effect.Effect<QueryPage, E>,
46 pageToken: string | undefined,
47 ): Effect.Effect<ReadonlyArray<bigquery.TableRow>, E> =>
48 pageToken
49 ? fetchPage(pageToken).pipe(
50 Effect.flatMap((page) =>
51 remainingRows(fetchPage, page.pageToken).pipe(
52 Effect.map((rest) => [...(page.rows ?? []), ...rest]),
53 ),
54 ),
55 )
56 : Effect.succeed([]);
57
58 /**
59 * Shared HTTP scaffolding for the BigQuery Read/Write/ReadWrite table
60 * bindings: resolves the distilled operations once at Layer construction
61 * and grants the level's roles at deploy time.
62 *
63 * NOT exported from `index.ts`.
64 */
65 export const makeBigQueryTableHelpers = Effect.gen(function* () {
66 const getTable = yield* GCPBindings.withCredentials(bigquery.getTables);
67 const listTabledata = yield* GCPBindings.withCredentials(bigquery.listTabledata);
68 const queryJobs = yield* GCPBindings.withCredentials(bigquery.queryJobs);
69 const getQueryResults = yield* GCPBindings.withCredentials(bigquery.getQueryResultsJobs);
70 const insertAll = yield* GCPBindings.withCredentials(bigquery.insertAllTabledata);
71
72 const makeRead = (table: Effect.Effect<TableRef>): ReadTableClient => ({
73 list: (
74 options,
75 ): Effect.Effect<ListTableResult, bigquery.GetTablesError | bigquery.ListTabledataError> =>
76 Effect.gen(function* () {
77 const ref = yield* table;
78 const key = {
79 projectId: ref.project,
80 datasetId: ref.datasetId,
81 tableId: ref.tableId,
82 };
83 const metadata = yield* getTable({ ...key, view: "BASIC" });
84 const page = yield* listTabledata({
85 ...key,
86 maxResults: options?.maxResults,
87 pageToken: options?.pageToken,
88 "formatOptions.useInt64Timestamp": true,
89 });
90 return {
91 rows: decodeRows(metadata.schema, page.rows),
92 nextPageToken: page.pageToken || undefined,
93 };
94 }),
95 query: (
96 sql,
97 params,
98 ): Effect.Effect<
99 Record<string, unknown>[],
100 bigquery.QueryJobsError | bigquery.GetQueryResultsJobsError | QueryNotComplete
101 > =>
102 Effect.gen(function* () {
103 const ref = yield* table;
104 const first = yield* queryJobs({
105 projectId: ref.project,
106 body: {
107 query: sql,
108 useLegacySql: false,
109 location: ref.location,
110 defaultDataset: {
111 projectId: ref.project,
112 datasetId: ref.datasetId,
113 },
114 timeoutMs: QUERY_WAIT_MS,
115 formatOptions: { useInt64Timestamp: true },
116 ...(params === undefined
117 ? {}
118 : {
119 parameterMode: "NAMED",
120 queryParameters: Object.entries(params).map(([name, value]) =>
121 queryParameter(name, value),
122 ),
123 }),
124 },
125 });
126 const jobId = first.jobReference?.jobId;
127 const location = first.jobReference?.location ?? ref.location;
128 const results = (
129 pageToken: string | undefined,
130 ): Effect.Effect<
131 bigquery.GetQueryResultsResponse,
132 bigquery.GetQueryResultsJobsError | QueryNotComplete
133 > =>
134 jobId === undefined
135 ? Effect.fail(new QueryNotComplete({ jobId }))
136 : getQueryResults({
137 projectId: ref.project,
138 jobId,
139 location,
140 pageToken,
141 timeoutMs: QUERY_WAIT_MS,
142 "formatOptions.useInt64Timestamp": true,
143 });
144
145 const settled: Effect.Effect<
146 QueryPage,
147 bigquery.GetQueryResultsJobsError | QueryNotComplete
148 > =
149 first.jobComplete === true
150 ? Effect.succeed(first)
151 : results(undefined).pipe(
152 Effect.repeat({
153 until: (r: bigquery.GetQueryResultsResponse) => r.jobComplete === true,
154 times: QUERY_POLLS,
155 }),
156 );
157 const complete = yield* settled;
158 if (complete.jobComplete !== true) {
159 return yield* new QueryNotComplete({ jobId });
160 }
161 const schema = complete.schema;
162 const rest: ReadonlyArray<bigquery.TableRow> = yield* remainingRows(
163 results,
164 complete.pageToken,
165 );
166 return decodeRows(schema, [...(complete.rows ?? []), ...rest]);
167 }),
168 });
169
170 const makeWrite = (table: Effect.Effect<TableRef>): WriteTableClient => ({
171 insert: (rows, options) =>
172 Effect.gen(function* () {
173 if (rows.length === 0) return;
174 const ref = yield* table;
175 const response = yield* insertAll({
176 projectId: ref.project,
177 datasetId: ref.datasetId,
178 tableId: ref.tableId,
179 body: {
180 rows: rows.map((row, index) => ({
181 json: encodeRow(row),
182 insertId: options?.insertIds?.[index],
183 })),
184 },
185 });
186 const insertErrors = response.insertErrors ?? [];
187 if (insertErrors.length > 0) {
188 return yield* new InsertRowsFailed({
189 insertErrors: insertErrors.map((item) => ({
190 index: item.index,
191 errors: item.errors ?? [],
192 })),
193 });
194 }
195 }),
196 });
197
198 return { makeRead, makeWrite };
199 });
200
201 /** Build a table binding that grants `iam` and returns `makeClient`'s client. */
202 export const makeBigQueryTableBinding = <Client>(options: {
203 tag: string;
204 iam: ReadonlyArray<BindingIam>;
205 makeClient: (
206 helpers: Effect.Success<typeof makeBigQueryTableHelpers>,
207 table: Effect.Effect<TableRef>,
208 ) => Client;
209 }) =>
210 Effect.gen(function* () {
211 const bind = yield* GCPBindings.Bind;
212 const helpers = yield* makeBigQueryTableHelpers;
213 return Effect.fn(function* (table: Table) {
214 if (!globalThis.__ALCHEMY_RUNTIME__) {
215 yield* bind`${options.tag}(${table})`({
216 iam: options.iam.map((iam) => grantFor(iam, table.name)),
217 });
218 }
219 const project = yield* table.project;
220 const datasetId = yield* table.datasetId;
221 const tableId = yield* table.tableId;
222 const location = yield* table.location;
223 const ref = Effect.all({
224 project,
225 datasetId,
226 tableId,
227 location,
228 });
229 return options.makeClient(helpers, ref);
230 });
231 });