The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as bigquery from "@distilled.cloud/gcp/bigquery_v2"; |
| 2 | import * as Effect from "effect/Effect"; |
| 3 | import * as GCPBindings from "../GCPBindings.ts"; |
| 4 | import { type BindingIam, grantFor } from "../HttpBinding.ts"; |
| 5 | import { type ListTableResult, QueryNotComplete, type ReadTableClient } from "./ReadTable.ts"; |
| 6 | import { decodeRows, encodeRow, queryParameter } from "./Rows.ts"; |
| 7 | import type { Table } from "./Table.ts"; |
| 8 | import { InsertRowsFailed, type WriteTableClient } from "./WriteTable.ts"; |
| 9 | |
| 10 | // bigquery.jobs.create is only grantable on the project; data access is |
| 11 | // granted on the table's own IAM policy. |
| 12 | export const readTableIam: ReadonlyArray<BindingIam> = [ |
| 13 | { role: "roles/bigquery.dataViewer", on: "bigquery.table" }, |
| 14 | { role: "roles/bigquery.jobUser" }, |
| 15 | ]; |
| 16 | export const writeTableIam: ReadonlyArray<BindingIam> = [ |
| 17 | { role: "roles/bigquery.dataEditor", on: "bigquery.table" }, |
| 18 | ]; |
| 19 | export const readWriteTableIam: ReadonlyArray<BindingIam> = [ |
| 20 | { role: "roles/bigquery.dataEditor", on: "bigquery.table" }, |
| 21 | { role: "roles/bigquery.jobUser" }, |
| 22 | ]; |
| 23 | |
| 24 | export interface TableRef { |
| 25 | project: string; |
| 26 | datasetId: string; |
| 27 | tableId: string; |
| 28 | location: string | undefined; |
| 29 | } |
| 30 | |
| 31 | // `getQueryResults` long-polls up to `timeoutMs` per call, so this bounds |
| 32 | // the total wait for a query job at ~2 minutes. |
| 33 | const QUERY_WAIT_MS = 10_000; |
| 34 | const QUERY_POLLS = 12; |
| 35 | |
| 36 | interface QueryPage { |
| 37 | jobComplete?: boolean; |
| 38 | schema?: bigquery.TableSchema; |
| 39 | rows?: bigquery.TableRow[]; |
| 40 | pageToken?: string; |
| 41 | } |
| 42 | |
| 43 | /** Fetch every result page after `pageToken`. */ |
| 44 | const remainingRows = <E>( |
| 45 | fetchPage: (pageToken: string) => Effect.Effect<QueryPage, E>, |
| 46 | pageToken: string | undefined, |
| 47 | ): Effect.Effect<ReadonlyArray<bigquery.TableRow>, E> => |
| 48 | pageToken |
| 49 | ? fetchPage(pageToken).pipe( |
| 50 | Effect.flatMap((page) => |
| 51 | remainingRows(fetchPage, page.pageToken).pipe( |
| 52 | Effect.map((rest) => [...(page.rows ?? []), ...rest]), |
| 53 | ), |
| 54 | ), |
| 55 | ) |
| 56 | : Effect.succeed([]); |
| 57 | |
| 58 | /** |
| 59 | * Shared HTTP scaffolding for the BigQuery Read/Write/ReadWrite table |
| 60 | * bindings: resolves the distilled operations once at Layer construction |
| 61 | * and grants the level's roles at deploy time. |
| 62 | * |
| 63 | * NOT exported from `index.ts`. |
| 64 | */ |
| 65 | export const makeBigQueryTableHelpers = Effect.gen(function* () { |
| 66 | const getTable = yield* GCPBindings.withCredentials(bigquery.getTables); |
| 67 | const listTabledata = yield* GCPBindings.withCredentials(bigquery.listTabledata); |
| 68 | const queryJobs = yield* GCPBindings.withCredentials(bigquery.queryJobs); |
| 69 | const getQueryResults = yield* GCPBindings.withCredentials(bigquery.getQueryResultsJobs); |
| 70 | const insertAll = yield* GCPBindings.withCredentials(bigquery.insertAllTabledata); |
| 71 | |
| 72 | const makeRead = (table: Effect.Effect<TableRef>): ReadTableClient => ({ |
| 73 | list: ( |
| 74 | options, |
| 75 | ): Effect.Effect<ListTableResult, bigquery.GetTablesError | bigquery.ListTabledataError> => |
| 76 | Effect.gen(function* () { |
| 77 | const ref = yield* table; |
| 78 | const key = { |
| 79 | projectId: ref.project, |
| 80 | datasetId: ref.datasetId, |
| 81 | tableId: ref.tableId, |
| 82 | }; |
| 83 | const metadata = yield* getTable({ ...key, view: "BASIC" }); |
| 84 | const page = yield* listTabledata({ |
| 85 | ...key, |
| 86 | maxResults: options?.maxResults, |
| 87 | pageToken: options?.pageToken, |
| 88 | "formatOptions.useInt64Timestamp": true, |
| 89 | }); |
| 90 | return { |
| 91 | rows: decodeRows(metadata.schema, page.rows), |
| 92 | nextPageToken: page.pageToken || undefined, |
| 93 | }; |
| 94 | }), |
| 95 | query: ( |
| 96 | sql, |
| 97 | params, |
| 98 | ): Effect.Effect< |
| 99 | Record<string, unknown>[], |
| 100 | bigquery.QueryJobsError | bigquery.GetQueryResultsJobsError | QueryNotComplete |
| 101 | > => |
| 102 | Effect.gen(function* () { |
| 103 | const ref = yield* table; |
| 104 | const first = yield* queryJobs({ |
| 105 | projectId: ref.project, |
| 106 | body: { |
| 107 | query: sql, |
| 108 | useLegacySql: false, |
| 109 | location: ref.location, |
| 110 | defaultDataset: { |
| 111 | projectId: ref.project, |
| 112 | datasetId: ref.datasetId, |
| 113 | }, |
| 114 | timeoutMs: QUERY_WAIT_MS, |
| 115 | formatOptions: { useInt64Timestamp: true }, |
| 116 | ...(params === undefined |
| 117 | ? {} |
| 118 | : { |
| 119 | parameterMode: "NAMED", |
| 120 | queryParameters: Object.entries(params).map(([name, value]) => |
| 121 | queryParameter(name, value), |
| 122 | ), |
| 123 | }), |
| 124 | }, |
| 125 | }); |
| 126 | const jobId = first.jobReference?.jobId; |
| 127 | const location = first.jobReference?.location ?? ref.location; |
| 128 | const results = ( |
| 129 | pageToken: string | undefined, |
| 130 | ): Effect.Effect< |
| 131 | bigquery.GetQueryResultsResponse, |
| 132 | bigquery.GetQueryResultsJobsError | QueryNotComplete |
| 133 | > => |
| 134 | jobId === undefined |
| 135 | ? Effect.fail(new QueryNotComplete({ jobId })) |
| 136 | : getQueryResults({ |
| 137 | projectId: ref.project, |
| 138 | jobId, |
| 139 | location, |
| 140 | pageToken, |
| 141 | timeoutMs: QUERY_WAIT_MS, |
| 142 | "formatOptions.useInt64Timestamp": true, |
| 143 | }); |
| 144 | |
| 145 | const settled: Effect.Effect< |
| 146 | QueryPage, |
| 147 | bigquery.GetQueryResultsJobsError | QueryNotComplete |
| 148 | > = |
| 149 | first.jobComplete === true |
| 150 | ? Effect.succeed(first) |
| 151 | : results(undefined).pipe( |
| 152 | Effect.repeat({ |
| 153 | until: (r: bigquery.GetQueryResultsResponse) => r.jobComplete === true, |
| 154 | times: QUERY_POLLS, |
| 155 | }), |
| 156 | ); |
| 157 | const complete = yield* settled; |
| 158 | if (complete.jobComplete !== true) { |
| 159 | return yield* new QueryNotComplete({ jobId }); |
| 160 | } |
| 161 | const schema = complete.schema; |
| 162 | const rest: ReadonlyArray<bigquery.TableRow> = yield* remainingRows( |
| 163 | results, |
| 164 | complete.pageToken, |
| 165 | ); |
| 166 | return decodeRows(schema, [...(complete.rows ?? []), ...rest]); |
| 167 | }), |
| 168 | }); |
| 169 | |
| 170 | const makeWrite = (table: Effect.Effect<TableRef>): WriteTableClient => ({ |
| 171 | insert: (rows, options) => |
| 172 | Effect.gen(function* () { |
| 173 | if (rows.length === 0) return; |
| 174 | const ref = yield* table; |
| 175 | const response = yield* insertAll({ |
| 176 | projectId: ref.project, |
| 177 | datasetId: ref.datasetId, |
| 178 | tableId: ref.tableId, |
| 179 | body: { |
| 180 | rows: rows.map((row, index) => ({ |
| 181 | json: encodeRow(row), |
| 182 | insertId: options?.insertIds?.[index], |
| 183 | })), |
| 184 | }, |
| 185 | }); |
| 186 | const insertErrors = response.insertErrors ?? []; |
| 187 | if (insertErrors.length > 0) { |
| 188 | return yield* new InsertRowsFailed({ |
| 189 | insertErrors: insertErrors.map((item) => ({ |
| 190 | index: item.index, |
| 191 | errors: item.errors ?? [], |
| 192 | })), |
| 193 | }); |
| 194 | } |
| 195 | }), |
| 196 | }); |
| 197 | |
| 198 | return { makeRead, makeWrite }; |
| 199 | }); |
| 200 | |
| 201 | /** Build a table binding that grants `iam` and returns `makeClient`'s client. */ |
| 202 | export const makeBigQueryTableBinding = <Client>(options: { |
| 203 | tag: string; |
| 204 | iam: ReadonlyArray<BindingIam>; |
| 205 | makeClient: ( |
| 206 | helpers: Effect.Success<typeof makeBigQueryTableHelpers>, |
| 207 | table: Effect.Effect<TableRef>, |
| 208 | ) => Client; |
| 209 | }) => |
| 210 | Effect.gen(function* () { |
| 211 | const bind = yield* GCPBindings.Bind; |
| 212 | const helpers = yield* makeBigQueryTableHelpers; |
| 213 | return Effect.fn(function* (table: Table) { |
| 214 | if (!globalThis.__ALCHEMY_RUNTIME__) { |
| 215 | yield* bind`${options.tag}(${table})`({ |
| 216 | iam: options.iam.map((iam) => grantFor(iam, table.name)), |
| 217 | }); |
| 218 | } |
| 219 | const project = yield* table.project; |
| 220 | const datasetId = yield* table.datasetId; |
| 221 | const tableId = yield* table.tableId; |
| 222 | const location = yield* table.location; |
| 223 | const ref = Effect.all({ |
| 224 | project, |
| 225 | datasetId, |
| 226 | tableId, |
| 227 | location, |
| 228 | }); |
| 229 | return options.makeClient(helpers, ref); |
| 230 | }); |
| 231 | }); |