@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 335 lines 13.1 KB
Raw
1 import * as Athena from "@alchemy.run/aws/Athena";
2 import * as Glue from "@alchemy.run/aws/Glue";
3 import * as Lambda from "@alchemy.run/aws/Lambda";
4 import * as S3 from "@alchemy.run/aws/S3";
5 import * as Output from "alchemy/Output";
6 import * as Duration from "effect/Duration";
7 import * as Effect from "effect/Effect";
8 import { HttpServerRequest } from "effect/http/HttpServerRequest";
9 import * as HttpServerResponse from "effect/http/HttpServerResponse";
10 import * as Layer from "effect/Layer";
11 import * as Schedule from "effect/Schedule";
12 import * as Stream from "effect/Stream";
13 import path from "pathe";
14
15 const main = path.resolve(import.meta.dirname, "handler.ts");
16
17 // Fixed lowercase Glue identifiers so the SQL below is deterministic. The
18 // bucket and workgroup are engine-named (unique); the database/table/catalog
19 // are scoped to this single Athena e2e suite.
20 const DATABASE = "alchemy_athena_e2e";
21 const TABLE = "people";
22 const CATALOG = "alchemy_athena_e2e_catalog";
23 const STATEMENT = "alchemy_athena_e2e_stmt";
24 const CSV = "1,alice\n2,bob\n3,carol\n";
25
26 export class AthenaTestFunction extends Lambda.Function<Lambda.Function>()("AthenaTestFunction") {}
27
28 export default AthenaTestFunction.make(
29 {
30 main,
31 functionUrl: true,
32 // startQueryExecution → poll getQueryExecution → getQueryResults fans out
33 // several SDK calls; AWS's 3s default would intermittently time out.
34 timeout: Duration.seconds(60),
35 },
36 Effect.gen(function* () {
37 // Single bucket holds the CSV source data (under data/), the Athena query
38 // results (under results/), and the event-source markers (under events/).
39 const bucket = yield* S3.Bucket("AthenaBucket", { forceDestroy: true });
40
41 const database = yield* Glue.Database("AthenaDatabase", {
42 databaseName: DATABASE,
43 });
44 yield* Glue.Table("AthenaTable", {
45 databaseName: database.databaseName,
46 tableName: TABLE,
47 tableType: "EXTERNAL_TABLE",
48 storageDescriptor: {
49 location: Output.interpolate`s3://${bucket.bucketName}/data/`,
50 columns: [
51 { name: "id", type: "string" },
52 { name: "name", type: "string" },
53 ],
54 inputFormat: "org.apache.hadoop.mapred.TextInputFormat",
55 outputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
56 serdeInfo: {
57 serializationLibrary: "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
58 parameters: { "field.delim": "," },
59 },
60 },
61 });
62
63 const workGroup = yield* Athena.WorkGroup("AthenaWorkGroup", {
64 outputLocation: Output.interpolate`s3://${bucket.bucketName}/results/`,
65 enforceWorkGroupConfiguration: true,
66 });
67
68 // A GLUE-type data catalog federating to this account's own Glue Data
69 // Catalog (catalog-id = the account id, derived from the workgroup ARN)
70 // — drives the catalog-metadata bindings against the fixture database.
71 const dataCatalog = yield* Athena.DataCatalog("AthenaGlueCatalog", {
72 name: CATALOG,
73 type: "GLUE",
74 parameters: {
75 "catalog-id": workGroup.workGroupArn.pipe(Output.map((arn) => arn.split(":")[4]!)),
76 },
77 });
78
79 // Saved-query + prepared-statement fixtures in the workgroup — the
80 // ListNamedQueries/ListPreparedStatements bindings enumerate them.
81 yield* Athena.NamedQuery("AthenaNamedQuery", {
82 queryString: `SELECT * FROM ${TABLE} LIMIT 1`,
83 database: DATABASE,
84 workGroup: workGroup.workGroupName,
85 });
86 yield* Athena.PreparedStatement("AthenaPreparedStatement", {
87 statementName: STATEMENT,
88 queryStatement: `SELECT * FROM ${DATABASE}.${TABLE} WHERE id = ?`,
89 workGroup: workGroup.workGroupName,
90 });
91
92 const putObject = yield* S3.PutObject(bucket);
93 const getObject = yield* S3.GetObject(bucket);
94 const query = yield* Athena.Query(workGroup, bucket);
95 const runQuery = Effect.fn((request: Athena.RunQueryRequest) =>
96 query(request).pipe(
97 Effect.retry({
98 while: (error) => error._tag === "AccessDeniedException",
99 schedule: Schedule.spaced("3 seconds"),
100 times: 8,
101 }),
102 ),
103 );
104
105 // --- workgroup-scoped query-execution bindings ---
106 const getQueryExecution = yield* Athena.GetQueryExecution(workGroup);
107 const getQueryResults = yield* Athena.GetQueryResults(workGroup);
108 const stopQueryExecution = yield* Athena.StopQueryExecution(workGroup);
109 const batchGetQueryExecution = yield* Athena.BatchGetQueryExecution(workGroup);
110 const listQueryExecutions = yield* Athena.ListQueryExecutions(workGroup);
111 const getQueryRuntimeStatistics = yield* Athena.GetQueryRuntimeStatistics(workGroup);
112 const listNamedQueries = yield* Athena.ListNamedQueries(workGroup);
113 const listPreparedStatements = yield* Athena.ListPreparedStatements(workGroup);
114
115 // --- catalog-metadata bindings ---
116 const listDatabases = yield* Athena.ListDatabases(dataCatalog);
117 const getDatabase = yield* Athena.GetDatabase(dataCatalog);
118 const listTableMetadata = yield* Athena.ListTableMetadata(dataCatalog);
119 const getTableMetadata = yield* Athena.GetTableMetadata(dataCatalog);
120
121 // --- event source ---
122 // Athena publishes every query state transition to the default bus; write
123 // a marker object per terminal transition so /events/probe can observe
124 // the delivery out-of-band (the event may arrive on another instance).
125 yield* Athena.consumeQueryStateChanges(
126 { states: ["SUCCEEDED", "FAILED", "CANCELLED"] },
127 (events) =>
128 Stream.runForEach(events, (event) =>
129 putObject({
130 Key: `events/${event.detail.queryExecutionId}`,
131 Body: JSON.stringify(event.detail),
132 ContentType: "application/json",
133 }).pipe(Effect.orDie, Effect.asVoid),
134 ),
135 );
136
137 return {
138 fetch: Effect.gen(function* () {
139 const request = yield* HttpServerRequest;
140 const url = new URL(request.originalUrl);
141 const pathname = url.pathname;
142 const param = (name: string) => url.searchParams.get(name) ?? "";
143
144 // Seed the CSV source data into s3://bucket/data/.
145 if (request.method === "POST" && pathname === "/seed") {
146 yield* putObject({
147 Key: "data/rows.csv",
148 Body: CSV,
149 ContentType: "text/csv",
150 });
151 return yield* HttpServerResponse.json({ seeded: true });
152 }
153
154 // Flagship: run a Glue-backed query end-to-end and read the result.
155 if (request.method === "GET" && pathname === "/count") {
156 const result = yield* runQuery({
157 QueryString: `SELECT COUNT(*) AS c FROM ${DATABASE}.${TABLE}`,
158 });
159 return yield* HttpServerResponse.json({
160 state: result.state,
161 columns: result.columns,
162 rows: result.rows,
163 });
164 }
165
166 // Glue-free path: proves start+poll+results without any table.
167 if (request.method === "GET" && pathname === "/select-one") {
168 const result = yield* runQuery({ QueryString: "SELECT 1" });
169 return yield* HttpServerResponse.json({
170 state: result.state,
171 columns: result.columns,
172 rows: result.rows,
173 });
174 }
175
176 // Run a trivial query to completion and hand back its execution id
177 // for the granular query-execution bindings below.
178 if (request.method === "GET" && pathname === "/exec/run") {
179 const result = yield* runQuery({ QueryString: "SELECT 1" });
180 return yield* HttpServerResponse.json({
181 id: result.queryExecutionId,
182 });
183 }
184
185 if (request.method === "GET" && pathname === "/exec/get") {
186 const res = yield* getQueryExecution({
187 QueryExecutionId: param("id"),
188 });
189 return yield* HttpServerResponse.json({
190 state: res.QueryExecution?.Status?.State,
191 workGroup: res.QueryExecution?.WorkGroup,
192 });
193 }
194
195 if (request.method === "GET" && pathname === "/exec/results") {
196 const res = yield* getQueryResults({
197 QueryExecutionId: param("id"),
198 });
199 return yield* HttpServerResponse.json({
200 rows: res.ResultSet?.Rows?.length ?? 0,
201 columns: (res.ResultSet?.ResultSetMetadata?.ColumnInfo ?? []).map((c) => c.Name),
202 });
203 }
204
205 if (request.method === "GET" && pathname === "/exec/stats") {
206 // Runtime statistics materialize asynchronously shortly after the
207 // query completes — retry briefly.
208 const res = yield* getQueryRuntimeStatistics({
209 QueryExecutionId: param("id"),
210 }).pipe(
211 Effect.retry({
212 while: (e): boolean => e._tag === "InvalidRequestException",
213 schedule: Schedule.spaced("1 second"),
214 times: 8,
215 }),
216 );
217 return yield* HttpServerResponse.json({
218 totalMillis: res.QueryRuntimeStatistics?.Timeline?.TotalExecutionTimeInMillis,
219 });
220 }
221
222 if (request.method === "GET" && pathname === "/exec/batch") {
223 const res = yield* batchGetQueryExecution({
224 QueryExecutionIds: [param("id")],
225 });
226 return yield* HttpServerResponse.json({
227 states: (res.QueryExecutions ?? []).map((qe) => qe.Status?.State),
228 });
229 }
230
231 if (request.method === "GET" && pathname === "/exec/list") {
232 const res = yield* listQueryExecutions({ MaxResults: 25 });
233 return yield* HttpServerResponse.json({
234 ids: res.QueryExecutionIds ?? [],
235 });
236 }
237
238 if (request.method === "POST" && pathname === "/exec/stop") {
239 // Stopping an already-completed query is a documented no-op, which
240 // makes this deterministic without racing a long-running query.
241 yield* stopQueryExecution({ QueryExecutionId: param("id") });
242 return yield* HttpServerResponse.json({ stopped: true });
243 }
244
245 if (request.method === "GET" && pathname === "/named/list") {
246 const res = yield* listNamedQueries({ MaxResults: 50 });
247 return yield* HttpServerResponse.json({
248 ids: res.NamedQueryIds ?? [],
249 });
250 }
251
252 if (request.method === "GET" && pathname === "/prepared/list") {
253 const res = yield* listPreparedStatements({ MaxResults: 50 });
254 return yield* HttpServerResponse.json({
255 names: (res.PreparedStatements ?? []).map((s) => s.StatementName),
256 });
257 }
258
259 if (request.method === "GET" && pathname === "/catalog/databases") {
260 const res = yield* listDatabases({});
261 return yield* HttpServerResponse.json({
262 names: (res.DatabaseList ?? []).map((db) => db.Name),
263 });
264 }
265
266 if (request.method === "GET" && pathname === "/catalog/database") {
267 const res = yield* getDatabase({ DatabaseName: param("name") });
268 return yield* HttpServerResponse.json({
269 name: res.Database?.Name,
270 });
271 }
272
273 if (request.method === "GET" && pathname === "/catalog/tables") {
274 const res = yield* listTableMetadata({ DatabaseName: param("db") });
275 return yield* HttpServerResponse.json({
276 names: (res.TableMetadataList ?? []).map((t) => t.Name),
277 });
278 }
279
280 if (request.method === "GET" && pathname === "/catalog/table") {
281 const res = yield* getTableMetadata({
282 DatabaseName: param("db"),
283 TableName: param("name"),
284 });
285 return yield* HttpServerResponse.json({
286 columns: (res.TableMetadata?.Columns ?? []).map((c) => c.Name),
287 });
288 }
289
290 // Run a query, then poll for the event-source marker the state-change
291 // handler wrote to S3 (bounded well under the 60s function timeout).
292 if (request.method === "GET" && pathname === "/events/probe") {
293 const result = yield* runQuery({ QueryString: "SELECT 1" });
294 const id = result.queryExecutionId;
295 const seen = yield* getObject({ Key: `events/${id}` }).pipe(
296 Effect.retry({
297 while: (e): boolean => e._tag === "NoSuchKey",
298 schedule: Schedule.spaced("2 seconds"),
299 times: 12,
300 }),
301 Effect.map(() => true),
302 Effect.catchTag("NoSuchKey", () => Effect.succeed(false)),
303 );
304 return yield* HttpServerResponse.json({ seen, id });
305 }
306
307 return yield* HttpServerResponse.json(
308 { error: "Not found", method: request.method, pathname },
309 { status: 404 },
310 );
311 }).pipe(Effect.orDie),
312 };
313 }).pipe(
314 Effect.provide(
315 Layer.mergeAll(
316 Lambda.EventSource,
317 Athena.QueryHttp,
318 Athena.GetQueryExecutionHttp,
319 Athena.GetQueryResultsHttp,
320 Athena.StopQueryExecutionHttp,
321 Athena.BatchGetQueryExecutionHttp,
322 Athena.ListQueryExecutionsHttp,
323 Athena.GetQueryRuntimeStatisticsHttp,
324 Athena.ListNamedQueriesHttp,
325 Athena.ListPreparedStatementsHttp,
326 Athena.ListDatabasesHttp,
327 Athena.GetDatabaseHttp,
328 Athena.ListTableMetadataHttp,
329 Athena.GetTableMetadataHttp,
330 S3.PutObjectHttp,
331 S3.GetObjectHttp,
332 ),
333 ),
334 ),
335 );