@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 475 lines 16.6 KB
Raw
1 import * as qbusiness from "@distilled.cloud/aws/qbusiness";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as EffectStream from "effect/Stream";
12 import type { Providers } from "../Providers.ts";
13
14 export class QBusinessDataSourceInvalidProps extends Data.TaggedError(
15 "QBusinessDataSourceInvalidProps",
16 )<{
17 readonly message: string;
18 readonly cause?: unknown;
19 }> {}
20
21 export class QBusinessDataSourceUnexpectedResponse extends Data.TaggedError(
22 "QBusinessDataSourceUnexpectedResponse",
23 )<{
24 readonly message: string;
25 readonly cause?: unknown;
26 }> {}
27
28 export type DataSourceStatus = qbusiness.DataSourceStatus;
29
30 export interface DataSourceProps {
31 /**
32 * The identifier of the Amazon Q Business application the data source
33 * attaches to. Changing it replaces the data source.
34 */
35 applicationId: string;
36 /**
37 * The identifier of the index the data source syncs documents into.
38 * Changing it replaces the data source.
39 */
40 indexId: string;
41 /**
42 * Display name of the data source.
43 * @default ${app}-${stage}-${id}
44 */
45 displayName?: string;
46 /**
47 * Connector-specific configuration (e.g. the S3 or Web Crawler connector
48 * template). See the Amazon Q Business connector documentation for each
49 * connector's schema.
50 */
51 configuration: unknown;
52 /**
53 * VPC configuration for connectors that reach into a VPC.
54 */
55 vpcConfiguration?: qbusiness.DataSourceVpcConfiguration;
56 /**
57 * A description of the data source.
58 */
59 description?: string;
60 /**
61 * Sync schedule as a cron expression (e.g. `cron(0 12 * * ? *)`). When
62 * omitted, syncs run only on demand.
63 */
64 syncSchedule?: string;
65 /**
66 * ARN of the IAM role the connector assumes to access the source
67 * content.
68 */
69 roleArn?: string;
70 /**
71 * Alter document metadata/content during ingestion.
72 */
73 documentEnrichmentConfiguration?: qbusiness.DocumentEnrichmentConfiguration;
74 /**
75 * Image/audio/video extraction settings for ingested media.
76 */
77 mediaExtractionConfiguration?: qbusiness.MediaExtractionConfiguration;
78 /**
79 * Tags to associate with the data source.
80 */
81 tags?: Record<string, string>;
82 }
83
84 export interface DataSource extends Resource<
85 "AWS.QBusiness.DataSource",
86 DataSourceProps,
87 {
88 /**
89 * Service-assigned unique identifier of the data source (unique within
90 * its index).
91 */
92 dataSourceId: string;
93 /**
94 * The identifier of the application the data source belongs to.
95 */
96 applicationId: string;
97 /**
98 * The identifier of the index the data source syncs into.
99 */
100 indexId: string;
101 /**
102 * ARN of the data source.
103 */
104 dataSourceArn: string;
105 /**
106 * The data source's display name.
107 */
108 displayName: string;
109 /**
110 * The connector type (e.g. `S3`, `WEBCRAWLERV2`, `CUSTOM`).
111 */
112 type: string | undefined;
113 /**
114 * Current lifecycle status of the data source.
115 */
116 status: DataSourceStatus | undefined;
117 /**
118 * Current tags reported for the data source.
119 */
120 tags: Record<string, string>;
121 },
122 never,
123 Providers
124 > {}
125
126 /**
127 * An Amazon Q Business data source — a connector that syncs documents from
128 * a repository (S3 bucket, website, SharePoint, ...) into an index.
129 *
130 * ### Creating Data Sources
131 * **Example:** S3 Data Source
132 * ```typescript
133 * import * as AWS from "@alchemy.run/aws";
134 *
135 * const source = yield* AWS.QBusiness.DataSource("Docs", {
136 * applicationId: app.applicationId,
137 * indexId: index.indexId,
138 * roleArn: dataSourceRole.roleArn,
139 * configuration: {
140 * type: "S3",
141 * syncMode: "FORCED_FULL_CRAWL",
142 * connectionConfiguration: {
143 * repositoryEndpointMetadata: { BucketName: bucket.bucketName },
144 * },
145 * repositoryConfigurations: {
146 * document: {
147 * fieldMappings: [{
148 * indexFieldName: "s3_document_id",
149 * indexFieldType: "STRING",
150 * dataSourceFieldName: "s3_document_id",
151 * }],
152 * },
153 * },
154 * },
155 * });
156 * ```
157 *
158 * **Example:** Scheduled Sync
159 * ```typescript
160 * const source = yield* AWS.QBusiness.DataSource("Docs", {
161 * applicationId: app.applicationId,
162 * indexId: index.indexId,
163 * roleArn: dataSourceRole.roleArn,
164 * syncSchedule: "cron(0 12 * * ? *)",
165 * configuration: { ... },
166 * });
167 * ```
168 *
169 * @resource
170 */
171 export const DataSource = Resource<DataSource>("AWS.QBusiness.DataSource");
172
173 const createDisplayName = (id: string, props: { displayName?: string | undefined }) =>
174 props.displayName
175 ? Effect.succeed(props.displayName)
176 : createPhysicalName({ id, maxLength: 100 });
177
178 const fetchTags = Effect.fn(function* (arn: string) {
179 const response = yield* qbusiness
180 .listTagsForResource({ resourceARN: arn })
181 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
182 return Object.fromEntries((response?.tags ?? []).map((tag) => [tag.key, tag.value]));
183 });
184
185 interface DataSourceState {
186 attrs: DataSource["Attributes"];
187 described: qbusiness.GetDataSourceResponse;
188 }
189
190 const readDataSourceById = Effect.fn(function* (
191 applicationId: string,
192 indexId: string,
193 dataSourceId: string,
194 ) {
195 const described = yield* qbusiness
196 .getDataSource({ applicationId, indexId, dataSourceId })
197 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
198 if (!described || described.status === "DELETING") return undefined;
199 const arn = described.dataSourceArn;
200 if (arn === undefined) return undefined;
201 const state: DataSourceState = {
202 described,
203 attrs: {
204 dataSourceId: described.dataSourceId ?? dataSourceId,
205 applicationId: described.applicationId ?? applicationId,
206 indexId: described.indexId ?? indexId,
207 dataSourceArn: arn,
208 displayName: described.displayName ?? "",
209 type: described.type,
210 status: described.status,
211 tags: yield* fetchTags(arn),
212 },
213 };
214 return state;
215 });
216
217 const findDataSourceByName = Effect.fn(function* (
218 applicationId: string,
219 indexId: string,
220 displayName: string,
221 ) {
222 const summaries = yield* qbusiness.listDataSources.pages({ applicationId, indexId }).pipe(
223 EffectStream.runCollect,
224 Effect.map((chunk) => Array.from(chunk).flatMap((page) => page.dataSources ?? [])),
225 // The parent application/index may itself be gone.
226 Effect.catchTag("ResourceNotFoundException", () =>
227 Effect.succeed([] as qbusiness.DataSource[]),
228 ),
229 );
230 const match = summaries.find(
231 (summary) => summary.displayName === displayName && summary.status !== "DELETING",
232 );
233 if (!match?.dataSourceId) return undefined;
234 return yield* readDataSourceById(applicationId, indexId, match.dataSourceId);
235 });
236
237 /**
238 * A data source still transitioning toward the awaited status — retried by
239 * {@link waitForDataSourceStatus}'s bounded schedule.
240 */
241 class DataSourceNotReady extends Data.TaggedError("QBusinessDataSourceNotReady")<{
242 readonly dataSourceId: string;
243 readonly status: string | undefined;
244 }> {}
245
246 /**
247 * A data source whose asynchronous provisioning converged to the terminal
248 * `FAILED` status.
249 */
250 export class DataSourceProvisioningFailed extends Data.TaggedError(
251 "QBusinessDataSourceProvisioningFailed",
252 )<{
253 readonly dataSourceId: string;
254 readonly message: string | undefined;
255 }> {}
256
257 // Explicitly-typed retry wrapper — an inline `Effect.retry` in provider
258 // lifecycle code leaks `Retry.Return`'s conditional type into declaration
259 // emit and widens the provider layer to `unknown` for every consumer of
260 // `AWS.providers()`.
261 const retryWhileNotReady = <A, E extends { readonly _tag: string }, R>(
262 self: Effect.Effect<A, E, R>,
263 ): Effect.Effect<A, E, R> =>
264 Effect.retry(self, {
265 while: (e) => e._tag === "QBusinessDataSourceNotReady",
266 // Data source provisioning is usually well under a minute; budget ~5 min.
267 schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(60)]),
268 });
269
270 // CreateDataSource validates the IAM role up front; a freshly-created role
271 // may not have propagated yet and surfaces as ValidationException. Bounded
272 // retry through the propagation window (~60s).
273 const retryThroughIamPropagation = <A, E extends { readonly _tag: string }, R>(
274 self: Effect.Effect<A, E, R>,
275 ): Effect.Effect<A, E, R> =>
276 Effect.retry(self, {
277 while: (e) => e._tag === "ValidationException",
278 schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(12)]),
279 });
280
281 const waitForDataSourceStatus = (
282 applicationId: string,
283 indexId: string,
284 dataSourceId: string,
285 target: "ACTIVE" | "DELETED",
286 ) =>
287 retryWhileNotReady(
288 Effect.gen(function* () {
289 const described = yield* qbusiness
290 .getDataSource({ applicationId, indexId, dataSourceId })
291 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
292 if (target === "DELETED") {
293 if (described === undefined) return;
294 return yield* new DataSourceNotReady({ dataSourceId, status: described.status });
295 }
296 if (described?.status === "ACTIVE") return;
297 if (described?.status === "FAILED") {
298 return yield* new DataSourceProvisioningFailed({
299 dataSourceId,
300 message: described.error?.errorMessage,
301 });
302 }
303 return yield* new DataSourceNotReady({ dataSourceId, status: described?.status });
304 }),
305 );
306
307 export const DataSourceProvider = () =>
308 Provider.effect(
309 DataSource,
310 Effect.gen(function* () {
311 return {
312 stables: ["dataSourceId", "applicationId", "indexId", "dataSourceArn"],
313 // Keyed by a parent application+index; cannot be enumerated
314 // account-wide without iterating every application — treated as a
315 // sub-resource per the factory list() convention.
316 list: () => Effect.succeed([]),
317 read: Effect.fn(function* ({ id, olds, output }) {
318 const applicationId = output?.applicationId ?? olds?.applicationId;
319 const indexId = output?.indexId ?? olds?.indexId;
320 if (applicationId === undefined || indexId === undefined) {
321 return undefined;
322 }
323 const state = output?.dataSourceId
324 ? yield* readDataSourceById(applicationId, indexId, output.dataSourceId)
325 : yield* findDataSourceByName(
326 applicationId,
327 indexId,
328 yield* createDisplayName(id, olds ?? {}),
329 );
330 if (!state) return undefined;
331 return (yield* hasAlchemyTags(id, state.attrs.tags as Tags))
332 ? state.attrs
333 : Unowned(state.attrs);
334 }),
335 diff: Effect.fn(function* ({ news, olds }) {
336 if (!isResolved(news)) return;
337 if (olds === undefined) return;
338 // The parent application and index are fixed at creation.
339 if (olds.applicationId !== news.applicationId || olds.indexId !== news.indexId) {
340 return { action: "replace" } as const;
341 }
342 }),
343 reconcile: Effect.fn(function* ({ id, news, output, session }) {
344 if (!news) {
345 return yield* new QBusinessDataSourceInvalidProps({
346 message: "QBusiness DataSource requires props",
347 });
348 }
349 const applicationId = news.applicationId;
350 const indexId = news.indexId;
351 const displayName = yield* createDisplayName(id, news);
352 const internalTags = yield* createInternalTags(id);
353 const desiredTags = { ...internalTags, ...news.tags };
354
355 // Observe — prefer the cached id; fall back to a name lookup so a
356 // create whose state failed to persist is adopted, not duplicated.
357 let state = output?.dataSourceId
358 ? yield* readDataSourceById(applicationId, indexId, output.dataSourceId)
359 : yield* findDataSourceByName(applicationId, indexId, displayName);
360
361 // Ensure — create if missing, then wait for ACTIVE.
362 if (state === undefined) {
363 const created = yield* retryThroughIamPropagation(
364 qbusiness.createDataSource({
365 applicationId,
366 indexId,
367 displayName,
368 configuration: news.configuration,
369 vpcConfiguration: news.vpcConfiguration,
370 description: news.description,
371 syncSchedule: news.syncSchedule,
372 roleArn: news.roleArn,
373 documentEnrichmentConfiguration: news.documentEnrichmentConfiguration,
374 mediaExtractionConfiguration: news.mediaExtractionConfiguration,
375 tags: Object.entries(desiredTags).map(([key, value]) => ({
376 key,
377 value,
378 })),
379 }),
380 );
381 if (!created.dataSourceId) {
382 return yield* new QBusinessDataSourceUnexpectedResponse({
383 message: `CreateDataSource for '${displayName}' returned no dataSourceId`,
384 });
385 }
386 yield* session.note(`Creating data source ${displayName} (${created.dataSourceId})...`);
387 yield* waitForDataSourceStatus(applicationId, indexId, created.dataSourceId, "ACTIVE");
388 state = yield* readDataSourceById(applicationId, indexId, created.dataSourceId);
389 if (state === undefined) {
390 return yield* new QBusinessDataSourceUnexpectedResponse({
391 message: `failed to read created data source ${displayName}`,
392 });
393 }
394 }
395
396 // Sync mutable settings via UpdateDataSource — only when drifted.
397 const described = state.described;
398 const needsUpdate =
399 displayName !== described.displayName ||
400 (news.description ?? "") !== (described.description ?? "") ||
401 (news.syncSchedule ?? "") !== (described.syncSchedule ?? "") ||
402 (news.roleArn !== undefined && news.roleArn !== described.roleArn) ||
403 news.configuration !== undefined ||
404 news.vpcConfiguration !== undefined ||
405 news.documentEnrichmentConfiguration !== undefined ||
406 news.mediaExtractionConfiguration !== undefined;
407 if (needsUpdate) {
408 yield* qbusiness.updateDataSource({
409 applicationId,
410 indexId,
411 dataSourceId: state.attrs.dataSourceId,
412 displayName,
413 configuration: news.configuration,
414 vpcConfiguration: news.vpcConfiguration,
415 description: news.description,
416 syncSchedule: news.syncSchedule,
417 roleArn: news.roleArn,
418 documentEnrichmentConfiguration: news.documentEnrichmentConfiguration,
419 mediaExtractionConfiguration: news.mediaExtractionConfiguration,
420 });
421 yield* waitForDataSourceStatus(
422 applicationId,
423 indexId,
424 state.attrs.dataSourceId,
425 "ACTIVE",
426 );
427 yield* session.note(`Updated data source ${displayName}`);
428 }
429
430 // Sync tags — diff against observed cloud tags.
431 const { removed, upsert } = diffTags(state.attrs.tags, desiredTags);
432 if (removed.length > 0) {
433 yield* qbusiness.untagResource({
434 resourceARN: state.attrs.dataSourceArn,
435 tagKeys: removed,
436 });
437 }
438 if (upsert.length > 0) {
439 yield* qbusiness.tagResource({
440 resourceARN: state.attrs.dataSourceArn,
441 tags: upsert.map(({ Key, Value }) => ({
442 key: Key,
443 value: Value,
444 })),
445 });
446 }
447
448 yield* session.note(state.attrs.dataSourceArn);
449
450 const final = yield* readDataSourceById(applicationId, indexId, state.attrs.dataSourceId);
451 if (!final) {
452 return yield* new QBusinessDataSourceUnexpectedResponse({
453 message: `failed to read reconciled data source ${displayName}`,
454 });
455 }
456 return final.attrs;
457 }),
458 delete: Effect.fn(function* ({ output }) {
459 yield* qbusiness
460 .deleteDataSource({
461 applicationId: output.applicationId,
462 indexId: output.indexId,
463 dataSourceId: output.dataSourceId,
464 })
465 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
466 yield* waitForDataSourceStatus(
467 output.applicationId,
468 output.indexId,
469 output.dataSourceId,
470 "DELETED",
471 );
472 }),
473 };
474 }),
475 );