@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 369 lines 13.3 KB
Raw
1 import type { Input, InputProps } from "alchemy/Input";
2 import * as CoreNamespace from "alchemy/Namespace";
3 import { isResource } from "alchemy/Resource";
4 import * as Effect from "effect/Effect";
5 import { AccountApiToken } from "../ApiToken/AccountApiToken.ts";
6 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
7 import type { Bucket } from "../R2/Bucket.ts";
8 import { SearchInstance, type SearchInstanceProps, type SourceParams } from "./SearchInstance.ts";
9 import type { SearchNamespace } from "./SearchNamespace.ts";
10 import { SearchToken } from "./SearchToken.ts";
11
12 type WebCrawlerParams = NonNullable<SourceParams["webCrawler"]>;
13
14 /**
15 * How a web crawler discovers and parses pages — Cloudflare's `parseType`
16 * folded together with its parse options. Defaults to `type: "sitemap"`.
17 */
18 export type Parse = {
19 /**
20 * How pages are discovered:
21 * - `"sitemap"` (default) — read `<seed>/sitemap.xml` (found via
22 * `robots.txt`) and index the URLs it lists.
23 * - `"discover"` — start at `source` and follow links.
24 * @default "sitemap"
25 */
26 type?: NonNullable<WebCrawlerParams["parseType"]>;
27 } & NonNullable<WebCrawlerParams["parseOptions"]>;
28
29 /**
30 * Where crawled content is stored. Cloudflare provisions managed storage by
31 * default; set this to store output in an R2 bucket you control.
32 */
33 export type Store = {
34 /** R2 bucket to store crawl output in. */
35 bucket: Bucket;
36 /** R2 data-residency jurisdiction for the store bucket. */
37 jurisdiction?: string;
38 };
39
40 /**
41 * Props common to every AI Search pipeline, regardless of data source. The
42 * underlying instance's `type`, `source`, and `sourceParams` are derived from
43 * the source-specific variant, so they're omitted here.
44 */
45 export type SharedProps = Omit<
46 InputProps<SearchInstanceProps, "type">,
47 "type" | "source" | "sourceParams" | "namespace"
48 > & {
49 /**
50 * Namespace to group this pipeline under. Pass an {@link SearchNamespace}
51 * resource — the engine orders this pipeline after the namespace on deploy
52 * and tears it down before the namespace on destroy. Omit to place the
53 * pipeline in the account-provided `default` namespace. The namespace is
54 * immutable — changing it triggers a replacement.
55 * @default the account-provided `default` namespace
56 */
57 namespace?: SearchNamespace;
58 };
59
60 /**
61 * An R2-backed AI Search pipeline. Passing an {@link Bucket} as `source`
62 * selects R2 as the data source.
63 */
64 export type R2Props = SharedProps & {
65 /**
66 * The R2 bucket to index. AI Search needs a service token to read it; the
67 * construct provisions one unless you pass your own `tokenId`.
68 */
69 source: Bucket;
70 /** Only index object keys under this prefix. */
71 prefix?: string;
72 /**
73 * Micromatch glob patterns; only objects matching at least one are
74 * indexed (`*` within a path segment, `**` across segments). Max 10.
75 */
76 include?: string[];
77 /**
78 * Micromatch glob patterns; objects matching any are skipped. Exclude
79 * takes precedence over `include`. Max 10.
80 */
81 exclude?: string[];
82 /** R2 data-residency jurisdiction of the source bucket. */
83 jurisdiction?: string;
84 parse?: never;
85 store?: never;
86 };
87
88 /**
89 * A web-crawler-backed AI Search pipeline. Passing a URL as `source` selects
90 * the web crawler as the data source (no service token needed).
91 */
92 export type WebCrawlerProps = SharedProps & {
93 /** Seed URL to crawl and index. */
94 source: Input<string>;
95 /** How pages are discovered and parsed. */
96 parse?: Parse;
97 /** Where crawl output is stored (defaults to managed storage). */
98 store?: Store;
99 prefix?: never;
100 include?: never;
101 exclude?: never;
102 jurisdiction?: never;
103 };
104
105 /**
106 * Props for the {@link Search} construct — a union discriminated by what you
107 * pass as `source`: an {@link Bucket} for an R2 source, or a URL string for
108 * a web crawl.
109 */
110 export type Props = R2Props | WebCrawlerProps;
111
112 /**
113 * The result of the {@link Search} construct. It *is* the underlying
114 * {@link SearchInstance}, augmented with the managed `serviceToken`, so it
115 * can be passed anywhere a `SearchInstance` is expected —
116 * `Cloudflare.AI.QuerySearch(search)`, a Worker's `env`, etc.
117 */
118 export type Search = SearchInstance & {
119 /**
120 * The managed AI Search service token minted for an R2 source, or
121 * `undefined` when the source is a web crawler (no token needed) or you
122 * supplied your own `tokenId`.
123 */
124 serviceToken: SearchToken | undefined;
125 };
126
127 /**
128 * A convenience construct over {@link SearchInstance} that auto-creates the
129 * sub-resources an AI Search instance typically needs, so a single call wires
130 * up a working pipeline. The data source is chosen by what you pass as
131 * `source` — an {@link Bucket} for R2, or a URL for a web crawl:
132 *
133 * - For an R2 source, it mints a least-privilege {@link AccountApiToken}
134 * (`AI Search Index Engine`, stable child `ApiToken`) and an
135 * {@link SearchToken} wrapping it (stable child `Token`), then passes
136 * that token to the instance.
137 * Cloudflare requires a service token to read an R2 bucket and only
138 * provisions one through the dashboard / Wrangler — never on a
139 * programmatic API create — so the construct provisions it for you. Pass
140 * your own `tokenId` to skip minting and reuse an existing token.
141 * - It creates the {@link SearchInstance} (child `SearchInstance`) with the
142 * remaining props.
143 *
144 * Drop down to the low-level resources directly when you need to share a
145 * token across instances, adopt an existing one, or bind a namespace.
146 *
147 * The returned value *is* an {@link SearchInstance} (augmented with the
148 * managed `serviceToken`, `undefined` for a web crawler), so a `Search`
149 * is usable anywhere a `SearchInstance` is expected — pass it straight to
150 * `Cloudflare.AI.QuerySearch(search)` or a Worker's `env`.
151 *
152 * ### Creating an AI Search pipeline
153 * **Example:** R2-backed instance (token provisioned for you)
154 * Pass an {@link Bucket} as `source` — its presence selects R2.
155 * ```typescript
156 * const bucket = yield* Cloudflare.R2.Bucket("docs");
157 * const search = yield* Cloudflare.AI.Search("docs-search", {
158 * source: bucket,
159 * });
160 * ```
161 *
162 * **Example:** Index only part of a bucket
163 * ```typescript
164 * const search = yield* Cloudflare.AI.Search("docs-search", {
165 * source: bucket,
166 * prefix: "docs/",
167 * include: ["/docs/**"],
168 * exclude: ["/docs/drafts/**"],
169 * });
170 * ```
171 *
172 * **Example:** Reuse an existing service token
173 * ```typescript
174 * const search = yield* Cloudflare.AI.Search("docs-search", {
175 * source: bucket,
176 * tokenId: existingToken.id,
177 * });
178 * ```
179 *
180 * **Example:** Web-crawler source
181 * Pass a URL as `source` to crawl and index a website (no service token
182 * needed). `parse.type` defaults to `"sitemap"`; use `"discover"` to follow
183 * links from the seed instead.
184 * ```typescript
185 * const search = yield* Cloudflare.AI.Search("site-search", {
186 * source: "https://example.com",
187 * parse: { type: "discover", contentSelector: [{ path: "/docs", selector: "main" }] },
188 * });
189 * ```
190 *
191 * **Example:** Store crawl output in your own bucket
192 * ```typescript
193 * const store = yield* Cloudflare.R2.Bucket("crawl-store");
194 * const search = yield* Cloudflare.AI.Search("site-search", {
195 * source: "https://example.com",
196 * parse: { type: "discover" },
197 * store: { bucket: store },
198 * });
199 * ```
200 *
201 * ### Binding to an Effect Worker
202 *
203 * The returned `search` is an {@link SearchInstance}. Bind it during the
204 * Worker's init phase with `Cloudflare.AI.QuerySearch(search)`, which
205 * attaches the single-instance `ai_search` binding and hands back an
206 * Effect-native client whose `search` / `chatCompletions` methods return
207 * `Effect`s. Provide `Cloudflare.AI.QuerySearchBinding` in the Worker's
208 * runtime layer.
209 *
210 * **Example:** Effect Worker that answers from AI Search
211 * ```typescript
212 * import * as Cloudflare from "@alchemy.run/cloudflare";
213 * import * as Effect from "effect/Effect";
214 * import { HttpServerRequest } from "effect/http/HttpServerRequest";
215 * import * as HttpServerResponse from "effect/http/HttpServerResponse";
216 *
217 * export default class Api extends Cloudflare.Worker<Api>()(
218 * "api",
219 * { main: import.meta.url },
220 * Effect.gen(function* () {
221 * const bucket = yield* Cloudflare.R2.Bucket("docs");
222 * const aiSearch = yield* Cloudflare.AI.Search("docs-search", {
223 * source: bucket,
224 * });
225 * const search = yield* Cloudflare.AI.QuerySearch(aiSearch);
226 *
227 * return {
228 * fetch: Effect.gen(function* () {
229 * const request = yield* HttpServerRequest;
230 * const query = new URL(request.url).searchParams.get("q") ?? "";
231 * const answer = yield* search.chatCompletions({
232 * messages: [{ role: "user", content: query }],
233 * });
234 * return yield* HttpServerResponse.json(answer);
235 * }),
236 * };
237 * }).pipe(Effect.provide(Cloudflare.AI.QuerySearchBinding)),
238 * ) {}
239 * ```
240 *
241 * ### Binding to an Async Worker
242 *
243 * For a vanilla `async fetch` Worker, pass the `search` under `Worker.env`.
244 * The engine attaches the same single-instance `ai_search` binding (see
245 * `toBinding` in `WorkerAsyncBindings.ts`), orders the deploy
246 * bucket → instance → worker, and `InferEnv` types `env.SEARCH` as the
247 * runtime `SearchInstance` handle — no hand-written types.
248 *
249 * **Example:** Async Worker that answers from AI Search
250 * ```typescript
251 * // stack.ts
252 * const bucket = yield* Cloudflare.R2.Bucket("docs");
253 * const search = yield* Cloudflare.AI.Search("docs-search", {
254 * source: bucket,
255 * });
256 *
257 * export const Api = Cloudflare.Worker("api", {
258 * main: "./worker.ts",
259 * env: { SEARCH: search },
260 * });
261 * export type ApiEnv = Cloudflare.InferEnv<typeof Api>;
262 *
263 * // worker.ts
264 * import type { ApiEnv } from "./stack.ts";
265 * export default {
266 * async fetch(request: Request, env: ApiEnv): Promise<Response> {
267 * const query = new URL(request.url).searchParams.get("q") ?? "";
268 * const answer = await env.SEARCH.chatCompletions({
269 * messages: [{ role: "user", content: query }],
270 * });
271 * return Response.json(answer);
272 * },
273 * };
274 * ```
275 *
276 * @see https://developers.cloudflare.com/ai-search/
277 *
278 * @resource
279 * @product AI Search
280 * @category AI
281 */
282 export const Search = (id: string, props: Props) =>
283 Effect.gen(function* () {
284 const { source, prefix, include, exclude, jurisdiction, parse, store, namespace, ...shared } =
285 props;
286
287 let tokenId = shared.tokenId;
288 let serviceToken: SearchToken | undefined;
289 let type: "r2" | "web-crawler";
290 let instanceSource: Input<string>;
291 let sourceParams: Input<SourceParams> | undefined;
292
293 // Discriminate the data source on what `source` is: an Bucket (a resource)
294 // indexes a bucket and needs a service token to read it; a URL crawls a seed
295 // and doesn't.
296 if (isResource(source)) {
297 const bucket = source as Bucket;
298 type = "r2";
299 instanceSource = bucket.bucketName;
300 sourceParams = clean({
301 prefix,
302 includeItems: include,
303 excludeItems: exclude,
304 r2Jurisdiction: jurisdiction,
305 }) as Input<SourceParams> | undefined;
306
307 // Cloudflare requires a service token to read an R2 source and only
308 // auto-creates one via the dashboard/Wrangler — not on a programmatic
309 // API create. Mint one ourselves unless the caller passed a `tokenId`.
310 if (tokenId === undefined) {
311 const { accountId } = yield* yield* CloudflareEnvironment;
312 const apiToken = yield* AccountApiToken("ApiToken", {
313 policies: [
314 {
315 effect: "allow",
316 permissionGroups: ["AI Search Index Engine"],
317 resources: {
318 [`com.cloudflare.api.account.${accountId}`]: "*",
319 },
320 },
321 ],
322 });
323 serviceToken = yield* SearchToken("Token", {
324 cfApiId: apiToken.tokenId,
325 cfApiKey: apiToken.value,
326 });
327 tokenId = serviceToken.id;
328 }
329 } else {
330 type = "web-crawler";
331 instanceSource = source as Input<string>;
332
333 const { type: parseType, ...parseOptions } = parse ?? {};
334 const webCrawler = clean({
335 parseType,
336 parseOptions: clean(parseOptions),
337 storeOptions: store
338 ? clean({
339 storageId: store.bucket.bucketName,
340 storageType: "r2" as const,
341 r2Jurisdiction: store.jurisdiction,
342 })
343 : undefined,
344 });
345 sourceParams = webCrawler ? ({ webCrawler } as Input<SourceParams>) : undefined;
346 }
347
348 const instance = yield* SearchInstance("Instance", {
349 ...shared,
350 // The instance is keyed by namespace name; pass the namespace's `name`
351 // output so the engine orders instance-after-namespace.
352 namespace: namespace?.name,
353 type,
354 source: instanceSource,
355 tokenId,
356 sourceParams,
357 });
358
359 // Return the instance itself (augmented with the managed `serviceToken`)
360 // so a `Search` is usable anywhere a `SearchInstance` is expected —
361 // `Cloudflare.AI.QuerySearch(search)`, `env: { SEARCH: search }`, etc.
362 return Object.assign(instance, { serviceToken }) as Search;
363 }).pipe(CoreNamespace.push(id));
364
365 /** Drop `undefined` entries; return `undefined` when nothing is left. */
366 const clean = <T extends object>(obj: T): { [K in keyof T]: T[K] } | undefined => {
367 const entries = Object.entries(obj).filter(([, v]) => v !== undefined);
368 return entries.length ? (Object.fromEntries(entries) as { [K in keyof T]: T[K] }) : undefined;
369 };