The best way to control Claude Code, Codex and more from any computer or phone

main
ts 349 lines 11.8 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { unownedUnlessPrevious } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 encodeOwnership,
16 hasOwnershipMarker,
17 normalizeLocation,
18 parentOf,
19 parseOwnership,
20 parseResourceName,
21 rfc1035,
22 } from "./internal.ts";
23 import { listLocations } from "./names.ts";
24 import type { EncryptionSpec } from "./shared.ts";
25
26 export type CachedContentPart = {
27 /** Text content of the part. */
28 text?: string;
29 };
30
31 export type CachedContentMessage = {
32 /**
33 * Producer of the content (`user` or `model`). Defaults to `user`.
34 */
35 role?: string;
36 /** Parts that make up the message. */
37 parts?: CachedContentPart[];
38 };
39
40 export type CachedContentProps = {
41 /**
42 * Region. Immutable — changing it replaces the cache.
43 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
44 */
45 location?: string;
46 /**
47 * Publisher model used for the cache, for example
48 * `projects/{project}/locations/{location}/publishers/google/models/gemini-2.0-flash-001`.
49 * Immutable — changing it replaces the cache.
50 */
51 model: string;
52 /**
53 * Content to cache. Immutable — changing it replaces the cache.
54 */
55 contents?: CachedContentMessage[];
56 /**
57 * Developer system instruction (text only). Immutable — changing it
58 * replaces the cache.
59 */
60 systemInstruction?: CachedContentMessage;
61 /**
62 * TTL such as `"3600s"`. The API computes `expireTime = now + ttl`.
63 * This is the only field that updates in place (along with
64 * `expireTime`).
65 */
66 ttl?: string;
67 /**
68 * Absolute expiration (RFC3339). Mutually exclusive with `ttl` on
69 * update; either field may be patched.
70 */
71 expireTime?: string;
72 /**
73 * User-facing display name. Alchemy stamps ownership here (cached
74 * content has no labels) so `list` / nuke can find the cache.
75 */
76 displayName?: string;
77 /**
78 * Customer-managed encryption. Immutable — changing it replaces the
79 * cache.
80 */
81 encryptionSpec?: EncryptionSpec;
82 };
83
84 export type CachedContent = Resource<
85 "GCP.AIPlatform.CachedContent",
86 CachedContentProps,
87 {
88 /** Full resource name `.../cachedContents/{cached_content}`. */
89 name: string;
90 /** Cached content id (last path segment). */
91 cachedContentId: string;
92 /** Project id. */
93 project: string;
94 /** Location id. */
95 location: string;
96 /** Model resource name. */
97 model: string | undefined;
98 /** User display name with the Alchemy ownership prefix stripped. */
99 displayName: string | undefined;
100 /** Absolute expiration (RFC3339). */
101 expireTime: string | undefined;
102 /** RFC3339 creation timestamp. */
103 createTime: string | undefined;
104 /** RFC3339 last-update timestamp. */
105 updateTime: string | undefined;
106 /** Total tokens consumed by the cache. */
107 totalTokenCount: number | undefined;
108 /** Customer-managed KMS key, if any. */
109 kmsKeyName: string | undefined;
110 },
111 never,
112 Providers
113 >;
114
115 /**
116 * Vertex AI cached content — explicit context cache for LLM queries.
117 *
118 * Cached content has no labels field, so Alchemy stamps ownership into
119 * `displayName` for `list` / nuke. Model, contents, system instruction,
120 * and CMEK are immutable. TTL / expire time update in place.
121 *
122 * ### Creating Cached Content
123 * **Example:** Cache a system prompt for Gemini
124 * ```typescript
125 * const cache = yield* GCP.AIPlatform.CachedContent("Style", {
126 * model:
127 * "projects/my-project/locations/us-central1/publishers/google/models/gemini-2.0-flash-001",
128 * ttl: "3600s",
129 * contents: [
130 * { role: "user", parts: [{ text: "You are a terse assistant." }] },
131 * ],
132 * });
133 * ```
134 *
135 * ### Updating expiry
136 * **Example:** Extend TTL
137 * ```typescript
138 * // Same logical id, changed TTL: the engine updates it in place.
139 * const cache = yield* GCP.AIPlatform.CachedContent("Style", {
140 * model:
141 * "projects/my-project/locations/us-central1/publishers/google/models/gemini-2.0-flash-001",
142 * ttl: "7200s",
143 * contents: [
144 * { role: "user", parts: [{ text: "You are a terse assistant." }] },
145 * ],
146 * });
147 * ```
148 *
149 * @resource
150 * @category AIPlatform
151 */
152 export const CachedContent = Resource<CachedContent>("GCP.AIPlatform.CachedContent");
153
154 export class CachedContentNotResolved extends Data.TaggedError(
155 "GCP.AIPlatform.CachedContentNotResolved",
156 )<{
157 resourceName: string;
158 }> {}
159
160 const toContents = (
161 contents: readonly CachedContentMessage[] | undefined,
162 ): aiplatform.GoogleCloudAiplatformV1Content[] | undefined => {
163 if (contents === undefined) return undefined;
164 return contents.map((content) => ({
165 role: content.role,
166 parts: (content.parts ?? [])
167 .filter((part) => part.text !== undefined)
168 .map((part) => ({ text: part.text })),
169 }));
170 };
171
172 const toAttrs = (cache: aiplatform.GoogleCloudAiplatformV1CachedContent, project: string) => {
173 const name = cache.name ?? "";
174 const parsed = parseResourceName(name, "cachedContents");
175 const ownership = parseOwnership(cache.displayName);
176 return {
177 name,
178 cachedContentId: parsed.id,
179 project: parsed.project || project,
180 location: parsed.location,
181 model: cache.model,
182 displayName: ownership.text,
183 expireTime: cache.expireTime,
184 createTime: cache.createTime,
185 updateTime: cache.updateTime,
186 totalTokenCount: cache.usageMetadata?.totalTokenCount,
187 kmsKeyName: cache.encryptionSpec?.kmsKeyName,
188 };
189 };
190
191 const getByName = (name: string) =>
192 aiplatform
193 .getProjectsLocationsCachedContents({ name })
194 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
195
196 const listCaches = (project: string, region: string) => {
197 const collect = (parent: string) =>
198 aiplatform.listProjectsLocationsCachedContents.pages({ parent, pageSize: 100 }).pipe(
199 Stream.flatMap((page) => Stream.fromIterable(page.cachedContents ?? [])),
200 Stream.runCollect,
201 Effect.map((chunk) => Array.from(chunk)),
202 );
203 const fallback = Effect.forEach(listLocations(region), (location) =>
204 collect(`projects/${project}/locations/${location}`),
205 ).pipe(Effect.map((pages) => pages.flat()));
206 // Vertex AI has no `locations/-` wildcard; scan known locations.
207 return fallback.pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
208 };
209
210 const findOwned = (id: string, project: string, region: string, hinted?: string) =>
211 Effect.gen(function* () {
212 if (hinted !== undefined && hinted.length > 0) {
213 const existing = yield* getByName(hinted);
214 if (existing !== undefined) return existing;
215 }
216 const caches = yield* listCaches(project, region);
217 for (const cache of caches) {
218 const { labels } = parseOwnership(cache.displayName);
219 if (yield* hasAlchemyLabels(id, labels)) return cache;
220 }
221 return undefined as aiplatform.GoogleCloudAiplatformV1CachedContent | undefined;
222 });
223
224 const fallbackDisplayName = (id: string) =>
225 Effect.gen(function* () {
226 return rfc1035(
227 yield* createPhysicalName({
228 id,
229 maxLength: 63,
230 lowercase: true,
231 }),
232 );
233 });
234
235 export const CachedContentProvider = () =>
236 Provider.succeed(CachedContent, {
237 stables: ["name", "cachedContentId", "project", "location", "model", "createTime"],
238
239 diff: Effect.fn(function* ({ news, olds, output }) {
240 const env = yield* GcpEnvironment.current;
241 if (!isResolved(news)) return undefined;
242 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
243 const nextLocation = normalizeLocation(
244 news.location ?? olds?.location ?? output?.location,
245 env.region,
246 );
247 const previousModel = olds?.model ?? output?.model ?? "";
248 const previousKey = olds?.encryptionSpec?.kmsKeyName ?? output?.kmsKeyName ?? "";
249 const nextKey = news.encryptionSpec?.kmsKeyName ?? previousKey;
250 const replace =
251 previousLocation !== nextLocation ||
252 news.model !== previousModel ||
253 nextKey !== previousKey;
254 if (!replace) return undefined;
255 return { action: "replace" as const, deleteFirst: true };
256 }),
257
258 read: Effect.fn(function* ({ id, output, previous }) {
259 const env = yield* GcpEnvironment.current;
260 const existing = yield* findOwned(id, env.project, env.region, output?.name);
261 if (existing === undefined) return undefined;
262 const attrs = toAttrs(existing, env.project);
263 const { labels } = parseOwnership(existing.displayName);
264 return (yield* hasAlchemyLabels(id, labels))
265 ? attrs
266 : unownedUnlessPrevious(attrs, previous, "name");
267 }),
268
269 list: () =>
270 Effect.gen(function* () {
271 const env = yield* GcpEnvironment.current;
272 const caches = yield* listCaches(env.project, env.region);
273 return caches
274 .filter((cache) => hasOwnershipMarker(cache.displayName))
275 .map((cache) => toAttrs(cache, env.project));
276 }),
277
278 reconcile: Effect.fn(function* ({ id, news, output }) {
279 const env = yield* GcpEnvironment.current;
280 const location = normalizeLocation(news.location ?? output?.location, env.region);
281 const internal = yield* createInternalLabels(id);
282 const desiredDisplayName = encodeOwnership(
283 internal,
284 news.displayName ?? (yield* fallbackDisplayName(id)),
285 " ",
286 );
287
288 let current = yield* findOwned(id, env.project, env.region, output?.name);
289
290 if (current === undefined) {
291 const created = yield* aiplatform
292 .createProjectsLocationsCachedContents({
293 parent: parentOf(env.project, location),
294 body: {
295 model: news.model,
296 contents: toContents(news.contents),
297 systemInstruction: news.systemInstruction
298 ? toContents([news.systemInstruction])?.[0]
299 : undefined,
300 ttl: news.ttl,
301 expireTime: news.expireTime,
302 displayName: desiredDisplayName,
303 encryptionSpec: news.encryptionSpec,
304 },
305 })
306 .pipe(
307 // Vertex AI intermittently counts large content as "1 tokens".
308 Effect.retry({
309 while: (error) => error._tag === "CachedContentTooFewTokens",
310 times: 4,
311 schedule: Schedule.spaced("3 seconds"),
312 }),
313 Effect.catchTag("Conflict", () => findOwned(id, env.project, env.region)),
314 );
315 current = created ?? undefined;
316 }
317
318 if (current === undefined) {
319 return yield* new CachedContentNotResolved({
320 resourceName: output?.name ?? parentOf(env.project, location),
321 });
322 }
323
324 const name = current.name ?? "";
325 const ttlChanged = news.ttl !== undefined && news.ttl.length > 0;
326 const expireChanged =
327 news.expireTime !== undefined && news.expireTime !== (current.expireTime ?? "");
328
329 if (ttlChanged || expireChanged) {
330 current = yield* aiplatform.patchProjectsLocationsCachedContents({
331 name,
332 updateMask: ttlChanged ? "ttl" : "expire_time",
333 body: {
334 name,
335 ttl: ttlChanged ? news.ttl : undefined,
336 expireTime: expireChanged ? news.expireTime : undefined,
337 },
338 });
339 }
340
341 return toAttrs(current, env.project);
342 }),
343
344 delete: Effect.fn(function* ({ output }) {
345 yield* aiplatform
346 .deleteProjectsLocationsCachedContents({ name: output.name })
347 .pipe(Effect.catchTag("NotFound", () => Effect.void));
348 }),
349 });