The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as Data from "effect/Data"; |
| 2 | import * as Schema from "effect/Schema"; |
| 3 | import * as AST from "effect/SchemaAST"; |
| 4 | |
| 5 | /** |
| 6 | * Scalar field types accepted by a structured stream schema. |
| 7 | */ |
| 8 | export type StreamFieldType = |
| 9 | | "int32" |
| 10 | | "int64" |
| 11 | | "float32" |
| 12 | | "float64" |
| 13 | | "bool" |
| 14 | | "string" |
| 15 | | "binary" |
| 16 | | "json"; |
| 17 | |
| 18 | /** |
| 19 | * Timestamp precision of a `timestamp` stream field. |
| 20 | */ |
| 21 | export type StreamTimestampUnit = "second" | "millisecond" | "microsecond" | "nanosecond"; |
| 22 | |
| 23 | /** |
| 24 | * The type of a stream field (or list element), without its name. |
| 25 | */ |
| 26 | export type StreamFieldItem = |
| 27 | | { |
| 28 | /** Scalar field type. */ |
| 29 | type: StreamFieldType; |
| 30 | /** Whether the value must be present. */ |
| 31 | required?: boolean; |
| 32 | } |
| 33 | | { |
| 34 | /** Timestamp field type. */ |
| 35 | type: "timestamp"; |
| 36 | /** Whether the value must be present. */ |
| 37 | required?: boolean; |
| 38 | /** |
| 39 | * Precision of numeric timestamps. |
| 40 | * @default "millisecond" |
| 41 | */ |
| 42 | unit?: StreamTimestampUnit; |
| 43 | } |
| 44 | | { |
| 45 | /** List (array) field type. */ |
| 46 | type: "list"; |
| 47 | /** Whether the value must be present. */ |
| 48 | required?: boolean; |
| 49 | /** Type of every element of the list. */ |
| 50 | items: StreamFieldItem; |
| 51 | } |
| 52 | | { |
| 53 | /** Nested object field type. */ |
| 54 | type: "struct"; |
| 55 | /** Whether the value must be present. */ |
| 56 | required?: boolean; |
| 57 | /** Fields of the nested object. */ |
| 58 | fields: StreamField[]; |
| 59 | }; |
| 60 | |
| 61 | /** |
| 62 | * A single named field of a structured stream schema. `timestamp` fields |
| 63 | * additionally accept a `unit`, `list` fields an `items` type and |
| 64 | * `struct` fields nested `fields`. |
| 65 | */ |
| 66 | export type StreamField = StreamFieldItem & { |
| 67 | /** Field name as it appears in ingested events. */ |
| 68 | name: string; |
| 69 | /** Whether the field must be present in every event. */ |
| 70 | required?: boolean; |
| 71 | /** Name to expose to SQL when it differs from `name`. */ |
| 72 | sqlName?: string; |
| 73 | /** Metadata key the field is populated from instead of the event body. */ |
| 74 | metadataKey?: string; |
| 75 | }; |
| 76 | |
| 77 | /** |
| 78 | * A structured stream schema as an explicit field list. |
| 79 | */ |
| 80 | export interface StreamFieldsSchema { |
| 81 | /** Fields of the structured schema. */ |
| 82 | fields: StreamField[]; |
| 83 | } |
| 84 | |
| 85 | /** |
| 86 | * Annotation key that overrides the stream field type derived from an |
| 87 | * Effect Schema, e.g. to store a number as `int32` or `float32`: |
| 88 | * |
| 89 | * ```typescript |
| 90 | * Schema.Number.annotate({ [StreamFieldTypeAnnotation]: "float32" }) |
| 91 | * ``` |
| 92 | */ |
| 93 | export const StreamFieldTypeAnnotation = "pipelinesFieldType" as const; |
| 94 | |
| 95 | /** |
| 96 | * Raised when an Effect Schema passed as a stream `schema` has no |
| 97 | * Pipelines representation (e.g. a union of different types). |
| 98 | */ |
| 99 | export class StreamSchemaUnsupported extends Data.TaggedError( |
| 100 | "Cloudflare.Pipelines.StreamSchemaUnsupported", |
| 101 | )<{ |
| 102 | message: string; |
| 103 | path: string; |
| 104 | }> {} |
| 105 | |
| 106 | const unsupported = (path: string, what: string) => |
| 107 | new StreamSchemaUnsupported({ |
| 108 | path, |
| 109 | message: |
| 110 | `Cloudflare.Pipelines.Stream: field '${path || "<root>"}' is ${what}, ` + |
| 111 | "which has no Pipelines stream field type. Use a field list or annotate the field " + |
| 112 | `with { ${StreamFieldTypeAnnotation}: "json" }.`, |
| 113 | }); |
| 114 | |
| 115 | const INT32_MIN = -2147483648; |
| 116 | const INT32_MAX = 2147483647; |
| 117 | |
| 118 | interface CheckRep { |
| 119 | id: string; |
| 120 | payload: unknown; |
| 121 | } |
| 122 | |
| 123 | const checkReps = (ast: AST.AST): CheckRep[] => { |
| 124 | const out: CheckRep[] = []; |
| 125 | const visit = (check: unknown) => { |
| 126 | const c = check as { |
| 127 | annotations?: { representation?: CheckRep }; |
| 128 | checks?: readonly unknown[]; |
| 129 | }; |
| 130 | if (c.annotations?.representation?.id) out.push(c.annotations.representation); |
| 131 | c.checks?.forEach(visit); |
| 132 | }; |
| 133 | ((ast as { checks?: readonly unknown[] }).checks ?? []).forEach(visit); |
| 134 | return out; |
| 135 | }; |
| 136 | |
| 137 | const annotationOverride = (ast: AST.AST): StreamFieldType | "timestamp" | undefined => { |
| 138 | const read = (annotations: unknown) => |
| 139 | (annotations as Record<string, unknown> | undefined)?.[StreamFieldTypeAnnotation]; |
| 140 | const candidates = [ |
| 141 | read((ast as { annotations?: unknown }).annotations), |
| 142 | ...((ast as { checks?: readonly { annotations?: unknown }[] }).checks ?? []).map((c) => |
| 143 | read(c.annotations), |
| 144 | ), |
| 145 | ]; |
| 146 | const found = candidates.find((v) => typeof v === "string"); |
| 147 | return found as StreamFieldType | "timestamp" | undefined; |
| 148 | }; |
| 149 | |
| 150 | const numberType = (ast: AST.AST): StreamFieldType => { |
| 151 | const reps = checkReps(ast); |
| 152 | if (!reps.some((r) => r.id === "effect/schema/isInt")) return "float64"; |
| 153 | const fitsInt32 = reps.some((r) => { |
| 154 | if (r.id !== "effect/schema/isBetween") return false; |
| 155 | const p = r.payload as { minimum?: number; maximum?: number } | null; |
| 156 | return ( |
| 157 | p?.minimum !== undefined && |
| 158 | p?.maximum !== undefined && |
| 159 | p.minimum >= INT32_MIN && |
| 160 | p.maximum <= INT32_MAX |
| 161 | ); |
| 162 | }); |
| 163 | return fitsInt32 ? "int32" : "int64"; |
| 164 | }; |
| 165 | |
| 166 | const isNullish = (ast: AST.AST) => |
| 167 | ast._tag === "Null" || ast._tag === "Undefined" || ast._tag === "Void"; |
| 168 | |
| 169 | const literalType = (value: unknown): StreamFieldType | undefined => |
| 170 | typeof value === "string" |
| 171 | ? "string" |
| 172 | : typeof value === "boolean" |
| 173 | ? "bool" |
| 174 | : typeof value === "number" |
| 175 | ? Number.isInteger(value) |
| 176 | ? "int64" |
| 177 | : "float64" |
| 178 | : undefined; |
| 179 | |
| 180 | /** |
| 181 | * Derive the stream field type of one (encoded-side) AST node. Returns the |
| 182 | * item plus whether the value may be absent (nullable union members). |
| 183 | */ |
| 184 | const toItem = (ast: AST.AST, path: string): { item: StreamFieldItem; nullable: boolean } => { |
| 185 | const override = annotationOverride(ast); |
| 186 | if (override) { |
| 187 | return { item: { type: override } as StreamFieldItem, nullable: false }; |
| 188 | } |
| 189 | switch (ast._tag) { |
| 190 | case "String": |
| 191 | case "TemplateLiteral": |
| 192 | return { item: { type: "string" }, nullable: false }; |
| 193 | case "Number": |
| 194 | return { item: { type: numberType(ast) }, nullable: false }; |
| 195 | case "Boolean": |
| 196 | return { item: { type: "bool" }, nullable: false }; |
| 197 | case "Literal": { |
| 198 | const type = literalType(ast.literal); |
| 199 | if (!type) throw unsupported(path, `the literal ${String(ast.literal)}`); |
| 200 | return { item: { type }, nullable: false }; |
| 201 | } |
| 202 | case "Enum": { |
| 203 | const types = new Set( |
| 204 | (ast as unknown as { enums: ReadonlyArray<readonly [string, unknown]> }).enums.map( |
| 205 | ([, v]) => literalType(v), |
| 206 | ), |
| 207 | ); |
| 208 | const [type] = [...types]; |
| 209 | if (types.size !== 1 || !type) throw unsupported(path, "a mixed enum"); |
| 210 | return { item: { type }, nullable: false }; |
| 211 | } |
| 212 | case "Unknown": |
| 213 | case "Any": |
| 214 | case "ObjectKeyword": |
| 215 | return { item: { type: "json" }, nullable: false }; |
| 216 | case "Declaration": { |
| 217 | const id = (ast.annotations?.representation as CheckRep | undefined)?.id; |
| 218 | if (id === "effect/schema/Date") return { item: { type: "timestamp" }, nullable: false }; |
| 219 | if (id === "effect/schema/Uint8Array") return { item: { type: "binary" }, nullable: false }; |
| 220 | if (id === "effect/schema/Json") return { item: { type: "json" }, nullable: false }; |
| 221 | throw unsupported(path, `a ${AST.resolveIdentifier(ast) ?? "declared"} value`); |
| 222 | } |
| 223 | case "Arrays": { |
| 224 | if (ast.elements.length > 0 || ast.rest.length !== 1) { |
| 225 | throw unsupported(path, "a tuple"); |
| 226 | } |
| 227 | const element = toItem(ast.rest[0]!, `${path}[]`); |
| 228 | return { |
| 229 | item: { |
| 230 | type: "list", |
| 231 | items: element.nullable ? { ...element.item, required: false } : element.item, |
| 232 | }, |
| 233 | nullable: false, |
| 234 | }; |
| 235 | } |
| 236 | case "Objects": { |
| 237 | if (ast.indexSignatures.length > 0 || ast.propertySignatures.length === 0) { |
| 238 | // Records and empty objects carry arbitrary keys — store as JSON. |
| 239 | return { item: { type: "json" }, nullable: false }; |
| 240 | } |
| 241 | return { |
| 242 | item: { type: "struct", fields: toFields(ast, path) }, |
| 243 | nullable: false, |
| 244 | }; |
| 245 | } |
| 246 | case "Union": { |
| 247 | const members = ast.types.filter((t) => !isNullish(t)); |
| 248 | const nullable = members.length !== ast.types.length; |
| 249 | if (members.length === 1) { |
| 250 | return { item: toItem(members[0]!, path).item, nullable }; |
| 251 | } |
| 252 | // A union of same-typed literals (e.g. `Schema.Literals(["a","b"])`). |
| 253 | if (members.length > 0 && members.every((m) => m._tag === "Literal")) { |
| 254 | const types = new Set(members.map((m) => literalType((m as AST.Literal).literal))); |
| 255 | const [type] = [...types]; |
| 256 | if (types.size === 1 && type) return { item: { type }, nullable }; |
| 257 | } |
| 258 | throw unsupported(path, "a union of different types"); |
| 259 | } |
| 260 | case "Suspend": |
| 261 | throw unsupported(path, "a recursive schema"); |
| 262 | default: |
| 263 | throw unsupported(path, `of type ${ast._tag}`); |
| 264 | } |
| 265 | }; |
| 266 | |
| 267 | const toFields = (ast: AST.Objects, path: string): StreamField[] => |
| 268 | ast.propertySignatures.map((ps) => { |
| 269 | const name = String(ps.name); |
| 270 | const fieldPath = path ? `${path}.${name}` : name; |
| 271 | const { item, nullable } = toItem(ps.type, fieldPath); |
| 272 | const optional = AST.isOptional(ps.type); |
| 273 | return { |
| 274 | ...item, |
| 275 | name, |
| 276 | required: !(optional || nullable), |
| 277 | } as StreamField; |
| 278 | }); |
| 279 | |
| 280 | /** |
| 281 | * Returns true if `value` is an Effect Schema (rather than a field list). |
| 282 | */ |
| 283 | export const isEffectSchema = (value: unknown): value is Schema.Top => Schema.isSchema(value); |
| 284 | |
| 285 | /** |
| 286 | * Convert an Effect Schema (`Schema.Struct` / `Schema.Class`) into the |
| 287 | * Pipelines stream field list. The schema's **encoded** side is used, so |
| 288 | * transformations (e.g. `NumberFromString`) map to their wire type, and |
| 289 | * records are sent with `Schema.toCodecJson(schema)`: |
| 290 | * |
| 291 | * - `String` → `string`, `Boolean` → `bool` |
| 292 | * - `Number` → `float64`, `Int` → `int64` (`int32` when bounded to 32 bits) |
| 293 | * - `Date` → `timestamp`, `Uint8Array` → `binary` |
| 294 | * - `Array` → `list`, `Struct` / `Class` → `struct` |
| 295 | * - `Record` / `Unknown` / `Json` → `json` |
| 296 | * - optional or nullable fields → `required: false` |
| 297 | * |
| 298 | * Unions of different types, tuples and recursive schemas have no stream |
| 299 | * representation and throw {@link StreamSchemaUnsupported}. Override any |
| 300 | * field's type with the {@link StreamFieldTypeAnnotation} annotation. |
| 301 | */ |
| 302 | export const streamFieldsFromSchema = (schema: Schema.Top): StreamField[] => { |
| 303 | const encoded = AST.toEncoded(schema.ast); |
| 304 | if (encoded._tag !== "Objects" || encoded.propertySignatures.length === 0) { |
| 305 | throw unsupported("", "not a Struct or Class with fields"); |
| 306 | } |
| 307 | return toFields(encoded, ""); |
| 308 | }; |
| 309 | |
| 310 | /** |
| 311 | * Resolve a stream `schema` prop (field list or Effect Schema) to the |
| 312 | * field-list form persisted in state and sent to Cloudflare. |
| 313 | */ |
| 314 | export const resolveStreamSchema = ( |
| 315 | schema: StreamFieldsSchema | Schema.Top | undefined, |
| 316 | ): StreamFieldsSchema | undefined => { |
| 317 | if (schema === undefined) return undefined; |
| 318 | if (isEffectSchema(schema)) return { fields: streamFieldsFromSchema(schema) }; |
| 319 | return schema as StreamFieldsSchema; |
| 320 | }; |
| 321 | |
| 322 | /** |
| 323 | * Canonical (default-filled, key-ordered) form of a field item, so |
| 324 | * equivalent spellings compare equal. |
| 325 | */ |
| 326 | const canonicalItem = (item: StreamFieldItem): Record<string, unknown> => { |
| 327 | const base: Record<string, unknown> = { |
| 328 | type: item.type, |
| 329 | required: item.required ?? false, |
| 330 | }; |
| 331 | if (item.type === "timestamp") base.unit = item.unit ?? "millisecond"; |
| 332 | if (item.type === "list") base.items = canonicalItem(item.items); |
| 333 | if (item.type === "struct") base.fields = canonicalStreamFields(item.fields); |
| 334 | return base; |
| 335 | }; |
| 336 | |
| 337 | /** |
| 338 | * Canonical form of a field list: defaults filled in (`required: false`, |
| 339 | * `unit: "millisecond"`), absent optional keys dropped. |
| 340 | */ |
| 341 | export const canonicalStreamFields = (fields: ReadonlyArray<StreamField>): unknown[] => |
| 342 | fields.map((f) => { |
| 343 | const out: Record<string, unknown> = { name: f.name, ...canonicalItem(f) }; |
| 344 | if (f.sqlName !== undefined) out.sqlName = f.sqlName; |
| 345 | if (f.metadataKey !== undefined) out.metadataKey = f.metadataKey; |
| 346 | return out; |
| 347 | }); |