The best way to control Claude Code, Codex and more from any computer or phone
| 1 | /** |
| 2 | * The `OpenAiClient` module defines the low-level Effect service used by the |
| 3 | * OpenAI integration for Responses API and embedding requests. It builds a |
| 4 | * configured HTTP client with authentication and OpenAI organization or project |
| 5 | * headers, exposes helpers for non-streaming responses, SSE response streams, |
| 6 | * WebSocket response streams, and embeddings, and maps transport or decoding |
| 7 | * failures into `AiError`. |
| 8 | * |
| 9 | * @stability unstable |
| 10 | * @since 4.0.0 |
| 11 | */ |
| 12 | import * as AiError from "effect/ai/AiError" |
| 13 | import * as ResponseIdTracker from "effect/ai/ResponseIdTracker" |
| 14 | import * as Array from "effect/Array" |
| 15 | import * as Cause from "effect/Cause" |
| 16 | import type * as Config from "effect/Config" |
| 17 | import * as Context from "effect/Context" |
| 18 | import * as Effect from "effect/Effect" |
| 19 | import * as Sse from "effect/encoding/Sse" |
| 20 | import { identity } from "effect/Function" |
| 21 | import * as Function from "effect/Function" |
| 22 | import * as Headers from "effect/http/Headers" |
| 23 | import * as HttpBody from "effect/http/HttpBody" |
| 24 | import * as HttpClient from "effect/http/HttpClient" |
| 25 | import * as HttpClientRequest from "effect/http/HttpClientRequest" |
| 26 | import * as HttpClientResponse from "effect/http/HttpClientResponse" |
| 27 | import * as Layer from "effect/Layer" |
| 28 | import * as Predicate from "effect/Predicate" |
| 29 | import * as Queue from "effect/Queue" |
| 30 | import * as RcRef from "effect/RcRef" |
| 31 | import * as Redacted from "effect/Redacted" |
| 32 | import * as Schema from "effect/Schema" |
| 33 | import * as Scope from "effect/Scope" |
| 34 | import * as Semaphore from "effect/Semaphore" |
| 35 | import * as Socket from "effect/socket/Socket" |
| 36 | import * as Stream from "effect/Stream" |
| 37 | import * as Errors from "./internal/errors.ts" |
| 38 | import { OpenAiConfig } from "./OpenAiConfig.ts" |
| 39 | import * as OpenAiSchema from "./OpenAiSchema.ts" |
| 40 | |
| 41 | const decodeResponseStreamEvent = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAiSchema.ResponseStreamEvent)) |
| 42 | |
| 43 | // ============================================================================= |
| 44 | // Service Interface |
| 45 | // ============================================================================= |
| 46 | |
| 47 | /** |
| 48 | * Effect service interface for the handwritten OpenAI client. |
| 49 | * |
| 50 | * **Details** |
| 51 | * |
| 52 | * Provides the configured HTTP client plus helpers for Responses API calls, streaming Responses events, and embeddings. Transport and schema decoding failures are mapped to `AiError`. |
| 53 | * |
| 54 | * @stability unstable |
| 55 | * @category services |
| 56 | * @since 4.0.0 |
| 57 | */ |
| 58 | export interface Service { |
| 59 | /** |
| 60 | * The transformed HTTP client used by this service. |
| 61 | */ |
| 62 | readonly client: HttpClient.HttpClient |
| 63 | |
| 64 | /** |
| 65 | * Create a response using the OpenAI responses endpoint. |
| 66 | */ |
| 67 | readonly createResponse: ( |
| 68 | options: typeof OpenAiSchema.CreateResponse.Encoded |
| 69 | ) => Effect.Effect< |
| 70 | readonly [body: typeof OpenAiSchema.Response.Type, response: HttpClientResponse.HttpClientResponse], |
| 71 | AiError.AiError |
| 72 | > |
| 73 | |
| 74 | /** |
| 75 | * Create a streaming response using the OpenAI responses endpoint. |
| 76 | */ |
| 77 | readonly createResponseStream: ( |
| 78 | options: Omit<typeof OpenAiSchema.CreateResponse.Encoded, "stream"> |
| 79 | ) => Effect.Effect< |
| 80 | readonly [ |
| 81 | response: HttpClientResponse.HttpClientResponse, |
| 82 | stream: Stream.Stream<typeof OpenAiSchema.ResponseStreamEvent.Type, AiError.AiError> |
| 83 | ], |
| 84 | AiError.AiError |
| 85 | > |
| 86 | |
| 87 | /** |
| 88 | * Create embeddings using the OpenAI embeddings endpoint. |
| 89 | */ |
| 90 | readonly createEmbedding: ( |
| 91 | options: typeof OpenAiSchema.CreateEmbeddingRequest.Encoded |
| 92 | ) => Effect.Effect<typeof OpenAiSchema.CreateEmbeddingResponse.Type, AiError.AiError> |
| 93 | |
| 94 | /** |
| 95 | * Answer decision questions using the OpenAI decisions endpoint. |
| 96 | */ |
| 97 | readonly createDecisions: ( |
| 98 | options: typeof OpenAiSchema.CreateDecisionsRequest.Encoded |
| 99 | ) => Effect.Effect<typeof OpenAiSchema.CreateDecisionsResponse.Type, AiError.AiError> |
| 100 | } |
| 101 | |
| 102 | // ============================================================================= |
| 103 | // Service Identifier |
| 104 | // ============================================================================= |
| 105 | |
| 106 | /** |
| 107 | * Service tag for the OpenAI client. |
| 108 | * |
| 109 | * **When to use** |
| 110 | * |
| 111 | * Use when accessing or providing the OpenAI client service through Effect's |
| 112 | * context. |
| 113 | * |
| 114 | * @see {@link make} for constructing an OpenAI client effectfully |
| 115 | * @see {@link layer} for providing a client from explicit options |
| 116 | * @see {@link layerConfig} for providing a client from `Config` |
| 117 | * |
| 118 | * @stability unstable |
| 119 | * @category services |
| 120 | * @since 4.0.0 |
| 121 | */ |
| 122 | export class OpenAiClient extends Context.Service<OpenAiClient, Service>()( |
| 123 | "@effect/ai-openai/OpenAiClient" |
| 124 | ) {} |
| 125 | |
| 126 | // ============================================================================= |
| 127 | // Options |
| 128 | // ============================================================================= |
| 129 | |
| 130 | /** |
| 131 | * Options for configuring the OpenAI client. |
| 132 | * |
| 133 | * @stability unstable |
| 134 | * @category options |
| 135 | * @since 4.0.0 |
| 136 | */ |
| 137 | export type Options = { |
| 138 | /** |
| 139 | * The OpenAI API key. |
| 140 | */ |
| 141 | readonly apiKey?: Redacted.Redacted<string> | undefined |
| 142 | |
| 143 | /** |
| 144 | * The base URL for the OpenAI API. |
| 145 | * |
| 146 | * @default "https://api.openai.com/v1" |
| 147 | */ |
| 148 | readonly apiUrl?: string | undefined |
| 149 | |
| 150 | /** |
| 151 | * Optional organization ID for multi-org accounts. |
| 152 | */ |
| 153 | readonly organizationId?: Redacted.Redacted<string> | undefined |
| 154 | |
| 155 | /** |
| 156 | * Optional project ID for project-scoped requests. |
| 157 | */ |
| 158 | readonly projectId?: Redacted.Redacted<string> | undefined |
| 159 | |
| 160 | /** |
| 161 | * Optional transformer for the HTTP client. |
| 162 | */ |
| 163 | readonly transformClient?: ((client: HttpClient.HttpClient) => HttpClient.HttpClient) | undefined |
| 164 | } |
| 165 | |
| 166 | // ============================================================================= |
| 167 | // Constructor |
| 168 | // ============================================================================= |
| 169 | |
| 170 | const RedactedOpenAiHeaders = { |
| 171 | OpenAiOrganization: "OpenAI-Organization", |
| 172 | OpenAiProject: "OpenAI-Project" |
| 173 | } |
| 174 | |
| 175 | const withRedactedHeaders = Effect.updateService( |
| 176 | Headers.CurrentRedactedNames, |
| 177 | Array.appendAll(Object.values(RedactedOpenAiHeaders)) |
| 178 | ) |
| 179 | |
| 180 | /** |
| 181 | * Creates an OpenAI client service with the given options. |
| 182 | * |
| 183 | * **When to use** |
| 184 | * |
| 185 | * Use when you need the OpenAI client service value inside an effect. |
| 186 | * |
| 187 | * **Details** |
| 188 | * |
| 189 | * The returned service uses the current `HttpClient`, prepends `apiUrl` or |
| 190 | * `https://api.openai.com/v1`, adds the bearer token and optional OpenAI |
| 191 | * organization/project headers, accepts JSON responses, filters for successful |
| 192 | * HTTP statuses, and applies `transformClient` when provided. |
| 193 | * |
| 194 | * **Gotchas** |
| 195 | * |
| 196 | * A scoped `OpenAiConfig.withClientTransform` is applied when request helpers |
| 197 | * run, after the `transformClient` option supplied to `make`. |
| 198 | * |
| 199 | * @see {@link layer} for providing this client from explicit options |
| 200 | * @see {@link layerConfig} for loading client settings from `Config` |
| 201 | * |
| 202 | * @stability unstable |
| 203 | * @category constructors |
| 204 | * @since 4.0.0 |
| 205 | */ |
| 206 | export const make = Effect.fnUntraced( |
| 207 | function*( |
| 208 | options: Options |
| 209 | ): Effect.fn.Return<Service, never, HttpClient.HttpClient> { |
| 210 | const baseClient = yield* HttpClient.HttpClient |
| 211 | const apiUrl = options.apiUrl ?? "https://api.openai.com/v1" |
| 212 | |
| 213 | const httpClient = baseClient.pipe( |
| 214 | HttpClient.mapRequest(Function.flow( |
| 215 | HttpClientRequest.prependUrl(apiUrl), |
| 216 | options.apiKey |
| 217 | ? HttpClientRequest.bearerToken(Redacted.value(options.apiKey)) |
| 218 | : identity, |
| 219 | options.organizationId |
| 220 | ? HttpClientRequest.setHeader( |
| 221 | RedactedOpenAiHeaders.OpenAiOrganization, |
| 222 | Redacted.value(options.organizationId) |
| 223 | ) |
| 224 | : identity, |
| 225 | options.projectId |
| 226 | ? HttpClientRequest.setHeader( |
| 227 | RedactedOpenAiHeaders.OpenAiProject, |
| 228 | Redacted.value(options.projectId) |
| 229 | ) |
| 230 | : identity, |
| 231 | HttpClientRequest.acceptJson |
| 232 | )), |
| 233 | HttpClient.filterStatusOk, |
| 234 | options.transformClient |
| 235 | ? options.transformClient |
| 236 | : identity |
| 237 | ) |
| 238 | |
| 239 | const resolveHttpClient = Effect.map( |
| 240 | OpenAiConfig.getOrUndefined, |
| 241 | (config) => |
| 242 | Predicate.isNotUndefined(config?.transformClient) |
| 243 | ? config.transformClient(httpClient) |
| 244 | : httpClient |
| 245 | ) |
| 246 | |
| 247 | const decodeResponse = HttpClientResponse.schemaBodyJson(OpenAiSchema.Response) |
| 248 | |
| 249 | const createResponse = ( |
| 250 | payload: typeof OpenAiSchema.CreateResponse.Encoded |
| 251 | ): Effect.Effect< |
| 252 | [body: typeof OpenAiSchema.Response.Type, response: HttpClientResponse.HttpClientResponse], |
| 253 | AiError.AiError |
| 254 | > => |
| 255 | resolveHttpClient.pipe( |
| 256 | Effect.flatMap((client) => |
| 257 | client.execute(HttpClientRequest.post("/responses", { |
| 258 | body: HttpBody.jsonUnsafe(payload) |
| 259 | })).pipe( |
| 260 | Effect.flatMap((response) => |
| 261 | decodeResponse(response).pipe( |
| 262 | Effect.map((body): [typeof OpenAiSchema.Response.Type, HttpClientResponse.HttpClientResponse] => [ |
| 263 | body, |
| 264 | response |
| 265 | ]) |
| 266 | ) |
| 267 | ), |
| 268 | Effect.catchTags({ |
| 269 | HttpClientError: (error) => Errors.mapHttpClientError(error, "createResponse"), |
| 270 | SchemaError: (error) => Effect.fail(Errors.mapSchemaError(error, "createResponse")) |
| 271 | }) |
| 272 | ) |
| 273 | ), |
| 274 | withRedactedHeaders |
| 275 | ) |
| 276 | |
| 277 | const buildResponseStream = ( |
| 278 | response: HttpClientResponse.HttpClientResponse |
| 279 | ): [ |
| 280 | HttpClientResponse.HttpClientResponse, |
| 281 | Stream.Stream<typeof OpenAiSchema.ResponseStreamEvent.Type, AiError.AiError> |
| 282 | ] => { |
| 283 | const stream = response.stream.pipe( |
| 284 | Stream.decodeText(), |
| 285 | Stream.pipeThroughChannel(Sse.decode()), |
| 286 | // Some OpenAI-compatible servers end the stream with `data: [DONE]` |
| 287 | Stream.takeWhile((event) => event.data !== "[DONE]"), |
| 288 | Stream.mapEffect((event) => decodeResponseStreamEvent(event.data)), |
| 289 | Stream.takeUntil((event) => |
| 290 | event.type === "response.completed" || |
| 291 | event.type === "response.incomplete" || |
| 292 | event.type === "response.failed" |
| 293 | ), |
| 294 | Stream.catchTags({ |
| 295 | // TODO: handle SSE retries |
| 296 | Retry: (error) => Stream.die(error), |
| 297 | SseError: (error) => Stream.fail(Errors.mapSseError(error, "createResponseStream")), |
| 298 | HttpClientError: (error) => Stream.fromEffect(Errors.mapHttpClientError(error, "createResponseStream")), |
| 299 | SchemaError: (error) => Stream.fail(Errors.mapSchemaError(error, "createResponseStream")) |
| 300 | }) |
| 301 | ) |
| 302 | return [response, stream] |
| 303 | } |
| 304 | |
| 305 | const createResponseStream: Service["createResponseStream"] = (payload) => |
| 306 | Effect.contextWith((services) => { |
| 307 | const socket = Context.getOrUndefined(services, OpenAiSocket) |
| 308 | if (socket) return socket.createResponseStream(payload) |
| 309 | return resolveHttpClient.pipe( |
| 310 | Effect.flatMap((client) => |
| 311 | client.execute(HttpClientRequest.post("/responses", { |
| 312 | body: HttpBody.jsonUnsafe({ ...payload, stream: true }) |
| 313 | })).pipe( |
| 314 | Effect.map(buildResponseStream), |
| 315 | Effect.catchTag( |
| 316 | "HttpClientError", |
| 317 | (error) => Errors.mapHttpClientError(error, "createResponseStream") |
| 318 | ) |
| 319 | ) |
| 320 | ), |
| 321 | withRedactedHeaders |
| 322 | ) |
| 323 | }) |
| 324 | |
| 325 | const postJson = <Response extends Schema.Decoder<unknown>>( |
| 326 | path: string, |
| 327 | schema: Response, |
| 328 | method: string |
| 329 | ) => { |
| 330 | const decode = HttpClientResponse.schemaBodyJson(schema) |
| 331 | return (payload: unknown): Effect.Effect<Response["Type"], AiError.AiError> => |
| 332 | resolveHttpClient.pipe( |
| 333 | Effect.flatMap((client) => |
| 334 | client.execute(HttpClientRequest.post(path, { |
| 335 | body: HttpBody.jsonUnsafe(payload) |
| 336 | })).pipe( |
| 337 | Effect.flatMap(decode), |
| 338 | Effect.catchTags({ |
| 339 | HttpClientError: (error) => Errors.mapHttpClientError(error, method), |
| 340 | SchemaError: (error) => Effect.fail(Errors.mapSchemaError(error, method)) |
| 341 | }) |
| 342 | ) |
| 343 | ), |
| 344 | withRedactedHeaders |
| 345 | ) |
| 346 | } |
| 347 | |
| 348 | const createEmbedding: Service["createEmbedding"] = postJson( |
| 349 | "/embeddings", |
| 350 | OpenAiSchema.CreateEmbeddingResponse, |
| 351 | "createEmbedding" |
| 352 | ) |
| 353 | |
| 354 | const createDecisions: Service["createDecisions"] = postJson( |
| 355 | "/decisions", |
| 356 | OpenAiSchema.CreateDecisionsResponse, |
| 357 | "createDecisions" |
| 358 | ) |
| 359 | |
| 360 | return OpenAiClient.of({ |
| 361 | client: httpClient, |
| 362 | createResponse, |
| 363 | createResponseStream, |
| 364 | createEmbedding, |
| 365 | createDecisions |
| 366 | }) |
| 367 | }, |
| 368 | withRedactedHeaders |
| 369 | ) |
| 370 | |
| 371 | // ============================================================================= |
| 372 | // Layers |
| 373 | // ============================================================================= |
| 374 | |
| 375 | /** |
| 376 | * Creates a layer for the OpenAI client with the given options. |
| 377 | * |
| 378 | * **When to use** |
| 379 | * |
| 380 | * Use when you already have explicit `Options` values, such as an API key or |
| 381 | * custom API URL, and want to provide `OpenAiClient` as a `Layer`. |
| 382 | * |
| 383 | * @see {@link make} for constructing the client service effectfully |
| 384 | * @see {@link layerConfig} for loading client settings from `Config` |
| 385 | * |
| 386 | * @stability unstable |
| 387 | * @category layers |
| 388 | * @since 4.0.0 |
| 389 | */ |
| 390 | export const layer = (options: Options): Layer.Layer<OpenAiClient, never, HttpClient.HttpClient> => |
| 391 | Layer.effect(OpenAiClient, make(options)) |
| 392 | |
| 393 | /** |
| 394 | * Creates a layer for the OpenAI client from provided `Config` values. |
| 395 | * |
| 396 | * **When to use** |
| 397 | * |
| 398 | * Use when you need client settings for OpenAI-compatible APIs to be read from |
| 399 | * Effect `Config` values while providing `OpenAiClient` as a `Layer`. |
| 400 | * |
| 401 | * **Details** |
| 402 | * |
| 403 | * Only config values supplied in `options` are loaded. Omitted fields are |
| 404 | * passed to `make` as `undefined`, and `transformClient` is forwarded as a |
| 405 | * plain option. |
| 406 | * |
| 407 | * @see {@link make} for constructing the client service effectfully |
| 408 | * @see {@link layer} for providing the client from already-resolved options |
| 409 | * |
| 410 | * @stability unstable |
| 411 | * @category layers |
| 412 | * @since 4.0.0 |
| 413 | */ |
| 414 | export const layerConfig = (options?: { |
| 415 | /** |
| 416 | * The config value to load for the API key. |
| 417 | */ |
| 418 | readonly apiKey?: Config.Config<Redacted.Redacted<string> | undefined> | undefined |
| 419 | |
| 420 | /** |
| 421 | * The config value to load for the API URL. |
| 422 | */ |
| 423 | readonly apiUrl?: Config.Config<string> | undefined |
| 424 | |
| 425 | /** |
| 426 | * The config value to load for the organization ID. |
| 427 | */ |
| 428 | readonly organizationId?: Config.Config<Redacted.Redacted<string> | undefined> | undefined |
| 429 | |
| 430 | /** |
| 431 | * The config value to load for the project ID. |
| 432 | */ |
| 433 | readonly projectId?: Config.Config<Redacted.Redacted<string> | undefined> | undefined |
| 434 | |
| 435 | /** |
| 436 | * Optional transformer for the HTTP client. |
| 437 | */ |
| 438 | readonly transformClient?: ((client: HttpClient.HttpClient) => HttpClient.HttpClient) | undefined |
| 439 | }): Layer.Layer<OpenAiClient, Config.ConfigError, HttpClient.HttpClient> => |
| 440 | Layer.effect( |
| 441 | OpenAiClient, |
| 442 | Effect.gen(function*() { |
| 443 | const apiKey = Predicate.isNotUndefined(options?.apiKey) |
| 444 | ? yield* options.apiKey : |
| 445 | undefined |
| 446 | const apiUrl = Predicate.isNotUndefined(options?.apiUrl) |
| 447 | ? yield* options.apiUrl : |
| 448 | undefined |
| 449 | const organizationId = Predicate.isNotUndefined(options?.organizationId) |
| 450 | ? yield* options.organizationId |
| 451 | : undefined |
| 452 | const projectId = Predicate.isNotUndefined(options?.projectId) |
| 453 | ? yield* options.projectId : |
| 454 | undefined |
| 455 | return yield* make({ |
| 456 | apiKey, |
| 457 | apiUrl, |
| 458 | organizationId, |
| 459 | projectId, |
| 460 | transformClient: options?.transformClient |
| 461 | }) |
| 462 | }) |
| 463 | ) |
| 464 | |
| 465 | // ============================================================================= |
| 466 | // Websocket mode |
| 467 | // ============================================================================= |
| 468 | |
| 469 | /** |
| 470 | * Response stream event emitted by the OpenAI Responses API. |
| 471 | * |
| 472 | * @stability unstable |
| 473 | * @category models |
| 474 | * @since 4.0.0 |
| 475 | */ |
| 476 | export type ResponseStreamEvent = typeof OpenAiSchema.ResponseStreamEvent.Type |
| 477 | |
| 478 | /** |
| 479 | * Service for creating OpenAI response streams over a WebSocket connection. |
| 480 | * |
| 481 | * **When to use** |
| 482 | * |
| 483 | * Use when you need direct access to the WebSocket-backed response streaming |
| 484 | * service rather than wrapping an effect with WebSocket mode. |
| 485 | * |
| 486 | * **Details** |
| 487 | * |
| 488 | * `createResponseStream` sends a `response.create` message over the WebSocket |
| 489 | * connection and returns an HTTP response together with a stream of |
| 490 | * `ResponseStreamEvent` values. |
| 491 | * |
| 492 | * **Gotchas** |
| 493 | * |
| 494 | * WebSocket response streams are serialized to one request at a time by the |
| 495 | * shared socket service. |
| 496 | * |
| 497 | * @see {@link withWebSocketMode} for enabling WebSocket mode for one effect |
| 498 | * @see {@link layerWebSocketMode} for providing WebSocket mode through a layer |
| 499 | * |
| 500 | * @stability unstable |
| 501 | * @category services |
| 502 | * @since 4.0.0 |
| 503 | */ |
| 504 | export class OpenAiSocket extends Context.Service<OpenAiSocket, { |
| 505 | /** |
| 506 | * Create a streaming response using the OpenAI responses endpoint. |
| 507 | */ |
| 508 | readonly createResponseStream: ( |
| 509 | options: Omit<typeof OpenAiSchema.CreateResponse.Encoded, "stream"> |
| 510 | ) => Effect.Effect< |
| 511 | readonly [ |
| 512 | response: HttpClientResponse.HttpClientResponse, |
| 513 | stream: Stream.Stream<ResponseStreamEvent, AiError.AiError> |
| 514 | ], |
| 515 | AiError.AiError |
| 516 | > |
| 517 | }>()("@effect/ai-openai/OpenAiClient/OpenAiSocket") {} |
| 518 | |
| 519 | const makeSocket = Effect.gen(function*() { |
| 520 | const client = yield* OpenAiClient |
| 521 | const tracker = yield* ResponseIdTracker.make |
| 522 | const socketScope = yield* Effect.scope |
| 523 | const makeRequest = Effect.flatMap( |
| 524 | OpenAiConfig.getOrUndefined, |
| 525 | (config) => { |
| 526 | const httpClient = Predicate.isNotUndefined(config?.transformClient) |
| 527 | ? config.transformClient(client.client) |
| 528 | : client.client |
| 529 | return Effect.orDie(httpClient.preprocess(HttpClientRequest.post("/responses"))) |
| 530 | } |
| 531 | ) |
| 532 | const makeWebSocket = yield* Socket.WebSocketConstructor |
| 533 | |
| 534 | const decoder = new TextDecoder() |
| 535 | |
| 536 | // The response currently in flight, or undefined between turns. Each turn |
| 537 | // owns its queue; the reader ends or fails it, so a turn that is still |
| 538 | // current when its consumer leaves was abandoned mid-response. |
| 539 | type Turn = Queue.Queue<ResponseStreamEvent, AiError.AiError | Cause.Done> |
| 540 | type SocketConnection = { |
| 541 | readonly send: (message: typeof OpenAiSchema.CreateResponse.Encoded) => Effect.Effect<void, AiError.AiError> |
| 542 | current: Turn | undefined |
| 543 | } |
| 544 | |
| 545 | const queueRef: RcRef.RcRef<SocketConnection> = yield* RcRef.make({ |
| 546 | idleTimeToLive: 60_000, |
| 547 | acquire: Effect.gen(function*() { |
| 548 | const scope = yield* Effect.scope |
| 549 | const request = yield* makeRequest |
| 550 | const socket = yield* Socket.makeWebSocket(request.url.replace(/^http/, "ws")).pipe( |
| 551 | Effect.provideService(Socket.WebSocketConstructor, (url) => |
| 552 | makeWebSocket(url, { |
| 553 | headers: request.headers |
| 554 | })) |
| 555 | ) |
| 556 | const writer = yield* socket.writer |
| 557 | |
| 558 | yield* Scope.addFinalizerExit(scope, () => { |
| 559 | tracker.clearUnsafe() |
| 560 | return Effect.void |
| 561 | }) |
| 562 | |
| 563 | const connection: SocketConnection = { |
| 564 | current: undefined, |
| 565 | send: (message) => |
| 566 | writer.write(JSON.stringify({ |
| 567 | type: "response.create", |
| 568 | ...message |
| 569 | })).pipe( |
| 570 | Effect.mapError((_error) => |
| 571 | AiError.make({ |
| 572 | module: "OpenAiClient", |
| 573 | method: "createResponseStream", |
| 574 | reason: new AiError.NetworkError({ |
| 575 | reason: "TransportError", |
| 576 | method: request.method, |
| 577 | url: request.url, |
| 578 | description: "Failed to send message over WebSocket" |
| 579 | }) |
| 580 | }) |
| 581 | ) |
| 582 | ) |
| 583 | } |
| 584 | |
| 585 | const handleMessage = (msg: Uint8Array | string): void => { |
| 586 | const turn = connection.current |
| 587 | if (turn === undefined) return |
| 588 | let event: typeof AllEvents.Type |
| 589 | try { |
| 590 | event = decodeEvent(typeof msg === "string" ? msg : decoder.decode(msg)) |
| 591 | } catch { |
| 592 | return |
| 593 | } |
| 594 | if (event.type === "error") { |
| 595 | const status = "status" in event ? Number(event.status) : NaN |
| 596 | const error = ("error" in event ? event.error : event) as { |
| 597 | readonly type?: string |
| 598 | readonly code?: string |
| 599 | readonly message: string |
| 600 | } |
| 601 | const errorType = error.code ?? |
| 602 | (error.type === "api_error" && /^gRPC error: Response with id=\S+ not found$/.test(error.message) |
| 603 | ? "previous_response_not_found" |
| 604 | : error.type ?? "unknown") |
| 605 | const json = JSON.stringify(error) |
| 606 | // LanguageModel retries `previous_response_not_found` with the full |
| 607 | // prompt on this socket. Other errors leave the turn current so its |
| 608 | // finalizer invalidates the connection, preserving the existing policy. |
| 609 | if (errorType === "previous_response_not_found") { |
| 610 | connection.current = undefined |
| 611 | } |
| 612 | Queue.failCauseUnsafe( |
| 613 | turn, |
| 614 | Cause.fail(AiError.make({ |
| 615 | module: "OpenAiClient", |
| 616 | method: "createResponseStream", |
| 617 | reason: AiError.reasonFromHttpStatus({ |
| 618 | description: json, |
| 619 | status: errorType === "previous_response_not_found" |
| 620 | ? 400 |
| 621 | : isNaN(status) |
| 622 | ? Object.hasOwn(errorTypeToStatus, errorType) |
| 623 | ? errorTypeToStatus[errorType] |
| 624 | : 500 |
| 625 | : status, |
| 626 | metadata: error as any, |
| 627 | http: { |
| 628 | body: json |
| 629 | } |
| 630 | }) |
| 631 | })) |
| 632 | ) |
| 633 | return |
| 634 | } |
| 635 | Queue.offerUnsafe(turn, event) |
| 636 | if ( |
| 637 | event.type === "response.completed" || event.type === "response.incomplete" || |
| 638 | event.type === "response.failed" |
| 639 | ) { |
| 640 | connection.current = undefined |
| 641 | Queue.endUnsafe(turn) |
| 642 | } |
| 643 | } |
| 644 | |
| 645 | yield* Effect.gen(function*() { |
| 646 | const { pull } = yield* socket.reader |
| 647 | while (true) { |
| 648 | const messages = yield* pull |
| 649 | for (let i = 0; i < messages.length; i++) { |
| 650 | handleMessage(messages[i]) |
| 651 | } |
| 652 | } |
| 653 | }).pipe( |
| 654 | Effect.scoped, |
| 655 | Effect.catchTag("SocketError", (error) => |
| 656 | AiError.make({ |
| 657 | module: "OpenAiClient", |
| 658 | method: "createResponseStream", |
| 659 | reason: new AiError.NetworkError({ |
| 660 | reason: "TransportError", |
| 661 | method: request.method, |
| 662 | url: request.url, |
| 663 | description: error.message |
| 664 | }) |
| 665 | })), |
| 666 | Effect.catchCause((cause) => |
| 667 | connection.current === undefined ? Effect.void : Queue.failCause(connection.current, cause) |
| 668 | ), |
| 669 | Effect.ensuring(Effect.forkIn(RcRef.invalidate(queueRef), socketScope, { |
| 670 | startImmediately: true |
| 671 | })), |
| 672 | Effect.forkScoped({ startImmediately: true }) |
| 673 | ) |
| 674 | |
| 675 | return connection |
| 676 | }) |
| 677 | }) |
| 678 | |
| 679 | // Prime the websocket |
| 680 | yield* Effect.scoped(RcRef.get(queueRef)) |
| 681 | |
| 682 | // Websocket mode only allows one request at a time |
| 683 | const semaphore = Semaphore.makeUnsafe(1) |
| 684 | const request = yield* makeRequest |
| 685 | |
| 686 | return OpenAiSocket.context({ |
| 687 | createResponseStream(options) { |
| 688 | const stream = Stream.unwrap(Effect.gen(function*() { |
| 689 | const scope = yield* Effect.scope |
| 690 | yield* Effect.acquireRelease( |
| 691 | semaphore.take(1), |
| 692 | () => semaphore.release(1), |
| 693 | { interruptible: true } |
| 694 | ) |
| 695 | const connection = yield* RcRef.get(queueRef) |
| 696 | const turn: Turn = yield* Queue.unbounded<ResponseStreamEvent, AiError.AiError | Cause.Done>() |
| 697 | connection.current = turn |
| 698 | |
| 699 | // Leaving while the response is still streaming makes the socket |
| 700 | // unusable for the next turn. |
| 701 | yield* Scope.addFinalizerExit( |
| 702 | scope, |
| 703 | () => connection.current === turn ? RcRef.invalidate(queueRef) : Effect.void |
| 704 | ) |
| 705 | |
| 706 | yield* connection.send(options).pipe( |
| 707 | Effect.forkScoped({ startImmediately: true }) |
| 708 | ) |
| 709 | |
| 710 | return Stream.fromQueue(turn) |
| 711 | })) |
| 712 | |
| 713 | return Effect.succeed([ |
| 714 | HttpClientResponse.fromWeb(request, new Response()), |
| 715 | stream |
| 716 | ]) |
| 717 | } |
| 718 | }).pipe( |
| 719 | Context.add(ResponseIdTracker.ResponseIdTracker, tracker) |
| 720 | ) |
| 721 | }) |
| 722 | |
| 723 | const ErrorEvent = Schema.Struct({ |
| 724 | type: Schema.Literal("error").pipe( |
| 725 | Schema.withDecodingDefault(Effect.succeed("error" as const)) |
| 726 | ), |
| 727 | status: Schema.optionalKey(Schema.Int), |
| 728 | // xAI sends `code` in place of `type`, e.g. `previous_response_not_found` |
| 729 | error: Schema.Struct({ |
| 730 | type: Schema.optional(Schema.String), |
| 731 | code: Schema.optional(Schema.String), |
| 732 | message: Schema.String |
| 733 | }) |
| 734 | }) |
| 735 | |
| 736 | const errorTypeToStatus: Record<string, number> = { |
| 737 | invalid_request_error: 400, |
| 738 | invalid_api_key_error: 401, |
| 739 | insufficient_quota_error: 429, |
| 740 | rate_limit_error: 429, |
| 741 | service_unavailable_error: 503 |
| 742 | } |
| 743 | |
| 744 | const AllEvents = Schema.Union([ErrorEvent, OpenAiSchema.ResponseStreamEvent]) |
| 745 | const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(AllEvents)) |
| 746 | |
| 747 | /** |
| 748 | * Uses OpenAI's WebSocket mode for response streams within the provided effect. |
| 749 | * |
| 750 | * **When to use** |
| 751 | * |
| 752 | * Use to enable WebSocket mode around one effect that creates OpenAI response |
| 753 | * streams. |
| 754 | * |
| 755 | * **Gotchas** |
| 756 | * |
| 757 | * This only works with the following WebSocket constructor layers: |
| 758 | * |
| 759 | * - `NodeSocket.layerWebSocketConstructor` |
| 760 | * - `NodeWS.layerWebSocketConstructor` |
| 761 | * - `BunSocket.layerWebSocketConstructor` |
| 762 | * |
| 763 | * These constructor layers support the non-standard options needed to set the |
| 764 | * Authorization header. |
| 765 | * |
| 766 | * @see {@link layerWebSocketMode} for providing WebSocket mode through a layer |
| 767 | * @see {@link OpenAiSocket} for direct access to the WebSocket-backed streaming service |
| 768 | * |
| 769 | * @stability unstable |
| 770 | * @category providing services |
| 771 | * @since 4.0.0 |
| 772 | */ |
| 773 | export const withWebSocketMode = <A, E, R>( |
| 774 | effect: Effect.Effect<A, E, R> |
| 775 | ): Effect.Effect< |
| 776 | A, |
| 777 | E, |
| 778 | Exclude<R, OpenAiSocket | ResponseIdTracker.ResponseIdTracker> | OpenAiClient | Socket.WebSocketConstructor |
| 779 | > => |
| 780 | Effect.scopedWith((scope) => |
| 781 | Effect.flatMap( |
| 782 | Scope.provide(makeSocket, scope), |
| 783 | (services) => Effect.provideContext(effect, services) |
| 784 | ) |
| 785 | ) |
| 786 | |
| 787 | /** |
| 788 | * Uses OpenAI's websocket mode for all responses that use the Layer. |
| 789 | * |
| 790 | * **When to use** |
| 791 | * |
| 792 | * Use to provide WebSocket mode through layer composition for effects that use |
| 793 | * OpenAI response streaming. |
| 794 | * |
| 795 | * **Gotchas** |
| 796 | * |
| 797 | * This only works with the following WebSocket constructor layers: |
| 798 | * |
| 799 | * - `NodeSocket.layerWebSocketConstructor` |
| 800 | * - `NodeWS.layerWebSocketConstructor` |
| 801 | * - `BunSocket.layerWebSocketConstructor` |
| 802 | * |
| 803 | * These constructor layers support the non-standard options needed to set the |
| 804 | * Authorization header. |
| 805 | * |
| 806 | * @see {@link withWebSocketMode} for enabling WebSocket mode around a single effect |
| 807 | * |
| 808 | * @stability unstable |
| 809 | * @category layers |
| 810 | * @since 4.0.0 |
| 811 | */ |
| 812 | export const layerWebSocketMode: Layer.Layer< |
| 813 | OpenAiSocket | ResponseIdTracker.ResponseIdTracker, |
| 814 | never, |
| 815 | OpenAiClient | Socket.WebSocketConstructor |
| 816 | > = Layer.effectContext(makeSocket) |