@samitouri / QOSami_Agents_Zero / commits / b04443be

Add native Responses API transport

Route Agent Zero turns through a LiteLLM transport layer that prefers the Responses API while preserving chat-completions fallback for providers without compatible endpoints. Persist Responses metadata in history and agent state so provider-state continuation, local replay, native function-call execution, and stored-response cleanup survive normal chat workflows. Normalize prompt caching by provider: OpenAI and Azure use prompt_cache_key and prompt_cache_retention, while Anthropic, Gemini, Bedrock, OpenRouter, and compatible chat providers keep block-level cache_control breakpoints and cached tool definitions.

Alessandro committed Jun 1, 2026 at 01:32 UTC b04443be1a23926b08c803d230ecd746ee559b45
9 files changed +4300 -158
agent.py
+539 -10
@@ -1,4 +1,4 @@
1 -import asyncio, random, string, threading
1 +import asyncio, json, random, re, string, threading
2
3 from collections import OrderedDict
4 from dataclasses import dataclass, field
@@ -32,6 +32,15 @@ from typing import Callable
32 from helpers.localization import Localization
33 from helpers import extension
34 from helpers.errors import RepairableException, InterventionException, HandledException
35 +from helpers.llm_result import (
36 + LLMResult,
37 + RESPONSE_METADATA_KEY,
38 + function_call_output_item,
39 + metadata_from_llm_result,
40 + result_from_metadata,
41 +)
42 +from helpers.litellm_transport import ResponsesTransport
43 +from helpers.responses_tools import build_responses_function_tools, original_tool_name
44
45 class AgentContextType(Enum):
46 USER = "user"
@@ -346,6 +355,9 @@ class Agent:
355 DATA_NAME_SUPERIOR = "_superior"
356 DATA_NAME_SUBORDINATE = "_subordinate"
357 DATA_NAME_CTX_WINDOW = "ctx_window"
358 + DATA_NAME_RESPONSES_STATE = "responses_state"
359 + DATA_NAME_RESPONSES_TOOL_NAME_MAP = "responses_tool_name_map"
360 + DATA_NAME_RESPONSES_COMPUTER_SESSION = "responses_computer_session_id"
361
362 @extension.extensible
363 def __init__(
@@ -468,11 +480,12 @@ class Agent:
480 return stop_response
481
482 # call main LLM
471 - agent_response, _reasoning = await self.call_chat_model(
483 + llm_result = await self.call_chat_model_turn(
484 messages=prompt,
485 response_callback=stream_callback,
486 reasoning_callback=reasoning_callback,
487 )
488 + agent_response = llm_result.response
489 await self.handle_intervention(agent_response)
490
491 # Notify extensions to finalize their stream filters
@@ -492,7 +505,12 @@ class Agent:
505 ): # if assistant_response is the same as last message in history, let him know
506 # Append the assistant's response to the history
507 log_item = self.loop_data.params_temporary.get("log_item_generating")
495 - self.hist_add_ai_response(agent_response, id=log_item.id if log_item else "")
508 + assistant_message = self.hist_add_ai_response(
509 + agent_response,
510 + id=log_item.id if log_item else "",
511 + llm_result=llm_result,
512 + )
513 + self._remember_llm_result_state(llm_result, assistant_message)
514 # Append warning message to the history
515 warning_msg = self.read_prompt("fw.msg_repeat.md")
516 wmsg = self.hist_add_warning(message=warning_msg)
@@ -504,9 +522,16 @@ class Agent:
522 else: # otherwise proceed with tool
523 # Append the assistant's response to the history
524 log_item = self.loop_data.params_temporary.get("log_item_generating")
507 - self.hist_add_ai_response(agent_response, id=log_item.id if log_item else "")
525 + assistant_message = self.hist_add_ai_response(
526 + agent_response,
527 + id=log_item.id if log_item else "",
528 + llm_result=llm_result,
529 + )
530 + self._remember_llm_result_state(llm_result, assistant_message)
531 # process tools requested in agent message
509 - tools_result = await self.process_tools(agent_response)
532 + tools_result = await self.process_llm_result_tools(
533 + llm_result
534 + )
535 if tools_result: # final response of message loop available
536 return tools_result # break the execution if the task is done
537
@@ -664,7 +689,12 @@ class Agent:
689
690 @extension.extensible
691 def hist_add_message(
667 - self, ai: bool, content: history.MessageContent, tokens: int = 0, id: str = ""
692 + self,
693 + ai: bool,
694 + content: history.MessageContent,
695 + tokens: int = 0,
696 + id: str = "",
697 + metadata: dict[str, Any] | None = None,
698 ):
699 self.last_message = Localization.get().now()
700 # Allow extensions to process content before adding to history
@@ -673,7 +703,11 @@ class Agent:
703 "hist_add_before", self, content_data=content_data, ai=ai
704 )
705 return self.history.add_message(
676 - ai=ai, content=content_data["content"], tokens=tokens, id=id
706 + ai=ai,
707 + content=content_data["content"],
708 + tokens=tokens,
709 + id=id,
710 + metadata=metadata,
711 )
712
713 @extension.extensible
@@ -706,10 +740,17 @@ class Agent:
740 return msg
741
742 @extension.extensible
709 - def hist_add_ai_response(self, message: str, id: str = ""):
743 + def hist_add_ai_response(
744 + self, message: str, id: str = "", llm_result: LLMResult | None = None
745 + ):
746 self.loop_data.last_response = message
747 content = self.parse_prompt("fw.ai_response.md", message=message)
712 - return self.hist_add_message(True, content=content, id=id)
748 + return self.hist_add_message(
749 + True,
750 + content=content,
751 + id=id,
752 + metadata=metadata_from_llm_result(llm_result),
753 + )
754
755 @extension.extensible
756 def hist_add_warning(self, message: history.MessageContent, id: str = ""):
@@ -719,13 +760,28 @@ class Agent:
760 @extension.extensible
761 def hist_add_tool_result(self, tool_name: str, tool_result: str, **kwargs):
762 msg_id = kwargs.pop("id", "")
763 + responses_item = kwargs.pop("_responses_output_item", None) or kwargs.pop(
764 + "responses_item", None
765 + )
766 + metadata = (
767 + {
768 + RESPONSE_METADATA_KEY: {
769 + "input_items": [responses_item],
770 + "output_items": [],
771 + "mode": "responses",
772 + "state": "provider",
773 + }
774 + }
775 + if isinstance(responses_item, dict)
776 + else None
777 + )
778 data = {
779 "tool_name": tool_name,
780 "tool_result": tool_result,
781 **kwargs,
782 }
783 extension.call_extensions_sync("hist_add_tool_result", self, data=data)
728 - return self.hist_add_message(False, content=data, id=msg_id)
784 + return self.hist_add_message(False, content=data, id=msg_id, metadata=metadata)
785
786 def concat_messages(
787 self, messages
@@ -830,6 +886,170 @@ class Agent:
886
887 return response, reasoning
888
889 + @extension.extensible
890 + async def call_chat_model_turn(
891 + self,
892 + messages: list[BaseMessage],
893 + response_callback: Callable[[str, str], Awaitable[str | None]] | None = None,
894 + reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,
895 + background: bool = False,
896 + explicit_caching: bool = True,
897 + ) -> LLMResult:
898 + model = self.get_chat_model()
899 + model_kwargs = getattr(model, "kwargs", {}) if model else {}
900 + if isinstance(model_kwargs, dict) and model_kwargs.get("responses_delete_on_chat_delete") is False:
901 + self.set_data("responses_delete_on_chat_delete", False)
902 + response_tools, name_map = build_responses_function_tools(self)
903 + self.set_data(Agent.DATA_NAME_RESPONSES_TOOL_NAME_MAP, name_map)
904 +
905 + call_data = {
906 + "model": model,
907 + "messages": messages,
908 + "response_callback": response_callback,
909 + "reasoning_callback": reasoning_callback,
910 + "background": background,
911 + "explicit_caching": explicit_caching,
912 + "a0_responses_function_tools": response_tools,
913 + }
914 +
915 + previous_state = self._responses_state_for_model(model)
916 + if previous_state:
917 + history_counter = int(previous_state.get("history_counter", 0) or 0)
918 + call_data["previous_response_id"] = previous_state.get("response_id", "")
919 + call_data["responses_input_items"] = self._responses_input_items_since(
920 + model,
921 + history_counter,
922 + )
923 + call_data["responses_local_input_items"] = (
924 + self._responses_static_prefix_items(model, messages)
925 + + self._responses_input_items_since(model, 0)
926 + )
927 +
928 + await extension.call_extensions_async(
929 + "chat_model_call_before", self, call_data=call_data
930 + )
931 +
932 + turn_kwargs = {
933 + "a0_responses_function_tools": call_data.get(
934 + "a0_responses_function_tools"
935 + ),
936 + "responses_local_input_items": call_data.get(
937 + "responses_local_input_items"
938 + ),
939 + }
940 + for key in (
941 + "responses_builtin_tools",
942 + "responses_state",
943 + "previous_response_id",
944 + "responses_input_items",
945 + ):
946 + if call_data.get(key) is not None:
947 + turn_kwargs[key] = call_data.get(key)
948 +
949 + llm_result = await call_data["model"].unified_turn(
950 + messages=call_data["messages"],
951 + reasoning_callback=call_data["reasoning_callback"],
952 + response_callback=call_data["response_callback"],
953 + rate_limiter_callback=(
954 + self.rate_limiter_callback if not call_data["background"] else None
955 + ),
956 + explicit_caching=call_data["explicit_caching"],
957 + **turn_kwargs,
958 + )
959 +
960 + downgraded = llm_result.capability.get("builtin_tool_downgrades")
961 + if downgraded:
962 + self.context.log.log(
963 + type="info",
964 + heading="Responses capability downgrade",
965 + content=(
966 + "Provider rejected Responses built-in tool(s); omitted: "
967 + + ", ".join(str(item) for item in downgraded)
968 + ),
969 + )
970 +
971 + await extension.call_extensions_async(
972 + "chat_model_call_after",
973 + self,
974 + call_data=call_data,
975 + response=llm_result.response,
976 + reasoning=llm_result.reasoning,
977 + )
978 +
979 + return llm_result
980 +
981 + def _responses_state_for_model(self, model: Any) -> dict[str, Any]:
982 + state = self.get_data(Agent.DATA_NAME_RESPONSES_STATE)
983 + if not isinstance(state, dict):
984 + return {}
985 + provider_model_key = str(getattr(model, "model_name", "") or "")
986 + if state.get("provider_model_key") != provider_model_key:
987 + return {}
988 + if not state.get("response_id"):
989 + return {}
990 + return state
991 +
992 + def _responses_input_items_since(
993 + self, model: Any, sequence: int
994 + ) -> list[dict[str, Any]]:
995 + items: list[dict[str, Any]] = []
996 + for message in self.history.messages_since(sequence):
997 + items.extend(self._responses_input_items_for_message(model, message))
998 + return items
999 +
1000 + def _responses_input_items_for_message(
1001 + self, model: Any, message: history.Message
1002 + ) -> list[dict[str, Any]]:
1003 + result = result_from_metadata(message.metadata)
1004 + if result:
1005 + if message.ai and result.output_items:
1006 + return [item.to_dict() for item in result.output_items]
1007 + if not message.ai and result.input_items:
1008 + return [dict(item) for item in result.input_items]
1009 +
1010 + output = message.output()
1011 + langchain_messages = history.output_langchain(output)
1012 + if hasattr(model, "_convert_messages"):
1013 + converted = model._convert_messages(langchain_messages)
1014 + return ResponsesTransport.input_from_messages(converted)
1015 + return []
1016 +
1017 + def _responses_static_prefix_items(
1018 + self, model: Any, messages: list[BaseMessage]
1019 + ) -> list[dict[str, Any]]:
1020 + prefix: list[BaseMessage] = []
1021 + for message in messages:
1022 + if isinstance(message, SystemMessage):
1023 + prefix.append(message)
1024 + continue
1025 + break
1026 + if not prefix or not hasattr(model, "_convert_messages"):
1027 + return []
1028 + converted = model._convert_messages(prefix)
1029 + return ResponsesTransport.input_from_messages(converted)
1030 +
1031 + def _remember_llm_result_state(
1032 + self, llm_result: LLMResult, history_message: history.Message
1033 + ) -> None:
1034 + if not llm_result.response_id:
1035 + return
1036 + current = self.get_data(Agent.DATA_NAME_RESPONSES_STATE)
1037 + response_ids = []
1038 + if isinstance(current, dict) and isinstance(current.get("response_ids"), list):
1039 + response_ids = [str(item) for item in current["response_ids"] if item]
1040 + if llm_result.response_id not in response_ids:
1041 + response_ids.append(llm_result.response_id)
1042 + self.set_data(
1043 + Agent.DATA_NAME_RESPONSES_STATE,
1044 + {
1045 + "response_id": llm_result.response_id,
1046 + "previous_response_id": llm_result.previous_response_id,
1047 + "provider_model_key": llm_result.provider_model_key,
1048 + "history_counter": history_message.sequence,
1049 + "response_ids": response_ids,
1050 + },
1051 + )
1052 +
1053 @extension.extensible
1054 async def rate_limiter_callback(
1055 self, message: str, key: str, total: int, limit: int
@@ -863,6 +1083,310 @@ class Agent:
1083 while self.context.paused:
1084 await asyncio.sleep(0.1)
1085
1086 + async def process_llm_result_tools(self, llm_result: LLMResult):
1087 + await self._log_response_builtin_items(llm_result)
1088 + if llm_result.function_calls:
1089 + for function_call in llm_result.function_calls:
1090 + name_map = self.get_data(Agent.DATA_NAME_RESPONSES_TOOL_NAME_MAP)
1091 + tool_name = original_tool_name(function_call.name, name_map)
1092 + response_item_factory = lambda response, call=function_call: function_call_output_item(
1093 + call.call_id,
1094 + response.message,
1095 + )
1096 + result = await self._execute_tool_request(
1097 + tool_name=tool_name,
1098 + tool_args=function_call.arguments,
1099 + message=llm_result.response,
1100 + raw_tool_name=tool_name,
1101 + responses_item_factory=response_item_factory,
1102 + )
1103 + if result:
1104 + return result
1105 + return None
1106 + if llm_result.builtin_items and not llm_result.response:
1107 + return None
1108 + if (
1109 + llm_result.mode == "responses"
1110 + and llm_result.response
1111 + and extract_tools.json_parse_dirty(llm_result.response) is None
1112 + ):
1113 + return llm_result.response
1114 + return await self.process_tools(llm_result.response)
1115 +
1116 + async def _execute_tool_request(
1117 + self,
1118 + tool_name: str,
1119 + tool_args: dict,
1120 + message: str,
1121 + raw_tool_name: str = "",
1122 + responses_item_factory: Callable[[Any], dict[str, Any]] | None = None,
1123 + ):
1124 + raw_tool_name = raw_tool_name or tool_name
1125 + tool_method = None
1126 + tool = None
1127 +
1128 + try:
1129 + import helpers.mcp_handler as mcp_helper
1130 +
1131 + mcp_tool_candidate = mcp_helper.MCPConfig.get_instance().get_tool(
1132 + self, tool_name
1133 + )
1134 + if mcp_tool_candidate:
1135 + tool = mcp_tool_candidate
1136 + except ImportError:
1137 + PrintStyle(
1138 + background_color="black", font_color="yellow", padding=True
1139 + ).print("MCP helper module not found. Skipping MCP tool lookup.")
1140 + except Exception as e:
1141 + PrintStyle(background_color="black", font_color="red", padding=True).print(
1142 + f"Failed to get MCP tool '{tool_name}': {e}"
1143 + )
1144 +
1145 + if not tool:
1146 + tool = self.get_tool(
1147 + name=tool_name,
1148 + method=tool_method,
1149 + args=tool_args,
1150 + message=message,
1151 + loop_data=self.loop_data,
1152 + )
1153 +
1154 + if not tool:
1155 + error_detail = (
1156 + f"Tool '{raw_tool_name}' not found or could not be initialized."
1157 + )
1158 + wmsg = self.hist_add_warning(error_detail)
1159 + PrintStyle(font_color="red", padding=True).print(error_detail)
1160 + self.context.log.log(
1161 + type="warning",
1162 + content=f"{self.agent_name}: {error_detail}",
1163 + id=wmsg.id,
1164 + )
1165 + return None
1166 +
1167 + self.loop_data.current_tool = tool # type: ignore
1168 + try:
1169 + await self.handle_intervention()
1170 +
1171 + await tool.before_execution(**tool_args)
1172 + await self.handle_intervention()
1173 +
1174 + await extension.call_extensions_async(
1175 + "tool_execute_before",
1176 + self,
1177 + tool_args=tool_args or {},
1178 + tool_name=tool_name,
1179 + )
1180 +
1181 + response = await tool.execute(**tool_args)
1182 + await self.handle_intervention()
1183 +
1184 + await extension.call_extensions_async(
1185 + "tool_execute_after",
1186 + self,
1187 + response=response,
1188 + tool_name=tool_name,
1189 + )
1190 +
1191 + if responses_item_factory:
1192 + response.additional = {
1193 + **(response.additional or {}),
1194 + "_responses_output_item": responses_item_factory(response),
1195 + }
1196 +
1197 + await tool.after_execution(response)
1198 + await self.handle_intervention()
1199 +
1200 + if response.break_loop:
1201 + self._clear_responses_pending_state()
1202 + return response.message
1203 + finally:
1204 + self.loop_data.current_tool = None
1205 + return None
1206 +
1207 + async def _log_response_builtin_items(self, llm_result: LLMResult) -> None:
1208 + for item in llm_result.builtin_items:
1209 + if item.type == "computer_call":
1210 + await self._handle_responses_computer_call(item.data)
1211 + continue
1212 + if item.type == "mcp_approval_request":
1213 + self._handle_responses_mcp_approval_request(item.data)
1214 + continue
1215 + self.context.log.log(
1216 + type="info",
1217 + heading=f"Responses tool item: {item.type}",
1218 + content=json.dumps(item.data, ensure_ascii=False, default=str),
1219 + )
1220 +
1221 + async def _handle_responses_computer_call(self, item: dict[str, Any]) -> None:
1222 + safety_checks = item.get("pending_safety_checks") or item.get("safety_checks")
1223 + if safety_checks:
1224 + message = (
1225 + "Responses computer_call requested safety-check acknowledgement. "
1226 + "Agent Zero requires explicit user acknowledgement before executing it."
1227 + )
1228 + output_item = {
1229 + "type": "computer_call_output",
1230 + "call_id": str(item.get("call_id") or item.get("id") or ""),
1231 + "output": {"type": "input_text", "text": message},
1232 + }
1233 + self.hist_add_tool_result(
1234 + "computer_call",
1235 + message,
1236 + responses_item=output_item,
1237 + )
1238 + self.context.log.log(type="warning", content=message)
1239 + return
1240 +
1241 + args = self._computer_call_args(item)
1242 + if not args:
1243 + message = "Responses computer_call action is unsupported by Agent Zero."
1244 + output_item = {
1245 + "type": "computer_call_output",
1246 + "call_id": str(item.get("call_id") or item.get("id") or ""),
1247 + "output": {"type": "input_text", "text": message},
1248 + }
1249 + self.hist_add_tool_result(
1250 + "computer_call",
1251 + message,
1252 + responses_item=output_item,
1253 + )
1254 + self.context.log.log(type="warning", content=message)
1255 + return
1256 +
1257 + if args.get("action") != "start_session" and not args.get("session_id"):
1258 + session_id = str(
1259 + self.get_data(Agent.DATA_NAME_RESPONSES_COMPUTER_SESSION) or ""
1260 + )
1261 + if session_id:
1262 + args["session_id"] = session_id
1263 +
1264 + response_item_factory = lambda response: self._computer_call_output_item(
1265 + item,
1266 + response,
1267 + )
1268 + result = await self._execute_tool_request(
1269 + tool_name="computer_use_remote",
1270 + tool_args=args,
1271 + message=json.dumps(item, ensure_ascii=False, default=str),
1272 + raw_tool_name="computer_call",
1273 + responses_item_factory=response_item_factory,
1274 + )
1275 + _ = result
1276 +
1277 + def _handle_responses_mcp_approval_request(self, item: dict[str, Any]) -> None:
1278 + request_id = str(
1279 + item.get("approval_request_id") or item.get("id") or item.get("call_id") or ""
1280 + )
1281 + message = (
1282 + "Responses MCP approval request received. Agent Zero denied it because "
1283 + "provider-hosted MCP approval requires explicit user approval."
1284 + )
1285 + output_item = {
1286 + "type": "mcp_approval_response",
1287 + "approval_request_id": request_id,
1288 + "approve": False,
1289 + }
1290 + self.hist_add_tool_result(
1291 + "mcp_approval_request",
1292 + message,
1293 + responses_item=output_item,
1294 + )
1295 + self.context.log.log(
1296 + type="warning",
1297 + heading="Responses MCP approval required",
1298 + content=message,
1299 + )
1300 +
1301 + def _computer_call_args(self, item: dict[str, Any]) -> dict[str, Any]:
1302 + action = item.get("action")
1303 + action_data = dict(action) if isinstance(action, dict) else {}
1304 + action_type = str(
1305 + action_data.get("type")
1306 + or action_data.get("action")
1307 + or item.get("action_type")
1308 + or ""
1309 + ).strip().lower()
1310 + args: dict[str, Any] = {}
1311 +
1312 + if action_type in {"screenshot", "capture"}:
1313 + args["action"] = "capture"
1314 + elif action_type in {"move", "mousemove"}:
1315 + args.update({"action": "move", "x": action_data.get("x"), "y": action_data.get("y")})
1316 + elif action_type in {"click", "double_click"}:
1317 + args.update(
1318 + {
1319 + "action": "click",
1320 + "x": action_data.get("x"),
1321 + "y": action_data.get("y"),
1322 + "button": action_data.get("button", "left"),
1323 + "count": 2 if action_type == "double_click" else action_data.get("count", 1),
1324 + }
1325 + )
1326 + elif action_type == "scroll":
1327 + args.update(
1328 + {
1329 + "action": "scroll",
1330 + "dx": action_data.get("dx", action_data.get("scroll_x", 0)),
1331 + "dy": action_data.get("dy", action_data.get("scroll_y", 0)),
1332 + }
1333 + )
1334 + elif action_type in {"keypress", "key"}:
1335 + args.update(
1336 + {
1337 + "action": "key",
1338 + "keys": action_data.get("keys") or action_data.get("key"),
1339 + }
1340 + )
1341 + elif action_type in {"type", "input_text"}:
1342 + args.update({"action": "type", "text": action_data.get("text", "")})
1343 + else:
1344 + return {}
1345 +
1346 + session_id = item.get("session_id") or action_data.get("session_id")
1347 + if session_id:
1348 + args["session_id"] = session_id
1349 + return args
1350 +
1351 + def _computer_call_output_item(
1352 + self, source_item: dict[str, Any], response: Any
1353 + ) -> dict[str, Any]:
1354 + output: dict[str, Any] = {
1355 + "type": "input_text",
1356 + "text": str(getattr(response, "message", "") or ""),
1357 + }
1358 + additional = getattr(response, "additional", None)
1359 + raw_content = additional.get("raw_content") if isinstance(additional, dict) else None
1360 + if isinstance(raw_content, list):
1361 + for content in raw_content:
1362 + if not isinstance(content, dict):
1363 + continue
1364 + if content.get("type") != "image_url":
1365 + continue
1366 + image_url = content.get("image_url")
1367 + url = image_url.get("url") if isinstance(image_url, dict) else image_url
1368 + if url:
1369 + output = {"type": "input_image", "image_url": url}
1370 + break
1371 +
1372 + session_id_match = re_search_session_id(str(getattr(response, "message", "") or ""))
1373 + if session_id_match:
1374 + self.set_data(Agent.DATA_NAME_RESPONSES_COMPUTER_SESSION, session_id_match)
1375 +
1376 + return {
1377 + "type": "computer_call_output",
1378 + "call_id": str(source_item.get("call_id") or source_item.get("id") or ""),
1379 + "output": output,
1380 + }
1381 +
1382 + def _clear_responses_pending_state(self) -> None:
1383 + state = self.get_data(Agent.DATA_NAME_RESPONSES_STATE)
1384 + if isinstance(state, dict):
1385 + state = dict(state)
1386 + state.pop("response_id", None)
1387 + state.pop("previous_response_id", None)
1388 + self.set_data(Agent.DATA_NAME_RESPONSES_STATE, state)
1389 +
1390 @extension.extensible
1391 async def process_tools(self, msg: str):
1392 # search for tool usage requests in agent message
@@ -1037,3 +1561,8 @@ class Agent:
1561 loop_data=loop_data,
1562 **kwargs,
1563 )
1564 +
1565 +
1566 +def re_search_session_id(text: str) -> str:
1567 + match = re.search(r"session_id=([A-Za-z0-9_.:-]+)", text or "")
1568 + return match.group(1) if match else ""
helpers/history.py
+104 -8
@@ -40,9 +40,12 @@ MessageContent = Union[
40 ]
41
42
43 -class OutputMessage(TypedDict):
43 +class OutputMessage(TypedDict, total=False):
44 ai: bool
45 content: MessageContent
46 + metadata: dict[str, Any]
47 + id: str
48 + sequence: int
49
50
51 class Record:
@@ -82,10 +85,20 @@ class Record:
85
86
87 class Message(Record):
85 - def __init__(self, ai: bool, content: MessageContent, tokens: int = 0, id: str = ""):
88 + def __init__(
89 + self,
90 + ai: bool,
91 + content: MessageContent,
92 + tokens: int = 0,
93 + id: str = "",
94 + metadata: dict[str, Any] | None = None,
95 + sequence: int = 0,
96 + ):
97 self.id = id or str(uuid.uuid4())
98 self.ai = ai
99 self.content = content
100 + self.metadata = metadata or {}
101 + self.sequence = sequence
102 self.summary: str = ""
103 self.tokens: int = tokens or self.calculate_tokens()
104
@@ -106,7 +119,15 @@ class Message(Record):
119 return False
120
121 def output(self):
109 - return [OutputMessage(ai=self.ai, content=self.summary or self.content)]
122 + return [
123 + OutputMessage(
124 + ai=self.ai,
125 + content=self.summary or self.content,
126 + metadata=self.metadata,
127 + id=self.id,
128 + sequence=self.sequence,
129 + )
130 + ]
131
132 def output_langchain(self):
133 return output_langchain(self.output())
@@ -120,6 +141,8 @@ class Message(Record):
141 "id": self.id,
142 "ai": self.ai,
143 "content": self.content,
144 + "metadata": self.metadata,
145 + "sequence": self.sequence,
146 "summary": self.summary,
147 "tokens": self.tokens,
148 }
@@ -127,7 +150,13 @@ class Message(Record):
150 @staticmethod
151 def from_dict(data: dict, history: "History"):
152 content = data.get("content", "Content lost")
130 - msg = Message(ai=data["ai"], content=content, id=data.get("id", ""))
153 + msg = Message(
154 + ai=data["ai"],
155 + content=content,
156 + id=data.get("id", ""),
157 + metadata=data.get("metadata", {}) if isinstance(data.get("metadata"), dict) else {},
158 + sequence=int(data.get("sequence", 0) or 0),
159 + )
160 msg.summary = data.get("summary", "")
161 msg.tokens = data.get("tokens", 0)
162 return msg
@@ -146,9 +175,22 @@ class Topic(Record):
175 return sum(msg.get_tokens() for msg in self.messages)
176
177 def add_message(
149 - self, ai: bool, content: MessageContent, tokens: int = 0, id: str = ""
178 + self,
179 + ai: bool,
180 + content: MessageContent,
181 + tokens: int = 0,
182 + id: str = "",
183 + metadata: dict[str, Any] | None = None,
184 + sequence: int = 0,
185 ) -> Message:
151 - msg = Message(ai=ai, content=content, tokens=tokens, id=id)
186 + msg = Message(
187 + ai=ai,
188 + content=content,
189 + tokens=tokens,
190 + id=id,
191 + metadata=metadata,
192 + sequence=sequence,
193 + )
194 self.messages.append(msg)
195 return msg
196
@@ -335,10 +377,22 @@ class History(Record):
377 return self.current.get_tokens()
378
379 def add_message(
338 - self, ai: bool, content: MessageContent, tokens: int = 0, id: str = ""
380 + self,
381 + ai: bool,
382 + content: MessageContent,
383 + tokens: int = 0,
384 + id: str = "",
385 + metadata: dict[str, Any] | None = None,
386 ) -> Message:
387 self.counter += 1
341 - return self.current.add_message(ai, content=content, tokens=tokens, id=id)
388 + return self.current.add_message(
389 + ai,
390 + content=content,
391 + tokens=tokens,
392 + id=id,
393 + metadata=metadata,
394 + sequence=self.counter,
395 + )
396
397 def new_topic(self):
398 if self.current.messages:
@@ -353,6 +407,35 @@ class History(Record):
407 result += self.current.output()
408 return result
409
410 + def messages_since(self, sequence: int) -> list[Message]:
411 + return [
412 + message
413 + for message in self.all_messages()
414 + if int(message.sequence or 0) > int(sequence or 0)
415 + ]
416 +
417 + def all_messages(self) -> list[Message]:
418 + messages: list[Message] = []
419 + for bulk in self.bulks:
420 + messages.extend(_messages_from_record(bulk))
421 + for topic in self.topics:
422 + messages.extend(topic.messages)
423 + messages.extend(self.current.messages)
424 + return messages
425 +
426 + def latest_llm_result_for_model(self, provider_model_key: str):
427 + from helpers.llm_result import result_from_metadata
428 +
429 + for message in reversed(self.all_messages()):
430 + if not message.ai:
431 + continue
432 + result = result_from_metadata(message.metadata)
433 + if not result:
434 + continue
435 + if result.provider_model_key == provider_model_key and result.response_id:
436 + return result
437 + return None
438 +
439 def trim_embeds(self, max_embeds: int) -> int:
440 if max_embeds == -1:
441 return 0
@@ -679,6 +762,19 @@ def _is_embedded_data(obj: object) -> bool:
762 return isinstance(obj, Mapping) and obj.get("type") == "image_url"
763
764
765 +def _messages_from_record(record: Record) -> list[Message]:
766 + if isinstance(record, Message):
767 + return [record]
768 + if isinstance(record, Topic):
769 + return list(record.messages)
770 + if isinstance(record, Bulk):
771 + messages: list[Message] = []
772 + for nested in record.records:
773 + messages.extend(_messages_from_record(nested))
774 + return messages
775 + return []
776 +
777 +
778 def _json_dumps(obj):
779 return json.dumps(obj, ensure_ascii=False)
780
helpers/litellm_transport.py new
+1676
@@ -0,0 +1,1676 @@
1 +from __future__ import annotations
2 +
3 +from dataclasses import dataclass, field
4 +from enum import Enum
5 +import hashlib
6 +import inspect
7 +import json
8 +from typing import Any, AsyncIterator, Iterator, Optional
9 +
10 +from litellm import (
11 + acompletion,
12 + adelete_responses,
13 + aresponses,
14 + completion,
15 + delete_responses,
16 + responses,
17 +)
18 +
19 +from helpers import images
20 +from helpers.llm_result import LLMResult
21 +
22 +
23 +ChatChunk = dict[str, str]
24 +
25 +
26 +class TransportMode(Enum):
27 + RESPONSES = "responses"
28 + CHAT_COMPLETIONS = "chat_completions"
29 +
30 +
31 +class TransportRecovery(Enum):
32 + RAISE = "raise"
33 + RETRY_RESPONSES = "retry_responses"
34 + RETRY_LOCAL_RESPONSES = "retry_local_responses"
35 + FALLBACK_TO_CHAT = "fallback_to_chat"
36 +
37 +
38 +CHAT_COMPLETIONS_ALIASES = {
39 + "chat",
40 + "chat_completion",
41 + "chat_completions",
42 + "completion",
43 + "completions",
44 +}
45 +RESPONSES_ALIASES = {"", "auto", "default", "response", "responses", "responses_api"}
46 +RESPONSES_REASONING_EFFORTS = {"minimal", "low", "medium", "high"}
47 +RESPONSES_REASONING_FALLBACK_EFFORT = "high"
48 +NO_REASONING_EFFORT_ALIASES = {"", "0", "false", "no", "none", "off", "disabled"}
49 +RESPONSES_UNSUPPORTED_CACHE: set[str] = set()
50 +RESPONSES_STATE_UNSUPPORTED_CACHE: set[str] = set()
51 +RESPONSES_BUILTIN_UNSUPPORTED_CACHE: dict[str, set[str]] = {}
52 +OPENAI_RESPONSES_EXTRA_BODY_PARAMS = {
53 + "context_management",
54 + "prompt_cache_retention",
55 +}
56 +CACHE_CONTROL_PROMPT_PROVIDERS = {
57 + "anthropic",
58 + "bedrock",
59 + "databricks",
60 + "dashscope",
61 + "gemini",
62 + "gemini_api_oauth",
63 + "minimax",
64 + "openrouter",
65 + "vertex_ai",
66 + "vertexai",
67 + "z_ai",
68 + "zai",
69 +}
70 +OPENAI_PROMPT_CACHE_PROVIDERS = {"openai", "azure"}
71 +RESPONSES_STATE_PROVIDER = "provider"
72 +RESPONSES_STATE_LOCAL = "local"
73 +RESPONSES_STATE_OFF = "off"
74 +RESPONSES_STATES = {
75 + RESPONSES_STATE_PROVIDER,
76 + RESPONSES_STATE_LOCAL,
77 + RESPONSES_STATE_OFF,
78 +}
79 +
80 +
81 +@dataclass
82 +class TransportPolicy:
83 + mode: TransportMode
84 + allow_fallback: bool = True
85 + retried_reasoning: bool = False
86 + fallback_error: Exception | None = None
87 + state_fallback_error: Exception | None = None
88 + cache_key: str = ""
89 + state: str = RESPONSES_STATE_PROVIDER
90 +
91 + @classmethod
92 + def from_request(
93 + cls,
94 + model: str,
95 + kwargs: dict[str, Any],
96 + messages: list[dict[str, Any]] | None = None,
97 + ) -> "TransportPolicy":
98 + mode = cls._pop_mode(kwargs)
99 + allow_fallback = _coerce_bool(
100 + kwargs.pop("a0_responses_fallback", True), default=True
101 + )
102 + cache_key = _responses_cache_key(model, kwargs)
103 + state = _normalize_responses_state(kwargs.get("responses_state"))
104 +
105 + if mode is TransportMode.CHAT_COMPLETIONS:
106 + _drop_responses_only_kwargs(kwargs)
107 + return cls(
108 + mode=mode,
109 + allow_fallback=allow_fallback,
110 + cache_key=cache_key,
111 + state=RESPONSES_STATE_OFF,
112 + )
113 +
114 + if (
115 + state == RESPONSES_STATE_PROVIDER
116 + and cache_key in RESPONSES_STATE_UNSUPPORTED_CACHE
117 + ):
118 + kwargs["responses_state"] = RESPONSES_STATE_LOCAL
119 + state = RESPONSES_STATE_LOCAL
120 +
121 + _filter_unsupported_builtin_tools(kwargs, cache_key)
122 +
123 + if _should_preserve_cache_control_on_chat(model, kwargs, messages or []):
124 + return cls(
125 + mode=TransportMode.CHAT_COMPLETIONS,
126 + allow_fallback=allow_fallback,
127 + cache_key=cache_key,
128 + state=RESPONSES_STATE_OFF,
129 + )
130 +
131 + if cache_key in RESPONSES_UNSUPPORTED_CACHE:
132 + return cls(
133 + mode=TransportMode.CHAT_COMPLETIONS,
134 + allow_fallback=allow_fallback,
135 + fallback_error=RuntimeError("Responses API previously failed"),
136 + cache_key=cache_key,
137 + state=RESPONSES_STATE_OFF,
138 + )
139 +
140 + return cls(
141 + mode=TransportMode.RESPONSES,
142 + allow_fallback=allow_fallback,
143 + cache_key=cache_key,
144 + state=state,
145 + )
146 +
147 + @staticmethod
148 + def _pop_mode(kwargs: dict[str, Any]) -> TransportMode:
149 + value = str(kwargs.pop("a0_api_mode", "responses") or "").lower().strip()
150 + if value in CHAT_COMPLETIONS_ALIASES:
151 + return TransportMode.CHAT_COMPLETIONS
152 + if value in RESPONSES_ALIASES:
153 + return TransportMode.RESPONSES
154 + return TransportMode.RESPONSES
155 +
156 + @property
157 + def using_responses(self) -> bool:
158 + return self.mode is TransportMode.RESPONSES
159 +
160 + def recover(self, exc: Exception, *, got_any_chunk: bool) -> TransportRecovery:
161 + if not self.using_responses or got_any_chunk:
162 + return TransportRecovery.RAISE
163 + if not self.retried_reasoning and _is_responses_reasoning_effort_error(exc):
164 + self.retried_reasoning = True
165 + return TransportRecovery.RETRY_RESPONSES
166 + if (
167 + self.state == RESPONSES_STATE_PROVIDER
168 + and _is_responses_state_unsupported_error(exc)
169 + ):
170 + self.state = RESPONSES_STATE_LOCAL
171 + self.state_fallback_error = exc
172 + if self.cache_key:
173 + RESPONSES_STATE_UNSUPPORTED_CACHE.add(self.cache_key)
174 + return TransportRecovery.RETRY_LOCAL_RESPONSES
175 + if self.allow_fallback and _is_responses_not_supported_error(exc):
176 + self.mode = TransportMode.CHAT_COMPLETIONS
177 + self.fallback_error = exc
178 + self.state = RESPONSES_STATE_OFF
179 + if self.cache_key:
180 + RESPONSES_UNSUPPORTED_CACHE.add(self.cache_key)
181 + return TransportRecovery.FALLBACK_TO_CHAT
182 + return TransportRecovery.RAISE
183 +
184 +
185 +@dataclass
186 +class LiteLLMTransport:
187 + model: str
188 + messages: list[dict[str, Any]]
189 + kwargs: dict[str, Any]
190 + stop: Optional[list[str]] = None
191 + policy: TransportPolicy = field(init=False)
192 + last_result: LLMResult | None = field(init=False, default=None)
193 + last_request_state: str = field(init=False, default=RESPONSES_STATE_PROVIDER)
194 + explicit_prompt_caching: bool = field(init=False, default=False)
195 +
196 + def __post_init__(self) -> None:
197 + self.kwargs = _without_stream_kwarg(dict(self.kwargs))
198 + self.explicit_prompt_caching = _coerce_bool(
199 + self.kwargs.pop("a0_explicit_prompt_caching", False), default=False
200 + )
201 + if self.explicit_prompt_caching:
202 + self.messages = apply_chat_prompt_cache_markers(
203 + self.messages,
204 + model=self.model,
205 + kwargs=self.kwargs,
206 + )
207 + self.policy = TransportPolicy.from_request(
208 + self.model,
209 + self.kwargs,
210 + messages=self.messages,
211 + )
212 +
213 + def complete(self) -> ChatChunk:
214 + while True:
215 + try:
216 + if self.policy.mode is TransportMode.CHAT_COMPLETIONS:
217 + parsed = ChatCompletionsTransport.parse(
218 + completion(**self._chat_request(stream=False))
219 + )
220 + self.last_result = self._llm_result_from_chat(parsed)
221 + return parsed
222 + request = self._responses_request(stream=False)
223 + raw_response = responses(**request)
224 + parsed = ResponsesTransport.parse_response(raw_response)
225 + self.last_result = self._llm_result_from_response(
226 + raw_response, request
227 + )
228 + return parsed
229 + except Exception as exc:
230 + if self._recover(exc, got_any_chunk=False):
231 + continue
232 + raise
233 +
234 + async def acomplete(self) -> ChatChunk:
235 + while True:
236 + try:
237 + if self.policy.mode is TransportMode.CHAT_COMPLETIONS:
238 + parsed = ChatCompletionsTransport.parse(
239 + await acompletion(**self._chat_request(stream=False))
240 + )
241 + self.last_result = self._llm_result_from_chat(parsed)
242 + return parsed
243 + request = self._responses_request(stream=False)
244 + raw_response = await aresponses(**request)
245 + parsed = ResponsesTransport.parse_response(raw_response)
246 + self.last_result = self._llm_result_from_response(
247 + raw_response, request
248 + )
249 + return parsed
250 + except Exception as exc:
251 + if self._recover(exc, got_any_chunk=False):
252 + continue
253 + raise
254 +
255 + def stream(self) -> Iterator[ChatChunk]:
256 + while True:
257 + iterator = None
258 + exhausted = False
259 + got_any_chunk = False
260 + try:
261 + if self.policy.mode is TransportMode.CHAT_COMPLETIONS:
262 + iterator = completion(**self._chat_request(stream=True))
263 + for chunk in iterator:
264 + parsed = ChatCompletionsTransport.parse(chunk)
265 + if _has_chunk_delta(parsed):
266 + got_any_chunk = True
267 + yield parsed
268 + else:
269 + request = self._responses_request(stream=True)
270 + iterator = responses(**request)
271 + parser = ResponsesEventParser()
272 + for event in iterator:
273 + parsed = parser.parse(event)
274 + if _has_chunk_delta(parsed):
275 + got_any_chunk = True
276 + yield parsed
277 + self.last_result = self._stream_result_from_parser(
278 + parser, request
279 + )
280 + exhausted = True
281 + return
282 + except Exception as exc:
283 + if self._recover(exc, got_any_chunk=got_any_chunk):
284 + continue
285 + raise
286 + finally:
287 + if iterator is not None and not exhausted:
288 + _close_sync_stream(iterator)
289 +
290 + async def astream(self) -> AsyncIterator[ChatChunk]:
291 + while True:
292 + iterator = None
293 + exhausted = False
294 + got_any_chunk = False
295 + try:
296 + if self.policy.mode is TransportMode.CHAT_COMPLETIONS:
297 + iterator = await acompletion(**self._chat_request(stream=True))
298 + async for chunk in iterator: # type: ignore[union-attr]
299 + parsed = ChatCompletionsTransport.parse(chunk)
300 + if _has_chunk_delta(parsed):
301 + got_any_chunk = True
302 + yield parsed
303 + else:
304 + request = self._responses_request(stream=True)
305 + iterator = await aresponses(**request)
306 + parser = ResponsesEventParser()
307 + async for event in iterator: # type: ignore[union-attr]
308 + parsed = parser.parse(event)
309 + if _has_chunk_delta(parsed):
310 + got_any_chunk = True
311 + yield parsed
312 + self.last_result = self._stream_result_from_parser(
313 + parser, request
314 + )
315 + exhausted = True
316 + return
317 + except Exception as exc:
318 + if self._recover(exc, got_any_chunk=got_any_chunk):
319 + continue
320 + raise
321 + finally:
322 + if iterator is not None and not exhausted:
323 + await _close_async_stream(iterator)
324 +
325 + def _recover(self, exc: Exception, *, got_any_chunk: bool) -> bool:
326 + if (
327 + self.policy.using_responses
328 + and not got_any_chunk
329 + and self.kwargs.get("responses_builtin_tools")
330 + and _is_responses_builtin_tool_error(exc)
331 + ):
332 + downgraded = _builtin_tool_types(self.kwargs.get("responses_builtin_tools"))
333 + if downgraded:
334 + RESPONSES_BUILTIN_UNSUPPORTED_CACHE.setdefault(
335 + self.policy.cache_key, set()
336 + ).update(downgraded)
337 + self.kwargs["_a0_responses_builtin_downgrades"] = sorted(downgraded)
338 + self.kwargs["responses_builtin_tools"] = []
339 + return True
340 +
341 + recovery = self.policy.recover(exc, got_any_chunk=got_any_chunk)
342 + if recovery is TransportRecovery.RETRY_RESPONSES:
343 + self.kwargs["reasoning"] = {
344 + "effort": RESPONSES_REASONING_FALLBACK_EFFORT
345 + }
346 + return True
347 + if recovery is TransportRecovery.RETRY_LOCAL_RESPONSES:
348 + self.kwargs["responses_state"] = RESPONSES_STATE_LOCAL
349 + self.kwargs.pop("previous_response_id", None)
350 + return True
351 + return recovery is TransportRecovery.FALLBACK_TO_CHAT
352 +
353 + def _chat_request(self, *, stream: bool) -> dict[str, Any]:
354 + chat_kwargs = ChatCompletionsTransport.prepare_kwargs(
355 + self.kwargs,
356 + fallback_error=self.policy.fallback_error,
357 + model=self.model,
358 + messages=self.messages,
359 + explicit_prompt_caching=self.explicit_prompt_caching,
360 + )
361 + request = {
362 + "model": self.model,
363 + "messages": ChatCompletionsTransport.prepare_messages(
364 + self.messages,
365 + model=self.model,
366 + kwargs=chat_kwargs,
367 + ),
368 + "stream": stream,
369 + **chat_kwargs,
370 + }
371 + if self.stop is not None:
372 + request["stop"] = self.stop
373 + return request
374 +
375 + def _responses_request(self, *, stream: bool) -> dict[str, Any]:
376 + response_kwargs = ResponsesTransport.from_chat(
377 + self.messages,
378 + self.kwargs,
379 + stop=self.stop,
380 + model=self.model,
381 + )
382 + self.last_request_state = _normalize_responses_state(
383 + self.kwargs.get("responses_state")
384 + )
385 + return {
386 + "model": self.model,
387 + "stream": stream,
388 + **response_kwargs,
389 + }
390 +
391 + def _llm_result_from_chat(self, parsed: ChatChunk) -> LLMResult:
392 + return LLMResult.from_chat(
393 + response=parsed["response_delta"],
394 + reasoning=parsed["reasoning_delta"],
395 + input_items=ResponsesTransport.input_from_messages(self.messages),
396 + provider_model_key=self.model,
397 + capability=self._capability_metadata(),
398 + )
399 +
400 + def _llm_result_from_response(
401 + self, response: Any, request: dict[str, Any]
402 + ) -> LLMResult:
403 + return LLMResult.from_response(
404 + response,
405 + input_items=_as_list(request.get("input")),
406 + previous_response_id=str(request.get("previous_response_id") or ""),
407 + provider_model_key=self.model,
408 + mode=TransportMode.RESPONSES.value,
409 + state=self.last_request_state,
410 + capability=self._capability_metadata(),
411 + )
412 +
413 + def _stream_result_from_parser(
414 + self, parser: "ResponsesEventParser", request: dict[str, Any]
415 + ) -> LLMResult | None:
416 + if parser.completed_response is None:
417 + return None
418 + return self._llm_result_from_response(parser.completed_response, request)
419 +
420 + def _capability_metadata(self) -> dict[str, Any]:
421 + return {
422 + "mode": self.policy.mode.value,
423 + "state": self.policy.state,
424 + "cache_key": self.policy.cache_key,
425 + "fallback_error": _exception_text(self.policy.fallback_error)
426 + if self.policy.fallback_error
427 + else "",
428 + "state_fallback_error": _exception_text(self.policy.state_fallback_error)
429 + if self.policy.state_fallback_error
430 + else "",
431 + "builtin_tool_downgrades": list(
432 + self.kwargs.get("_a0_responses_builtin_downgrades") or []
433 + ),
434 + }
435 +
436 +
437 +class ChatCompletionsTransport:
438 + @staticmethod
439 + def prepare_messages(
440 + messages: list[dict[str, Any]],
441 + *,
442 + model: str = "",
443 + kwargs: dict[str, Any] | None = None,
444 + ) -> list[dict[str, Any]]:
445 + if _is_openai_prompt_cache_provider(model, kwargs or {}):
446 + stripped = _without_cache_control(messages)
447 + return stripped if isinstance(stripped, list) else messages
448 + return messages
449 +
450 + @staticmethod
451 + def prepare_kwargs(
452 + kwargs: dict[str, Any],
453 + fallback_error: Exception | None = None,
454 + *,
455 + model: str = "",
456 + messages: list[dict[str, Any]] | None = None,
457 + explicit_prompt_caching: bool = False,
458 + ) -> dict[str, Any]:
459 + chat_kwargs = dict(kwargs)
460 + _drop_internal_transport_kwargs(chat_kwargs)
461 + if not _has_tools(chat_kwargs.get("tools")):
462 + chat_kwargs.pop("tool_choice", None)
463 + chat_kwargs.pop("parallel_tool_calls", None)
464 + if _is_openai_prompt_cache_provider(model, chat_kwargs):
465 + _prepare_openai_prompt_cache_params(
466 + chat_kwargs,
467 + messages or [],
468 + model=model,
469 + )
470 + elif _supports_cache_control_markers(model, chat_kwargs) and (
471 + explicit_prompt_caching or _has_cache_control(messages or [])
472 + ):
473 + _apply_tool_cache_control(chat_kwargs)
474 + if fallback_error is not None:
475 + chat_kwargs.setdefault("drop_params", True)
476 + return {key: value for key, value in chat_kwargs.items() if value is not None}
477 +
478 + @staticmethod
479 + def parse(chunk: Any) -> ChatChunk:
480 + choice = _first_choice(chunk)
481 + delta = _get_value(choice, "delta") or {}
482 + message = _get_value(choice, "message") or _get_value(
483 + _get_value(choice, "model_extra") or {}, "message"
484 + ) or {}
485 + response_delta = _get_value(delta, "content") or _get_value(
486 + message, "content"
487 + ) or ""
488 + reasoning_delta = _get_value(delta, "reasoning_content") or _get_value(
489 + message, "reasoning_content"
490 + ) or ""
491 + return {"reasoning_delta": reasoning_delta, "response_delta": response_delta}
492 +
493 +
494 +class ResponsesTransport:
495 + @classmethod
496 + def from_chat(
497 + cls,
498 + messages: list[dict[str, Any]],
499 + kwargs: dict[str, Any],
500 + stop: Optional[list[str]] = None,
501 + model: str = "",
502 + ) -> dict[str, Any]:
503 + request = cls.prepare_kwargs(kwargs, stop=stop, model=model, messages=messages)
504 + state = _normalize_responses_state(kwargs.get("responses_state"))
505 + input_items = cls._select_input_items(kwargs, messages, state)
506 + request["input"] = input_items or ""
507 + cls.apply_state(request, kwargs, state=state)
508 + return request
509 +
510 + @classmethod
511 + def from_input(
512 + cls,
513 + input_items: list[dict[str, Any]],
514 + kwargs: dict[str, Any],
515 + stop: Optional[list[str]] = None,
516 + model: str = "",
517 + messages: list[dict[str, Any]] | None = None,
518 + ) -> dict[str, Any]:
519 + request = cls.prepare_kwargs(kwargs, stop=stop, model=model, messages=messages)
520 + state = _normalize_responses_state(kwargs.get("responses_state"))
521 + request["input"] = list(input_items or []) or ""
522 + cls.apply_state(request, kwargs, state=state)
523 + return request
524 +
525 + @classmethod
526 + def prepare_kwargs(
527 + cls,
528 + kwargs: dict[str, Any],
529 + stop: Optional[list[str]] = None,
530 + model: str = "",
531 + messages: list[dict[str, Any]] | None = None,
532 + ) -> dict[str, Any]:
533 + request = dict(kwargs)
534 + response_function_tools = request.pop("a0_responses_function_tools", None)
535 + response_builtin_tools = request.pop("responses_builtin_tools", None)
536 + _drop_responses_only_kwargs(request)
537 + _drop_legacy_transport_kwargs(request)
538 + request.pop("stop", None)
539 +
540 + max_completion_tokens = request.pop("max_completion_tokens", None)
541 + max_tokens = request.pop("max_tokens", None)
542 + if "max_output_tokens" not in request:
543 + request["max_output_tokens"] = max_completion_tokens or max_tokens
544 +
545 + reasoning_effort = request.pop("reasoning_effort", None)
546 + if "reasoning" in request:
547 + request["reasoning"] = cls.normalize_reasoning(request["reasoning"])
548 + elif reasoning_effort is not None:
549 + request["reasoning"] = cls.normalize_reasoning(reasoning_effort)
550 +
551 + response_format = request.pop("response_format", None)
552 + if response_format is not None:
553 + text_param, text_format = cls.text_from_response_format(response_format)
554 + if text_param is not None and "text" not in request:
555 + request["text"] = text_param
556 + if text_format is not None and "text_format" not in request:
557 + request["text_format"] = text_format
558 +
559 + functions = request.pop("functions", None)
560 + if functions and "tools" not in request:
561 + request["tools"] = [
562 + {"type": "function", **function}
563 + for function in functions
564 + if isinstance(function, dict)
565 + ]
566 +
567 + tools = cls.tools_from_chat(request.pop("tools", None))
568 + tools = cls.merge_response_tools(
569 + tools,
570 + response_function_tools=response_function_tools,
571 + response_builtin_tools=response_builtin_tools,
572 + )
573 + if _has_tools(tools):
574 + request["tools"] = tools
575 + else:
576 + request.pop("tools", None)
577 +
578 + function_call = request.pop("function_call", None)
579 + if function_call is not None and "tool_choice" not in request:
580 + request["tool_choice"] = cls.tool_choice_from_function_call(function_call)
581 + elif "tool_choice" in request:
582 + request["tool_choice"] = cls.tool_choice_from_chat(request["tool_choice"])
583 +
584 + if not _has_tools(request.get("tools")):
585 + request.pop("tool_choice", None)
586 + request.pop("parallel_tool_calls", None)
587 +
588 + cls.prepare_prompt_caching(request, messages or [], model=model)
589 +
590 + _ = stop
591 + return {key: value for key, value in request.items() if value is not None}
592 +
593 + @classmethod
594 + def _select_input_items(
595 + cls,
596 + kwargs: dict[str, Any],
597 + messages: list[dict[str, Any]],
598 + state: str,
599 + ) -> list[dict[str, Any]]:
600 + provider_items = kwargs.get("responses_input_items")
601 + local_items = kwargs.get("responses_local_input_items")
602 + previous_response_id = kwargs.get("previous_response_id")
603 +
604 + if (
605 + state == RESPONSES_STATE_PROVIDER
606 + and previous_response_id
607 + and isinstance(provider_items, list)
608 + ):
609 + return [dict(item) for item in provider_items if isinstance(item, dict)]
610 +
611 + if state == RESPONSES_STATE_LOCAL and isinstance(local_items, list):
612 + return [dict(item) for item in local_items if isinstance(item, dict)]
613 +
614 + return cls.input_from_messages(messages)
615 +
616 + @staticmethod
617 + def apply_state(
618 + request: dict[str, Any],
619 + kwargs: dict[str, Any],
620 + *,
621 + state: str,
622 + ) -> None:
623 + if state == RESPONSES_STATE_PROVIDER:
624 + request.setdefault("store", True)
625 + previous_response_id = str(kwargs.get("previous_response_id") or "")
626 + if previous_response_id:
627 + request["previous_response_id"] = previous_response_id
628 + elif state == RESPONSES_STATE_LOCAL:
629 + request.setdefault("store", False)
630 + else:
631 + request.setdefault("store", False)
632 +
633 + @classmethod
634 + def merge_response_tools(
635 + cls,
636 + tools: Any,
637 + *,
638 + response_function_tools: Any = None,
639 + response_builtin_tools: Any = None,
640 + ) -> list[Any]:
641 + merged: list[Any] = []
642 + if isinstance(tools, list):
643 + merged.extend(tools)
644 + elif tools:
645 + merged.append(tools)
646 +
647 + for source in (response_function_tools, response_builtin_tools):
648 + for tool in _as_list(source):
649 + normalized = cls.normalize_response_tool(tool)
650 + if normalized:
651 + merged.append(normalized)
652 + return merged
653 +
654 + @staticmethod
655 + def normalize_response_tool(tool: Any) -> dict[str, Any] | None:
656 + if isinstance(tool, str):
657 + tool = {"type": tool}
658 + if not isinstance(tool, dict):
659 + return None
660 + return dict(tool)
661 +
662 + @staticmethod
663 + def prepare_prompt_caching(
664 + request: dict[str, Any],
665 + messages: list[dict[str, Any]],
666 + model: str = "",
667 + ) -> None:
668 + if not _is_openai_prompt_cache_provider(model, request):
669 + return
670 +
671 + _prepare_openai_prompt_cache_params(request, messages, model=model)
672 +
673 + for key in OPENAI_RESPONSES_EXTRA_BODY_PARAMS:
674 + if key not in request:
675 + continue
676 + extra_body = request.get("extra_body")
677 + if not isinstance(extra_body, dict):
678 + extra_body = {}
679 + extra_body.setdefault(key, request.pop(key))
680 + request["extra_body"] = extra_body
681 +
682 + @classmethod
683 + def input_from_messages(
684 + cls, messages: list[dict[str, Any]]
685 + ) -> list[dict[str, Any]]:
686 + response_input: list[dict[str, Any]] = []
687 +
688 + for message in messages:
689 + role = str(message.get("role") or "user")
690 + content = message.get("content", "")
691 +
692 + if role == "tool":
693 + response_input.append(
694 + {
695 + "type": "function_call_output",
696 + "call_id": str(message.get("tool_call_id") or ""),
697 + "output": _content_to_text(content),
698 + }
699 + )
700 + continue
701 +
702 + tool_calls = message.get("tool_calls")
703 + if role == "assistant" and isinstance(tool_calls, list) and tool_calls:
704 + if _has_real_content(content):
705 + response_input.append(
706 + {
707 + "role": "assistant",
708 + "content": cls.content_from_chat(content, role=role),
709 + }
710 + )
711 + response_input.extend(cls.tool_calls_from_chat(tool_calls))
712 + continue
713 +
714 + response_input.append(
715 + {
716 + "role": role
717 + if role in {"user", "assistant", "system", "developer"}
718 + else "user",
719 + "content": cls.content_from_chat(content, role=role),
720 + }
721 + )
722 +
723 + return response_input
724 +
725 + @classmethod
726 + def content_from_chat(cls, content: Any, role: str = "user") -> Any:
727 + content = images.prepare_content(content)
728 + if not isinstance(content, list):
729 + return content
730 + return [
731 + converted
732 + for item in content
733 + if (converted := cls.content_part_from_chat(item, role=role)) is not None
734 + ]
735 +
736 + @staticmethod
737 + def content_part_from_chat(item: Any, role: str = "user") -> Any:
738 + if not isinstance(item, dict):
739 + return item
740 +
741 + item_type = item.get("type")
742 + if item_type in {"input_text", "output_text", "input_image", "input_file"}:
743 + return dict(item)
744 + if item_type == "text":
745 + return {
746 + "type": "output_text" if role == "assistant" else "input_text",
747 + "text": item.get("text", ""),
748 + }
749 + if item_type == "image_url":
750 + image_url = item.get("image_url")
751 + if isinstance(image_url, dict):
752 + url = image_url.get("url", "")
753 + detail = image_url.get("detail")
754 + else:
755 + url = image_url or ""
756 + detail = item.get("detail")
757 + result = {"type": "input_image", "image_url": url}
758 + if detail:
759 + result["detail"] = detail
760 + return result
761 +
762 + return dict(item)
763 +
764 + @staticmethod
765 + def tool_calls_from_chat(tool_calls: list[Any]) -> list[dict[str, Any]]:
766 + response_input: list[dict[str, Any]] = []
767 + for tool_call in tool_calls:
768 + if not isinstance(tool_call, dict):
769 + continue
770 + function = tool_call.get("function") or {}
771 + if not isinstance(function, dict):
772 + function = {}
773 + response_input.append(
774 + {
775 + "type": "function_call",
776 + "call_id": str(tool_call.get("id") or ""),
777 + "id": str(tool_call.get("id") or ""),
778 + "name": str(function.get("name") or tool_call.get("name") or ""),
779 + "arguments": str(function.get("arguments") or ""),
780 + "status": "completed",
781 + }
782 + )
783 + return response_input
784 +
785 + @staticmethod
786 + def tools_from_chat(tools: Any) -> Any:
787 + if not isinstance(tools, list):
788 + return tools
789 + response_tools: list[Any] = []
790 + for tool in tools:
791 + if not isinstance(tool, dict):
792 + response_tools.append(tool)
793 + continue
794 + if tool.get("type") == "function" and isinstance(
795 + tool.get("function"), dict
796 + ):
797 + function = tool["function"]
798 + response_tool = {
799 + "type": "function",
800 + "name": function.get("name", ""),
801 + "description": function.get("description", ""),
802 + "parameters": function.get("parameters", {}),
803 + }
804 + if "strict" in function:
805 + response_tool["strict"] = function["strict"]
806 + response_tools.append(response_tool)
807 + else:
808 + response_tools.append(dict(tool))
809 + return response_tools
810 +
811 + @staticmethod
812 + def tool_choice_from_function_call(function_call: Any) -> Any:
813 + if isinstance(function_call, str):
814 + return function_call
815 + if isinstance(function_call, dict) and function_call.get("name"):
816 + return {"type": "function", "name": function_call["name"]}
817 + return function_call
818 +
819 + @staticmethod
820 + def tool_choice_from_chat(tool_choice: Any) -> Any:
821 + if (
822 + isinstance(tool_choice, dict)
823 + and tool_choice.get("type") == "function"
824 + and isinstance(tool_choice.get("function"), dict)
825 + ):
826 + return {"type": "function", "name": tool_choice["function"].get("name", "")}
827 + return tool_choice
828 +
829 + @staticmethod
830 + def text_from_response_format(response_format: Any) -> tuple[Any, Any]:
831 + if isinstance(response_format, type):
832 + return None, response_format
833 + if not isinstance(response_format, dict):
834 + return response_format, None
835 +
836 + format_type = response_format.get("type")
837 + if format_type == "json_schema":
838 + schema = response_format.get("json_schema") or {}
839 + return (
840 + {
841 + "format": {
842 + "type": "json_schema",
843 + "name": schema.get("name", "response_schema"),
844 + "schema": schema.get("schema", {}),
845 + "strict": schema.get("strict", False),
846 + }
847 + },
848 + None,
849 + )
850 + if format_type:
851 + return {"format": {"type": format_type}}, None
852 + return response_format, None
853 +
854 + @staticmethod
855 + def normalize_reasoning(reasoning: Any) -> Any:
856 + if isinstance(reasoning, dict):
857 + normalized = dict(reasoning)
858 + if "effort" in normalized:
859 + effort = _normalize_reasoning_effort(normalized.get("effort"))
860 + if effort is None:
861 + normalized.pop("effort", None)
862 + else:
863 + normalized["effort"] = effort
864 + return normalized or None
865 + if reasoning is None:
866 + return None
867 + effort = _normalize_reasoning_effort(reasoning)
868 + return {"effort": effort} if effort is not None else None
869 +
870 + @classmethod
871 + def parse_response(cls, response: Any) -> ChatChunk:
872 + response_delta = cls.output_text(response)
873 + reasoning_delta = cls.reasoning_text(response)
874 + if not response_delta:
875 + response_delta = cls.function_calls_text(response)
876 + return {"reasoning_delta": reasoning_delta, "response_delta": response_delta}
877 +
878 + @classmethod
879 + def parse_event(cls, event: Any) -> ChatChunk:
880 + return ResponsesEventParser().parse(event)
881 +
882 + @classmethod
883 + def output_text(cls, response: Any) -> str:
884 + output_text = _get_value(response, "output_text")
885 + if isinstance(output_text, str):
886 + return output_text
887 +
888 + pieces: list[str] = []
889 + for item in _as_list(_get_value(response, "output")):
890 + if _get_value(item, "type") != "message":
891 + continue
892 + for block in _as_list(_get_value(item, "content")):
893 + block_type = _get_value(block, "type")
894 + if block_type in {"output_text", "text"}:
895 + text = _get_value(block, "text")
896 + if isinstance(text, str):
897 + pieces.append(text)
898 + elif block_type == "refusal":
899 + refusal = _get_value(block, "refusal")
900 + if isinstance(refusal, str):
901 + pieces.append(refusal)
902 + return "".join(pieces)
903 +
904 + @staticmethod
905 + def reasoning_text(response: Any) -> str:
906 + pieces: list[str] = []
907 + for item in _as_list(_get_value(response, "output")):
908 + if _get_value(item, "type") != "reasoning":
909 + continue
910 + for block in _as_list(_get_value(item, "summary")):
911 + text = _get_value(block, "text") or _get_value(block, "reasoning")
912 + if isinstance(text, str):
913 + pieces.append(text)
914 + return "".join(pieces)
915 +
916 + @classmethod
917 + def function_calls_text(cls, response: Any) -> str:
918 + calls = [
919 + cls.function_call_object(item)
920 + for item in _as_list(_get_value(response, "output"))
921 + ]
922 + calls = [call for call in calls if call]
923 + if not calls:
924 + return ""
925 + if len(calls) == 1:
926 + return json.dumps(calls[0])
927 + return json.dumps(
928 + {"tool_name": "parallel_tool_calls", "tool_args": {"calls": calls}}
929 + )
930 +
931 + @classmethod
932 + def function_call_text(cls, item: Any) -> str:
933 + call = cls.function_call_object(item)
934 + if not call:
935 + return ""
936 + return json.dumps(call)
937 +
938 + @staticmethod
939 + def function_call_object(item: Any) -> dict[str, Any]:
940 + if _get_value(item, "type") != "function_call":
941 + return {}
942 + name = _get_value(item, "name")
943 + if not name:
944 + return {}
945 + raw_arguments = _get_value(item, "arguments") or "{}"
946 + if isinstance(raw_arguments, str):
947 + try:
948 + args = json.loads(raw_arguments or "{}")
949 + except Exception:
950 + args = {"arguments": raw_arguments}
951 + elif isinstance(raw_arguments, dict):
952 + args = raw_arguments
953 + else:
954 + args = {"arguments": raw_arguments}
955 + if not isinstance(args, dict):
956 + args = {"arguments": args}
957 + return {
958 + "tool_name": str(name),
959 + "tool_args": args,
960 + }
961 +
962 +
963 +class ResponsesEventParser:
964 + """Stateful parser for Responses streaming events."""
965 +
966 + def __init__(self) -> None:
967 + self.function_calls: dict[str, dict[str, Any]] = {}
968 + self.output_index_keys: dict[str, str] = {}
969 + self.emitted_function_calls: set[str] = set()
970 + self.seen_response_delta = False
971 + self.seen_reasoning_delta = False
972 + self.completed_response: Any = None
973 +
974 + def parse(self, event: Any) -> ChatChunk:
975 + event_type = _get_value(event, "type") or ""
976 + response_delta = ""
977 + reasoning_delta = ""
978 +
979 + if event_type in {
980 + "response.output_text.delta",
981 + "response.refusal.delta",
982 + "response.text.delta",
983 + }:
984 + response_delta = str(_get_value(event, "delta") or "")
985 + elif event_type in {
986 + "response.reasoning_summary_text.delta",
987 + "response.reasoning_text.delta",
988 + }:
989 + reasoning_delta = str(_get_value(event, "delta") or "")
990 + elif event_type == "response.output_item.added":
991 + self._remember_function_call(_get_value(event, "item"), event)
992 + elif event_type == "response.function_call_arguments.delta":
993 + self._append_function_call_arguments(event)
994 + elif event_type == "response.function_call_arguments.done":
995 + response_delta = self._complete_function_call(event)
996 + elif event_type == "response.output_item.done":
997 + response_delta = self._complete_output_item(_get_value(event, "item"), event)
998 + elif event_type == "response.completed":
999 + response_delta, reasoning_delta = self._complete_response(event)
1000 + elif event_type == "response.failed":
1001 + raise RuntimeError(self._response_error_message(event))
1002 + elif event_type == "error":
1003 + error = _get_value(event, "error")
1004 + message = _get_value(error, "message") or error
1005 + raise RuntimeError(str(message))
1006 +
1007 + if response_delta:
1008 + self.seen_response_delta = True
1009 + if reasoning_delta:
1010 + self.seen_reasoning_delta = True
1011 +
1012 + return {"reasoning_delta": reasoning_delta, "response_delta": response_delta}
1013 +
1014 + def _remember_function_call(self, item: Any, event: Any) -> str:
1015 + if _get_value(item, "type") != "function_call":
1016 + return ""
1017 + key = self._event_key(event, item)
1018 + if not key:
1019 + return ""
1020 + current = self.function_calls.get(key, {})
1021 + merged = {**current, **_object_to_dict(item)}
1022 + self.function_calls[key] = merged
1023 + output_index = _get_value(event, "output_index")
1024 + if output_index is not None:
1025 + self.output_index_keys[str(output_index)] = key
1026 + return key
1027 +
1028 + def _append_function_call_arguments(self, event: Any) -> None:
1029 + key = self._event_key(event)
1030 + if not key:
1031 + return
1032 + current = self.function_calls.setdefault(key, {"type": "function_call"})
1033 + current["arguments"] = str(current.get("arguments") or "") + str(
1034 + _get_value(event, "delta") or ""
1035 + )
1036 +
1037 + def _complete_function_call(self, event: Any) -> str:
1038 + key = self._event_key(event)
1039 + if not key:
1040 + return ""
1041 + current = self.function_calls.setdefault(key, {"type": "function_call"})
1042 + if _get_value(event, "arguments") is not None:
1043 + current["arguments"] = _get_value(event, "arguments")
1044 + if _get_value(event, "name"):
1045 + current["name"] = _get_value(event, "name")
1046 + return self._emit_function_call(key, current)
1047 +
1048 + def _complete_output_item(self, item: Any, event: Any) -> str:
1049 + key = self._remember_function_call(item, event)
1050 + if not key:
1051 + return ""
1052 + return self._emit_function_call(key, self.function_calls[key])
1053 +
1054 + def _complete_response(self, event: Any) -> tuple[str, str]:
1055 + self.completed_response = _get_value(event, "response")
1056 + if self.seen_response_delta or self.emitted_function_calls:
1057 + return "", ""
1058 + parsed = ResponsesTransport.parse_response(self.completed_response)
1059 + if self.seen_reasoning_delta:
1060 + parsed["reasoning_delta"] = ""
1061 + return parsed["response_delta"], parsed["reasoning_delta"]
1062 +
1063 + def _emit_function_call(self, key: str, item: Any) -> str:
1064 + if key in self.emitted_function_calls:
1065 + return ""
1066 + text = ResponsesTransport.function_call_text(item)
1067 + if text:
1068 + self.emitted_function_calls.add(key)
1069 + return text
1070 +
1071 + def _event_key(self, event: Any, item: Any = None) -> str:
1072 + key = _get_value(event, "item_id") or _get_value(item, "id")
1073 + if key:
1074 + return str(key)
1075 + output_index = _get_value(event, "output_index")
1076 + if output_index is not None:
1077 + output_key = self.output_index_keys.get(str(output_index))
1078 + if output_key:
1079 + return output_key
1080 + return f"output:{output_index}"
1081 + return ""
1082 +
1083 + def _response_error_message(self, event: Any) -> str:
1084 + response = _get_value(event, "response") or {}
1085 + error = _get_value(response, "error") or _get_value(event, "error")
1086 + message = _get_value(error, "message") or error
1087 + return str(message or "Responses API request failed")
1088 +
1089 +
1090 +def clear_transport_capability_cache() -> None:
1091 + RESPONSES_UNSUPPORTED_CACHE.clear()
1092 + RESPONSES_STATE_UNSUPPORTED_CACHE.clear()
1093 + RESPONSES_BUILTIN_UNSUPPORTED_CACHE.clear()
1094 +
1095 +
1096 +def delete_stored_response_ids(
1097 + response_ids: list[str], **kwargs: Any
1098 +) -> list[tuple[str, str]]:
1099 + errors: list[tuple[str, str]] = []
1100 + for response_id in response_ids:
1101 + try:
1102 + delete_responses(response_id=response_id, **kwargs)
1103 + except Exception as exc:
1104 + errors.append((response_id, _exception_text(exc)))
1105 + return errors
1106 +
1107 +
1108 +async def adelete_stored_response_ids(
1109 + response_ids: list[str], **kwargs: Any
1110 +) -> list[tuple[str, str]]:
1111 + errors: list[tuple[str, str]] = []
1112 + for response_id in response_ids:
1113 + try:
1114 + await adelete_responses(response_id=response_id, **kwargs)
1115 + except Exception as exc:
1116 + errors.append((response_id, _exception_text(exc)))
1117 + return errors
1118 +
1119 +
1120 +def _coerce_bool(value: Any, default: bool = False) -> bool:
1121 + if value is None:
1122 + return default
1123 + if isinstance(value, bool):
1124 + return value
1125 + if isinstance(value, str):
1126 + normalized = value.strip().lower()
1127 + if normalized in {"1", "true", "yes", "on"}:
1128 + return True
1129 + if normalized in {"0", "false", "no", "off", "none"}:
1130 + return False
1131 + return bool(value)
1132 +
1133 +
1134 +def _responses_cache_key(model: str, kwargs: dict[str, Any]) -> str:
1135 + api_base = (
1136 + kwargs.get("api_base")
1137 + or kwargs.get("base_url")
1138 + or kwargs.get("api_base_url")
1139 + or ""
1140 + )
1141 + custom_provider = kwargs.get("custom_llm_provider") or ""
1142 + return "|".join(str(part) for part in (model, custom_provider, api_base))
1143 +
1144 +
1145 +def _drop_legacy_transport_kwargs(kwargs: dict[str, Any]) -> None:
1146 + kwargs.pop("a0_api_mode", None)
1147 + kwargs.pop("a0_responses_fallback", None)
1148 +
1149 +
1150 +def _drop_responses_only_kwargs(kwargs: dict[str, Any]) -> None:
1151 + kwargs.pop("responses_state", None)
1152 + kwargs.pop("responses_delete_on_chat_delete", None)
1153 + kwargs.pop("responses_input_items", None)
1154 + kwargs.pop("responses_local_input_items", None)
1155 + kwargs.pop("previous_response_id", None)
1156 + kwargs.pop("_a0_responses_builtin_downgrades", None)
1157 +
1158 +
1159 +def _drop_internal_transport_kwargs(kwargs: dict[str, Any]) -> None:
1160 + _drop_legacy_transport_kwargs(kwargs)
1161 + kwargs.pop("a0_explicit_prompt_caching", None)
1162 + kwargs.pop("a0_responses_function_tools", None)
1163 + kwargs.pop("responses_builtin_tools", None)
1164 + _drop_responses_only_kwargs(kwargs)
1165 +
1166 +
1167 +def _normalize_responses_state(value: Any) -> str:
1168 + normalized = str(value or RESPONSES_STATE_PROVIDER).strip().lower()
1169 + return normalized if normalized in RESPONSES_STATES else RESPONSES_STATE_PROVIDER
1170 +
1171 +
1172 +def _filter_unsupported_builtin_tools(kwargs: dict[str, Any], cache_key: str) -> None:
1173 + unsupported = RESPONSES_BUILTIN_UNSUPPORTED_CACHE.get(cache_key)
1174 + if not unsupported:
1175 + return
1176 + tools = kwargs.get("responses_builtin_tools")
1177 + if not isinstance(tools, list):
1178 + return
1179 + filtered = []
1180 + downgraded = []
1181 + for tool in tools:
1182 + tool_type = _response_tool_type(tool)
1183 + if tool_type in unsupported:
1184 + downgraded.append(tool_type)
1185 + continue
1186 + filtered.append(tool)
1187 + if downgraded:
1188 + kwargs["responses_builtin_tools"] = filtered
1189 + kwargs["_a0_responses_builtin_downgrades"] = sorted(set(downgraded))
1190 +
1191 +
1192 +def _builtin_tool_types(tools: Any) -> set[str]:
1193 + return {
1194 + tool_type
1195 + for tool in _as_list(tools)
1196 + if (tool_type := _response_tool_type(tool))
1197 + }
1198 +
1199 +
1200 +def _response_tool_type(tool: Any) -> str:
1201 + if isinstance(tool, str):
1202 + return tool
1203 + if isinstance(tool, dict):
1204 + return str(tool.get("type") or "")
1205 + return ""
1206 +
1207 +
1208 +def apply_chat_prompt_cache_markers(
1209 + messages: list[dict[str, Any]],
1210 + *,
1211 + model: str = "",
1212 + kwargs: dict[str, Any] | None = None,
1213 +) -> list[dict[str, Any]]:
1214 + if not _supports_cache_control_markers(model, kwargs or {}):
1215 + return [dict(message) for message in messages]
1216 +
1217 + prepared = [_strip_message_cache_control(message) for message in messages]
1218 + for index in _prompt_cache_message_indexes(prepared):
1219 + prepared[index] = _message_with_cache_control(prepared[index])
1220 + return prepared
1221 +
1222 +
1223 +def _prompt_cache_message_indexes(messages: list[dict[str, Any]]) -> list[int]:
1224 + indexes: list[int] = []
1225 +
1226 + leading_context: list[int] = []
1227 + for index, message in enumerate(messages):
1228 + role = str(message.get("role") or "")
1229 + if role in {"system", "developer"}:
1230 + leading_context.append(index)
1231 + continue
1232 + break
1233 + if leading_context:
1234 + indexes.append(leading_context[-1])
1235 +
1236 + user_indexes = [
1237 + index
1238 + for index, message in enumerate(messages)
1239 + if str(message.get("role") or "") == "user"
1240 + ]
1241 + indexes.extend(user_indexes[-2:])
1242 +
1243 + deduplicated: list[int] = []
1244 + for index in indexes:
1245 + if index not in deduplicated:
1246 + deduplicated.append(index)
1247 + return deduplicated[:3]
1248 +
1249 +
1250 +def _strip_message_cache_control(message: dict[str, Any]) -> dict[str, Any]:
1251 + result = dict(message)
1252 + result.pop("cache_control", None)
1253 + return result
1254 +
1255 +
1256 +def _message_with_cache_control(message: dict[str, Any]) -> dict[str, Any]:
1257 + result = dict(message)
1258 + result["content"] = _content_with_cache_control(result.get("content", ""))
1259 + return result
1260 +
1261 +
1262 +def _content_with_cache_control(content: Any) -> Any:
1263 + marker = _cache_control_marker()
1264 + if isinstance(content, list):
1265 + blocks = [_copy_content_block(block) for block in content]
1266 + if not blocks:
1267 + return [{"type": "text", "text": "", "cache_control": marker}]
1268 + for index in range(len(blocks) - 1, -1, -1):
1269 + block = blocks[index]
1270 + if isinstance(block, dict):
1271 + block["cache_control"] = marker
1272 + return blocks
1273 + if isinstance(block, str):
1274 + blocks[index] = {
1275 + "type": "text",
1276 + "text": block,
1277 + "cache_control": marker,
1278 + }
1279 + return blocks
1280 + return blocks
1281 +
1282 + if isinstance(content, dict):
1283 + block = dict(content)
1284 + block["cache_control"] = marker
1285 + return block
1286 +
1287 + return [
1288 + {
1289 + "type": "text",
1290 + "text": _content_to_text(content),
1291 + "cache_control": marker,
1292 + }
1293 + ]
1294 +
1295 +
1296 +def _copy_content_block(block: Any) -> Any:
1297 + if isinstance(block, dict):
1298 + return dict(block)
1299 + if isinstance(block, list):
1300 + return [_copy_content_block(item) for item in block]
1301 + return block
1302 +
1303 +
1304 +def _apply_tool_cache_control(kwargs: dict[str, Any]) -> None:
1305 + tools = kwargs.get("tools")
1306 + if not isinstance(tools, list) or not tools or _has_cache_control(tools):
1307 + return
1308 +
1309 + prepared = [dict(tool) if isinstance(tool, dict) else tool for tool in tools]
1310 + for index in range(len(prepared) - 1, -1, -1):
1311 + tool = prepared[index]
1312 + if not isinstance(tool, dict):
1313 + continue
1314 + if tool.get("type") == "function" and isinstance(tool.get("function"), dict):
1315 + function = dict(tool["function"])
1316 + function["cache_control"] = _cache_control_marker()
1317 + tool = dict(tool)
1318 + tool["function"] = function
1319 + prepared[index] = tool
1320 + else:
1321 + tool = dict(tool)
1322 + tool["cache_control"] = _cache_control_marker()
1323 + prepared[index] = tool
1324 + kwargs["tools"] = prepared
1325 + return
1326 +
1327 +
1328 +def _cache_control_marker() -> dict[str, str]:
1329 + return {"type": "ephemeral"}
1330 +
1331 +
1332 +def _should_preserve_cache_control_on_chat(
1333 + model: str,
1334 + kwargs: dict[str, Any],
1335 + messages: list[dict[str, Any]],
1336 +) -> bool:
1337 + if not (_has_cache_control(messages) or _has_cache_control(kwargs.get("tools"))):
1338 + return False
1339 + return not _is_native_responses_provider(model, kwargs)
1340 +
1341 +
1342 +def _is_native_responses_provider(model: str, kwargs: dict[str, Any]) -> bool:
1343 + api_base = _api_base(kwargs)
1344 + if "openrouter.ai" in api_base or "anthropic.com" in api_base:
1345 + return False
1346 +
1347 + provider = _normalized_provider(model, kwargs)
1348 + return provider in {"openai", "azure", "azure_ai", "xai"}
1349 +
1350 +
1351 +def _supports_cache_control_markers(
1352 + model: str,
1353 + kwargs: dict[str, Any],
1354 +) -> bool:
1355 + api_base = _api_base(kwargs)
1356 + if "openrouter.ai" in api_base or "anthropic.com" in api_base:
1357 + return True
1358 + provider = _normalized_provider(model, kwargs)
1359 + return provider in CACHE_CONTROL_PROMPT_PROVIDERS
1360 +
1361 +
1362 +def _is_openai_prompt_cache_provider(model: str, kwargs: dict[str, Any]) -> bool:
1363 + api_base = _api_base(kwargs)
1364 + provider = _normalized_provider(model, kwargs)
1365 +
1366 + if api_base:
1367 + if "api.openai.com" in api_base:
1368 + return provider in {"", "openai"}
1369 + if "openai.azure.com" in api_base:
1370 + return provider in {"", "azure", "openai"}
1371 + return False
1372 +
1373 + if not provider and str(model):
1374 + provider = "openai"
1375 + return provider in OPENAI_PROMPT_CACHE_PROVIDERS
1376 +
1377 +
1378 +def _api_base(kwargs: dict[str, Any]) -> str:
1379 + return str(
1380 + kwargs.get("api_base")
1381 + or kwargs.get("base_url")
1382 + or kwargs.get("api_base_url")
1383 + or ""
1384 + ).lower()
1385 +
1386 +
1387 +def _normalized_provider(model: str, kwargs: dict[str, Any]) -> str:
1388 + provider = str(kwargs.get("custom_llm_provider") or "").strip().lower()
1389 + if not provider and "/" in str(model):
1390 + provider = str(model).split("/", 1)[0].strip().lower()
1391 + return provider.replace("-", "_")
1392 +
1393 +
1394 +def _prepare_openai_prompt_cache_params(
1395 + request: dict[str, Any],
1396 + messages: list[dict[str, Any]],
1397 + *,
1398 + model: str = "",
1399 +) -> None:
1400 + if "prompt_cache_key" in request:
1401 + return
1402 +
1403 + sanitized_messages = _without_cache_control(messages)
1404 + sanitized_request = _without_cache_control(request)
1405 + prompt_cache_key = _default_prompt_cache_key(
1406 + model,
1407 + sanitized_messages if isinstance(sanitized_messages, list) else messages,
1408 + sanitized_request if isinstance(sanitized_request, dict) else request,
1409 + )
1410 + if prompt_cache_key:
1411 + request["prompt_cache_key"] = prompt_cache_key
1412 +
1413 +
1414 +def _default_prompt_cache_key(
1415 + model: str,
1416 + messages: list[dict[str, Any]],
1417 + request: dict[str, Any],
1418 +) -> str:
1419 + material = _prompt_cache_key_material(messages, request)
1420 + if not material:
1421 + return ""
1422 + digest = hashlib.sha256(
1423 + json.dumps(
1424 + {
1425 + "model": model,
1426 + "material": material,
1427 + },
1428 + sort_keys=True,
1429 + default=str,
1430 + separators=(",", ":"),
1431 + ).encode("utf-8")
1432 + ).hexdigest()[:32]
1433 + return f"a0-{digest}"
1434 +
1435 +
1436 +def _prompt_cache_key_material(
1437 + messages: list[dict[str, Any]],
1438 + request: dict[str, Any],
1439 +) -> dict[str, Any]:
1440 + material: dict[str, Any] = {}
1441 +
1442 + leading_messages: list[dict[str, Any]] = []
1443 + for message in messages:
1444 + role = str(message.get("role") or "")
1445 + if role not in {"system", "developer"}:
1446 + break
1447 + leading_messages.append(
1448 + {
1449 + "role": role,
1450 + "content": message.get("content"),
1451 + }
1452 + )
1453 + if leading_messages:
1454 + material["messages"] = leading_messages
1455 +
1456 + if request.get("instructions"):
1457 + material["instructions"] = request["instructions"]
1458 + if request.get("prompt"):
1459 + material["prompt"] = request["prompt"]
1460 + if request.get("tools"):
1461 + material["tools"] = request["tools"]
1462 +
1463 + return material
1464 +
1465 +
1466 +def _has_cache_control(value: Any) -> bool:
1467 + if isinstance(value, dict):
1468 + if value.get("cache_control") is not None:
1469 + return True
1470 + return any(_has_cache_control(item) for item in value.values())
1471 + if isinstance(value, list):
1472 + return any(_has_cache_control(item) for item in value)
1473 + return False
1474 +
1475 +
1476 +def _without_cache_control(value: Any) -> Any:
1477 + if isinstance(value, dict):
1478 + return {
1479 + key: _without_cache_control(item)
1480 + for key, item in value.items()
1481 + if key != "cache_control"
1482 + }
1483 + if isinstance(value, list):
1484 + return [_without_cache_control(item) for item in value]
1485 + return value
1486 +
1487 +
1488 +def _object_to_dict(obj: Any) -> dict[str, Any]:
1489 + if isinstance(obj, dict):
1490 + return dict(obj)
1491 + if hasattr(obj, "model_dump"):
1492 + dumped = obj.model_dump()
1493 + return dict(dumped) if isinstance(dumped, dict) else {}
1494 + if hasattr(obj, "dict"):
1495 + dumped = obj.dict()
1496 + return dict(dumped) if isinstance(dumped, dict) else {}
1497 + return {}
1498 +
1499 +
1500 +def _normalize_reasoning_effort(effort: Any) -> str | None:
1501 + if isinstance(effort, str):
1502 + normalized = effort.strip().lower()
1503 + else:
1504 + normalized = str(effort).strip().lower() if effort is not None else ""
1505 + if normalized in RESPONSES_REASONING_EFFORTS:
1506 + return normalized
1507 + if normalized in NO_REASONING_EFFORT_ALIASES:
1508 + return None
1509 + return RESPONSES_REASONING_FALLBACK_EFFORT
1510 +
1511 +
1512 +def _is_responses_reasoning_effort_error(exc: Exception) -> bool:
1513 + text = _exception_text(exc).lower()
1514 + return (
1515 + "response.reasoning.effort" in text
1516 + and "minimal" in text
1517 + and "high" in text
1518 + and "none" in text
1519 + )
1520 +
1521 +
1522 +def _is_responses_not_supported_error(exc: Exception) -> bool:
1523 + text = _exception_text(exc).lower()
1524 + if any(marker in text for marker in ("429", "too many requests", "rate limit")):
1525 + return False
1526 + if "/v1/responses" in text and any(
1527 + marker in text for marker in ("404", "not found")
1528 + ):
1529 + return True
1530 + return any(
1531 + marker in text
1532 + for marker in (
1533 + "responses api",
1534 + "does not support responses",
1535 + "not support responses",
1536 + "unsupportedparamserror",
1537 + "does not support parameters",
1538 + "no 'tools' defined while 'tool_choice' is specified",
1539 + )
1540 + )
1541 +
1542 +
1543 +def _is_responses_state_unsupported_error(exc: Exception) -> bool:
1544 + text = _exception_text(exc).lower()
1545 + if any(marker in text for marker in ("429", "too many requests", "rate limit")):
1546 + return False
1547 + if "404" in text and "/v1/responses/" in text:
1548 + return True
1549 + return any(
1550 + marker in text
1551 + for marker in (
1552 + "previous_response_id",
1553 + "store",
1554 + "stored response",
1555 + "response not found",
1556 + "no response found",
1557 + "does not support response storage",
1558 + "doesn't support response storage",
1559 + "response storage is not supported",
1560 + "state is not supported",
1561 + )
1562 + )
1563 +
1564 +
1565 +def _is_responses_builtin_tool_error(exc: Exception) -> bool:
1566 + text = _exception_text(exc).lower()
1567 + if any(marker in text for marker in ("429", "too many requests", "rate limit")):
1568 + return False
1569 + return any(
1570 + marker in text
1571 + for marker in (
1572 + "unsupported tool",
1573 + "unsupported tools",
1574 + "invalid tool",
1575 + "tool type",
1576 + "tools[",
1577 + "web_search",
1578 + "file_search",
1579 + "code_interpreter",
1580 + "image_generation",
1581 + "computer_use_preview",
1582 + "mcp",
1583 + )
1584 + )
1585 +
1586 +
1587 +def _exception_text(exc: Exception | None) -> str:
1588 + if exc is None:
1589 + return ""
1590 + parts = [exc.__class__.__name__, str(exc)]
1591 + cause = getattr(exc, "__cause__", None)
1592 + context = getattr(exc, "__context__", None)
1593 + if cause is not None:
1594 + parts.append(str(cause))
1595 + if context is not None and context is not cause:
1596 + parts.append(str(context))
1597 + return "\n".join(parts)
1598 +
1599 +
1600 +def _close_sync_stream(stream: Any) -> None:
1601 + for method_name in ("close", "aclose"):
1602 + close = getattr(stream, method_name, None)
1603 + if close is None:
1604 + continue
1605 + result = close()
1606 + if inspect.isawaitable(result):
1607 + result.close()
1608 + return
1609 +
1610 +
1611 +async def _close_async_stream(stream: Any) -> None:
1612 + for method_name in ("aclose", "close"):
1613 + close = getattr(stream, method_name, None)
1614 + if close is None:
1615 + continue
1616 + result = close()
1617 + if inspect.isawaitable(result):
1618 + await result
1619 + return
1620 +
1621 +
1622 +def _without_stream_kwarg(kwargs: dict[str, Any]) -> dict[str, Any]:
1623 + kwargs.pop("stream", None)
1624 + return kwargs
1625 +
1626 +
1627 +def _first_choice(chunk: Any) -> Any:
1628 + choices = _get_value(chunk, "choices") or []
1629 + return choices[0] if choices else {}
1630 +
1631 +
1632 +def _get_value(obj: Any, key: str) -> Any:
1633 + if isinstance(obj, dict):
1634 + return obj.get(key)
1635 + return getattr(obj, key, None)
1636 +
1637 +
1638 +def _as_list(value: Any) -> list[Any]:
1639 + return value if isinstance(value, list) else []
1640 +
1641 +
1642 +def _has_tools(tools: Any) -> bool:
1643 + if isinstance(tools, list):
1644 + return bool(tools)
1645 + return bool(tools)
1646 +
1647 +
1648 +def _has_chunk_delta(chunk: ChatChunk) -> bool:
1649 + return bool(chunk.get("response_delta") or chunk.get("reasoning_delta"))
1650 +
1651 +
1652 +def _has_real_content(content: Any) -> bool:
1653 + if content == "empty":
1654 + return False
1655 + if isinstance(content, str):
1656 + return bool(content.strip())
1657 + if isinstance(content, list):
1658 + return len(content) > 0
1659 + return content is not None
1660 +
1661 +
1662 +def _content_to_text(content: Any) -> str:
1663 + content = images.prepare_content(content)
1664 + if isinstance(content, str):
1665 + return content
1666 + if isinstance(content, list):
1667 + pieces: list[str] = []
1668 + for item in content:
1669 + if isinstance(item, str):
1670 + pieces.append(item)
1671 + elif isinstance(item, dict):
1672 + text = item.get("text")
1673 + if isinstance(text, str):
1674 + pieces.append(text)
1675 + return "\n".join(pieces)
1676 + return "" if content is None else str(content)
helpers/llm_result.py new
+311
@@ -0,0 +1,311 @@
1 +from __future__ import annotations
2 +
3 +from dataclasses import dataclass, field
4 +import json
5 +from typing import Any
6 +
7 +
8 +RESPONSE_METADATA_KEY = "responses"
9 +LOCAL_FUNCTION_TOOL_TYPES = {"function_call"}
10 +TEXT_OUTPUT_TYPES = {"message"}
11 +REASONING_OUTPUT_TYPES = {"reasoning"}
12 +
13 +
14 +@dataclass
15 +class ResponseItem:
16 + type: str
17 + data: dict[str, Any] = field(default_factory=dict)
18 +
19 + @classmethod
20 + def from_any(cls, item: Any) -> "ResponseItem":
21 + data = object_to_dict(item)
22 + return cls(type=str(data.get("type") or ""), data=data)
23 +
24 + def to_dict(self) -> dict[str, Any]:
25 + return dict(self.data)
26 +
27 +
28 +@dataclass
29 +class ResponseFunctionCall:
30 + name: str
31 + arguments: dict[str, Any]
32 + call_id: str
33 + item_id: str = ""
34 + raw: dict[str, Any] = field(default_factory=dict)
35 +
36 + @classmethod
37 + def from_item(cls, item: ResponseItem) -> "ResponseFunctionCall | None":
38 + if item.type != "function_call":
39 + return None
40 + name = str(item.data.get("name") or "")
41 + if not name:
42 + return None
43 + return cls(
44 + name=name,
45 + arguments=parse_arguments(item.data.get("arguments")),
46 + call_id=str(item.data.get("call_id") or item.data.get("id") or ""),
47 + item_id=str(item.data.get("id") or ""),
48 + raw=dict(item.data),
49 + )
50 +
51 +
52 +@dataclass
53 +class LLMResult:
54 + response: str = ""
55 + reasoning: str = ""
56 + response_id: str = ""
57 + previous_response_id: str = ""
58 + input_items: list[dict[str, Any]] = field(default_factory=list)
59 + output_items: list[ResponseItem] = field(default_factory=list)
60 + provider_model_key: str = ""
61 + mode: str = "responses"
62 + state: str = "provider"
63 + usage: dict[str, Any] = field(default_factory=dict)
64 + raw: dict[str, Any] = field(default_factory=dict)
65 + capability: dict[str, Any] = field(default_factory=dict)
66 +
67 + @classmethod
68 + def from_dict(cls, data: dict[str, Any] | None) -> "LLMResult":
69 + data = data or {}
70 + return cls(
71 + response=str(data.get("response") or ""),
72 + reasoning=str(data.get("reasoning") or ""),
73 + response_id=str(data.get("response_id") or ""),
74 + previous_response_id=str(data.get("previous_response_id") or ""),
75 + input_items=list(data.get("input_items") or []),
76 + output_items=[
77 + ResponseItem.from_any(item) for item in data.get("output_items") or []
78 + ],
79 + provider_model_key=str(data.get("provider_model_key") or ""),
80 + mode=str(data.get("mode") or "responses"),
81 + state=str(data.get("state") or "provider"),
82 + usage=object_to_dict(data.get("usage") or {}),
83 + raw=object_to_dict(data.get("raw") or {}),
84 + capability=object_to_dict(data.get("capability") or {}),
85 + )
86 +
87 + @classmethod
88 + def from_response(
89 + cls,
90 + response: Any,
91 + *,
92 + input_items: list[dict[str, Any]] | None = None,
93 + previous_response_id: str = "",
94 + provider_model_key: str = "",
95 + mode: str = "responses",
96 + state: str = "provider",
97 + capability: dict[str, Any] | None = None,
98 + ) -> "LLMResult":
99 + raw = object_to_dict(response)
100 + output_items = [ResponseItem.from_any(item) for item in as_list(raw.get("output"))]
101 + result = cls(
102 + response_id=str(raw.get("id") or ""),
103 + previous_response_id=str(
104 + raw.get("previous_response_id") or previous_response_id or ""
105 + ),
106 + input_items=list(input_items or []),
107 + output_items=output_items,
108 + provider_model_key=provider_model_key,
109 + mode=mode,
110 + state=state,
111 + usage=object_to_dict(raw.get("usage") or {}),
112 + raw=raw,
113 + capability=dict(capability or {}),
114 + )
115 + result.response = output_text(raw, output_items)
116 + result.reasoning = reasoning_text(output_items)
117 + if not result.response and result.function_calls:
118 + result.response = result.function_calls_text()
119 + return result
120 +
121 + @classmethod
122 + def from_chat(
123 + cls,
124 + *,
125 + response: str,
126 + reasoning: str = "",
127 + input_items: list[dict[str, Any]] | None = None,
128 + provider_model_key: str = "",
129 + capability: dict[str, Any] | None = None,
130 + ) -> "LLMResult":
131 + output_items = []
132 + if response:
133 + output_items.append(
134 + ResponseItem(
135 + type="message",
136 + data={
137 + "type": "message",
138 + "role": "assistant",
139 + "content": [{"type": "output_text", "text": response}],
140 + },
141 + )
142 + )
143 + if reasoning:
144 + output_items.insert(
145 + 0,
146 + ResponseItem(
147 + type="reasoning",
148 + data={
149 + "type": "reasoning",
150 + "summary": [{"type": "summary_text", "text": reasoning}],
151 + },
152 + ),
153 + )
154 + return cls(
155 + response=response,
156 + reasoning=reasoning,
157 + input_items=list(input_items or []),
158 + output_items=output_items,
159 + provider_model_key=provider_model_key,
160 + mode="chat_completions",
161 + state="off",
162 + capability=dict(capability or {}),
163 + )
164 +
165 + @property
166 + def function_calls(self) -> list[ResponseFunctionCall]:
167 + calls: list[ResponseFunctionCall] = []
168 + for item in self.output_items:
169 + call = ResponseFunctionCall.from_item(item)
170 + if call:
171 + calls.append(call)
172 + return calls
173 +
174 + @property
175 + def builtin_items(self) -> list[ResponseItem]:
176 + return [
177 + item
178 + for item in self.output_items
179 + if item.type
180 + and item.type not in TEXT_OUTPUT_TYPES
181 + and item.type not in REASONING_OUTPUT_TYPES
182 + and item.type not in LOCAL_FUNCTION_TOOL_TYPES
183 + ]
184 +
185 + def function_calls_text(self) -> str:
186 + calls = [
187 + {"tool_name": call.name, "tool_args": call.arguments}
188 + for call in self.function_calls
189 + ]
190 + if not calls:
191 + return ""
192 + if len(calls) == 1:
193 + return json.dumps(calls[0])
194 + return json.dumps(
195 + {"tool_name": "parallel_tool_calls", "tool_args": {"calls": calls}}
196 + )
197 +
198 + def to_dict(self) -> dict[str, Any]:
199 + return {
200 + "response": self.response,
201 + "reasoning": self.reasoning,
202 + "response_id": self.response_id,
203 + "previous_response_id": self.previous_response_id,
204 + "input_items": self.input_items,
205 + "output_items": [item.to_dict() for item in self.output_items],
206 + "provider_model_key": self.provider_model_key,
207 + "mode": self.mode,
208 + "state": self.state,
209 + "usage": self.usage,
210 + "raw": self.raw,
211 + "capability": self.capability,
212 + }
213 +
214 + def metadata(self) -> dict[str, Any]:
215 + return {RESPONSE_METADATA_KEY: self.to_dict()}
216 +
217 +
218 +def function_call_output_item(
219 + call_id: str,
220 + output: str,
221 + *,
222 + acknowledged_safety_checks: list[dict[str, Any]] | None = None,
223 +) -> dict[str, Any]:
224 + item: dict[str, Any] = {
225 + "type": "function_call_output",
226 + "call_id": str(call_id or ""),
227 + "output": output,
228 + }
229 + if acknowledged_safety_checks:
230 + item["acknowledged_safety_checks"] = acknowledged_safety_checks
231 + return item
232 +
233 +
234 +def metadata_from_llm_result(result: LLMResult | None) -> dict[str, Any]:
235 + return result.metadata() if result else {}
236 +
237 +
238 +def result_from_metadata(metadata: dict[str, Any] | None) -> LLMResult | None:
239 + if not isinstance(metadata, dict):
240 + return None
241 + data = metadata.get(RESPONSE_METADATA_KEY)
242 + if not isinstance(data, dict):
243 + return None
244 + return LLMResult.from_dict(data)
245 +
246 +
247 +def object_to_dict(obj: Any) -> dict[str, Any]:
248 + if isinstance(obj, dict):
249 + return dict(obj)
250 + if hasattr(obj, "model_dump"):
251 + dumped = obj.model_dump()
252 + return dict(dumped) if isinstance(dumped, dict) else {}
253 + if hasattr(obj, "dict"):
254 + dumped = obj.dict()
255 + return dict(dumped) if isinstance(dumped, dict) else {}
256 + return {}
257 +
258 +
259 +def as_list(value: Any) -> list[Any]:
260 + return value if isinstance(value, list) else []
261 +
262 +
263 +def output_text(raw: dict[str, Any], output_items: list[ResponseItem]) -> str:
264 + direct = raw.get("output_text")
265 + if isinstance(direct, str):
266 + return direct
267 + pieces: list[str] = []
268 + for item in output_items:
269 + if item.type != "message":
270 + continue
271 + for block in as_list(item.data.get("content")):
272 + if not isinstance(block, dict):
273 + continue
274 + block_type = block.get("type")
275 + if block_type in {"output_text", "text", "input_text"}:
276 + text = block.get("text")
277 + if isinstance(text, str):
278 + pieces.append(text)
279 + elif block_type == "refusal":
280 + refusal = block.get("refusal")
281 + if isinstance(refusal, str):
282 + pieces.append(refusal)
283 + return "".join(pieces)
284 +
285 +
286 +def reasoning_text(output_items: list[ResponseItem]) -> str:
287 + pieces: list[str] = []
288 + for item in output_items:
289 + if item.type != "reasoning":
290 + continue
291 + for block in as_list(item.data.get("summary")):
292 + if isinstance(block, dict):
293 + text = block.get("text") or block.get("reasoning")
294 + if isinstance(text, str):
295 + pieces.append(text)
296 + elif isinstance(block, str):
297 + pieces.append(block)
298 + return "".join(pieces)
299 +
300 +
301 +def parse_arguments(raw_arguments: Any) -> dict[str, Any]:
302 + if isinstance(raw_arguments, dict):
303 + return raw_arguments
304 + if isinstance(raw_arguments, str):
305 + try:
306 + parsed = json.loads(raw_arguments or "{}")
307 + except Exception:
308 + parsed = {"arguments": raw_arguments}
309 + else:
310 + parsed = {"arguments": raw_arguments}
311 + return parsed if isinstance(parsed, dict) else {"arguments": parsed}
helpers/persist_chat.py
+69
@@ -4,6 +4,7 @@ from typing import Any
4 import uuid
5 from agent import Agent, AgentConfig, AgentContext, AgentContextType
6 from helpers import files, history
7 +from helpers.litellm_transport import delete_stored_response_ids
8 from helpers.localization import Localization
9 import json
10 from initialize import initialize_agent
@@ -118,6 +119,7 @@ def export_json_chat(context: AgentContext):
119
120 def remove_chat(ctxid):
121 """Remove a chat or task context"""
122 + _delete_provider_responses_for_chat(ctxid)
123 path = get_chat_folder_path(ctxid)
124 files.delete_dir(path)
125
@@ -324,3 +326,70 @@ def _safe_json_serialize(obj, **kwargs):
326 return False
327
328 return json.dumps(obj, default=serializer, **kwargs)
329 +
330 +
331 +def _delete_provider_responses_for_chat(ctxid: str) -> None:
332 + try:
333 + data = json.loads(files.read_file(_get_chat_file_path(ctxid)))
334 + except Exception:
335 + return
336 + if _responses_delete_disabled(data):
337 + return
338 + response_ids = _collect_response_ids(data)
339 + if not response_ids:
340 + return
341 + delete_stored_response_ids(response_ids)
342 +
343 +
344 +def _responses_delete_disabled(data: dict[str, Any]) -> bool:
345 + if data.get("responses_delete_on_chat_delete") is False:
346 + return True
347 + context_data = data.get("data")
348 + if isinstance(context_data, dict) and context_data.get("responses_delete_on_chat_delete") is False:
349 + return True
350 + for agent_data in data.get("agents", []) or []:
351 + if not isinstance(agent_data, dict):
352 + continue
353 + state = agent_data.get("data")
354 + if isinstance(state, dict) and state.get("responses_delete_on_chat_delete") is False:
355 + return True
356 + return False
357 +
358 +
359 +def _collect_response_ids(data: Any) -> list[str]:
360 + found: list[str] = []
361 + seen: set[str] = set()
362 +
363 + def add(value: Any) -> None:
364 + response_id = str(value or "").strip()
365 + if response_id and response_id not in seen:
366 + seen.add(response_id)
367 + found.append(response_id)
368 +
369 + def walk(obj: Any) -> None:
370 + if isinstance(obj, dict):
371 + state = obj.get(Agent.DATA_NAME_RESPONSES_STATE)
372 + if isinstance(state, dict):
373 + add(state.get("response_id"))
374 + for response_id in state.get("response_ids") or []:
375 + add(response_id)
376 +
377 + metadata = obj.get("metadata")
378 + if isinstance(metadata, dict):
379 + responses = metadata.get("responses")
380 + if isinstance(responses, dict):
381 + add(responses.get("response_id"))
382 +
383 + for value in obj.values():
384 + walk(value)
385 + elif isinstance(obj, list):
386 + for value in obj:
387 + walk(value)
388 + elif isinstance(obj, str) and '"response_id"' in obj:
389 + try:
390 + walk(json.loads(obj))
391 + except Exception:
392 + return
393 +
394 + walk(data)
395 + return found
helpers/responses_tools.py new
+231
@@ -0,0 +1,231 @@
1 +from __future__ import annotations
2 +
3 +import hashlib
4 +import json
5 +import os
6 +import re
7 +from typing import Any
8 +
9 +from helpers import files, subagents
10 +
11 +
12 +FUNCTION_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
13 +TOOL_PROMPT_PREFIX = "agent.system.tool."
14 +TOOL_PROMPT_SUFFIX = ".md"
15 +MAX_TOOL_DESCRIPTION_CHARS = 1024
16 +
17 +
18 +def build_responses_function_tools(agent: Any) -> tuple[list[dict[str, Any]], dict[str, str]]:
19 + """Build permissive Responses function tools from A0 tool prompts and MCP schemas."""
20 +
21 + tools: list[dict[str, Any]] = []
22 + name_map: dict[str, str] = {}
23 +
24 + for tool_name, prompt in _local_tool_prompts(agent):
25 + native_name = _native_tool_name(tool_name)
26 + name_map[native_name] = tool_name
27 + tools.append(
28 + {
29 + "type": "function",
30 + "name": native_name,
31 + "description": _description_from_prompt(prompt, fallback=tool_name),
32 + "parameters": _schema_from_prompt(prompt),
33 + }
34 + )
35 +
36 + for tool_name, tool in _mcp_tools(agent):
37 + native_name = _native_tool_name(tool_name)
38 + name_map[native_name] = tool_name
39 + tools.append(
40 + {
41 + "type": "function",
42 + "name": native_name,
43 + "description": _truncate(str(tool.get("description") or tool_name)),
44 + "parameters": _schema_from_any(tool.get("input_schema")),
45 + }
46 + )
47 +
48 + return _dedupe_tools(tools), name_map
49 +
50 +
51 +def original_tool_name(native_name: str, name_map: dict[str, str] | None) -> str:
52 + if not name_map:
53 + return native_name
54 + return name_map.get(native_name, native_name)
55 +
56 +
57 +def _local_tool_prompts(agent: Any) -> list[tuple[str, str]]:
58 + prompt_dirs = subagents.get_paths(agent, "prompts")
59 + tool_files = files.get_unique_filenames_in_dirs(
60 + prompt_dirs, f"{TOOL_PROMPT_PREFIX}*{TOOL_PROMPT_SUFFIX}"
61 + )
62 + result: list[tuple[str, str]] = []
63 + for tool_file in tool_files:
64 + basename = os.path.basename(tool_file)
65 + tool_name = _tool_name_from_prompt_basename(basename)
66 + if not tool_name:
67 + continue
68 + try:
69 + prompt = agent.read_prompt(basename)
70 + except Exception:
71 + try:
72 + prompt = files.read_file(tool_file)
73 + except Exception:
74 + prompt = ""
75 + result.append((tool_name, prompt))
76 + return result
77 +
78 +
79 +def _mcp_tools(agent: Any) -> list[tuple[str, dict[str, Any]]]:
80 + try:
81 + import helpers.mcp_handler as mcp_helper
82 +
83 + raw_tools = mcp_helper.MCPConfig.get_instance().get_tools()
84 + except Exception:
85 + return []
86 +
87 + result: list[tuple[str, dict[str, Any]]] = []
88 + for entry in raw_tools or []:
89 + if not isinstance(entry, dict):
90 + continue
91 + for tool_name, tool in entry.items():
92 + if isinstance(tool, dict):
93 + result.append((str(tool_name), tool))
94 + return result
95 +
96 +
97 +def _tool_name_from_prompt_basename(basename: str) -> str:
98 + if not basename.startswith(TOOL_PROMPT_PREFIX) or not basename.endswith(TOOL_PROMPT_SUFFIX):
99 + return ""
100 + name = basename[len(TOOL_PROMPT_PREFIX) : -len(TOOL_PROMPT_SUFFIX)]
101 + if not name or name in {"tools", "tools_vision"}:
102 + return ""
103 + return name
104 +
105 +
106 +def _native_tool_name(tool_name: str) -> str:
107 + if FUNCTION_NAME_PATTERN.fullmatch(tool_name):
108 + return tool_name
109 + slug = re.sub(r"[^A-Za-z0-9_-]+", "_", tool_name).strip("_")
110 + digest = hashlib.sha1(tool_name.encode("utf-8")).hexdigest()[:8]
111 + native = f"{slug[:52]}_{digest}" if slug else f"a0_tool_{digest}"
112 + return native[:64]
113 +
114 +
115 +def _description_from_prompt(prompt: str, *, fallback: str) -> str:
116 + lines: list[str] = []
117 + in_fence = False
118 + for raw_line in (prompt or "").splitlines():
119 + line = raw_line.strip()
120 + if line.startswith("```"):
121 + in_fence = not in_fence
122 + continue
123 + if in_fence or not line:
124 + continue
125 + if line.startswith("#"):
126 + line = line.lstrip("#").strip()
127 + if line.lower() == fallback.lower():
128 + continue
129 + lines.append(line)
130 + if sum(len(part) for part in lines) >= MAX_TOOL_DESCRIPTION_CHARS:
131 + break
132 + description = " ".join(lines).strip() or fallback
133 + return _truncate(description)
134 +
135 +
136 +def _schema_from_prompt(prompt: str) -> dict[str, Any]:
137 + schema = _schema_from_embedded_json(prompt)
138 + if schema:
139 + return schema
140 + return _schema_from_args_line(prompt)
141 +
142 +
143 +def _schema_from_embedded_json(prompt: str) -> dict[str, Any]:
144 + marker = "Input schema for tool_args:"
145 + index = (prompt or "").find(marker)
146 + if index == -1:
147 + return {}
148 + tail = prompt[index + len(marker) :].strip()
149 + match = re.search(r"\{(?:[^{}]|(?R))*\}", tail, flags=re.DOTALL) if hasattr(re, "VERSION1") else None
150 + candidate = match.group(0) if match else _balanced_json_object(tail)
151 + if not candidate:
152 + return {}
153 + try:
154 + return _schema_from_any(json.loads(candidate))
155 + except Exception:
156 + return {}
157 +
158 +
159 +def _schema_from_args_line(prompt: str) -> dict[str, Any]:
160 + properties: dict[str, Any] = {}
161 + for line in (prompt or "").splitlines():
162 + normalized = line.strip()
163 + if "args:" not in normalized.lower() and "argument:" not in normalized.lower():
164 + continue
165 + for name in re.findall(r"`([A-Za-z_][A-Za-z0-9_-]*)`", normalized):
166 + properties.setdefault(name, {"type": "string"})
167 + if properties:
168 + return {
169 + "type": "object",
170 + "properties": properties,
171 + "additionalProperties": True,
172 + }
173 + return _permissive_schema()
174 +
175 +
176 +def _schema_from_any(schema: Any) -> dict[str, Any]:
177 + if isinstance(schema, dict):
178 + normalized = dict(schema)
179 + normalized.setdefault("type", "object")
180 + normalized.setdefault("additionalProperties", True)
181 + return normalized
182 + return _permissive_schema()
183 +
184 +
185 +def _permissive_schema() -> dict[str, Any]:
186 + return {"type": "object", "additionalProperties": True}
187 +
188 +
189 +def _balanced_json_object(text: str) -> str:
190 + start = text.find("{")
191 + if start == -1:
192 + return ""
193 + depth = 0
194 + in_string = False
195 + escape = False
196 + for index, char in enumerate(text[start:], start=start):
197 + if in_string:
198 + if escape:
199 + escape = False
200 + elif char == "\\":
201 + escape = True
202 + elif char == '"':
203 + in_string = False
204 + continue
205 + if char == '"':
206 + in_string = True
207 + elif char == "{":
208 + depth += 1
209 + elif char == "}":
210 + depth -= 1
211 + if depth == 0:
212 + return text[start : index + 1]
213 + return ""
214 +
215 +
216 +def _dedupe_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
217 + seen: set[str] = set()
218 + result: list[dict[str, Any]] = []
219 + for tool in tools:
220 + name = str(tool.get("name") or "")
221 + if not name or name in seen:
222 + continue
223 + seen.add(name)
224 + result.append(tool)
225 + return result
226 +
227 +
228 +def _truncate(text: str) -> str:
229 + if len(text) <= MAX_TOOL_DESCRIPTION_CHARS:
230 + return text
231 + return text[: MAX_TOOL_DESCRIPTION_CHARS - 3].rstrip() + "..."
models.py
+211 -134
@@ -14,19 +14,19 @@ from typing import (
14 TypedDict,
15 )
16
17 -from litellm import completion, acompletion, embedding
17 +from litellm import embedding
18 import litellm
19 import openai
20 -from litellm.types.utils import ModelResponse
20
21 from helpers import dotenv
23 -from helpers import settings, dirty_json, images
22 +from helpers import settings, images
23 from helpers.dotenv import load_dotenv
24 from helpers.providers import ModelType as ProviderModelType, get_provider_config
25 from helpers.rate_limiter import RateLimiter
26 from helpers.tokens import approximate_tokens
28 -from helpers import dirty_json
27 from helpers.extension import extensible # extensible: allows plugins to intercept get_api_key()
28 +from helpers.litellm_transport import LiteLLMTransport, ResponsesTransport
29 +from helpers.llm_result import LLMResult
30
31 from langchain_core.language_models.chat_models import SimpleChatModel
32 from langchain_core.outputs.chat_generation import ChatGenerationChunk
@@ -155,6 +155,7 @@ class ChatChunk(TypedDict):
155 response_delta: str
156 reasoning_delta: str
157
158 +
159 class ChatGenerationResult:
160 """Chat generation result object"""
161 def __init__(self, chunk: ChatChunk|None = None):
@@ -434,14 +435,6 @@ class LiteLLMChatWrapper(SimpleChatModel):
435
436 result.append(message_dict)
437
437 - if explicit_caching and result:
438 - if result[0]["role"] == "system":
439 - result[0]["cache_control"] = {"type": "ephemeral"}
440 - for i in range(len(result) - 1, -1, -1):
441 - if result[i]["role"] == "assistant":
442 - result[i]["cache_control"] = {"type": "ephemeral"}
443 - break
444 -
438 return result
439
440 def _call(
@@ -451,24 +444,20 @@ class LiteLLMChatWrapper(SimpleChatModel):
444 run_manager: Optional[CallbackManagerForLLMRun] = None,
445 **kwargs: Any,
446 ) -> str:
454 - import asyncio
455 -
447 configure_litellm()
448 msgs = self._convert_messages(messages)
449
450 # Apply rate limiting if configured
451 apply_rate_limiter_sync(self.a0_model_conf, str(msgs))
452
462 - # Call the model
463 - call_kwargs = _without_stream_kwarg(
464 - _merge_litellm_call_kwargs(self.kwargs, kwargs)
465 - )
466 - resp = completion(
467 - model=self.model_name, messages=msgs, stop=stop, **call_kwargs
453 + call_kwargs = _merge_litellm_call_kwargs(self.kwargs, kwargs)
454 + transport = LiteLLMTransport(
455 + model=self.model_name,
456 + messages=msgs,
457 + kwargs=call_kwargs,
458 + stop=stop,
459 )
469 -
470 - # Parse output
471 - parsed = _parse_chunk(resp)
460 + parsed = transport.complete()
461 output = ChatGenerationResult(parsed).output()
462 return output["response_delta"]
463
@@ -479,8 +468,6 @@ class LiteLLMChatWrapper(SimpleChatModel):
468 run_manager: Optional[CallbackManagerForLLMRun] = None,
469 **kwargs: Any,
470 ) -> Iterator[ChatGenerationChunk]:
482 - import asyncio
483 -
471 configure_litellm()
472 msgs = self._convert_messages(messages)
473
@@ -488,22 +475,15 @@ class LiteLLMChatWrapper(SimpleChatModel):
475 apply_rate_limiter_sync(self.a0_model_conf, str(msgs))
476
477 result = ChatGenerationResult()
491 - call_kwargs = _without_stream_kwarg(
492 - _merge_litellm_call_kwargs(self.kwargs, kwargs)
493 - )
494 -
495 - for chunk in completion(
478 + call_kwargs = _merge_litellm_call_kwargs(self.kwargs, kwargs)
479 + transport = LiteLLMTransport(
480 model=self.model_name,
481 messages=msgs,
498 - stream=True,
482 + kwargs=call_kwargs,
483 stop=stop,
500 - **call_kwargs,
501 - ):
502 - # parse chunk
503 - parsed = _parse_chunk(chunk) # chunk parsing
504 - output = result.add_chunk(parsed) # chunk processing
505 -
506 - # Only yield chunks with non-None content
484 + )
485 + for parsed in transport.stream():
486 + output = result.add_chunk(parsed)
487 if output["response_delta"]:
488 yield ChatGenerationChunk(
489 message=AIMessageChunk(content=output["response_delta"])
@@ -523,23 +503,15 @@ class LiteLLMChatWrapper(SimpleChatModel):
503 await apply_rate_limiter(self.a0_model_conf, str(msgs))
504
505 result = ChatGenerationResult()
526 - call_kwargs = _without_stream_kwarg(
527 - _merge_litellm_call_kwargs(self.kwargs, kwargs)
528 - )
529 -
530 - response = await acompletion(
506 + call_kwargs = _merge_litellm_call_kwargs(self.kwargs, kwargs)
507 + transport = LiteLLMTransport(
508 model=self.model_name,
509 messages=msgs,
533 - stream=True,
510 + kwargs=call_kwargs,
511 stop=stop,
535 - **call_kwargs,
512 )
537 - async for chunk in response: # type: ignore
538 - # parse chunk
539 - parsed = _parse_chunk(chunk) # chunk parsing
540 - output = result.add_chunk(parsed) # chunk processing
541 -
542 - # Only yield chunks with non-None content
513 + async for parsed in transport.astream():
514 + output = result.add_chunk(parsed)
515 if output["response_delta"]:
516 yield ChatGenerationChunk(
517 message=AIMessageChunk(content=output["response_delta"])
@@ -579,12 +551,19 @@ class LiteLLMChatWrapper(SimpleChatModel):
551 )
552
553 # Prepare call kwargs and retry config (strip A0-only params before calling LiteLLM)
582 - call_kwargs: dict[str, Any] = _without_stream_kwarg(
583 - _merge_litellm_call_kwargs(self.kwargs, kwargs)
554 + call_kwargs: dict[str, Any] = _merge_litellm_call_kwargs(
555 + self.kwargs, kwargs
556 )
557 + if explicit_caching:
558 + call_kwargs["a0_explicit_prompt_caching"] = True
559 max_retries: int = int(call_kwargs.pop("a0_retry_attempts", 2))
560 retry_delay_s: float = float(call_kwargs.pop("a0_retry_delay_seconds", 1.5))
561 stream = reasoning_callback is not None or response_callback is not None or tokens_callback is not None
562 + transport = LiteLLMTransport(
563 + model=self.model_name,
564 + messages=msgs_conv,
565 + kwargs=call_kwargs,
566 + )
567
568 # results
569 result = ChatGenerationResult()
@@ -593,60 +572,45 @@ class LiteLLMChatWrapper(SimpleChatModel):
572 while True:
573 got_any_chunk = False
574 try:
596 - # call model
597 - _completion = await acompletion(
598 - model=self.model_name,
599 - messages=msgs_conv,
600 - stream=stream,
601 - **call_kwargs,
602 - )
603 -
575 if stream:
605 - # iterate over chunks
576 stop_response: str | None = None
607 - try:
608 - async for chunk in _completion: # type: ignore
609 - got_any_chunk = True
610 - # parse chunk
611 - parsed = _parse_chunk(chunk)
612 - output = result.add_chunk(parsed)
613 -
614 - # collect reasoning delta and call callbacks
615 - if output["reasoning_delta"]:
616 - if reasoning_callback:
617 - await reasoning_callback(output["reasoning_delta"], result.reasoning)
618 - if tokens_callback:
619 - await tokens_callback(
620 - output["reasoning_delta"],
621 - approximate_tokens(output["reasoning_delta"]),
622 - )
623 - # Add output tokens to rate limiter if configured
624 - if limiter:
625 - limiter.add(output=approximate_tokens(output["reasoning_delta"]))
626 - # collect response delta and call callbacks
627 - if output["response_delta"]:
628 - if response_callback:
629 - stop_response = await response_callback(
630 - output["response_delta"], result.response
631 - )
632 - if tokens_callback:
633 - await tokens_callback(
634 - output["response_delta"],
635 - approximate_tokens(output["response_delta"]),
636 - )
637 - # Add output tokens to rate limiter if configured
638 - if limiter:
639 - limiter.add(output=approximate_tokens(output["response_delta"]))
640 - if stop_response is not None:
641 - result.response = stop_response
642 - break
643 - finally:
644 - if stop_response is not None and hasattr(_completion, "aclose"):
645 - await _completion.aclose() # type: ignore[attr-defined]
577 + async for parsed in transport.astream():
578 + got_any_chunk = True
579 + output = result.add_chunk(parsed)
580 +
581 + # collect reasoning delta and call callbacks
582 + if output["reasoning_delta"]:
583 + if reasoning_callback:
584 + await reasoning_callback(output["reasoning_delta"], result.reasoning)
585 + if tokens_callback:
586 + await tokens_callback(
587 + output["reasoning_delta"],
588 + approximate_tokens(output["reasoning_delta"]),
589 + )
590 + # Add output tokens to rate limiter if configured
591 + if limiter:
592 + limiter.add(output=approximate_tokens(output["reasoning_delta"]))
593 + # collect response delta and call callbacks
594 + if output["response_delta"]:
595 + if response_callback:
596 + stop_response = await response_callback(
597 + output["response_delta"], result.response
598 + )
599 + if tokens_callback:
600 + await tokens_callback(
601 + output["response_delta"],
602 + approximate_tokens(output["response_delta"]),
603 + )
604 + # Add output tokens to rate limiter if configured
605 + if limiter:
606 + limiter.add(output=approximate_tokens(output["response_delta"]))
607 + if stop_response is not None:
608 + result.response = stop_response
609 + break
610
611 # non-stream response
612 else:
649 - parsed = _parse_chunk(_completion)
613 + parsed = await transport.acomplete()
614 output = result.add_chunk(parsed)
615 if limiter:
616 if output["response_delta"]:
@@ -666,6 +630,151 @@ class LiteLLMChatWrapper(SimpleChatModel):
630 attempt += 1
631 await asyncio.sleep(retry_delay_s)
632
633 + async def unified_turn(
634 + self,
635 + system_message="",
636 + user_message="",
637 + messages: List[BaseMessage] | None = None,
638 + response_callback: Callable[[str, str], Awaitable[str | None]] | None = None,
639 + reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,
640 + tokens_callback: Callable[[str, int], Awaitable[None]] | None = None,
641 + rate_limiter_callback: (
642 + Callable[[str, str, int, int], Awaitable[bool]] | None
643 + ) = None,
644 + explicit_caching: bool = False,
645 + **kwargs: Any,
646 + ) -> LLMResult:
647 + """Canonical internal LLM turn with Responses metadata.
648 +
649 + Public plugin-facing callers should keep using ``unified_call``. Core
650 + orchestration uses this method when it needs response ids, native output
651 + items, and state/capability metadata.
652 + """
653 +
654 + configure_litellm()
655 +
656 + if not messages:
657 + messages = []
658 + if system_message:
659 + messages.insert(0, SystemMessage(content=system_message))
660 + if user_message:
661 + messages.append(HumanMessage(content=user_message))
662 +
663 + msgs_conv = self._convert_messages(messages, explicit_caching=explicit_caching)
664 +
665 + limiter = await apply_rate_limiter(
666 + self.a0_model_conf, str(msgs_conv), rate_limiter_callback
667 + )
668 +
669 + call_kwargs: dict[str, Any] = _merge_litellm_call_kwargs(
670 + self.kwargs, kwargs
671 + )
672 + if explicit_caching:
673 + call_kwargs["a0_explicit_prompt_caching"] = True
674 + max_retries: int = int(call_kwargs.pop("a0_retry_attempts", 2))
675 + retry_delay_s: float = float(call_kwargs.pop("a0_retry_delay_seconds", 1.5))
676 + stream = (
677 + reasoning_callback is not None
678 + or response_callback is not None
679 + or tokens_callback is not None
680 + )
681 + transport = LiteLLMTransport(
682 + model=self.model_name,
683 + messages=msgs_conv,
684 + kwargs=call_kwargs,
685 + )
686 +
687 + result = ChatGenerationResult()
688 +
689 + attempt = 0
690 + while True:
691 + got_any_chunk = False
692 + try:
693 + if stream:
694 + stop_response: str | None = None
695 + async for parsed in transport.astream():
696 + got_any_chunk = True
697 + output = result.add_chunk(parsed)
698 +
699 + if output["reasoning_delta"]:
700 + if reasoning_callback:
701 + await reasoning_callback(
702 + output["reasoning_delta"], result.reasoning
703 + )
704 + if tokens_callback:
705 + await tokens_callback(
706 + output["reasoning_delta"],
707 + approximate_tokens(output["reasoning_delta"]),
708 + )
709 + if limiter:
710 + limiter.add(
711 + output=approximate_tokens(
712 + output["reasoning_delta"]
713 + )
714 + )
715 +
716 + if output["response_delta"]:
717 + if response_callback:
718 + stop_response = await response_callback(
719 + output["response_delta"], result.response
720 + )
721 + if tokens_callback:
722 + await tokens_callback(
723 + output["response_delta"],
724 + approximate_tokens(output["response_delta"]),
725 + )
726 + if limiter:
727 + limiter.add(
728 + output=approximate_tokens(
729 + output["response_delta"]
730 + )
731 + )
732 + if (
733 + stop_response is not None
734 + and not transport.policy.using_responses
735 + ):
736 + result.response = stop_response
737 + break
738 + if stop_response is not None:
739 + result.response = stop_response
740 + else:
741 + parsed = await transport.acomplete()
742 + output = result.add_chunk(parsed)
743 + if limiter:
744 + if output["response_delta"]:
745 + limiter.add(
746 + output=approximate_tokens(output["response_delta"])
747 + )
748 + if output["reasoning_delta"]:
749 + limiter.add(
750 + output=approximate_tokens(output["reasoning_delta"])
751 + )
752 +
753 + llm_result = transport.last_result or LLMResult.from_chat(
754 + response=result.output()["response_delta"],
755 + reasoning=result.output()["reasoning_delta"],
756 + input_items=ResponsesTransport.input_from_messages(msgs_conv),
757 + provider_model_key=self.model_name,
758 + capability=transport._capability_metadata(),
759 + )
760 + if result.output()["response_delta"]:
761 + llm_result.response = result.output()["response_delta"]
762 + if result.output()["reasoning_delta"]:
763 + llm_result.reasoning = result.output()["reasoning_delta"]
764 + return llm_result
765 +
766 + except Exception as e:
767 + import asyncio
768 +
769 + if (
770 + got_any_chunk
771 + or not _is_transient_litellm_error(e)
772 + or attempt >= max_retries
773 + ):
774 + raise
775 + attempt += 1
776 + await asyncio.sleep(retry_delay_s)
777 +
778
779 class LiteLLMEmbeddingWrapper(Embeddings):
780 model_name: str
@@ -820,38 +929,6 @@ def _get_litellm_embedding(
929 )
930
931
823 -def _parse_chunk(chunk: Any) -> ChatChunk:
824 - delta = chunk["choices"][0].get("delta", {})
825 - message = chunk["choices"][0].get("message", {}) or chunk["choices"][0].get(
826 - "model_extra", {}
827 - ).get("message", {})
828 - response_delta = (
829 - delta.get("content", "")
830 - if isinstance(delta, dict)
831 - else getattr(delta, "content", "")
832 - ) or (
833 - message.get("content", "")
834 - if isinstance(message, dict)
835 - else getattr(message, "content", "")
836 - ) or ""
837 - reasoning_delta = (
838 - delta.get("reasoning_content", "")
839 - if isinstance(delta, dict)
840 - else getattr(delta, "reasoning_content", "")
841 - ) or (
842 - message.get("reasoning_content", "")
843 - if isinstance(message, dict)
844 - else getattr(message, "reasoning_content", "")
845 - ) or ""
846 -
847 - return ChatChunk(reasoning_delta=reasoning_delta, response_delta=response_delta)
848 -
849 -
850 -def _without_stream_kwarg(kwargs: dict[str, Any]) -> dict[str, Any]:
851 - kwargs.pop("stream", None)
852 - return kwargs
853 -
854 -
932
933 def _adjust_call_args(provider_name: str, model_name: str, kwargs: dict):
934
tests/test_responses_architecture.py new
+385
@@ -0,0 +1,385 @@
1 +import sys
2 +from pathlib import Path
3 +
4 +import pytest
5 +from langchain_core.messages import HumanMessage
6 +
7 +
8 +PROJECT_ROOT = Path(__file__).resolve().parents[1]
9 +if str(PROJECT_ROOT) not in sys.path:
10 + sys.path.insert(0, str(PROJECT_ROOT))
11 +
12 +import models
13 +from agent import Agent, AgentConfig, LoopData
14 +from helpers import history, litellm_transport
15 +from helpers.log import Log
16 +from helpers.llm_result import LLMResult, result_from_metadata
17 +from helpers.persist_chat import _collect_response_ids
18 +from helpers.tool import Response
19 +
20 +
21 +@pytest.fixture(autouse=True)
22 +def _clear_transport_capability_cache():
23 + litellm_transport.clear_transport_capability_cache()
24 +
25 +
26 +class _AsyncEventStream:
27 + def __init__(self, events: list[dict]):
28 + self.events = events
29 + self.index = 0
30 + self.closed = False
31 +
32 + def __aiter__(self):
33 + return self
34 +
35 + async def __anext__(self):
36 + if self.index >= len(self.events):
37 + raise StopAsyncIteration
38 + event = self.events[self.index]
39 + self.index += 1
40 + return event
41 +
42 + async def aclose(self):
43 + self.closed = True
44 +
45 +
46 +def test_llm_result_round_trips_responses_metadata():
47 + result = LLMResult.from_response(
48 + {
49 + "id": "resp_123",
50 + "usage": {"input_tokens": 10},
51 + "output": [
52 + {"type": "reasoning", "summary": [{"text": "because"}]},
53 + {
54 + "type": "function_call",
55 + "id": "fc_1",
56 + "call_id": "call_1",
57 + "name": "lookup",
58 + "arguments": '{"q":"a0"}',
59 + },
60 + {
61 + "type": "web_search_call",
62 + "id": "ws_1",
63 + "status": "completed",
64 + },
65 + ],
66 + },
67 + input_items=[{"role": "user", "content": "question"}],
68 + previous_response_id="resp_prev",
69 + provider_model_key="openai/gpt-5.4",
70 + )
71 +
72 + loaded = result_from_metadata(result.metadata())
73 +
74 + assert loaded is not None
75 + assert loaded.response_id == "resp_123"
76 + assert loaded.previous_response_id == "resp_prev"
77 + assert loaded.function_calls[0].name == "lookup"
78 + assert loaded.function_calls[0].arguments == {"q": "a0"}
79 + assert loaded.builtin_items[0].type == "web_search_call"
80 +
81 +
82 +def test_history_serializes_metadata_and_migrates_old_messages():
83 + class DummyAgent:
84 + pass
85 +
86 + hist = history.History(DummyAgent())
87 + result = LLMResult.from_response(
88 + {"id": "resp_1", "output": [{"type": "message", "content": [{"type": "output_text", "text": "ok"}]}]},
89 + provider_model_key="openai/gpt-5.4",
90 + )
91 +
92 + message = hist.add_message(True, "ok", metadata=result.metadata())
93 + restored = history.deserialize_history(hist.serialize(), DummyAgent())
94 +
95 + restored_message = restored.all_messages()[0]
96 + assert restored_message.sequence == message.sequence
97 + assert result_from_metadata(restored_message.metadata).response_id == "resp_1"
98 +
99 + old = history.Message.from_dict({"_cls": "Message", "ai": False, "content": "old"}, restored)
100 + assert old.metadata == {}
101 + assert old.sequence == 0
102 +
103 +
104 +def test_responses_provider_state_uses_previous_response_and_new_items():
105 + new_items = [{"type": "function_call_output", "call_id": "call_1", "output": "done"}]
106 + local_items = [{"role": "user", "content": "full replay"}]
107 +
108 + request = litellm_transport.ResponsesTransport.from_chat(
109 + [{"role": "user", "content": "ignored while continuing provider state"}],
110 + {
111 + "previous_response_id": "resp_1",
112 + "responses_input_items": new_items,
113 + "responses_local_input_items": local_items,
114 + },
115 + model="openai/gpt-5.4",
116 + )
117 +
118 + assert request["store"] is True
119 + assert request["previous_response_id"] == "resp_1"
120 + assert request["input"] == new_items
121 +
122 + local_request = litellm_transport.ResponsesTransport.from_chat(
123 + [{"role": "user", "content": "ignored"}],
124 + {
125 + "responses_state": "local",
126 + "previous_response_id": "resp_1",
127 + "responses_input_items": new_items,
128 + "responses_local_input_items": local_items,
129 + },
130 + model="openai/gpt-5.4",
131 + )
132 +
133 + assert local_request["store"] is False
134 + assert "previous_response_id" not in local_request
135 + assert local_request["input"] == local_items
136 +
137 +
138 +@pytest.mark.asyncio
139 +async def test_transport_retries_provider_state_as_local_replay(monkeypatch):
140 + calls: list[dict] = []
141 +
142 + async def fake_aresponses(*args, **kwargs):
143 + calls.append(kwargs)
144 + if len(calls) == 1:
145 + raise RuntimeError("previous_response_id is not supported by this provider")
146 + return {
147 + "id": "resp_local",
148 + "output": [
149 + {
150 + "type": "message",
151 + "content": [{"type": "output_text", "text": "ok"}],
152 + }
153 + ],
154 + }
155 +
156 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
157 +
158 + transport = litellm_transport.LiteLLMTransport(
159 + model="openai/gpt-5.4",
160 + messages=[{"role": "user", "content": "new"}],
161 + kwargs={
162 + "previous_response_id": "resp_1",
163 + "responses_input_items": [{"role": "user", "content": "new"}],
164 + "responses_local_input_items": [{"role": "user", "content": "full"}],
165 + },
166 + )
167 +
168 + parsed = await transport.acomplete()
169 +
170 + assert parsed["response_delta"] == "ok"
171 + assert calls[0]["store"] is True
172 + assert calls[0]["previous_response_id"] == "resp_1"
173 + assert calls[1]["store"] is False
174 + assert "previous_response_id" not in calls[1]
175 + assert calls[1]["input"] == [{"role": "user", "content": "full"}]
176 + assert transport.last_result.response_id == "resp_local"
177 +
178 +
179 +@pytest.mark.asyncio
180 +async def test_transport_downgrades_unsupported_builtin_tools(monkeypatch):
181 + calls: list[dict] = []
182 +
183 + async def fake_aresponses(*args, **kwargs):
184 + calls.append(kwargs)
185 + if len(calls) == 1:
186 + raise RuntimeError("unsupported tool type: web_search")
187 + return {
188 + "id": "resp_no_builtin",
189 + "output": [
190 + {
191 + "type": "message",
192 + "content": [{"type": "output_text", "text": "ok"}],
193 + }
194 + ],
195 + }
196 +
197 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
198 +
199 + transport = litellm_transport.LiteLLMTransport(
200 + model="openai/gpt-5.4",
201 + messages=[{"role": "user", "content": "new"}],
202 + kwargs={"responses_builtin_tools": [{"type": "web_search"}]},
203 + )
204 +
205 + parsed = await transport.acomplete()
206 +
207 + assert parsed["response_delta"] == "ok"
208 + assert calls[0]["tools"] == [{"type": "web_search"}]
209 + assert "tools" not in calls[1]
210 + assert transport.last_result.capability["builtin_tool_downgrades"] == [
211 + "web_search"
212 + ]
213 +
214 + next_transport = litellm_transport.LiteLLMTransport(
215 + model="openai/gpt-5.4",
216 + messages=[{"role": "user", "content": "again"}],
217 + kwargs={"responses_builtin_tools": [{"type": "web_search"}]},
218 + )
219 + request = next_transport._responses_request(stream=False)
220 + assert "tools" not in request
221 +
222 +
223 +@pytest.mark.asyncio
224 +async def test_unified_turn_keeps_stream_open_to_capture_response_id(monkeypatch):
225 + stream = _AsyncEventStream(
226 + [
227 + {
228 + "type": "response.output_item.added",
229 + "output_index": 0,
230 + "item": {
231 + "type": "function_call",
232 + "id": "fc_1",
233 + "call_id": "call_1",
234 + "name": "lookup",
235 + "arguments": "",
236 + },
237 + },
238 + {
239 + "type": "response.function_call_arguments.done",
240 + "item_id": "fc_1",
241 + "output_index": 0,
242 + "name": "lookup",
243 + "arguments": '{"q":"a0"}',
244 + },
245 + {
246 + "type": "response.completed",
247 + "response": {
248 + "id": "resp_1",
249 + "output": [
250 + {
251 + "type": "function_call",
252 + "id": "fc_1",
253 + "call_id": "call_1",
254 + "name": "lookup",
255 + "arguments": '{"q":"a0"}',
256 + }
257 + ],
258 + },
259 + },
260 + ]
261 + )
262 +
263 + async def fake_aresponses(*args, **kwargs):
264 + return stream
265 +
266 + async def fake_rate_limiter(*args, **kwargs):
267 + return None
268 +
269 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
270 + monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
271 +
272 + wrapper = models.LiteLLMChatWrapper(
273 + model="test-model",
274 + provider="openai",
275 + model_config=None,
276 + )
277 +
278 + async def response_callback(chunk: str, full: str):
279 + return full
280 +
281 + result = await wrapper.unified_turn(
282 + messages=[HumanMessage(content="hi")],
283 + response_callback=response_callback,
284 + )
285 +
286 + assert stream.index == 3
287 + assert stream.closed is False
288 + assert result.response_id == "resp_1"
289 + assert result.function_calls[0].call_id == "call_1"
290 +
291 +
292 +def test_collect_response_ids_from_agent_state_and_history_metadata():
293 + payload = {
294 + "agents": [
295 + {
296 + "data": {
297 + "responses_state": {
298 + "response_id": "resp_latest",
299 + "response_ids": ["resp_old", "resp_latest"],
300 + }
301 + },
302 + "history": '{"current":{"messages":[{"metadata":{"responses":{"response_id":"resp_history"}}}]}}',
303 + }
304 + ]
305 + }
306 +
307 + assert _collect_response_ids(payload) == [
308 + "resp_latest",
309 + "resp_old",
310 + "resp_history",
311 + ]
312 +
313 +
314 +@pytest.mark.asyncio
315 +async def test_agent_executes_native_responses_function_call_and_records_output():
316 + class DummyContext:
317 + paused = False
318 + log = Log()
319 +
320 + def get_data(self, key, recursive=True):
321 + return None
322 +
323 + class DummyTool:
324 + name = "lookup"
325 + progress = ""
326 +
327 + def __init__(self, agent):
328 + self.agent = agent
329 +
330 + async def before_execution(self, **kwargs):
331 + self.args = kwargs
332 +
333 + async def execute(self, **kwargs):
334 + return Response(message=f"done:{kwargs['q']}", break_loop=False)
335 +
336 + async def after_execution(self, response):
337 + self.agent.hist_add_tool_result(
338 + self.name,
339 + response.message,
340 + **(response.additional or {}),
341 + )
342 +
343 + agent = object.__new__(Agent)
344 + agent.data = {Agent.DATA_NAME_RESPONSES_TOOL_NAME_MAP: {}}
345 + agent.context = DummyContext()
346 + agent.config = AgentConfig(mcp_servers="")
347 + agent.loop_data = LoopData()
348 + agent.history = history.History(agent)
349 + agent.intervention = None
350 + agent.agent_name = "A0"
351 + agent.number = 0
352 +
353 + def get_tool(**kwargs):
354 + return DummyTool(agent)
355 +
356 + agent.get_tool = get_tool
357 +
358 + result = LLMResult.from_response(
359 + {
360 + "id": "resp_1",
361 + "output": [
362 + {
363 + "type": "function_call",
364 + "id": "fc_1",
365 + "call_id": "call_1",
366 + "name": "lookup",
367 + "arguments": '{"q":"a0"}',
368 + }
369 + ],
370 + },
371 + provider_model_key="openai/gpt-5.4",
372 + )
373 +
374 + assert await Agent.process_llm_result_tools(agent, result) is None
375 +
376 + recorded = agent.history.all_messages()[0]
377 + metadata = result_from_metadata(recorded.metadata)
378 + assert recorded.content["tool_result"] == "done:a0"
379 + assert metadata.input_items == [
380 + {
381 + "type": "function_call_output",
382 + "call_id": "call_1",
383 + "output": "done:a0",
384 + }
385 + ]
tests/test_stream_tool_early_stop.py
+774 -6
@@ -2,6 +2,7 @@ import sys
2 from pathlib import Path
3
4 import pytest
5 +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
6
7
8 PROJECT_ROOT = Path(__file__).resolve().parents[1]
@@ -10,16 +11,27 @@ if str(PROJECT_ROOT) not in sys.path:
11
12 import models
13 from helpers import extract_tools
14 +from helpers import litellm_transport
15 +
16 +
17 +@pytest.fixture(autouse=True)
18 +def _clear_transport_capability_cache():
19 + litellm_transport.clear_transport_capability_cache()
20
21
22 def _chunk(content: str) -> dict:
23 return {"choices": [{"delta": {"content": content}, "message": {}}]}
24
25
26 +def _response_event(delta: str) -> dict:
27 + return {"type": "response.output_text.delta", "delta": delta}
28 +
29 +
30 class _AsyncChunkStream:
31 def __init__(self, chunks: list[dict]):
32 self._chunks = chunks
33 self.index = 0
34 + self.closed = False
35
36 def __aiter__(self):
37 return self
@@ -31,6 +43,24 @@ class _AsyncChunkStream:
43 self.index += 1
44 return chunk
45
46 + async def aclose(self):
47 + self.closed = True
48 +
49 +
50 +class _FailingAsyncChunkStream:
51 + def __init__(self, exc: Exception):
52 + self.exc = exc
53 + self.closed = False
54 +
55 + def __aiter__(self):
56 + return self
57 +
58 + async def __anext__(self):
59 + raise self.exc
60 +
61 + async def aclose(self):
62 + self.closed = True
63 +
64
65 def test_extract_json_root_string_returns_canonical_snapshot():
66 text = (
@@ -94,22 +124,24 @@ def test_litellm_global_kwargs_merge_defaults_and_config(monkeypatch):
124 async def test_unified_call_stops_after_canonical_root_snapshot(monkeypatch):
125 stream = _AsyncChunkStream(
126 [
97 - _chunk(
127 + {"type": "response.created"},
128 + _response_event(
129 '{"tool_name":"response","tool_args":{"text":"hello"}} trailing text'
130 ),
100 - _chunk(" unreachable"),
131 + _response_event(" unreachable"),
132 ]
133 )
134
104 - async def fake_acompletion(*args, **kwargs):
135 + async def fake_aresponses(*args, **kwargs):
136 assert kwargs["stream"] is True
106 - assert kwargs["drop_params"] is True
137 + assert kwargs["input"] == ""
138 + assert kwargs["store"] is True
139 return stream
140
141 async def fake_rate_limiter(*args, **kwargs):
142 return None
143
112 - monkeypatch.setattr(models, "acompletion", fake_acompletion)
144 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
145 monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
146 monkeypatch.setattr(
147 models.settings,
@@ -139,6 +171,742 @@ async def test_unified_call_stops_after_canonical_root_snapshot(monkeypatch):
171
172 assert response == '{"tool_name":"response","tool_args":{"text":"hello"}}'
173 assert reasoning == ""
142 - assert stream.index == 1
174 + assert stream.index == 2
175 + assert stream.closed is True
176 assert len(seen) == 1
177 assert seen[0][1] == '{"tool_name":"response","tool_args":{"text":"hello"}} trailing text'
178 +
179 +
180 +@pytest.mark.asyncio
181 +async def test_unified_call_closes_responses_stream_when_callback_raises(monkeypatch):
182 + stream = _AsyncChunkStream([_response_event("interrupt me")])
183 +
184 + class ExpectedIntervention(Exception):
185 + pass
186 +
187 + async def fake_aresponses(*args, **kwargs):
188 + assert kwargs["stream"] is True
189 + return stream
190 +
191 + async def fake_rate_limiter(*args, **kwargs):
192 + return None
193 +
194 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
195 + monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
196 +
197 + wrapper = models.LiteLLMChatWrapper(
198 + model="test-model",
199 + provider="openai",
200 + model_config=None,
201 + )
202 +
203 + async def response_callback(chunk: str, full: str):
204 + raise ExpectedIntervention()
205 +
206 + with pytest.raises(ExpectedIntervention):
207 + await wrapper.unified_call(
208 + messages=[],
209 + response_callback=response_callback,
210 + )
211 +
212 + assert stream.closed is True
213 +
214 +
215 +@pytest.mark.asyncio
216 +async def test_chat_completions_escape_hatch_still_uses_acompletion(monkeypatch):
217 + stream = _AsyncChunkStream([_chunk("hello")])
218 + calls: list[str] = []
219 +
220 + async def fake_acompletion(*args, **kwargs):
221 + calls.append("chat")
222 + assert kwargs["stream"] is True
223 + assert "a0_api_mode" not in kwargs
224 + return stream
225 +
226 + async def fake_aresponses(*args, **kwargs):
227 + raise AssertionError("Responses path should not be used")
228 +
229 + async def fake_rate_limiter(*args, **kwargs):
230 + return None
231 +
232 + monkeypatch.setattr(litellm_transport, "acompletion", fake_acompletion)
233 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
234 + monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
235 +
236 + wrapper = models.LiteLLMChatWrapper(
237 + model="test-model",
238 + provider="openai",
239 + model_config=None,
240 + a0_api_mode="chat_completions",
241 + )
242 +
243 + async def response_callback(chunk: str, full: str):
244 + return None
245 +
246 + response, reasoning = await wrapper.unified_call(
247 + messages=[],
248 + response_callback=response_callback,
249 + )
250 +
251 + assert response == "hello"
252 + assert reasoning == ""
253 + assert calls == ["chat"]
254 +
255 +
256 +@pytest.mark.asyncio
257 +async def test_unified_call_retries_responses_with_high_reasoning(monkeypatch):
258 + validation_error = ValueError(
259 + "1 validation error for ResponseCreatedEvent\n"
260 + "response.reasoning.effort\n"
261 + "Input should be 'minimal', 'low', 'medium' or 'high' "
262 + "[type=literal_error, input_value='none', input_type=str]"
263 + )
264 + failing_stream = _FailingAsyncChunkStream(validation_error)
265 + working_stream = _AsyncChunkStream([_response_event("ok")])
266 + calls: list[dict] = []
267 +
268 + async def fake_aresponses(*args, **kwargs):
269 + calls.append(kwargs)
270 + if len(calls) == 1:
271 + return failing_stream
272 + return working_stream
273 +
274 + async def fake_rate_limiter(*args, **kwargs):
275 + return None
276 +
277 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
278 + monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
279 +
280 + wrapper = models.LiteLLMChatWrapper(
281 + model="gpt-5.4",
282 + provider="openai",
283 + model_config=None,
284 + )
285 +
286 + async def response_callback(chunk: str, full: str):
287 + return None
288 +
289 + response, reasoning = await wrapper.unified_call(
290 + messages=[],
291 + response_callback=response_callback,
292 + )
293 +
294 + assert response == "ok"
295 + assert reasoning == ""
296 + assert failing_stream.closed is True
297 + assert len(calls) == 2
298 + assert "reasoning" not in calls[0]
299 + assert calls[1]["reasoning"] == {"effort": "high"}
300 +
301 +
302 +@pytest.mark.asyncio
303 +async def test_unified_call_falls_back_to_chat_when_responses_endpoint_missing(
304 + monkeypatch,
305 +):
306 + calls: list[str] = []
307 +
308 + async def fake_aresponses(*args, **kwargs):
309 + calls.append("responses")
310 + raise RuntimeError(
311 + "Client error '404 Not Found' for url "
312 + "'https://llm.agent-zero.ai/v1/responses'"
313 + )
314 +
315 + async def fake_acompletion(*args, **kwargs):
316 + calls.append("chat")
317 + assert kwargs["stream"] is True
318 + assert kwargs["drop_params"] is True
319 + assert "tool_choice" not in kwargs
320 + assert "parallel_tool_calls" not in kwargs
321 + return _AsyncChunkStream([_chunk("fallback")])
322 +
323 + async def fake_rate_limiter(*args, **kwargs):
324 + return None
325 +
326 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
327 + monkeypatch.setattr(litellm_transport, "acompletion", fake_acompletion)
328 + monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
329 +
330 + wrapper = models.LiteLLMChatWrapper(
331 + model="claude-opus-4.7",
332 + provider="openai",
333 + model_config=None,
334 + tool_choice="auto",
335 + parallel_tool_calls=True,
336 + )
337 +
338 + async def response_callback(chunk: str, full: str):
339 + return None
340 +
341 + response, reasoning = await wrapper.unified_call(
342 + messages=[],
343 + response_callback=response_callback,
344 + )
345 +
346 + assert response == "fallback"
347 + assert reasoning == ""
348 + assert calls == ["responses", "chat"]
349 +
350 + response, reasoning = await wrapper.unified_call(
351 + messages=[],
352 + response_callback=response_callback,
353 + )
354 +
355 + assert response == "fallback"
356 + assert reasoning == ""
357 + assert calls == ["responses", "chat", "chat"]
358 +
359 +
360 +@pytest.mark.asyncio
361 +async def test_unified_call_preserves_cache_control_with_chat_for_non_native_responses(
362 + monkeypatch,
363 +):
364 + calls: list[str] = []
365 +
366 + async def fake_aresponses(*args, **kwargs):
367 + raise AssertionError("cache_control should keep Anthropic-family calls on chat")
368 +
369 + async def fake_acompletion(*args, **kwargs):
370 + calls.append("chat")
371 + assert kwargs["stream"] is True
372 + messages = kwargs["messages"]
373 + assert "cache_control" not in messages[0]
374 + assert messages[0]["content"][-1]["cache_control"] == {
375 + "type": "ephemeral"
376 + }
377 + assert messages[1]["content"][-1]["cache_control"] == {
378 + "type": "ephemeral"
379 + }
380 + assert "cache_control" not in messages[2]
381 + assert messages[3]["content"][-1]["cache_control"] == {
382 + "type": "ephemeral"
383 + }
384 + return _AsyncChunkStream([_chunk("cached")])
385 +
386 + async def fake_rate_limiter(*args, **kwargs):
387 + return None
388 +
389 + monkeypatch.setattr(litellm_transport, "aresponses", fake_aresponses)
390 + monkeypatch.setattr(litellm_transport, "acompletion", fake_acompletion)
391 + monkeypatch.setattr(models, "apply_rate_limiter", fake_rate_limiter)
392 +
393 + wrapper = models.LiteLLMChatWrapper(
394 + model="claude-sonnet-4-5",
395 + provider="anthropic",
396 + model_config=None,
397 + )
398 +
399 + async def response_callback(chunk: str, full: str):
400 + return None
401 +
402 + response, reasoning = await wrapper.unified_call(
403 + messages=[
404 + SystemMessage(content="static instructions"),
405 + HumanMessage(content="question"),
406 + AIMessage(content="previous answer"),
407 + HumanMessage(content="follow up"),
408 + ],
409 + response_callback=response_callback,
410 + explicit_caching=True,
411 + )
412 +
413 + assert response == "cached"
414 + assert reasoning == ""
415 + assert calls == ["chat"]
416 +
417 +
418 +def test_responses_request_translates_messages_and_params():
419 + messages = [
420 + {"role": "system", "content": "You are precise."},
421 + {
422 + "role": "user",
423 + "content": [
424 + {"type": "text", "text": "Inspect this."},
425 + {
426 + "type": "image_url",
427 + "image_url": {"url": "https://example.test/a.png"},
428 + },
429 + ],
430 + },
431 + {
432 + "role": "assistant",
433 + "content": "empty",
434 + "tool_calls": [
435 + {
436 + "id": "call_1",
437 + "type": "function",
438 + "function": {"name": "lookup", "arguments": '{"q":"a0"}'},
439 + }
440 + ],
441 + },
442 + {"role": "tool", "tool_call_id": "call_1", "content": "done"},
443 + ]
444 + kwargs = {
445 + "max_tokens": 42,
446 + "reasoning_effort": "high",
447 + "response_format": {
448 + "type": "json_schema",
449 + "json_schema": {
450 + "name": "answer",
451 + "schema": {"type": "object"},
452 + "strict": True,
453 + },
454 + },
455 + "tools": [
456 + {
457 + "type": "function",
458 + "function": {
459 + "name": "lookup",
460 + "description": "Search",
461 + "parameters": {"type": "object"},
462 + "strict": True,
463 + },
464 + }
465 + ],
466 + }
467 +
468 + request = litellm_transport.ResponsesTransport.from_chat(messages, kwargs)
469 +
470 + assert "instructions" not in request
471 + assert request["store"] is True
472 + assert request["max_output_tokens"] == 42
473 + assert request["reasoning"] == {"effort": "high"}
474 + assert request["text"] == {
475 + "format": {
476 + "type": "json_schema",
477 + "name": "answer",
478 + "schema": {"type": "object"},
479 + "strict": True,
480 + }
481 + }
482 + assert request["tools"] == [
483 + {
484 + "type": "function",
485 + "name": "lookup",
486 + "description": "Search",
487 + "parameters": {"type": "object"},
488 + "strict": True,
489 + }
490 + ]
491 + assert request["input"] == [
492 + {"role": "system", "content": "You are precise."},
493 + {
494 + "role": "user",
495 + "content": [
496 + {"type": "input_text", "text": "Inspect this."},
497 + {
498 + "type": "input_image",
499 + "image_url": "https://example.test/a.png",
500 + },
501 + ],
502 + },
503 + {
504 + "type": "function_call",
505 + "call_id": "call_1",
506 + "id": "call_1",
507 + "name": "lookup",
508 + "arguments": '{"q":"a0"}',
509 + "status": "completed",
510 + },
511 + {"type": "function_call_output", "call_id": "call_1", "output": "done"},
512 + ]
513 +
514 +
515 +def test_responses_request_normalizes_reasoning_and_orphan_tool_choice():
516 + request = litellm_transport.ResponsesTransport.from_chat(
517 + [],
518 + {
519 + "reasoning_effort": "none",
520 + "tools": [],
521 + "tool_choice": "auto",
522 + "parallel_tool_calls": True,
523 + },
524 + )
525 +
526 + assert "reasoning" not in request
527 + assert "tools" not in request
528 + assert "tool_choice" not in request
529 + assert "parallel_tool_calls" not in request
530 +
531 + request = litellm_transport.ResponsesTransport.from_chat(
532 + [],
533 + {"reasoning": {"effort": "xhigh"}},
534 + )
535 +
536 + assert request["reasoning"] == {"effort": "high"}
537 +
538 + request = litellm_transport.ResponsesTransport.from_chat(
539 + [],
540 + {"reasoning_effort": "off"},
541 + )
542 +
543 + assert "reasoning" not in request
544 +
545 +
546 +def test_responses_request_adds_openai_prompt_cache_key_for_static_prefix():
547 + request = litellm_transport.ResponsesTransport.from_chat(
548 + [
549 + {"role": "system", "content": "stable system prompt"},
550 + {"role": "user", "content": "dynamic question"},
551 + ],
552 + {
553 + "tools": [
554 + {
555 + "type": "function",
556 + "function": {
557 + "name": "lookup",
558 + "description": "Search",
559 + "parameters": {"type": "object"},
560 + },
561 + }
562 + ],
563 + },
564 + model="openai/gpt-5.4",
565 + )
566 +
567 + assert request["prompt_cache_key"].startswith("a0-")
568 + assert len(request["prompt_cache_key"]) == 35
569 + assert "stable system prompt" not in request["prompt_cache_key"]
570 +
571 + request_again = litellm_transport.ResponsesTransport.from_chat(
572 + [
573 + {"role": "system", "content": "stable system prompt"},
574 + {"role": "user", "content": "different dynamic question"},
575 + ],
576 + {
577 + "tools": [
578 + {
579 + "type": "function",
580 + "function": {
581 + "name": "lookup",
582 + "description": "Search",
583 + "parameters": {"type": "object"},
584 + },
585 + }
586 + ],
587 + },
588 + model="openai/gpt-5.4",
589 + )
590 +
591 + assert request_again["prompt_cache_key"] == request["prompt_cache_key"]
592 +
593 +
594 +def test_responses_request_respects_explicit_prompt_cache_and_retention():
595 + request = litellm_transport.ResponsesTransport.from_chat(
596 + [{"role": "system", "content": "stable system prompt"}],
597 + {
598 + "prompt_cache_key": "user-provided-key",
599 + "prompt_cache_retention": "24h",
600 + "extra_body": {"prompt_cache_retention": "in_memory"},
601 + },
602 + model="openai/gpt-5.4",
603 + )
604 +
605 + assert request["prompt_cache_key"] == "user-provided-key"
606 + assert "prompt_cache_retention" not in request
607 + assert request["extra_body"]["prompt_cache_retention"] == "in_memory"
608 +
609 +
610 +def test_responses_request_adds_azure_prompt_cache_params():
611 + request = litellm_transport.ResponsesTransport.from_chat(
612 + [{"role": "system", "content": "stable system prompt"}],
613 + {"prompt_cache_retention": "24h"},
614 + model="azure/gpt-4.1",
615 + )
616 +
617 + assert request["prompt_cache_key"].startswith("a0-")
618 + assert "prompt_cache_retention" not in request
619 + assert request["extra_body"]["prompt_cache_retention"] == "24h"
620 +
621 +
622 +def test_responses_request_does_not_add_openai_cache_key_to_custom_api_base():
623 + request = litellm_transport.ResponsesTransport.from_chat(
624 + [{"role": "system", "content": "stable system prompt"}],
625 + {"api_base": "https://llm.agent-zero.ai/v1"},
626 + model="openai/gpt-5.4",
627 + )
628 +
629 + assert "prompt_cache_key" not in request
630 +
631 +
632 +def test_chat_kwargs_add_openai_prompt_cache_key_for_chat_completions():
633 + kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs(
634 + {"max_tokens": 10},
635 + model="openai/gpt-5.4",
636 + messages=[
637 + {"role": "system", "content": "stable system prompt"},
638 + {"role": "user", "content": "dynamic question"},
639 + ],
640 + )
641 +
642 + assert kwargs["prompt_cache_key"].startswith("a0-")
643 + assert kwargs["max_tokens"] == 10
644 +
645 +
646 +def test_chat_messages_strip_cache_control_for_openai_prompt_cache():
647 + messages = [
648 + {
649 + "role": "system",
650 + "cache_control": {"type": "ephemeral"},
651 + "content": [
652 + {
653 + "type": "text",
654 + "text": "stable system prompt",
655 + "cache_control": {"type": "ephemeral"},
656 + }
657 + ],
658 + }
659 + ]
660 +
661 + prepared = litellm_transport.ChatCompletionsTransport.prepare_messages(
662 + messages,
663 + model="openai/gpt-5.4",
664 + kwargs={},
665 + )
666 +
667 + assert "cache_control" not in prepared[0]
668 + assert "cache_control" not in prepared[0]["content"][0]
669 + assert messages[0]["content"][0]["cache_control"] == {"type": "ephemeral"}
670 +
671 +
672 +def test_chat_kwargs_mark_cached_tools_for_cache_control_providers():
673 + kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs(
674 + {
675 + "tools": [
676 + {
677 + "type": "function",
678 + "function": {
679 + "name": "lookup",
680 + "description": "Search",
681 + "parameters": {"type": "object"},
682 + },
683 + }
684 + ],
685 + },
686 + model="anthropic/claude-sonnet-4-5",
687 + messages=[
688 + {
689 + "role": "system",
690 + "content": [
691 + {
692 + "type": "text",
693 + "text": "static instructions",
694 + "cache_control": {"type": "ephemeral"},
695 + }
696 + ],
697 + }
698 + ],
699 + explicit_prompt_caching=True,
700 + )
701 +
702 + assert kwargs["tools"][0]["function"]["cache_control"] == {
703 + "type": "ephemeral"
704 + }
705 +
706 +
707 +def test_chat_kwargs_strip_orphan_tool_choice_and_enable_fallback_drop_params():
708 + kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs(
709 + {
710 + "tool_choice": "auto",
711 + "parallel_tool_calls": True,
712 + "max_tokens": 10,
713 + },
714 + fallback_error=RuntimeError("This model does not support Responses API"),
715 + )
716 +
717 + assert kwargs["max_tokens"] == 10
718 + assert kwargs["drop_params"] is True
719 + assert "tool_choice" not in kwargs
720 + assert "parallel_tool_calls" not in kwargs
721 +
722 +
723 +def test_cache_control_policy_keeps_native_responses_first():
724 + messages = [
725 + {
726 + "role": "system",
727 + "content": "static instructions",
728 + "cache_control": {"type": "ephemeral"},
729 + }
730 + ]
731 +
732 + openai_policy = litellm_transport.TransportPolicy.from_request(
733 + "openai/gpt-5.4",
734 + {},
735 + messages=messages,
736 + )
737 + anthropic_policy = litellm_transport.TransportPolicy.from_request(
738 + "anthropic/claude-sonnet-4-5",
739 + {},
740 + messages=messages,
741 + )
742 +
743 + assert openai_policy.mode is litellm_transport.TransportMode.RESPONSES
744 + assert anthropic_policy.mode is litellm_transport.TransportMode.CHAT_COMPLETIONS
745 +
746 +
747 +def test_responses_fallback_does_not_mask_rate_limits():
748 + exc = RuntimeError(
749 + "RateLimitError: 429 Too Many Requests for url "
750 + "https://api.openai.com/v1/responses"
751 + )
752 +
753 + policy = litellm_transport.TransportPolicy(
754 + mode=litellm_transport.TransportMode.RESPONSES
755 + )
756 +
757 + assert (
758 + policy.recover(exc, got_any_chunk=False)
759 + is litellm_transport.TransportRecovery.RAISE
760 + )
761 +
762 +
763 +def test_responses_response_parser_extracts_text_reasoning_and_function_calls():
764 + text_response = {
765 + "output": [
766 + {"type": "reasoning", "summary": [{"text": "because"}]},
767 + {
768 + "type": "message",
769 + "content": [{"type": "output_text", "text": "answer"}],
770 + },
771 + ]
772 + }
773 +
774 + parsed = litellm_transport.ResponsesTransport.parse_response(text_response)
775 +
776 + assert parsed == {"response_delta": "answer", "reasoning_delta": "because"}
777 +
778 + tool_response = {
779 + "output": [
780 + {
781 + "type": "function_call",
782 + "name": "lookup",
783 + "arguments": '{"q":"a0"}',
784 + }
785 + ]
786 + }
787 +
788 + parsed_tool = litellm_transport.ResponsesTransport.parse_response(tool_response)
789 +
790 + assert extract_tools.json_parse_dirty(parsed_tool["response_delta"]) == {
791 + "tool_name": "lookup",
792 + "tool_args": {"q": "a0"},
793 + }
794 +
795 +
796 +def test_responses_stream_parser_accumulates_function_call_arguments():
797 + parser = litellm_transport.ResponsesEventParser()
798 +
799 + assert parser.parse(
800 + {
801 + "type": "response.output_item.added",
802 + "output_index": 0,
803 + "item": {
804 + "type": "function_call",
805 + "id": "fc_1",
806 + "call_id": "call_1",
807 + "name": "lookup",
808 + "arguments": "",
809 + },
810 + }
811 + ) == {"reasoning_delta": "", "response_delta": ""}
812 + assert parser.parse(
813 + {
814 + "type": "response.function_call_arguments.delta",
815 + "item_id": "fc_1",
816 + "output_index": 0,
817 + "delta": '{"q":',
818 + }
819 + ) == {"reasoning_delta": "", "response_delta": ""}
820 +
821 + parsed = parser.parse(
822 + {
823 + "type": "response.function_call_arguments.done",
824 + "item_id": "fc_1",
825 + "output_index": 0,
826 + "name": "lookup",
827 + "arguments": '{"q":"a0"}',
828 + }
829 + )
830 +
831 + assert extract_tools.json_parse_dirty(parsed["response_delta"]) == {
832 + "tool_name": "lookup",
833 + "tool_args": {"q": "a0"},
834 + }
835 + assert parser.parse(
836 + {
837 + "type": "response.output_item.done",
838 + "output_index": 0,
839 + "item": {
840 + "type": "function_call",
841 + "id": "fc_1",
842 + "call_id": "call_1",
843 + "name": "lookup",
844 + "arguments": '{"q":"a0"}',
845 + },
846 + }
847 + ) == {"reasoning_delta": "", "response_delta": ""}
848 +
849 +
850 +def test_responses_stream_parser_uses_completed_response_when_no_deltas_arrive():
851 + parser = litellm_transport.ResponsesEventParser()
852 +
853 + parsed = parser.parse(
854 + {
855 + "type": "response.completed",
856 + "response": {
857 + "output": [
858 + {
859 + "type": "message",
860 + "content": [{"type": "output_text", "text": "done"}],
861 + }
862 + ]
863 + },
864 + }
865 + )
866 +
867 + assert parsed == {"reasoning_delta": "", "response_delta": "done"}
868 +
869 +
870 +def test_responses_stream_parser_handles_refusal_and_failed_events():
871 + parser = litellm_transport.ResponsesEventParser()
872 +
873 + assert parser.parse(
874 + {"type": "response.refusal.delta", "delta": "no"}
875 + ) == {"reasoning_delta": "", "response_delta": "no"}
876 +
877 + with pytest.raises(RuntimeError, match="policy"):
878 + parser.parse(
879 + {
880 + "type": "response.failed",
881 + "response": {"error": {"message": "policy"}},
882 + }
883 + )
884 +
885 +
886 +def test_responses_response_parser_groups_parallel_function_calls():
887 + response = {
888 + "output": [
889 + {
890 + "type": "function_call",
891 + "name": "lookup",
892 + "arguments": '{"q":"a0"}',
893 + },
894 + {
895 + "type": "function_call",
896 + "name": "rank",
897 + "arguments": '{"limit":2}',
898 + },
899 + ]
900 + }
901 +
902 + parsed = litellm_transport.ResponsesTransport.parse_response(response)
903 +
904 + assert extract_tools.json_parse_dirty(parsed["response_delta"]) == {
905 + "tool_name": "parallel_tool_calls",
906 + "tool_args": {
907 + "calls": [
908 + {"tool_name": "lookup", "tool_args": {"q": "a0"}},
909 + {"tool_name": "rank", "tool_args": {"limit": 2}},
910 + ]
911 + },
912 + }