fix (browser-use): monkeypatch for browser-use and Gemini schemas
Gemini JSON-pure outputs + action normalization
Alessandro committed
Aug 26, 2025 at 21:58 UTC
73c3d179d467cf825bf5499396f50971cdcecf7c
2 files changed
+177
-3
models.py
+15
-3
@@ -23,7 +23,7 @@ from python.helpers.dotenv import load_dotenv
23
from python.helpers.providers import get_provider_config
24
from python.helpers.rate_limiter import RateLimiter
25
from python.helpers.tokens import approximate_tokens
26
-from python.helpers import dirty_json
26
+from python.helpers import dirty_json, browser_use_monkeypatch
27
28
from langchain_core.language_models.chat_models import SimpleChatModel
29
from langchain_core.outputs.chat_generation import ChatGenerationChunk
@@ -54,6 +54,7 @@ def turn_off_logging():
54
# init
55
load_dotenv()
56
turn_off_logging()
57
+browser_use_monkeypatch.apply()
58
59
60
class ModelType(Enum):
@@ -450,9 +451,9 @@ class BrowserCompatibleChatWrapper(ChatOpenRouter):
451
model = kwargs.pop("model", None)
452
kwrgs = {**self._wrapper.kwargs, **kwargs}
453
453
- # hack from browser-use to fix json schema for gemini
454
+ # hack from browser-use to fix json schema for gemini (additionalProperties, $defs, $ref)
455
if "response_format" in kwrgs and "json_schema" in kwrgs["response_format"] and model.startswith("gemini/"):
455
- kwrgs["response_format"]["json_schema"] = ChatGoogle("")._fix_gemini_schema(self._wrapper.kwargs)
456
+ kwrgs["response_format"]["json_schema"] = ChatGoogle("")._fix_gemini_schema(kwrgs["response_format"]["json_schema"])
457
458
resp = await acompletion(
459
model=self._wrapper.model_name,
@@ -460,6 +461,17 @@ class BrowserCompatibleChatWrapper(ChatOpenRouter):
461
stop=stop,
462
**kwrgs,
463
)
464
+
465
+ # Gemini: strip triple backticks and conform schema
466
+ try:
467
+ msg = resp.choices[0].message
468
+ if self.provider == "gemini" and isinstance(getattr(msg, "content", None), str):
469
+ cleaned = browser_use_monkeypatch.gemini_clean_and_conform(msg.content)
470
+ if cleaned:
471
+ msg.content = cleaned
472
+ except Exception:
473
+ pass
474
+
475
except Exception as e:
476
raise e
477
python/helpers/browser_use_monkeypatch.py
new
+162
@@ -0,0 +1,162 @@
1
+from typing import Any
2
+from browser_use.llm import ChatGoogle
3
+from python.helpers import dirty_json
4
+
5
+
6
+# ------------------------------------------------------------------------------
7
+# Gemini Helper for Output Conformance
8
+# ------------------------------------------------------------------------------
9
+# This function sanitizes and conforms the JSON output from Gemini to match
10
+# the specific schema expectations of the browser-use library. It handles
11
+# markdown fences, aliases actions (like 'complete_task' to 'done'), and
12
+# intelligently constructs a valid 'data' object for the final action.
13
+
14
+def gemini_clean_and_conform(text: str):
15
+ obj = None
16
+ try:
17
+ # dirty_json parser is robust enough to handle markdown fences
18
+ obj = dirty_json.parse(text)
19
+ except Exception:
20
+ return None # return None if parsing fails
21
+
22
+ if not isinstance(obj, dict):
23
+ return None
24
+
25
+ # Conform actions to browser-use expectations
26
+ if isinstance(obj.get("action"), list):
27
+ normalized_actions = []
28
+ for item in obj["action"]:
29
+ if not isinstance(item, dict):
30
+ continue # Skip non-dict items
31
+
32
+ action_key, action_value = next(iter(item.items()), (None, None))
33
+ if not action_key:
34
+ continue
35
+
36
+ # Alias 'complete_task' to 'done' to handle inconsistencies
37
+ if action_key == "complete_task":
38
+ action_key = "done"
39
+
40
+ # Create a mutable copy of the value
41
+ v = (action_value or {}).copy()
42
+
43
+ if action_key in ("scroll_down", "scroll_up", "scroll"):
44
+ is_down = action_key != "scroll_up"
45
+ v.setdefault("down", is_down)
46
+ v.setdefault("num_pages", 1.0)
47
+ normalized_actions.append({"scroll": v})
48
+ elif action_key == "go_to_url":
49
+ v.setdefault("new_tab", False)
50
+ normalized_actions.append({action_key: v})
51
+ elif action_key == "done":
52
+ # If `data` is missing, construct it from other keys
53
+ if "data" not in v:
54
+ # Pop fields from the top-level `done` object
55
+ response_text = v.pop("response", None)
56
+ summary_text = v.pop("page_summary", None)
57
+ title_text = v.pop("title", "Task Completed")
58
+
59
+ final_response = response_text or "Task completed successfully." # browser-use expects string
60
+ final_summary = summary_text or "No page summary available." # browser-use expects string
61
+
62
+ v["data"] = {
63
+ "title": title_text,
64
+ "response": final_response,
65
+ "page_summary": final_summary,
66
+ }
67
+
68
+ v.setdefault("success", True)
69
+ normalized_actions.append({action_key: v})
70
+ else:
71
+ normalized_actions.append(item)
72
+ obj["action"] = normalized_actions
73
+
74
+ return dirty_json.stringify(obj)
75
+
76
+# ------------------------------------------------------------------------------
77
+# Monkey-patch for browser-use Gemini schema issue
78
+# ------------------------------------------------------------------------------
79
+# The original _fix_gemini_schema in browser_use.llm.google.chat.ChatGoogle
80
+# removes the 'title' property but fails to remove it from the 'required' list,
81
+# causing a validation error with the Gemini API. This patch corrects that behavior.
82
+
83
+def _patched_fix_gemini_schema(self, schema: dict[str, Any]) -> dict[str, Any]:
84
+ """
85
+ Convert a Pydantic model to a Gemini-compatible schema.
86
+
87
+ This function removes unsupported properties like 'additionalProperties' and resolves
88
+ $ref references that Gemini doesn't support.
89
+ """
90
+
91
+ # Handle $defs and $ref resolution
92
+ if '$defs' in schema:
93
+ defs = schema.pop('$defs')
94
+
95
+ def resolve_refs(obj: Any) -> Any:
96
+ if isinstance(obj, dict):
97
+ if '$ref' in obj:
98
+ ref = obj.pop('$ref')
99
+ ref_name = ref.split('/')[-1]
100
+ if ref_name in defs:
101
+ # Replace the reference with the actual definition
102
+ resolved = defs[ref_name].copy()
103
+ # Merge any additional properties from the reference
104
+ for key, value in obj.items():
105
+ if key != '$ref':
106
+ resolved[key] = value
107
+ return resolve_refs(resolved)
108
+ return obj
109
+ else:
110
+ # Recursively process all dictionary values
111
+ return {k: resolve_refs(v) for k, v in obj.items()}
112
+ elif isinstance(obj, list):
113
+ return [resolve_refs(item) for item in obj]
114
+ return obj
115
+
116
+ schema = resolve_refs(schema)
117
+
118
+ # Remove unsupported properties
119
+ def clean_schema(obj: Any) -> Any:
120
+ if isinstance(obj, dict):
121
+ # Remove unsupported properties
122
+ cleaned = {}
123
+ for key, value in obj.items():
124
+ if key not in ['additionalProperties', 'title', 'default']:
125
+ cleaned_value = clean_schema(value)
126
+ # Handle empty object properties - Gemini doesn't allow empty OBJECT types
127
+ if (
128
+ key == 'properties'
129
+ and isinstance(cleaned_value, dict)
130
+ and len(cleaned_value) == 0
131
+ and isinstance(obj.get('type', ''), str)
132
+ and obj.get('type', '').upper() == 'OBJECT'
133
+ ):
134
+ # Convert empty object to have at least one property
135
+ cleaned['properties'] = {'_placeholder': {'type': 'string'}}
136
+ else:
137
+ cleaned[key] = cleaned_value
138
+
139
+ # If this is an object type with empty properties, add a placeholder
140
+ if (
141
+ isinstance(cleaned.get('type', ''), str)
142
+ and cleaned.get('type', '').upper() == 'OBJECT'
143
+ and 'properties' in cleaned
144
+ and isinstance(cleaned['properties'], dict)
145
+ and len(cleaned['properties']) == 0
146
+ ):
147
+ cleaned['properties'] = {'_placeholder': {'type': 'string'}}
148
+
149
+ # PATCH: Also remove 'title' from the required list if it exists
150
+ if 'required' in cleaned and isinstance(cleaned.get('required'), list):
151
+ cleaned['required'] = [p for p in cleaned['required'] if p != 'title']
152
+
153
+ return cleaned
154
+ elif isinstance(obj, list):
155
+ return [clean_schema(item) for item in obj]
156
+ return obj
157
+
158
+ return clean_schema(schema)
159
+
160
+def apply():
161
+ """Applies the monkey-patch to ChatGoogle."""
162
+ ChatGoogle._fix_gemini_schema = _patched_fix_gemini_schema