| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import argparse |
| 4 | import json |
| 5 | from pathlib import Path |
| 6 | from unittest.mock import MagicMock, patch |
| 7 | |
| 8 | import pytest |
| 9 | from transformers import AutoTokenizer, PreTrainedTokenizerBase |
| 10 | |
| 11 | from vllm.benchmarks.datasets import BFCLDataset, get_samples |
| 12 | |
| 13 | |
| 14 | def _patch_hf_api(side_effect): |
| 15 | """Return a patch context that swaps `hf_api()` to a stub whose |
| 16 | `.hf_hub_download` attribute uses `side_effect`.""" |
| 17 | fake_api = MagicMock() |
| 18 | fake_api.hf_hub_download.side_effect = side_effect |
| 19 | return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api) |
| 20 | |
| 21 | |
| 22 | @pytest.fixture(scope="session") |
| 23 | def hf_tokenizer() -> PreTrainedTokenizerBase: |
| 24 | return AutoTokenizer.from_pretrained("openai-community/gpt2") |
| 25 | |
| 26 | |
| 27 | _FAKE_ROWS = { |
| 28 | "simple": [ |
| 29 | { |
| 30 | "id": "simple_0", |
| 31 | "question": [ |
| 32 | [ |
| 33 | { |
| 34 | "role": "user", |
| 35 | "content": "What is 2+2?", |
| 36 | } |
| 37 | ] |
| 38 | ], |
| 39 | "function": [ |
| 40 | { |
| 41 | "name": "add", |
| 42 | "description": "Add two numbers.", |
| 43 | "parameters": { |
| 44 | "type": "dict", |
| 45 | "properties": { |
| 46 | "a": {"type": "integer", "description": "first"}, |
| 47 | "b": {"type": "float", "description": "second"}, |
| 48 | }, |
| 49 | "required": ["a", "b"], |
| 50 | }, |
| 51 | } |
| 52 | ], |
| 53 | }, |
| 54 | ], |
| 55 | "live_simple": [ |
| 56 | { |
| 57 | "id": "live_simple_0", |
| 58 | "question": [[{"role": "user", "content": "Tell me the weather."}]], |
| 59 | "function": [ |
| 60 | { |
| 61 | "name": "get_weather", |
| 62 | "description": "Get weather.", |
| 63 | "parameters": { |
| 64 | "type": "dict", |
| 65 | "properties": { |
| 66 | "city": {"type": "any", "description": "city"}, |
| 67 | "coords": {"type": "tuple", "description": "coords"}, |
| 68 | }, |
| 69 | "required": ["city"], |
| 70 | }, |
| 71 | } |
| 72 | ], |
| 73 | }, |
| 74 | ], |
| 75 | } |
| 76 | |
| 77 | |
| 78 | def _write_fake_files(tmp_path: Path) -> dict[str, Path]: |
| 79 | """Write fake BFCL JSONL files mimicking the HF repo layout.""" |
| 80 | paths = {} |
| 81 | for category, rows in _FAKE_ROWS.items(): |
| 82 | p = tmp_path / f"BFCL_v3_{category}.json" |
| 83 | with p.open("w") as f: |
| 84 | for row in rows: |
| 85 | f.write(json.dumps(row) + "\n") |
| 86 | paths[category] = p |
| 87 | return paths |
| 88 | |
| 89 | |
| 90 | def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace: |
| 91 | return argparse.Namespace( |
| 92 | dataset_name="hf", |
| 93 | dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard", |
| 94 | hf_name=None, |
| 95 | hf_subset=None, |
| 96 | hf_split=None, |
| 97 | hf_output_len=64, |
| 98 | disable_shuffle=True, |
| 99 | num_prompts=2, |
| 100 | no_oversample=False, |
| 101 | no_stream=True, |
| 102 | seed=0, |
| 103 | request_id_prefix="", |
| 104 | trust_remote_code=False, |
| 105 | skip_chat_template=False, |
| 106 | enable_multimodal_chat=False, |
| 107 | backend="openai-chat", |
| 108 | bfcl_categories=categories, |
| 109 | ) |
| 110 | |
| 111 | |
| 112 | @pytest.mark.benchmark |
| 113 | def test_bfcl_dataset_translates_schema_and_attaches_tools( |
| 114 | hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path |
| 115 | ) -> None: |
| 116 | """BFCLDataset should translate schemas to OpenAI tool format, set |
| 117 | `messages` directly on SampleRequest, and attach tools/tool_choice via |
| 118 | request_overrides.""" |
| 119 | paths = _write_fake_files(tmp_path) |
| 120 | |
| 121 | def fake_download(_repo, filename, **_kwargs): |
| 122 | category = filename.removeprefix("BFCL_v3_").removesuffix(".json") |
| 123 | return str(paths[category]) |
| 124 | |
| 125 | args = _args_for_bfcl(categories=["simple", "live_simple"]) |
| 126 | |
| 127 | with _patch_hf_api(fake_download): |
| 128 | samples = get_samples(args, hf_tokenizer) |
| 129 | |
| 130 | assert len(samples) == 2 |
| 131 | for s in samples: |
| 132 | assert s.chat_messages is not None |
| 133 | assert isinstance(s.chat_messages, list) |
| 134 | assert s.chat_messages[0]["role"] == "user" |
| 135 | assert s.request_overrides is not None |
| 136 | assert "tools" in s.request_overrides |
| 137 | assert s.request_overrides["tool_choice"] == "auto" |
| 138 | # messages must NOT leak into request_overrides — it has its own |
| 139 | # typed field on SampleRequest. |
| 140 | assert "messages" not in s.request_overrides |
| 141 | tools = s.request_overrides["tools"] |
| 142 | assert len(tools) == 1 |
| 143 | tool = tools[0] |
| 144 | assert tool["type"] == "function" |
| 145 | # Translated schema: dict -> object, float -> number, |
| 146 | # any -> string, tuple -> array. |
| 147 | params = tool["function"]["parameters"] |
| 148 | assert params["type"] == "object" |
| 149 | for prop in params["properties"].values(): |
| 150 | assert prop["type"] in {"integer", "number", "string", "array"} |
| 151 | |
| 152 | |
| 153 | @pytest.mark.benchmark |
| 154 | def test_bfcl_dataset_requires_openai_chat_backend( |
| 155 | hf_tokenizer: PreTrainedTokenizerBase, |
| 156 | ) -> None: |
| 157 | args = _args_for_bfcl(categories=["simple"]) |
| 158 | args.backend = "openai" |
| 159 | |
| 160 | with pytest.raises(ValueError, match="openai-chat"): |
| 161 | get_samples(args, hf_tokenizer) |
| 162 | |
| 163 | |
| 164 | @pytest.mark.benchmark |
| 165 | def test_bfcl_dataset_missing_category_raises_clear_error( |
| 166 | hf_tokenizer: PreTrainedTokenizerBase, |
| 167 | ) -> None: |
| 168 | """A typo'd category should produce an actionable ValueError, not an |
| 169 | opaque huggingface_hub exception.""" |
| 170 | from huggingface_hub.errors import EntryNotFoundError |
| 171 | |
| 172 | args = _args_for_bfcl(categories=["simpl"]) # typo |
| 173 | |
| 174 | def raise_missing(_repo, filename, **_kwargs): |
| 175 | raise EntryNotFoundError(f"404 Not Found: {filename}") |
| 176 | |
| 177 | with ( |
| 178 | _patch_hf_api(raise_missing), |
| 179 | pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"), |
| 180 | ): |
| 181 | get_samples(args, hf_tokenizer) |
| 182 | |
| 183 | |
| 184 | @pytest.mark.benchmark |
| 185 | def test_chat_backend_uses_messages_field_when_set() -> None: |
| 186 | """When RequestFuncInput.chat_messages is set, the chat backend must use |
| 187 | it verbatim and skip default content construction from `prompt`.""" |
| 188 | import asyncio |
| 189 | |
| 190 | from vllm.benchmarks.lib.endpoint_request_func import ( |
| 191 | RequestFuncInput, |
| 192 | async_request_openai_chat_completions, |
| 193 | ) |
| 194 | |
| 195 | captured: dict = {} |
| 196 | |
| 197 | class _FakeResp: |
| 198 | status = 500 |
| 199 | reason = "stop-after-capture" |
| 200 | content = None |
| 201 | |
| 202 | async def __aenter__(self): |
| 203 | return self |
| 204 | |
| 205 | async def __aexit__(self, *exc): |
| 206 | return False |
| 207 | |
| 208 | class _FakeSession: |
| 209 | def post(self, url, json, headers): # noqa: A002 |
| 210 | captured["url"] = url |
| 211 | captured["payload"] = json |
| 212 | return _FakeResp() |
| 213 | |
| 214 | messages = [ |
| 215 | {"role": "system", "content": "you are helpful"}, |
| 216 | {"role": "user", "content": "call add(3, 4)"}, |
| 217 | ] |
| 218 | req = RequestFuncInput( |
| 219 | prompt="IGNORED", |
| 220 | api_url="http://localhost:0/v1/chat/completions", |
| 221 | prompt_len=10, |
| 222 | output_len=16, |
| 223 | model="test-model", |
| 224 | chat_messages=messages, |
| 225 | extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]}, |
| 226 | ) |
| 227 | |
| 228 | asyncio.run( |
| 229 | async_request_openai_chat_completions( |
| 230 | request_func_input=req, session=_FakeSession() |
| 231 | ) |
| 232 | ) |
| 233 | |
| 234 | payload = captured["payload"] |
| 235 | assert payload["messages"] is messages, ( |
| 236 | "chat backend must forward RequestFuncInput.chat_messages verbatim " |
| 237 | "instead of constructing a default user message from `prompt`" |
| 238 | ) |
| 239 | # extra_body still merges in as before (shallow, per-request wins). |
| 240 | assert payload["tools"][0]["function"]["name"] == "add" |
| 241 | |
| 242 | |
| 243 | @pytest.mark.benchmark |
| 244 | def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None: |
| 245 | """prompt_len must reflect tokens from both messages *and* tool schemas, |
| 246 | so percentile buckets and input-distribution summaries aren't biased |
| 247 | low for tool-heavy traffic.""" |
| 248 | paths = _write_fake_files(tmp_path) |
| 249 | |
| 250 | def fake_download(_repo, filename, **_kwargs): |
| 251 | category = filename.removeprefix("BFCL_v3_").removesuffix(".json") |
| 252 | return str(paths[category]) |
| 253 | |
| 254 | captured: dict = {} |
| 255 | |
| 256 | class _FakeTokenizer: |
| 257 | def apply_chat_template( |
| 258 | self, messages, tools=None, tokenize=False, add_generation_prompt=True |
| 259 | ): |
| 260 | captured["tools"] = tools |
| 261 | base = " ".join(m.get("content", "") for m in messages) |
| 262 | tool_text = json.dumps(tools) if tools else "" |
| 263 | return base + " " + tool_text |
| 264 | |
| 265 | def __call__(self, text): |
| 266 | # 1 "token" per whitespace-separated word. |
| 267 | return type("Enc", (), {"input_ids": text.split()})() |
| 268 | |
| 269 | fake = _FakeTokenizer() |
| 270 | args = _args_for_bfcl(categories=["simple"]) |
| 271 | args.num_prompts = 1 |
| 272 | |
| 273 | with _patch_hf_api(fake_download): |
| 274 | samples = get_samples(args, fake) |
| 275 | |
| 276 | assert len(samples) == 1 |
| 277 | assert captured["tools"] is not None, ( |
| 278 | "apply_chat_template must be called with tools= so the schema " |
| 279 | "contributes to the prompt-length estimate" |
| 280 | ) |
| 281 | assert len(captured["tools"]) == 1 |
| 282 | assert captured["tools"][0]["function"]["name"] == "add" |
| 283 | |
| 284 | # Sanity: prompt_len exceeds a messages-only estimate. The fake row's |
| 285 | # user message is "What is 2+2?" (3 whitespace-separated tokens). |
| 286 | assert samples[0].prompt_len > 3 |
| 287 | |
| 288 | |
| 289 | @pytest.mark.benchmark |
| 290 | def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools( |
| 291 | tmp_path: Path, |
| 292 | ) -> None: |
| 293 | """Older tokenizers don't accept tools=; fallback must still produce a |
| 294 | non-zero prompt_len without crashing.""" |
| 295 | paths = _write_fake_files(tmp_path) |
| 296 | |
| 297 | def fake_download(_repo, filename, **_kwargs): |
| 298 | category = filename.removeprefix("BFCL_v3_").removesuffix(".json") |
| 299 | return str(paths[category]) |
| 300 | |
| 301 | class _LegacyTokenizer: |
| 302 | def apply_chat_template(self, messages, **kwargs): |
| 303 | if "tools" in kwargs: |
| 304 | raise TypeError("unexpected keyword argument 'tools'") |
| 305 | return " ".join(m.get("content", "") for m in messages) |
| 306 | |
| 307 | def __call__(self, text): |
| 308 | return type("Enc", (), {"input_ids": text.split()})() |
| 309 | |
| 310 | args = _args_for_bfcl(categories=["simple"]) |
| 311 | args.num_prompts = 1 |
| 312 | |
| 313 | with _patch_hf_api(fake_download): |
| 314 | samples = get_samples(args, _LegacyTokenizer()) |
| 315 | |
| 316 | assert len(samples) == 1 |
| 317 | assert samples[0].prompt_len > 0 |
| 318 | |
| 319 | |
| 320 | @pytest.mark.benchmark |
| 321 | def test_bfcl_schema_translation_is_recursive() -> None: |
| 322 | """_translate_schema must recurse into nested properties.""" |
| 323 | input_schema = { |
| 324 | "type": "dict", |
| 325 | "properties": { |
| 326 | "nested": { |
| 327 | "type": "dict", |
| 328 | "properties": { |
| 329 | "value": {"type": "float"}, |
| 330 | "tags": {"type": "tuple", "items": {"type": "any"}}, |
| 331 | }, |
| 332 | } |
| 333 | }, |
| 334 | } |
| 335 | out = BFCLDataset._translate_schema(input_schema) |
| 336 | assert out["type"] == "object" |
| 337 | assert out["properties"]["nested"]["type"] == "object" |
| 338 | assert out["properties"]["nested"]["properties"]["value"]["type"] == "number" |
| 339 | assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array" |
| 340 | nested_props = out["properties"]["nested"]["properties"] |
| 341 | assert nested_props["tags"]["items"]["type"] == "string" |