A high-throughput and memory-efficient inference and serving engine for LLMs
[Bench] Add BFCL dataset for vllm bench serve tool-calling workloads (#42457)
Signed-off-by: Li Zhang <lzhanga@amazon.com> Co-authored-by: Li Zhang <lzhanga@amazon.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Li committed
Jun 9, 2026 at 23:59 UTC
89c6a410017ce2a330e4ed5c3d9cbac355225b63
7 files changed
+666
-4
docs/benchmarking/cli.md
+41
@@ -405,6 +405,47 @@ vllm bench serve \
405
406
Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card.
407
408
+#### BFCL (Tool-Calling) Benchmark
409
+
410
+The Berkeley Function Calling Leaderboard (BFCL) dataset measures serving
411
+latency and throughput on realistic tool-calling traffic. Each request
412
+carries a per-sample `tools` schema and chat history, so the server must
413
+expose `/v1/chat/completions` with an auto-tool-choice parser enabled.
414
+The benchmark client always uses the `openai-chat` backend.
415
+
416
+Start a tool-parser-enabled server, then run the bench. For example, with
417
+`gpt-oss-20b`:
418
+
419
+```bash
420
+# Server
421
+vllm serve openai/gpt-oss-20b \
422
+ --enable-auto-tool-choice \
423
+ --tool-call-parser openai \
424
+ --reasoning-parser openai_gptoss
425
+
426
+# Client
427
+vllm bench serve \
428
+ --backend openai-chat \
429
+ --endpoint /v1/chat/completions \
430
+ --model openai/gpt-oss-20b \
431
+ --dataset-name hf \
432
+ --dataset-path gorilla-llm/Berkeley-Function-Calling-Leaderboard \
433
+ --bfcl-categories simple,live_simple,multiple \
434
+ --num-prompts 200
435
+```
436
+
437
+`--bfcl-categories` is a comma-separated list of BFCL v3 category names
438
+(without the `BFCL_v3_` prefix or `.json` suffix). Defaults to
439
+`simple,live_simple,multiple`. Other supported non-multi-turn categories
440
+include `parallel`, `live_parallel`, `parallel_multiple`,
441
+`live_parallel_multiple`, `irrelevance`, `live_irrelevance`,
442
+`live_relevance`, `java`, `javascript`, and `rest`. Multi-turn categories
443
+are not yet supported.
444
+
445
+The dataset class normalizes BFCL's loose schema dialect (`dict` →
446
+`object`, `float` → `number`, `tuple` → `array`, `any` → `string`) so
447
+modern grammar backends accept the translated tool definitions.
448
+
449
#### Other HuggingFaceDataset Examples
450
451
```bash
docs/features/tool_calling.md
+7
@@ -504,6 +504,13 @@ Flags: `--tool-call-parser pythonic --chat-template {see_above}`
504
!!! warning
505
Llama's smaller models frequently fail to emit tool calls in the correct format. Results may vary depending on the model.
506
507
+## Benchmarking Tool-Calling Performance
508
+
509
+To measure serving latency and throughput on realistic tool-calling traffic,
510
+use the BFCL (Berkeley Function Calling Leaderboard) dataset with
511
+`vllm bench serve`. See the [BFCL benchmark example](../benchmarking/cli.md#bfcl-tool-calling-benchmark)
512
+for the full server + client commands.
513
+
514
## How to Write a Tool Parser Plugin
515
516
A tool parser plugin is a Python file containing one or more ToolParser implementations. You can write a ToolParser similar to the `Hermes2ProToolParser` in [vllm/tool_parsers/hermes_tool_parser.py](../../vllm/tool_parsers/hermes_tool_parser.py).
tests/benchmarks/test_bfcl_dataset.py
new
+341
@@ -0,0 +1,341 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import argparse
4
+import json
5
+from pathlib import Path
6
+from unittest.mock import MagicMock, patch
7
+
8
+import pytest
9
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
10
+
11
+from vllm.benchmarks.datasets import BFCLDataset, get_samples
12
+
13
+
14
+def _patch_hf_api(side_effect):
15
+ """Return a patch context that swaps `hf_api()` to a stub whose
16
+ `.hf_hub_download` attribute uses `side_effect`."""
17
+ fake_api = MagicMock()
18
+ fake_api.hf_hub_download.side_effect = side_effect
19
+ return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
20
+
21
+
22
+@pytest.fixture(scope="session")
23
+def hf_tokenizer() -> PreTrainedTokenizerBase:
24
+ return AutoTokenizer.from_pretrained("gpt2")
25
+
26
+
27
+_FAKE_ROWS = {
28
+ "simple": [
29
+ {
30
+ "id": "simple_0",
31
+ "question": [
32
+ [
33
+ {
34
+ "role": "user",
35
+ "content": "What is 2+2?",
36
+ }
37
+ ]
38
+ ],
39
+ "function": [
40
+ {
41
+ "name": "add",
42
+ "description": "Add two numbers.",
43
+ "parameters": {
44
+ "type": "dict",
45
+ "properties": {
46
+ "a": {"type": "integer", "description": "first"},
47
+ "b": {"type": "float", "description": "second"},
48
+ },
49
+ "required": ["a", "b"],
50
+ },
51
+ }
52
+ ],
53
+ },
54
+ ],
55
+ "live_simple": [
56
+ {
57
+ "id": "live_simple_0",
58
+ "question": [[{"role": "user", "content": "Tell me the weather."}]],
59
+ "function": [
60
+ {
61
+ "name": "get_weather",
62
+ "description": "Get weather.",
63
+ "parameters": {
64
+ "type": "dict",
65
+ "properties": {
66
+ "city": {"type": "any", "description": "city"},
67
+ "coords": {"type": "tuple", "description": "coords"},
68
+ },
69
+ "required": ["city"],
70
+ },
71
+ }
72
+ ],
73
+ },
74
+ ],
75
+}
76
+
77
+
78
+def _write_fake_files(tmp_path: Path) -> dict[str, Path]:
79
+ """Write fake BFCL JSONL files mimicking the HF repo layout."""
80
+ paths = {}
81
+ for category, rows in _FAKE_ROWS.items():
82
+ p = tmp_path / f"BFCL_v3_{category}.json"
83
+ with p.open("w") as f:
84
+ for row in rows:
85
+ f.write(json.dumps(row) + "\n")
86
+ paths[category] = p
87
+ return paths
88
+
89
+
90
+def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace:
91
+ return argparse.Namespace(
92
+ dataset_name="hf",
93
+ dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard",
94
+ hf_name=None,
95
+ hf_subset=None,
96
+ hf_split=None,
97
+ hf_output_len=64,
98
+ disable_shuffle=True,
99
+ num_prompts=2,
100
+ no_oversample=False,
101
+ no_stream=True,
102
+ seed=0,
103
+ request_id_prefix="",
104
+ trust_remote_code=False,
105
+ skip_chat_template=False,
106
+ enable_multimodal_chat=False,
107
+ backend="openai-chat",
108
+ bfcl_categories=categories,
109
+ )
110
+
111
+
112
+@pytest.mark.benchmark
113
+def test_bfcl_dataset_translates_schema_and_attaches_tools(
114
+ hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
115
+) -> None:
116
+ """BFCLDataset should translate schemas to OpenAI tool format, set
117
+ `messages` directly on SampleRequest, and attach tools/tool_choice via
118
+ request_overrides."""
119
+ paths = _write_fake_files(tmp_path)
120
+
121
+ def fake_download(_repo, filename, **_kwargs):
122
+ category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
123
+ return str(paths[category])
124
+
125
+ args = _args_for_bfcl(categories=["simple", "live_simple"])
126
+
127
+ with _patch_hf_api(fake_download):
128
+ samples = get_samples(args, hf_tokenizer)
129
+
130
+ assert len(samples) == 2
131
+ for s in samples:
132
+ assert s.chat_messages is not None
133
+ assert isinstance(s.chat_messages, list)
134
+ assert s.chat_messages[0]["role"] == "user"
135
+ assert s.request_overrides is not None
136
+ assert "tools" in s.request_overrides
137
+ assert s.request_overrides["tool_choice"] == "auto"
138
+ # messages must NOT leak into request_overrides — it has its own
139
+ # typed field on SampleRequest.
140
+ assert "messages" not in s.request_overrides
141
+ tools = s.request_overrides["tools"]
142
+ assert len(tools) == 1
143
+ tool = tools[0]
144
+ assert tool["type"] == "function"
145
+ # Translated schema: dict -> object, float -> number,
146
+ # any -> string, tuple -> array.
147
+ params = tool["function"]["parameters"]
148
+ assert params["type"] == "object"
149
+ for prop in params["properties"].values():
150
+ assert prop["type"] in {"integer", "number", "string", "array"}
151
+
152
+
153
+@pytest.mark.benchmark
154
+def test_bfcl_dataset_requires_openai_chat_backend(
155
+ hf_tokenizer: PreTrainedTokenizerBase,
156
+) -> None:
157
+ args = _args_for_bfcl(categories=["simple"])
158
+ args.backend = "openai"
159
+
160
+ with pytest.raises(ValueError, match="openai-chat"):
161
+ get_samples(args, hf_tokenizer)
162
+
163
+
164
+@pytest.mark.benchmark
165
+def test_bfcl_dataset_missing_category_raises_clear_error(
166
+ hf_tokenizer: PreTrainedTokenizerBase,
167
+) -> None:
168
+ """A typo'd category should produce an actionable ValueError, not an
169
+ opaque huggingface_hub exception."""
170
+ from huggingface_hub.errors import EntryNotFoundError
171
+
172
+ args = _args_for_bfcl(categories=["simpl"]) # typo
173
+
174
+ def raise_missing(_repo, filename, **_kwargs):
175
+ raise EntryNotFoundError(f"404 Not Found: {filename}")
176
+
177
+ with (
178
+ _patch_hf_api(raise_missing),
179
+ pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"),
180
+ ):
181
+ get_samples(args, hf_tokenizer)
182
+
183
+
184
+@pytest.mark.benchmark
185
+def test_chat_backend_uses_messages_field_when_set() -> None:
186
+ """When RequestFuncInput.chat_messages is set, the chat backend must use
187
+ it verbatim and skip default content construction from `prompt`."""
188
+ import asyncio
189
+
190
+ from vllm.benchmarks.lib.endpoint_request_func import (
191
+ RequestFuncInput,
192
+ async_request_openai_chat_completions,
193
+ )
194
+
195
+ captured: dict = {}
196
+
197
+ class _FakeResp:
198
+ status = 500
199
+ reason = "stop-after-capture"
200
+ content = None
201
+
202
+ async def __aenter__(self):
203
+ return self
204
+
205
+ async def __aexit__(self, *exc):
206
+ return False
207
+
208
+ class _FakeSession:
209
+ def post(self, url, json, headers): # noqa: A002
210
+ captured["url"] = url
211
+ captured["payload"] = json
212
+ return _FakeResp()
213
+
214
+ messages = [
215
+ {"role": "system", "content": "you are helpful"},
216
+ {"role": "user", "content": "call add(3, 4)"},
217
+ ]
218
+ req = RequestFuncInput(
219
+ prompt="IGNORED",
220
+ api_url="http://localhost:0/v1/chat/completions",
221
+ prompt_len=10,
222
+ output_len=16,
223
+ model="test-model",
224
+ chat_messages=messages,
225
+ extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]},
226
+ )
227
+
228
+ asyncio.run(
229
+ async_request_openai_chat_completions(
230
+ request_func_input=req, session=_FakeSession()
231
+ )
232
+ )
233
+
234
+ payload = captured["payload"]
235
+ assert payload["messages"] is messages, (
236
+ "chat backend must forward RequestFuncInput.chat_messages verbatim "
237
+ "instead of constructing a default user message from `prompt`"
238
+ )
239
+ # extra_body still merges in as before (shallow, per-request wins).
240
+ assert payload["tools"][0]["function"]["name"] == "add"
241
+
242
+
243
+@pytest.mark.benchmark
244
+def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
245
+ """prompt_len must reflect tokens from both messages *and* tool schemas,
246
+ so percentile buckets and input-distribution summaries aren't biased
247
+ low for tool-heavy traffic."""
248
+ paths = _write_fake_files(tmp_path)
249
+
250
+ def fake_download(_repo, filename, **_kwargs):
251
+ category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
252
+ return str(paths[category])
253
+
254
+ captured: dict = {}
255
+
256
+ class _FakeTokenizer:
257
+ def apply_chat_template(
258
+ self, messages, tools=None, tokenize=False, add_generation_prompt=True
259
+ ):
260
+ captured["tools"] = tools
261
+ base = " ".join(m.get("content", "") for m in messages)
262
+ tool_text = json.dumps(tools) if tools else ""
263
+ return base + " " + tool_text
264
+
265
+ def __call__(self, text):
266
+ # 1 "token" per whitespace-separated word.
267
+ return type("Enc", (), {"input_ids": text.split()})()
268
+
269
+ fake = _FakeTokenizer()
270
+ args = _args_for_bfcl(categories=["simple"])
271
+ args.num_prompts = 1
272
+
273
+ with _patch_hf_api(fake_download):
274
+ samples = get_samples(args, fake)
275
+
276
+ assert len(samples) == 1
277
+ assert captured["tools"] is not None, (
278
+ "apply_chat_template must be called with tools= so the schema "
279
+ "contributes to the prompt-length estimate"
280
+ )
281
+ assert len(captured["tools"]) == 1
282
+ assert captured["tools"][0]["function"]["name"] == "add"
283
+
284
+ # Sanity: prompt_len exceeds a messages-only estimate. The fake row's
285
+ # user message is "What is 2+2?" (3 whitespace-separated tokens).
286
+ assert samples[0].prompt_len > 3
287
+
288
+
289
+@pytest.mark.benchmark
290
+def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
291
+ tmp_path: Path,
292
+) -> None:
293
+ """Older tokenizers don't accept tools=; fallback must still produce a
294
+ non-zero prompt_len without crashing."""
295
+ paths = _write_fake_files(tmp_path)
296
+
297
+ def fake_download(_repo, filename, **_kwargs):
298
+ category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
299
+ return str(paths[category])
300
+
301
+ class _LegacyTokenizer:
302
+ def apply_chat_template(self, messages, **kwargs):
303
+ if "tools" in kwargs:
304
+ raise TypeError("unexpected keyword argument 'tools'")
305
+ return " ".join(m.get("content", "") for m in messages)
306
+
307
+ def __call__(self, text):
308
+ return type("Enc", (), {"input_ids": text.split()})()
309
+
310
+ args = _args_for_bfcl(categories=["simple"])
311
+ args.num_prompts = 1
312
+
313
+ with _patch_hf_api(fake_download):
314
+ samples = get_samples(args, _LegacyTokenizer())
315
+
316
+ assert len(samples) == 1
317
+ assert samples[0].prompt_len > 0
318
+
319
+
320
+@pytest.mark.benchmark
321
+def test_bfcl_schema_translation_is_recursive() -> None:
322
+ """_translate_schema must recurse into nested properties."""
323
+ input_schema = {
324
+ "type": "dict",
325
+ "properties": {
326
+ "nested": {
327
+ "type": "dict",
328
+ "properties": {
329
+ "value": {"type": "float"},
330
+ "tags": {"type": "tuple", "items": {"type": "any"}},
331
+ },
332
+ }
333
+ },
334
+ }
335
+ out = BFCLDataset._translate_schema(input_schema)
336
+ assert out["type"] == "object"
337
+ assert out["properties"]["nested"]["type"] == "object"
338
+ assert out["properties"]["nested"]["properties"]["value"]["type"] == "number"
339
+ assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array"
340
+ nested_props = out["properties"]["nested"]["properties"]
341
+ assert nested_props["tags"]["items"]["type"] == "string"
vllm/benchmarks/datasets/__init__.py
+2
@@ -6,6 +6,7 @@ from vllm.benchmarks.datasets.datasets import (
6
AIMODataset,
7
ASRDataset,
8
BenchmarkDataset,
9
+ BFCLDataset,
10
BlazeditDataset,
11
BurstGPTDataset,
12
ConversationDataset,
@@ -49,6 +50,7 @@ __all__ = [
50
"AIMODataset",
51
"ASRDataset",
52
"BenchmarkDataset",
53
+ "BFCLDataset",
54
"BlazeditDataset",
55
"BurstGPTDataset",
56
"ConversationDataset",
vllm/benchmarks/datasets/datasets.py
+250
@@ -86,6 +86,14 @@ class SampleRequest:
86
lora_request: LoRARequest | None = None
87
request_id: str | None = None
88
timestamp: float | None = None
89
+ # Pre-built chat messages. When set, the chat backend uses this list
90
+ # directly and skips constructing messages from `prompt` + multimodal
91
+ # content. Mutually exclusive with the `prompt`-based path.
92
+ chat_messages: list[dict[str, Any]] | None = None
93
+ # Per-request fields merged into the request body (e.g. tools,
94
+ # tool_choice, response_format). Shallow-merged with --extra-body at
95
+ # dispatch time; per-request keys win.
96
+ request_overrides: dict | None = None
97
98
99
# -----------------------------------------------------------------------------
@@ -1822,6 +1830,19 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1830
"from the sampled HF dataset.",
1831
)
1832
1833
+ bfcl_group = parser.add_argument_group(
1834
+ "BFCL dataset options", description=BFCLDataset.__doc__
1835
+ )
1836
+ bfcl_group.add_argument(
1837
+ "--bfcl-categories",
1838
+ type=lambda s: [c.strip() for c in s.split(",") if c.strip()],
1839
+ default=None,
1840
+ help="Comma-separated list of BFCL v3 category names (without the "
1841
+ "'BFCL_v3_' prefix or '.json' suffix) to sample from, e.g. "
1842
+ "'simple,live_simple,multiple'. Defaults to "
1843
+ f"'{','.join(BFCLDataset.DEFAULT_CATEGORIES)}'.",
1844
+ )
1845
+
1846
prefix_repetition_group = parser.add_argument_group(
1847
"prefix repetition dataset options"
1848
)
@@ -2249,6 +2270,20 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
2270
dataset_class = MMStarDataset
2271
args.hf_split = args.hf_split if args.hf_split else "val"
2272
args.hf_subset = None
2273
+ elif (
2274
+ args.dataset_path in BFCLDataset.SUPPORTED_DATASET_PATHS
2275
+ or args.hf_name in BFCLDataset.SUPPORTED_DATASET_PATHS
2276
+ ):
2277
+ if args.backend != "openai-chat":
2278
+ raise ValueError(
2279
+ "BFCL dataset requires the 'openai-chat' backend because "
2280
+ "it sends per-request tool schemas via chat completions."
2281
+ )
2282
+ dataset_class = BFCLDataset
2283
+ # BFCL does not use HF splits/subsets; stub values for base init.
2284
+ args.hf_split = args.hf_split if args.hf_split else "train"
2285
+ args.hf_subset = None
2286
+ hf_kwargs = {"categories": args.bfcl_categories}
2287
else:
2288
supported_datasets = set(
2289
[
@@ -4320,6 +4355,221 @@ class MMStarDataset(HuggingFaceDataset):
4355
return sampled_requests
4356
4357
4358
+# -----------------------------------------------------------------------------
4359
+# BFCL (Berkeley Function Calling Leaderboard) Dataset Implementation
4360
+# -----------------------------------------------------------------------------
4361
+
4362
+
4363
+class BFCLDataset(HuggingFaceDataset):
4364
+ """Berkeley Function Calling Leaderboard dataset.
4365
+
4366
+ https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard
4367
+
4368
+ BFCL ships one JSON-lines file per category at the repo root (e.g.
4369
+ ``BFCL_v3_simple.json``, ``BFCL_v3_live_simple.json``) rather than a
4370
+ single HuggingFace split. Each record has ``{id, question, function}``
4371
+ where ``function`` uses a non-OpenAI schema dialect (``"type": "dict"``).
4372
+
4373
+ This dataset loader:
4374
+ - downloads the selected per-category files via ``hf_hub_download``
4375
+ and interleaves rows round-robin so sampling is balanced
4376
+ - translates BFCL function schemas to OpenAI tool format
4377
+ - sets :attr:`SampleRequest.chat_messages` directly and attaches
4378
+ ``tools`` / ``tool_choice`` via :attr:`SampleRequest.request_overrides`,
4379
+ producing production-alike tool calling traffic when used with an
4380
+ ``openai-chat`` backend
4381
+ """
4382
+
4383
+ DEFAULT_OUTPUT_LEN = 512
4384
+ DEFAULT_CATEGORIES = ("simple", "live_simple", "multiple")
4385
+ SUPPORTED_DATASET_PATHS = {
4386
+ "gorilla-llm/Berkeley-Function-Calling-Leaderboard",
4387
+ }
4388
+ IS_MULTIMODAL = False
4389
+
4390
+ # BFCL primitive type names that are not valid JSON Schema types.
4391
+ # Map them to the closest JSON Schema equivalent so that grammar
4392
+ # backends (xgrammar, outlines) accept the translated tool schema.
4393
+ _TYPE_REMAP = {
4394
+ "dict": "object",
4395
+ "float": "number",
4396
+ "tuple": "array",
4397
+ "any": "string",
4398
+ }
4399
+
4400
+ def load_data(self) -> None:
4401
+ """Defer loading to :meth:`sample` where categories are known."""
4402
+ self.data = None
4403
+
4404
+ def _resolve_categories(self, categories: list[str] | None) -> list[str]:
4405
+ if not categories:
4406
+ return list(self.DEFAULT_CATEGORIES)
4407
+ resolved: list[str] = []
4408
+ for c in categories:
4409
+ c = c.strip()
4410
+ if not c:
4411
+ continue
4412
+ resolved.append(c)
4413
+ return resolved or list(self.DEFAULT_CATEGORIES)
4414
+
4415
+ def _load_category(self, category: str) -> list[dict]:
4416
+ # Local import: huggingface_hub.errors is a small module and
4417
+ # importing at call site keeps module import cheap for users who
4418
+ # never touch BFCL.
4419
+ from huggingface_hub.errors import EntryNotFoundError
4420
+
4421
+ filename = f"BFCL_v3_{category}.json"
4422
+ try:
4423
+ path = hf_api().hf_hub_download(
4424
+ self.dataset_path, filename, repo_type="dataset"
4425
+ )
4426
+ except EntryNotFoundError as e:
4427
+ defaults = ", ".join(self.DEFAULT_CATEGORIES)
4428
+ raise ValueError(
4429
+ f"BFCL category '{category}' not found: file '{filename}' "
4430
+ f"does not exist in {self.dataset_path}. Check --bfcl-categories "
4431
+ f"(defaults: {defaults})."
4432
+ ) from e
4433
+ rows: list[dict] = []
4434
+ with open(path) as f:
4435
+ for line in f:
4436
+ line = line.strip()
4437
+ if not line:
4438
+ continue
4439
+ rows.append(json.loads(line))
4440
+ return rows
4441
+
4442
+ @classmethod
4443
+ def _translate_schema(cls, node: Any) -> Any:
4444
+ """Recursively translate BFCL-flavored JSON schema to strict JSON Schema."""
4445
+ if isinstance(node, dict):
4446
+ translated = {k: cls._translate_schema(v) for k, v in node.items()}
4447
+ t = translated.get("type")
4448
+ if isinstance(t, str) and t in cls._TYPE_REMAP:
4449
+ translated["type"] = cls._TYPE_REMAP[t]
4450
+ return translated
4451
+ if isinstance(node, list):
4452
+ return [cls._translate_schema(v) for v in node]
4453
+ return node
4454
+
4455
+ @classmethod
4456
+ def _to_openai_tools(cls, functions: list[dict]) -> list[dict]:
4457
+ tools: list[dict] = []
4458
+ for fn in functions:
4459
+ translated = cls._translate_schema(fn)
4460
+ tools.append({"type": "function", "function": translated})
4461
+ return tools
4462
+
4463
+ def sample(
4464
+ self,
4465
+ tokenizer: TokenizerLike,
4466
+ num_requests: int,
4467
+ request_id_prefix: str = "",
4468
+ no_oversample: bool = False,
4469
+ output_len: int | None = None,
4470
+ categories: list[str] | None = None,
4471
+ **kwargs,
4472
+ ) -> list[SampleRequest]:
4473
+ output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
4474
+ categories = self._resolve_categories(categories)
4475
+
4476
+ per_category_rows: list[list[dict]] = [
4477
+ self._load_category(c) for c in categories
4478
+ ]
4479
+ # Round-robin interleave so that when --disable-shuffle is set,
4480
+ # taking the first num_requests rows still yields balanced category
4481
+ # coverage. When shuffle is on (the default) this ordering is
4482
+ # randomized away, which is fine — the subsequent random sample is
4483
+ # already balanced in expectation.
4484
+ interleaved: list[dict] = []
4485
+ max_len = max((len(rows) for rows in per_category_rows), default=0)
4486
+ for i in range(max_len):
4487
+ for rows in per_category_rows:
4488
+ if i < len(rows):
4489
+ interleaved.append(rows[i])
4490
+
4491
+ if not self.disable_shuffle:
4492
+ rng = random.Random(self.random_seed)
4493
+ rng.shuffle(interleaved)
4494
+
4495
+ sampled_requests: list[SampleRequest] = []
4496
+ for row in interleaved:
4497
+ if len(sampled_requests) >= num_requests:
4498
+ break
4499
+ question = row.get("question")
4500
+ functions = row.get("function")
4501
+ if not question or not functions:
4502
+ continue
4503
+ # BFCL question is list[list[dict]] — outer is turns. Use the
4504
+ # first turn only; skip multi-turn categories in this loader.
4505
+ if not isinstance(question, list) or not question:
4506
+ continue
4507
+ first_turn = question[0]
4508
+ if not isinstance(first_turn, list) or not first_turn:
4509
+ continue
4510
+ messages = first_turn
4511
+ if not isinstance(functions, list):
4512
+ functions = [functions]
4513
+
4514
+ tools = self._to_openai_tools(functions)
4515
+
4516
+ # Best-effort prompt length for percentile bucketing. Pass tools=
4517
+ # so modern chat templates (Llama 3.1+, Qwen, gpt-oss harmony,
4518
+ # Hermes) render the tool schemas — without this, the estimate
4519
+ # misses a significant chunk of the true input for BFCL traffic.
4520
+ # Older tokenizers reject the kwarg; fall back to tools-free.
4521
+ try:
4522
+ rendered = tokenizer.apply_chat_template(
4523
+ messages,
4524
+ tools=tools,
4525
+ tokenize=False,
4526
+ add_generation_prompt=True,
4527
+ )
4528
+ except TypeError:
4529
+ rendered = tokenizer.apply_chat_template(
4530
+ messages, tokenize=False, add_generation_prompt=True
4531
+ )
4532
+ except Exception as e:
4533
+ # Unexpected template failure — prompt_len will fall back to a
4534
+ # plain-text concatenation. Log so the degraded estimate is
4535
+ # visible instead of silently skewing latency buckets.
4536
+ logger.warning(
4537
+ "BFCL: apply_chat_template failed for a sample, falling "
4538
+ "back to plain-text prompt length: %s",
4539
+ e,
4540
+ exc_info=True,
4541
+ )
4542
+ rendered = None
4543
+ if rendered is not None:
4544
+ prompt_len = len(tokenizer(rendered).input_ids)
4545
+ else:
4546
+ text = "\n".join(m.get("content", "") for m in messages)
4547
+ prompt_len = len(tokenizer(text).input_ids)
4548
+
4549
+ # The chat backend uses `messages` directly; `prompt` is only
4550
+ # kept as a fallback string for display/debug.
4551
+ prompt_text = messages[-1].get("content", "") if messages else ""
4552
+
4553
+ sampled_requests.append(
4554
+ SampleRequest(
4555
+ prompt=prompt_text,
4556
+ prompt_len=prompt_len,
4557
+ expected_output_len=output_len,
4558
+ request_id=request_id_prefix + str(len(sampled_requests)),
4559
+ chat_messages=messages,
4560
+ request_overrides={
4561
+ "tools": tools,
4562
+ "tool_choice": "auto",
4563
+ },
4564
+ )
4565
+ )
4566
+
4567
+ self.maybe_oversample_requests(
4568
+ sampled_requests, num_requests, request_id_prefix, no_oversample
4569
+ )
4570
+ return sampled_requests
4571
+
4572
+
4573
# -----------------------------------------------------------------------------
4574
# Speed Bench Dataset Implementation
4575
# -----------------------------------------------------------------------------
vllm/benchmarks/lib/endpoint_request_func.py
+8
-1
@@ -79,6 +79,10 @@ class RequestFuncInput:
79
ignore_eos: bool = False
80
language: str | None = None
81
request_id: str | None = None
82
+ # Pre-built chat messages. When set, `async_request_openai_chat_completions`
83
+ # uses this list directly and skips building messages from `prompt` and
84
+ # `multi_modal_content`.
85
+ chat_messages: list[dict[str, Any]] | None = None
86
87
88
@dataclass
@@ -343,7 +347,10 @@ async def async_request_openai_chat_completions(
347
api_url = request_func_input.api_url
348
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
349
346
- messages = _get_chat_messages(request_func_input, mm_position=mm_position)
350
+ if request_func_input.chat_messages is not None:
351
+ messages = request_func_input.chat_messages
352
+ else:
353
+ messages = _get_chat_messages(request_func_input, mm_position=mm_position)
354
355
payload = {
356
"model": request_func_input.model_name
vllm/benchmarks/serve.py
+17
-3
@@ -57,6 +57,14 @@ from vllm.utils.network_utils import join_host_port
57
58
MILLISECONDS_TO_SECONDS_CONVERSION = 1000
59
60
+
61
+def _merge_overrides(base: dict | None, override: dict | None) -> dict | None:
62
+ """Shallow merge; per-request wins. Returns None if both are empty."""
63
+ if not base and not override:
64
+ return None
65
+ return {**(base or {}), **(override or {})}
66
+
67
+
68
TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and (
69
shutil.which("gnuplot") is not None
70
)
@@ -753,6 +761,8 @@ async def benchmark(
761
input_requests[0].expected_output_len,
762
input_requests[0].multi_modal_data,
763
)
764
+ test_extra_body = _merge_overrides(extra_body, input_requests[0].request_overrides)
765
+ test_chat_messages = input_requests[0].chat_messages
766
767
assert (
768
test_mm_content is None
@@ -773,7 +783,8 @@ async def benchmark(
783
multi_modal_content=test_mm_content,
784
ignore_eos=ignore_eos,
785
extra_headers=extra_headers,
776
- extra_body=extra_body,
786
+ extra_body=test_extra_body,
787
+ chat_messages=test_chat_messages,
788
)
789
790
if ready_check_timeout_sec > 0:
@@ -850,7 +861,8 @@ async def benchmark(
861
multi_modal_content=test_mm_content,
862
ignore_eos=ignore_eos,
863
extra_headers=extra_headers,
853
- extra_body=extra_body,
864
+ extra_body=test_extra_body,
865
+ chat_messages=test_chat_messages,
866
)
867
profile_output = await request_func(
868
request_func_input=profile_input, session=session
@@ -927,6 +939,7 @@ async def benchmark(
939
request.multi_modal_data,
940
request.request_id,
941
)
942
+ per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
943
req_model_id, req_model_name = model_id, model_name
944
if lora_modules:
945
req_lora_module = next(lora_modules)
@@ -943,8 +956,9 @@ async def benchmark(
956
multi_modal_content=mm_content,
957
ignore_eos=ignore_eos,
958
extra_headers=extra_headers,
946
- extra_body=extra_body,
959
+ extra_body=per_request_extra_body,
960
request_id=request_id,
961
+ chat_messages=request.chat_messages,
962
)
963
tasks.append(
964
asyncio.create_task(