[Bugfix] Correct prompt lengths for timed_traces benchmark (#45423)
Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com> Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
Stan Wozniak committed
Aug 13, 2026 at 23:51 UTC
63550515b498261c820a1540be86a96e50802725
3 files changed
+8
-4
vllm/benchmarks/datasets/datasets.py
+2
-3
@@ -79,7 +79,7 @@ class SampleRequest:
79
Represents a single inference request for benchmarking.
80
"""
81
82
- prompt: str | list[str] | list[dict]
82
+ prompt: str | list[str] | list[int] | list[dict]
83
prompt_len: int
84
expected_output_len: int = 0
85
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
@@ -1572,7 +1572,6 @@ class TimedTrace(BenchmarkDataset):
1572
prompt_ids = self._expand_prompt(
1573
entry.get(self.label_hash_ids, []), input_length, tokenizer
1574
)
1575
- prompt = tokenizer.decode(prompt_ids)
1575
1576
# Get timestamp with proper error handling
1577
ts_value = entry.get(self.label_ts)
@@ -1588,7 +1587,7 @@ class TimedTrace(BenchmarkDataset):
1587
1588
samples.append(
1589
SampleRequest(
1591
- prompt=prompt,
1590
+ prompt=prompt_ids,
1591
prompt_len=prompt_len,
1592
expected_output_len=new_output_len,
1593
lora_request=None,
vllm/benchmarks/lib/endpoint_request_func.py
+1
-1
@@ -66,7 +66,7 @@ class StreamedResponseHandler:
66
class RequestFuncInput:
67
"""The input for the request function."""
68
69
- prompt: str | list[str] | list[dict[str, Any]]
69
+ prompt: str | list[str] | list[int] | list[dict[str, Any]]
70
api_url: str
71
prompt_len: int
72
output_len: int
vllm/benchmarks/serve.py
+5
@@ -2097,6 +2097,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
2097
args.ignore_eos = True
2098
2099
if args.dataset_name == "timed_trace":
2100
+ if args.backend not in ("vllm", "openai"):
2101
+ raise ValueError(
2102
+ "timed_trace dataset passes pre-tokenized prompts (list[int])"
2103
+ " and requires a completions backend ('vllm' or 'openai')."
2104
+ )
2105
# timed_trace carries per-request timestamps;
2106
# ignore EOS so generation runs to the trace's specified output length,
2107
# and default to using those timestamps for scheduling unless the user