A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fixes prefix-repetition benchmark script (#26828)
Signed-off-by: Kourosh Hakhamaneshi <Kourosh@anyscale.com>
kourosh hakhamaneshi committed
Oct 14, 2025 at 19:54 UTC
a2986b3e337cb83676700fd4e76a6548e96e874b
1 file changed
+4
-3
vllm/benchmarks/datasets.py
+4
-3
@@ -2979,13 +2979,14 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):
2979
requests = []
2980
token_mismatch_total = 0
2981
for _ in range(num_prefixes):
2982
- prefix_tokens = _generate_exact_length_tokens(prefix_len)
2982
+ prefix_tokens, prefix_mismatch = _generate_exact_length_tokens(prefix_len)
2983
+ token_mismatch_total += prefix_mismatch
2984
2985
for _ in range(prompts_per_prefix):
2985
- suffix_tokens, token_mistmatch = _generate_exact_length_tokens(
2986
+ suffix_tokens, suffix_mismatch = _generate_exact_length_tokens(
2987
suffix_len
2988
)
2988
- token_mismatch_total += token_mistmatch
2989
+ token_mismatch_total += suffix_mismatch
2990
combined_tokens = prefix_tokens + suffix_tokens
2991
prompt = tokenizer.decode(combined_tokens)
2992
prompt_len = len(combined_tokens)