A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 08:15 UTC
Fix: Correctly filter special tokens in benchmark_prefix_caching (#28615)
Signed-off-by: Di Wu <dw2761@nyu.edu>
Di Wu committed
Nov 13, 2025 at 18:57 UTC
e63fd445605b442a81a4eb2f402206cc337ab8dd
1 file changed
+1
-1
benchmarks/benchmark_prefix_caching.py
+1
-1
@@ -69,7 +69,7 @@ def sample_tokens(tokenizer: PreTrainedTokenizerBase, length: int) -> list[int]:
69
70
# Remove the special tokens.
71
return random.choices(
72
- [v for k, v in vocab.items() if k not in all_special_ids],
72
+ [v for v in vocab.values() if v not in all_special_ids],
73
k=length,
74
)
75