A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Misc]Fix benchmarks/README.md for speculative decoding (#18897)
Signed-off-by: rabi <ramishra@redhat.com>
Rabi Mishra committed
May 30, 2025 at 13:28 UTC
6acb7a62855a037a9cb50344e692ca23ed8782ea
1 file changed
+6
-6
benchmarks/README.md
+6
-6
@@ -146,9 +146,9 @@ python3 vllm/benchmarks/benchmark_serving.py \
146
147
``` bash
148
VLLM_USE_V1=1 vllm serve meta-llama/Meta-Llama-3-8B-Instruct \
149
- --ngram_prompt_lookup_min 2 \
150
- --ngram-prompt-lookup-max 5 \
151
- --speculative_config '{"model": "[ngram]", "num_speculative_tokens": 5}
149
+ --speculative-config $'{"method": "ngram",
150
+ "num_speculative_tokens": 5, "prompt_lookup_max": 5,
151
+ "prompt_lookup_min": 2}'
152
```
153
154
``` bash
@@ -273,9 +273,9 @@ python3 vllm/benchmarks/benchmark_throughput.py \
273
--output-len=100 \
274
--num-prompts=2048 \
275
--async-engine \
276
- --ngram_prompt_lookup_min=2 \
277
- --ngram-prompt-lookup-max=5 \
278
- --speculative_config '{"model": "[ngram]", "num_speculative_tokens": 5}
276
+ --speculative-config $'{"method": "ngram",
277
+ "num_speculative_tokens": 5, "prompt_lookup_max": 5,
278
+ "prompt_lookup_min": 2}'
279
```
280
281
```