A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 19:15 UTC
[Bugfix][Spec Decode] Avoid double call of Ngram CPU (#36952)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Ekagra Ranjan committed
Mar 13, 2026 at 16:33 UTC
d0b402974ffa2c26090ab0d816288b4bcd09f761
1 file changed
-9
vllm/v1/worker/gpu_model_runner.py
-9
@@ -4247,15 +4247,6 @@ class GPUModelRunner(
4247
self.input_batch.token_ids_cpu,
4248
slot_mappings=slot_mappings,
4249
)
4250
- if isinstance(self.drafter, NgramProposer):
4251
- assert isinstance(sampled_token_ids, list), (
4252
- "sampled_token_ids should be a python list when ngram is used."
4253
- )
4254
- draft_token_ids = self.drafter.propose(
4255
- sampled_token_ids,
4256
- self.input_batch.num_tokens_no_spec,
4257
- self.input_batch.token_ids_cpu,
4258
- )
4250
elif spec_config.use_ngram_gpu():
4251
assert isinstance(self.drafter, NgramProposerGPU)
4252
(