A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 12:30 UTC
[Bugfix] Lazy import NgramProposer in GPU model runner (#32821)
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com>
22quinn committed
Jan 27, 2026 at 21:07 UTC
a2b877df6c95b539ef23585051ab60bfa3ad177a
1 file changed
+6
-2
vllm/v1/worker/gpu_model_runner.py
+6
-2
@@ -150,7 +150,6 @@ from vllm.v1.spec_decode.draft_model import DraftModelProposer
150
from vllm.v1.spec_decode.eagle import EagleProposer
151
from vllm.v1.spec_decode.medusa import MedusaProposer
152
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
153
-from vllm.v1.spec_decode.ngram_proposer import NgramProposer
153
from vllm.v1.spec_decode.suffix_decoding import SuffixDecodingProposer
154
from vllm.v1.structured_output.utils import apply_grammar_bitmask
155
from vllm.v1.utils import CpuGpuBuffer, record_function_or_nullcontext
@@ -185,6 +184,7 @@ from .utils import (
184
if TYPE_CHECKING:
185
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
186
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
187
+ from vllm.v1.spec_decode.ngram_proposer import NgramProposer
188
189
logger = init_logger(__name__)
190
@@ -439,13 +439,15 @@ class GPUModelRunner(
439
# layers in the draft model.
440
if self.speculative_config and get_pp_group().is_last_rank:
441
self.drafter: (
442
- NgramProposer
442
+ NgramProposer # noqa: F823
443
| SuffixDecodingProposer
444
| EagleProposer
445
| DraftModelProposer
446
| MedusaProposer
447
)
448
if self.speculative_config.method == "ngram":
449
+ from vllm.v1.spec_decode.ngram_proposer import NgramProposer
450
+
451
self.drafter = NgramProposer(self.vllm_config)
452
elif self.speculative_config.uses_draft_model():
453
self.drafter = DraftModelProposer(
@@ -3848,6 +3850,8 @@ class GPUModelRunner(
3850
spec_config = self.speculative_config
3851
assert spec_config is not None
3852
if spec_config.method == "ngram":
3853
+ from vllm.v1.spec_decode.ngram_proposer import NgramProposer
3854
+
3855
assert isinstance(sampled_token_ids, list)
3856
assert isinstance(self.drafter, NgramProposer)
3857
draft_token_ids = self.drafter.propose(