@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a74c68bd1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Minor patch for draft model runner (#6523)

Cody Yu committed Jul 17, 2024 at 23:06 UTC 8a74c68bd1ae48cb71e4c3bf9d7ff9a2ef8f9dae
1 file changed +6 -2
vllm/spec_decode/draft_model_runner.py
+6 -2
@@ -15,8 +15,12 @@ from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
15
16 logger = init_logger(__name__)
17
18 +# A flag to enable debug prints for the updated input tensors
19 +# before each step.
20 debug_advance_input = False
19 -enable_gpu_advance_step = True
21 +# A flag to allow GPU advance step for draft model runner.
22 +# Set to False for debugging.
23 +allow_gpu_advance_step = True
24
25
26 class TP1DraftModelRunner(ModelRunner):
@@ -196,7 +200,7 @@ class TP1DraftModelRunner(ModelRunner):
200 3. No LORA
201 4. No prompt_adapter_config
202 """
199 - if not enable_gpu_advance_step:
203 + if not allow_gpu_advance_step:
204 return False
205
206 # We allow multi-step GPU only in decode mode