A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Minor patch for draft model runner (#6523)
Cody Yu committed
Jul 17, 2024 at 23:06 UTC
8a74c68bd1ae48cb71e4c3bf9d7ff9a2ef8f9dae
1 file changed
+6
-2
vllm/spec_decode/draft_model_runner.py
+6
-2
@@ -15,8 +15,12 @@ from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
15
16
logger = init_logger(__name__)
17
18
+# A flag to enable debug prints for the updated input tensors
19
+# before each step.
20
debug_advance_input = False
19
-enable_gpu_advance_step = True
21
+# A flag to allow GPU advance step for draft model runner.
22
+# Set to False for debugging.
23
+allow_gpu_advance_step = True
24
25
26
class TP1DraftModelRunner(ModelRunner):
@@ -196,7 +200,7 @@ class TP1DraftModelRunner(ModelRunner):
200
3. No LORA
201
4. No prompt_adapter_config
202
"""
199
- if not enable_gpu_advance_step:
203
+ if not allow_gpu_advance_step:
204
return False
205
206
# We allow multi-step GPU only in decode mode