A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Enable multi-step output streaming by default (#9047)
Michael Goin committed
Oct 3, 2024 at 22:55 UTC
303d44790a2ccab86257f1b6097e67795f0845d4
1 file changed
+9
-5
vllm/engine/arg_utils.py
+9
-5
@@ -145,7 +145,7 @@ class EngineArgs:
145
max_cpu_loras: Optional[int] = None
146
device: str = 'auto'
147
num_scheduler_steps: int = 1
148
- multi_step_stream_outputs: bool = False
148
+ multi_step_stream_outputs: bool = True
149
ray_workers_use_nsight: bool = False
150
num_gpu_blocks_override: Optional[int] = None
151
num_lookahead_slots: int = 0
@@ -603,13 +603,17 @@ class EngineArgs:
603
604
parser.add_argument(
605
'--multi-step-stream-outputs',
606
- action='store_true',
607
- help='If True, then multi-step will stream outputs for every step')
606
+ action=StoreBoolean,
607
+ default=EngineArgs.multi_step_stream_outputs,
608
+ nargs="?",
609
+ const="True",
610
+ help='If False, then multi-step will stream outputs at the end '
611
+ 'of all steps')
612
parser.add_argument(
613
'--scheduler-delay-factor',
614
type=float,
615
default=EngineArgs.scheduler_delay_factor,
612
- help='Apply a delay (of delay factor multiplied by previous'
616
+ help='Apply a delay (of delay factor multiplied by previous '
617
'prompt latency) before scheduling next prompt.')
618
parser.add_argument(
619
'--enable-chunked-prefill',
@@ -632,7 +636,7 @@ class EngineArgs:
636
type=nullable_str,
637
choices=[*QUANTIZATION_METHODS, None],
638
default=EngineArgs.speculative_model_quantization,
635
- help='Method used to quantize the weights of speculative model.'
639
+ help='Method used to quantize the weights of speculative model. '
640
'If None, we first check the `quantization_config` '
641
'attribute in the model config file. If that is '
642
'None, we assume the model weights are not '