@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 303d44790a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Enable multi-step output streaming by default (#9047)

Michael Goin committed Oct 3, 2024 at 22:55 UTC 303d44790a2ccab86257f1b6097e67795f0845d4
1 file changed +9 -5
vllm/engine/arg_utils.py
+9 -5
@@ -145,7 +145,7 @@ class EngineArgs:
145 max_cpu_loras: Optional[int] = None
146 device: str = 'auto'
147 num_scheduler_steps: int = 1
148 - multi_step_stream_outputs: bool = False
148 + multi_step_stream_outputs: bool = True
149 ray_workers_use_nsight: bool = False
150 num_gpu_blocks_override: Optional[int] = None
151 num_lookahead_slots: int = 0
@@ -603,13 +603,17 @@ class EngineArgs:
603
604 parser.add_argument(
605 '--multi-step-stream-outputs',
606 - action='store_true',
607 - help='If True, then multi-step will stream outputs for every step')
606 + action=StoreBoolean,
607 + default=EngineArgs.multi_step_stream_outputs,
608 + nargs="?",
609 + const="True",
610 + help='If False, then multi-step will stream outputs at the end '
611 + 'of all steps')
612 parser.add_argument(
613 '--scheduler-delay-factor',
614 type=float,
615 default=EngineArgs.scheduler_delay_factor,
612 - help='Apply a delay (of delay factor multiplied by previous'
616 + help='Apply a delay (of delay factor multiplied by previous '
617 'prompt latency) before scheduling next prompt.')
618 parser.add_argument(
619 '--enable-chunked-prefill',
@@ -632,7 +636,7 @@ class EngineArgs:
636 type=nullable_str,
637 choices=[*QUANTIZATION_METHODS, None],
638 default=EngineArgs.speculative_model_quantization,
635 - help='Method used to quantize the weights of speculative model.'
639 + help='Method used to quantize the weights of speculative model. '
640 'If None, we first check the `quantization_config` '
641 'attribute in the model config file. If that is '
642 'None, we assume the model weights are not '