A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Move missed `SchedulerConfig` args into scheduler config group in `EngineArgs` (#17131)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 25, 2025 at 06:48 UTC
6ca02344780a745ccb31511ee97f4a1e59228bbf
2 files changed
+9
-17
vllm/config.py
+2
-1
@@ -1778,6 +1778,7 @@ class ParallelConfig:
1778
"worker_extension_cls must be a string (qualified class name).")
1779
1780
1781
+PreemptionMode = Literal["swap", "recompute"]
1782
SchedulerPolicy = Literal["fcfs", "priority"]
1783
1784
@@ -1854,7 +1855,7 @@ class SchedulerConfig:
1855
NOTE: This is not currently configurable. It will be overridden by
1856
max_num_batched_tokens in case max multimodal embedding size is larger."""
1857
1857
- preemption_mode: Optional[str] = None
1858
+ preemption_mode: Optional[PreemptionMode] = None
1859
"""Whether to perform preemption by swapping or
1860
recomputation. If not specified, we determine the mode as follows:
1861
We use recomputation by default since it incurs lower overhead than
vllm/engine/arg_utils.py
+7
-16
@@ -753,12 +753,6 @@ class EngineArgs:
753
)
754
device_group.add_argument("--device", **device_kwargs["device"])
755
756
- parser.add_argument('--num-scheduler-steps',
757
- type=int,
758
- default=1,
759
- help=('Maximum number of forward steps per '
760
- 'scheduler call.'))
761
-
756
# Speculative arguments
757
speculative_group = parser.add_argument_group(
758
title="SpeculativeConfig",
@@ -779,13 +773,6 @@ class EngineArgs:
773
help="The pattern(s) to ignore when loading the model."
774
"Default to `original/**/*` to avoid repeated loading of llama's "
775
"checkpoints.")
782
- parser.add_argument(
783
- '--preemption-mode',
784
- type=str,
785
- default=None,
786
- help='If \'recompute\', the engine performs preemption by '
787
- 'recomputing; If \'swap\', the engine performs preemption by '
788
- 'block swapping.')
776
777
parser.add_argument(
778
"--served-model-name",
@@ -865,14 +852,18 @@ class EngineArgs:
852
**scheduler_kwargs["num_lookahead_slots"])
853
scheduler_group.add_argument('--scheduler-delay-factor',
854
**scheduler_kwargs["delay_factor"])
868
- scheduler_group.add_argument(
869
- '--enable-chunked-prefill',
870
- **scheduler_kwargs["enable_chunked_prefill"])
855
+ scheduler_group.add_argument('--preemption-mode',
856
+ **scheduler_kwargs["preemption_mode"])
857
+ scheduler_group.add_argument('--num-scheduler-steps',
858
+ **scheduler_kwargs["num_scheduler_steps"])
859
scheduler_group.add_argument(
860
'--multi-step-stream-outputs',
861
**scheduler_kwargs["multi_step_stream_outputs"])
862
scheduler_group.add_argument('--scheduling-policy',
863
**scheduler_kwargs["policy"])
864
+ scheduler_group.add_argument(
865
+ '--enable-chunked-prefill',
866
+ **scheduler_kwargs["enable_chunked_prefill"])
867
scheduler_group.add_argument(
868
"--disable-chunked-mm-input",
869
**scheduler_kwargs["disable_chunked_mm_input"])