A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core] Make scheduling policy settable via EngineArgs (#8956)
Sebastian Schoennenbeck committed
Sep 30, 2024 at 14:28 UTC
be76e5aabf8c026e1a82028ad70167e8c652cee9
1 file changed
+14
-2
vllm/engine/arg_utils.py
+14
-2
@@ -2,8 +2,8 @@ import argparse
2
import dataclasses
3
import json
4
from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
6
- Type, Union)
5
+from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
6
+ Tuple, Type, Union)
7
8
import torch
9
@@ -177,6 +177,7 @@ class EngineArgs:
177
disable_async_output_proc: bool = False
178
override_neuron_config: Optional[Dict[str, Any]] = None
179
mm_processor_kwargs: Optional[Dict[str, Any]] = None
180
+ scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
181
182
def __post_init__(self):
183
if self.tokenizer is None:
@@ -797,6 +798,16 @@ class EngineArgs:
798
default=None,
799
help="override or set neuron device configuration.")
800
801
+ parser.add_argument(
802
+ '--scheduling-policy',
803
+ choices=['fcfs', 'priority'],
804
+ default="fcfs",
805
+ help='The scheduling policy to use. "fcfs" (first come first served'
806
+ ', i.e. requests are handled in order of arrival; default) '
807
+ 'or "priority" (requests are handled based on given '
808
+ 'priority (lower value means earlier handling) and time of '
809
+ 'arrival deciding any ties).')
810
+
811
return parser
812
813
@classmethod
@@ -1011,6 +1022,7 @@ class EngineArgs:
1022
multi_step_stream_outputs=self.multi_step_stream_outputs,
1023
send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
1024
and parallel_config.use_ray),
1025
+ policy=self.scheduling_policy,
1026
)
1027
lora_config = LoRAConfig(
1028
max_lora_rank=self.max_lora_rank,