@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / be76e5aabf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] Make scheduling policy settable via EngineArgs (#8956)

Sebastian Schoennenbeck committed Sep 30, 2024 at 14:28 UTC be76e5aabf8c026e1a82028ad70167e8c652cee9
1 file changed +14 -2
vllm/engine/arg_utils.py
+14 -2
@@ -2,8 +2,8 @@ import argparse
2 import dataclasses
3 import json
4 from dataclasses import dataclass
5 -from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
6 - Type, Union)
5 +from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
6 + Tuple, Type, Union)
7
8 import torch
9
@@ -177,6 +177,7 @@ class EngineArgs:
177 disable_async_output_proc: bool = False
178 override_neuron_config: Optional[Dict[str, Any]] = None
179 mm_processor_kwargs: Optional[Dict[str, Any]] = None
180 + scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
181
182 def __post_init__(self):
183 if self.tokenizer is None:
@@ -797,6 +798,16 @@ class EngineArgs:
798 default=None,
799 help="override or set neuron device configuration.")
800
801 + parser.add_argument(
802 + '--scheduling-policy',
803 + choices=['fcfs', 'priority'],
804 + default="fcfs",
805 + help='The scheduling policy to use. "fcfs" (first come first served'
806 + ', i.e. requests are handled in order of arrival; default) '
807 + 'or "priority" (requests are handled based on given '
808 + 'priority (lower value means earlier handling) and time of '
809 + 'arrival deciding any ties).')
810 +
811 return parser
812
813 @classmethod
@@ -1011,6 +1022,7 @@ class EngineArgs:
1022 multi_step_stream_outputs=self.multi_step_stream_outputs,
1023 send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
1024 and parallel_config.use_ray),
1025 + policy=self.scheduling_policy,
1026 )
1027 lora_config = LoRAConfig(
1028 max_lora_rank=self.max_lora_rank,