@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e51929ebca

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

Improve configs - `SchedulerConfig` (#16533)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Apr 14, 2025 at 10:24 UTC e51929ebca5833a5e6eed484775178d33a014faa

4 files changed +283 -222

vllm/config.py

+102 -55

@@ -1522,6 +1522,9 @@ class LoadConfig:

1522 self.ignore_patterns = ["original/**/*"]

1523

1524

1525 +DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]

1526 +

1527 +

1528 @config

1529 @dataclass

1530 class ParallelConfig:

@@ -1563,7 +1566,7 @@ class ParallelConfig:

1566 placement_group: Optional["PlacementGroup"] = None

1567 """ray distributed model workers placement group."""

1568

1566 - distributed_executor_backend: Optional[Union[str,

1569 + distributed_executor_backend: Optional[Union[DistributedExecutorBackend,

1570 type["ExecutorBase"]]] = None

1571 """Backend to use for distributed model

1572 workers, either "ray" or "mp" (multiprocessing). If the product

@@ -1687,7 +1690,7 @@ class ParallelConfig:

1690 # current node and we aren't in a ray placement group.

1691

1692 from vllm.executor import ray_utils

1690 - backend = "mp"

1693 + backend: DistributedExecutorBackend = "mp"

1694 ray_found = ray_utils.ray_is_available()

1695 if current_platform.is_neuron():

1696 # neuron uses single process to control multiple devices

@@ -1755,92 +1758,124 @@ class ParallelConfig:

1758 "worker_extension_cls must be a string (qualified class name).")

1759

1760

1761 +SchedulerPolicy = Literal["fcfs", "priority"]

1762 +

1763 +

1764 +@config

1765 @dataclass

1766 class SchedulerConfig:

1767 """Scheduler configuration."""

1768

1762 - runner_type: str = "generate" # The runner type to launch for the model.

1769 + runner_type: RunnerType = "generate"

1770 + """The runner type to launch for the model."""

1771

1764 - # Maximum number of tokens to be processed in a single iteration.

1765 - max_num_batched_tokens: int = field(default=None) # type: ignore

1772 + max_num_batched_tokens: int = None # type: ignore

1773 + """Maximum number of tokens to be processed in a single iteration.

1774 +

1775 + This config has no static default. If left unspecified by the user, it will

1776 + be set in `EngineArgs.create_engine_config` based on the usage context."""

1777

1767 - # Maximum number of sequences to be processed in a single iteration.

1768 - max_num_seqs: int = 128

1778 + max_num_seqs: int = None # type: ignore

1779 + """Maximum number of sequences to be processed in a single iteration.

1780 +

1781 + This config has no static default. If left unspecified by the user, it will

1782 + be set in `EngineArgs.create_engine_config` based on the usage context."""

1783

1770 - # Maximum length of a sequence (including prompt and generated text).

1771 - max_model_len: int = 8192

1784 + max_model_len: int = None # type: ignore

1785 + """Maximum length of a sequence (including prompt and generated text). This

1786 + is primarily set in `ModelConfig` and that value should be manually

1787 + duplicated here."""

1788

1773 - # Maximum number of sequences that can be partially prefilled concurrently

1789 max_num_partial_prefills: int = 1

1790 + """For chunked prefill, the maximum number of sequences that can be

1791 + partially prefilled concurrently."""

1792

1776 - # Maximum number of "very long prompt" sequences that can be prefilled

1777 - # concurrently (long is defined by long_prefill_threshold)

1793 max_long_partial_prefills: int = 1

1794 + """For chunked prefill, the maximum number of prompts longer than

1795 + long_prefill_token_threshold that will be prefilled concurrently. Setting

1796 + this less than max_num_partial_prefills will allow shorter prompts to jump

1797 + the queue in front of longer prompts in some cases, improving latency."""

1798

1780 - # calculate context length that determines which sequences are

1781 - # considered "long"

1799 long_prefill_token_threshold: int = 0

1800 + """For chunked prefill, a request is considered long if the prompt is

1801 + longer than this number of tokens."""

1802

1784 - # The number of slots to allocate per sequence per

1785 - # step, beyond the known token ids. This is used in speculative

1786 - # decoding to store KV activations of tokens which may or may not be

1787 - # accepted.

1803 num_lookahead_slots: int = 0

1804 + """The number of slots to allocate per sequence per

1805 + step, beyond the known token ids. This is used in speculative

1806 + decoding to store KV activations of tokens which may or may not be

1807 + accepted.

1808 +

1809 + NOTE: This will be replaced by speculative config in the future; it is

1810 + present to enable correctness tests until then."""

1811

1790 - # Apply a delay (of delay factor multiplied by previous

1791 - # prompt latency) before scheduling next prompt.

1812 delay_factor: float = 0.0

1813 + """Apply a delay (of delay factor multiplied by previous

1814 + prompt latency) before scheduling next prompt."""

1815

1794 - # If True, prefill requests can be chunked based

1795 - # on the remaining max_num_batched_tokens.

1796 - enable_chunked_prefill: bool = False

1816 + enable_chunked_prefill: bool = None # type: ignore

1817 + """If True, prefill requests can be chunked based

1818 + on the remaining max_num_batched_tokens."""

1819

1820 is_multimodal_model: bool = False

1821 + """True if the model is multimodal."""

1822 +

1823 + # TODO (ywang96): Make this configurable.

1824 + max_num_encoder_input_tokens: int = field(init=False)

1825 + """Multimodal encoder compute budget, only used in V1.

1826 +

1827 + NOTE: This is not currently configurable. It will be overridden by

1828 + max_num_batched_tokens in case max multimodal embedding size is larger."""

1829 +

1830 + # TODO (ywang96): Make this configurable.

1831 + encoder_cache_size: int = field(init=False)

1832 + """Multimodal encoder cache size, only used in V1.

1833 +

1834 + NOTE: This is not currently configurable. It will be overridden by

1835 + max_num_batched_tokens in case max multimodal embedding size is larger."""

1836

1800 - # NOTE: The following multimodal encoder budget will be initialized to

1801 - # max_num_batched_tokens and overridden in case max multimodal embedding

1802 - # size is larger.

1803 - # TODO (ywang96): Make these configurable.

1804 - # Multimodal encoder compute budget, only used in V1

1805 - max_num_encoder_input_tokens: int = field(default=None) # type: ignore

1806 -

1807 - # Multimodal encoder cache size, only used in V1

1808 - encoder_cache_size: int = field(default=None) # type: ignore

1809 -

1810 - # Whether to perform preemption by swapping or

1811 - # recomputation. If not specified, we determine the mode as follows:

1812 - # We use recomputation by default since it incurs lower overhead than

1813 - # swapping. However, when the sequence group has multiple sequences

1814 - # (e.g., beam search), recomputation is not currently supported. In

1815 - # such a case, we use swapping instead.

1837 preemption_mode: Optional[str] = None

1838 + """Whether to perform preemption by swapping or

1839 + recomputation. If not specified, we determine the mode as follows:

1840 + We use recomputation by default since it incurs lower overhead than

1841 + swapping. However, when the sequence group has multiple sequences

1842 + (e.g., beam search), recomputation is not currently supported. In

1843 + such a case, we use swapping instead."""

1844

1845 num_scheduler_steps: int = 1

1846 + """Maximum number of forward steps per scheduler call."""

1847

1820 - multi_step_stream_outputs: bool = False

1848 + multi_step_stream_outputs: bool = True

1849 + """If False, then multi-step will stream outputs at the end of all steps"""

1850

1822 - # Private API. If used, scheduler sends delta data to

1823 - # workers instead of an entire data. It should be enabled only

1824 - # when SPMD worker architecture is enabled. I.e.,

1825 - # VLLM_USE_RAY_SPMD_WORKER=1

1851 send_delta_data: bool = False

1827 -

1828 - # The scheduling policy to use. "fcfs" (default) or "priority".

1829 - policy: str = "fcfs"

1852 + """Private API. If used, scheduler sends delta data to

1853 + workers instead of an entire data. It should be enabled only

1854 + when SPMD worker architecture is enabled. I.e.,

1855 + VLLM_USE_RAY_SPMD_WORKER=1"""

1856 +

1857 + policy: SchedulerPolicy = "fcfs"

1858 + """The scheduling policy to use:\n

1859 + - "fcfs" means first come first served, i.e. requests are handled in order

1860 + of arrival.\n

1861 + - "priority" means requests are handled based on given priority (lower

1862 + value means earlier handling) and time of arrival deciding any ties)."""

1863

1864 chunked_prefill_enabled: bool = field(init=False)

1865 + """True if chunked prefill is enabled."""

1866

1833 - # If set to true and chunked prefill is enabled, we do not want to

1834 - # partially schedule a multimodal item. Only used in V1

1835 - # This ensures that if a request has a mixed prompt

1836 - # (like text tokens TTTT followed by image tokens IIIIIIIIII) where only

1837 - # some image tokens can be scheduled (like TTTTIIIII, leaving IIIII),

1838 - # it will be scheduled as TTTT in one step and IIIIIIIIII in the next.

1867 disable_chunked_mm_input: bool = False

1868 + """If set to true and chunked prefill is enabled, we do not want to

1869 + partially schedule a multimodal item. Only used in V1

1870 + This ensures that if a request has a mixed prompt

1871 + (like text tokens TTTT followed by image tokens IIIIIIIIII) where only

1872 + some image tokens can be scheduled (like TTTTIIIII, leaving IIIII),

1873 + it will be scheduled as TTTT in one step and IIIIIIIIII in the next."""

1874

1841 - # scheduler class or path. "vllm.core.scheduler.Scheduler" (default)

1842 - # or "mod.custom_class".

1875 scheduler_cls: Union[str, type[object]] = "vllm.core.scheduler.Scheduler"

1876 + """The scheduler class to use. "vllm.core.scheduler.Scheduler" is the

1877 + default scheduler. Can be a class directly or the path to a class of form

1878 + "mod.custom_class"."""

1879

1880 def compute_hash(self) -> str:

1881 """

@@ -1862,6 +1897,18 @@ class SchedulerConfig:

1897 return hash_str

1898

1899 def __post_init__(self) -> None:

1900 + if self.max_model_len is None:

1901 + self.max_model_len = 8192

1902 + logger.warning(

1903 + "max_model_len was is not set. Defaulting to arbitrary value "

1904 + "of %d.", self.max_model_len)

1905 +

1906 + if self.max_num_seqs is None:

1907 + self.max_num_seqs = 128

1908 + logger.warning(

1909 + "max_num_seqs was is not set. Defaulting to arbitrary value "

1910 + "of %d.", self.max_num_seqs)

1911 +

1912 if self.max_num_batched_tokens is None:

1913 if self.enable_chunked_prefill:

1914 if self.num_scheduler_steps > 1:

vllm/engine/arg_utils.py

+167 -153

@@ -1,25 +1,30 @@

1 # SPDX-License-Identifier: Apache-2.0

3 +# yapf: disable

4 import argparse

5 import dataclasses

6 import json

7 import re

8 import threading

9 from dataclasses import MISSING, dataclass, fields

9 -from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,

10 - Tuple, Type, Union, cast, get_args, get_origin)

10 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, Mapping,

11 + Optional, Tuple, Type, TypeVar, Union, cast, get_args,

12 + get_origin)

14 import torch

15 +from typing_extensions import TypeIs

17 import vllm.envs as envs

18 from vllm import version

19 from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,

17 - DecodingConfig, DeviceConfig, HfOverrides,

20 + DecodingConfig, DeviceConfig,

21 + DistributedExecutorBackend, HfOverrides,

22 KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig,

23 ModelConfig, ModelImpl, ObservabilityConfig,

24 ParallelConfig, PoolerConfig, PromptAdapterConfig,

21 - SchedulerConfig, SpeculativeConfig, TaskOption,

22 - TokenizerPoolConfig, VllmConfig, get_attr_docs)

25 + SchedulerConfig, SchedulerPolicy, SpeculativeConfig,

26 + TaskOption, TokenizerPoolConfig, VllmConfig,

27 + get_attr_docs)

28 from vllm.executor.executor_base import ExecutorBase

29 from vllm.logger import init_logger

30 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS

@@ -28,7 +33,9 @@ from vllm.reasoning import ReasoningParserManager

33 from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3

34 from vllm.transformers_utils.utils import check_gguf_file

35 from vllm.usage.usage_lib import UsageContext

31 -from vllm.utils import FlexibleArgumentParser, StoreBoolean, is_in_ray_actor

36 +from vllm.utils import FlexibleArgumentParser, is_in_ray_actor

37 +

38 +# yapf: enable

40 if TYPE_CHECKING:

41 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup

@@ -47,11 +54,32 @@ DEVICE_OPTIONS = [

54 "hpu",

55 ]

57 +# object is used to allow for special typing forms

58 +T = TypeVar("T")

59 +TypeHint = Union[type[Any], object]

60 +TypeHintT = Union[type[T], object]

61 +

51 -def nullable_str(val: str):

52 - if not val or val == "None":

63 +def optional_arg(val: str, return_type: type[T]) -> Optional[T]:

64 + if val == "" or val == "None":

65 return None

54 - return val

66 + try:

67 + return cast(Callable, return_type)(val)

68 + except ValueError as e:

69 + raise argparse.ArgumentTypeError(

70 + f"Value {val} cannot be converted to {return_type}.") from e

71 +

72 +

73 +def optional_str(val: str) -> Optional[str]:

74 + return optional_arg(val, str)

75 +

76 +

77 +def optional_int(val: str) -> Optional[int]:

78 + return optional_arg(val, int)

79 +

80 +

81 +def optional_float(val: str) -> Optional[float]:

82 + return optional_arg(val, float)

85 def nullable_kvs(val: str) -> Optional[Mapping[str, int]]:

@@ -112,7 +140,8 @@ class EngineArgs:

140 # is intended for expert use only. The API may change without

141 # notice.

142 distributed_executor_backend: Optional[Union[

115 - str, Type[ExecutorBase]]] = ParallelConfig.distributed_executor_backend

143 + DistributedExecutorBackend,

144 + Type[ExecutorBase]]] = ParallelConfig.distributed_executor_backend

145 # number of P/D disaggregation (or other disaggregation) workers

146 pipeline_parallel_size: int = ParallelConfig.pipeline_parallel_size

147 tensor_parallel_size: int = ParallelConfig.tensor_parallel_size

@@ -129,11 +158,13 @@ class EngineArgs:

158 swap_space: float = 4 # GiB

159 cpu_offload_gb: float = 0 # GiB

160 gpu_memory_utilization: float = 0.90

132 - max_num_batched_tokens: Optional[int] = None

133 - max_num_partial_prefills: Optional[int] = 1

134 - max_long_partial_prefills: Optional[int] = 1

135 - long_prefill_token_threshold: Optional[int] = 0

136 - max_num_seqs: Optional[int] = None

161 + max_num_batched_tokens: Optional[

162 + int] = SchedulerConfig.max_num_batched_tokens

163 + max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills

164 + max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills

165 + long_prefill_token_threshold: int = \

166 + SchedulerConfig.long_prefill_token_threshold

167 + max_num_seqs: Optional[int] = SchedulerConfig.max_num_seqs

168 max_logprobs: int = 20 # Default value for OpenAI Chat Completions API

169 disable_log_stats: bool = False

170 revision: Optional[str] = None

@@ -169,20 +200,21 @@ class EngineArgs:

200 lora_dtype: Optional[Union[str, torch.dtype]] = 'auto'

201 max_cpu_loras: Optional[int] = None

202 device: str = 'auto'

172 - num_scheduler_steps: int = 1

173 - multi_step_stream_outputs: bool = True

203 + num_scheduler_steps: int = SchedulerConfig.num_scheduler_steps

204 + multi_step_stream_outputs: bool = SchedulerConfig.multi_step_stream_outputs

205 ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight

206 num_gpu_blocks_override: Optional[int] = None

176 - num_lookahead_slots: int = 0

207 + num_lookahead_slots: int = SchedulerConfig.num_lookahead_slots

208 model_loader_extra_config: Optional[

209 dict] = LoadConfig.model_loader_extra_config

210 ignore_patterns: Optional[Union[str,

211 List[str]]] = LoadConfig.ignore_patterns

181 - preemption_mode: Optional[str] = None

212 + preemption_mode: Optional[str] = SchedulerConfig.preemption_mode

213

183 - scheduler_delay_factor: float = 0.0

184 - enable_chunked_prefill: Optional[bool] = None

185 - disable_chunked_mm_input: bool = False

214 + scheduler_delay_factor: float = SchedulerConfig.delay_factor

215 + enable_chunked_prefill: Optional[

216 + bool] = SchedulerConfig.enable_chunked_prefill

217 + disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input

218

219 guided_decoding_backend: str = DecodingConfig.guided_decoding_backend

220 logits_processor_pattern: Optional[str] = None

@@ -194,8 +226,8 @@ class EngineArgs:

226 otlp_traces_endpoint: Optional[str] = None

227 collect_detailed_traces: Optional[str] = None

228 disable_async_output_proc: bool = False

197 - scheduling_policy: Literal["fcfs", "priority"] = "fcfs"

198 - scheduler_cls: Union[str, Type[object]] = "vllm.core.scheduler.Scheduler"

229 + scheduling_policy: SchedulerPolicy = SchedulerConfig.policy

230 + scheduler_cls: Union[str, Type[object]] = SchedulerConfig.scheduler_cls

231

232 override_neuron_config: Optional[Dict[str, Any]] = None

233 override_pooler_config: Optional[PoolerConfig] = None

@@ -236,15 +268,33 @@ class EngineArgs:

268 def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

269 """Shared CLI arguments for vLLM engine."""

270

239 - def is_type_in_union(cls: type[Any], type: type[Any]) -> bool:

271 + def is_type_in_union(cls: TypeHint, type: TypeHint) -> bool:

272 """Check if the class is a type in a union type."""

241 - return get_origin(cls) is Union and type in get_args(cls)

242 -

243 - def is_optional(cls: type[Any]) -> bool:

273 + is_union = get_origin(cls) is Union

274 + type_in_union = type in [get_origin(a) or a for a in get_args(cls)]

275 + return is_union and type_in_union

276 +

277 + def get_type_from_union(cls: TypeHint, type: TypeHintT) -> TypeHintT:

278 + """Get the type in a union type."""

279 + for arg in get_args(cls):

280 + if (get_origin(arg) or arg) is type:

281 + return arg

282 + raise ValueError(f"Type {type} not found in union type {cls}.")

283 +

284 + def is_optional(cls: TypeHint) -> TypeIs[Union[Any, None]]:

285 """Check if the class is an optional type."""

286 return is_type_in_union(cls, type(None))

287

247 - def get_kwargs(cls: type[Any]) -> Dict[str, Any]:

288 + def can_be_type(cls: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]:

289 + """Check if the class can be of type."""

290 + return cls is type or get_origin(cls) is type or is_type_in_union(

291 + cls, type)

292 +

293 + def is_custom_type(cls: TypeHint) -> bool:

294 + """Check if the class is a custom type."""

295 + return cls.__module__ != "builtins"

296 +

297 + def get_kwargs(cls: type[Any]) -> dict[str, Any]:

298 cls_docs = get_attr_docs(cls)

299 kwargs = {}

300 for field in fields(cls):

@@ -253,19 +303,41 @@ class EngineArgs:

303 default = (field.default_factory

304 if field.default is MISSING else field.default)

305 kwargs[name] = {"default": default, "help": cls_docs[name]}

256 - # When using action="store_true"

257 - # add_argument doesn't accept type

258 - if field.type is bool:

259 - continue

260 - # Handle optional fields

261 - if is_optional(field.type):

262 - kwargs[name]["type"] = nullable_str

263 - continue

264 - # Handle str in union fields

265 - if is_type_in_union(field.type, str):

266 - kwargs[name]["type"] = str

267 - continue

268 - kwargs[name]["type"] = field.type

306 +

307 + # Make note of if the field is optional and get the actual

308 + # type of the field if it is

309 + optional = is_optional(field.type)

310 + field_type = get_args(

311 + field.type)[0] if optional else field.type

312 +

313 + if can_be_type(field_type, bool):

314 + # Creates --no-<name> and --<name> flags

315 + kwargs[name]["action"] = argparse.BooleanOptionalAction

316 + kwargs[name]["type"] = bool

317 + elif can_be_type(field_type, Literal):

318 + # Creates choices from Literal arguments

319 + if is_type_in_union(field_type, Literal):

320 + field_type = get_type_from_union(field_type, Literal)

321 + choices = get_args(field_type)

322 + kwargs[name]["choices"] = choices

323 + choice_type = type(choices[0])

324 + assert all(type(c) is choice_type for c in choices), (

325 + f"All choices must be of the same type. "

326 + f"Got {choices} with types {[type(c) for c in choices]}"

327 + )

328 + kwargs[name]["type"] = choice_type

329 + elif can_be_type(field_type, int):

330 + kwargs[name]["type"] = optional_int if optional else int

331 + elif can_be_type(field_type, float):

332 + kwargs[name][

333 + "type"] = optional_float if optional else float

334 + elif (can_be_type(field_type, str)

335 + or can_be_type(field_type, dict)

336 + or is_custom_type(field_type)):

337 + kwargs[name]["type"] = optional_str if optional else str

338 + else:

339 + raise ValueError(

340 + f"Unsupported type {field.type} for argument {name}. ")

341 return kwargs

342

343 # Model arguments

@@ -285,13 +357,13 @@ class EngineArgs:

357 'which task to use.')

358 parser.add_argument(

359 '--tokenizer',

288 - type=nullable_str,

360 + type=optional_str,

361 default=EngineArgs.tokenizer,

362 help='Name or path of the huggingface tokenizer to use. '

363 'If unspecified, model name or path will be used.')

364 parser.add_argument(

365 "--hf-config-path",

294 - type=nullable_str,

366 + type=optional_str,

367 default=EngineArgs.hf_config_path,

368 help='Name or path of the huggingface config to use. '

369 'If unspecified, model name or path will be used.')

@@ -303,21 +375,21 @@ class EngineArgs:

375 'the input. The generated output will contain token ids.')

376 parser.add_argument(

377 '--revision',

306 - type=nullable_str,

378 + type=optional_str,

379 default=None,

380 help='The specific model version to use. It can be a branch '

381 'name, a tag name, or a commit id. If unspecified, will use '

382 'the default version.')

383 parser.add_argument(

384 '--code-revision',

313 - type=nullable_str,

385 + type=optional_str,

386 default=None,

387 help='The specific revision to use for the model code on '

388 'Hugging Face Hub. It can be a branch name, a tag name, or a '

389 'commit id. If unspecified, will use the default version.')

390 parser.add_argument(

391 '--tokenizer-revision',

320 - type=nullable_str,

392 + type=optional_str,

393 default=None,

394 help='Revision of the huggingface tokenizer to use. '

395 'It can be a branch name, a tag name, or a commit id. '

@@ -357,7 +429,6 @@ class EngineArgs:

429 load_group.add_argument('--model-loader-extra-config',

430 **load_kwargs["model_loader_extra_config"])

431 load_group.add_argument('--use-tqdm-on-load',

360 - action=argparse.BooleanOptionalAction,

432 **load_kwargs["use_tqdm_on_load"])

433

434 parser.add_argument(

@@ -413,7 +484,7 @@ class EngineArgs:

484 'the behavior is subject to change in each release.')

485 parser.add_argument(

486 '--logits-processor-pattern',

416 - type=nullable_str,

487 + type=optional_str,

488 default=None,

489 help='Optional regex pattern specifying valid logits processor '

490 'qualified names that can be passed with the `logits_processors` '

@@ -439,7 +510,6 @@ class EngineArgs:

510 )

511 parallel_group.add_argument(

512 '--distributed-executor-backend',

442 - choices=['ray', 'mp', 'uni', 'external_launcher'],

513 **parallel_kwargs["distributed_executor_backend"])

514 parallel_group.add_argument(

515 '--pipeline-parallel-size', '-pp',

@@ -450,18 +520,15 @@ class EngineArgs:

520 **parallel_kwargs["data_parallel_size"])

521 parallel_group.add_argument(

522 '--enable-expert-parallel',

453 - action='store_true',

523 **parallel_kwargs["enable_expert_parallel"])

524 parallel_group.add_argument(

525 '--max-parallel-loading-workers',

526 **parallel_kwargs["max_parallel_loading_workers"])

527 parallel_group.add_argument(

528 '--ray-workers-use-nsight',

460 - action='store_true',

529 **parallel_kwargs["ray_workers_use_nsight"])

530 parallel_group.add_argument(

531 '--disable-custom-all-reduce',

464 - action='store_true',

532 **parallel_kwargs["disable_custom_all_reduce"])

533 # KV cache arguments

534 parser.add_argument('--block-size',

@@ -502,14 +569,6 @@ class EngineArgs:

569 'block manager v2) is now the default. '

570 'Setting this flag to True or False'

571 ' has no effect on vLLM behavior.')

505 - parser.add_argument(

506 - '--num-lookahead-slots',

507 - type=int,

508 - default=EngineArgs.num_lookahead_slots,

509 - help='Experimental scheduling config necessary for '

510 - 'speculative decoding. This will be replaced by '

511 - 'speculative config in the future; it is present '

512 - 'to enable correctness tests until then.')

572

573 parser.add_argument('--seed',

574 type=int,

@@ -552,36 +611,6 @@ class EngineArgs:

611 default=None,

612 help='If specified, ignore GPU profiling result and use this number'

613 ' of GPU blocks. Used for testing preemption.')

555 - parser.add_argument('--max-num-batched-tokens',

556 - type=int,

557 - default=EngineArgs.max_num_batched_tokens,

558 - help='Maximum number of batched tokens per '

559 - 'iteration.')

560 - parser.add_argument(

561 - "--max-num-partial-prefills",

562 - type=int,

563 - default=EngineArgs.max_num_partial_prefills,

564 - help="For chunked prefill, the max number of concurrent \

565 - partial prefills.")

566 - parser.add_argument(

567 - "--max-long-partial-prefills",

568 - type=int,

569 - default=EngineArgs.max_long_partial_prefills,

570 - help="For chunked prefill, the maximum number of prompts longer "

571 - "than --long-prefill-token-threshold that will be prefilled "

572 - "concurrently. Setting this less than --max-num-partial-prefills "

573 - "will allow shorter prompts to jump the queue in front of longer "

574 - "prompts in some cases, improving latency.")

575 - parser.add_argument(

576 - "--long-prefill-token-threshold",

577 - type=float,

578 - default=EngineArgs.long_prefill_token_threshold,

579 - help="For chunked prefill, a request is considered long if the "

580 - "prompt is longer than this number of tokens.")

581 - parser.add_argument('--max-num-seqs',

582 - type=int,

583 - default=EngineArgs.max_num_seqs,

584 - help='Maximum number of sequences per iteration.')

614 parser.add_argument(

615 '--max-logprobs',

616 type=int,

@@ -594,7 +623,7 @@ class EngineArgs:

623 # Quantization settings.

624 parser.add_argument('--quantization',

625 '-q',

597 - type=nullable_str,

626 + type=optional_str,

627 choices=[*QUANTIZATION_METHODS, None],

628 default=EngineArgs.quantization,

629 help='Method used to quantize the weights. If '

@@ -658,7 +687,7 @@ class EngineArgs:

687 'asynchronous tokenization. Ignored '

688 'if tokenizer_pool_size is 0.')

689 parser.add_argument('--tokenizer-pool-extra-config',

661 - type=nullable_str,

690 + type=optional_str,

691 default=EngineArgs.tokenizer_pool_extra_config,

692 help='Extra config for tokenizer pool. '

693 'This should be a JSON string that will be '

@@ -721,7 +750,7 @@ class EngineArgs:

750 'base model dtype.'))

751 parser.add_argument(

752 '--long-lora-scaling-factors',

724 - type=nullable_str,

753 + type=optional_str,

754 default=EngineArgs.long_lora_scaling_factors,

755 help=('Specify multiple scaling factors (which can '

756 'be different from base model scaling factor '

@@ -766,28 +795,6 @@ class EngineArgs:

795 help=('Maximum number of forward steps per '

796 'scheduler call.'))

797

769 - parser.add_argument(

770 - '--multi-step-stream-outputs',

771 - action=StoreBoolean,

772 - default=EngineArgs.multi_step_stream_outputs,

773 - nargs="?",

774 - const="True",

775 - help='If False, then multi-step will stream outputs at the end '

776 - 'of all steps')

777 - parser.add_argument(

778 - '--scheduler-delay-factor',

779 - type=float,

780 - default=EngineArgs.scheduler_delay_factor,

781 - help='Apply a delay (of delay factor multiplied by previous '

782 - 'prompt latency) before scheduling next prompt.')

783 - parser.add_argument(

784 - '--enable-chunked-prefill',

785 - action=StoreBoolean,

786 - default=EngineArgs.enable_chunked_prefill,

787 - nargs="?",

788 - const="True",

789 - help='If set, the prefill requests can be chunked based on the '

790 - 'max_num_batched_tokens.')

798 parser.add_argument('--speculative-config',

799 type=json.loads,

800 default=None,

@@ -863,22 +870,43 @@ class EngineArgs:

870 help="Disable async output processing. This may result in "

871 "lower performance.")

872

866 - parser.add_argument(

867 - '--scheduling-policy',

868 - choices=['fcfs', 'priority'],

869 - default="fcfs",

870 - help='The scheduling policy to use. "fcfs" (first come first served'

871 - ', i.e. requests are handled in order of arrival; default) '

872 - 'or "priority" (requests are handled based on given '

873 - 'priority (lower value means earlier handling) and time of '

874 - 'arrival deciding any ties).')

875 -

876 - parser.add_argument(

877 - '--scheduler-cls',

878 - default=EngineArgs.scheduler_cls,

879 - help='The scheduler class to use. "vllm.core.scheduler.Scheduler" '

880 - 'is the default scheduler. Can be a class directly or the path to '

881 - 'a class of form "mod.custom_class".')

873 + # Scheduler arguments

874 + scheduler_kwargs = get_kwargs(SchedulerConfig)

875 + scheduler_group = parser.add_argument_group(

876 + title="SchedulerConfig",

877 + description=SchedulerConfig.__doc__,

878 + )

879 + scheduler_group.add_argument(

880 + '--max-num-batched-tokens',

881 + **scheduler_kwargs["max_num_batched_tokens"])

882 + scheduler_group.add_argument('--max-num-seqs',

883 + **scheduler_kwargs["max_num_seqs"])

884 + scheduler_group.add_argument(

885 + "--max-num-partial-prefills",

886 + **scheduler_kwargs["max_num_partial_prefills"])

887 + scheduler_group.add_argument(

888 + "--max-long-partial-prefills",

889 + **scheduler_kwargs["max_long_partial_prefills"])

890 + scheduler_group.add_argument(

891 + "--long-prefill-token-threshold",

892 + **scheduler_kwargs["long_prefill_token_threshold"])

893 + scheduler_group.add_argument('--num-lookahead-slots',

894 + **scheduler_kwargs["num_lookahead_slots"])

895 + scheduler_group.add_argument('--scheduler-delay-factor',

896 + **scheduler_kwargs["delay_factor"])

897 + scheduler_group.add_argument(

898 + '--enable-chunked-prefill',

899 + **scheduler_kwargs["enable_chunked_prefill"])

900 + scheduler_group.add_argument(

901 + '--multi-step-stream-outputs',

902 + **scheduler_kwargs["multi_step_stream_outputs"])

903 + scheduler_group.add_argument('--scheduling-policy',

904 + **scheduler_kwargs["policy"])

905 + scheduler_group.add_argument(

906 + "--disable-chunked-mm-input",

907 + **scheduler_kwargs["disable_chunked_mm_input"])

908 + parser.add_argument('--scheduler-cls',

909 + **scheduler_kwargs["scheduler_cls"])

910

911 parser.add_argument(

912 '--override-neuron-config',

@@ -930,7 +958,7 @@ class EngineArgs:

958 'class without changing the existing functions.')

959 parser.add_argument(

960 "--generation-config",

933 - type=nullable_str,

961 + type=optional_str,

962 default="auto",

963 help="The folder path to the generation config. "

964 "Defaults to 'auto', the generation config will be loaded from "

@@ -1003,20 +1031,6 @@ class EngineArgs:

1031 "Note that even if this is set to False, cascade attention will be "

1032 "only used when the heuristic tells that it's beneficial.")

1033

1006 - parser.add_argument(

1007 - "--disable-chunked-mm-input",

1008 - action=StoreBoolean,

1009 - default=EngineArgs.disable_chunked_mm_input,

1010 - nargs="?",

1011 - const="True",

1012 - help="Disable multimodal input chunking attention for V1. "

1013 - "If set to true and chunked prefill is enabled, we do not want to"

1014 - " partially schedule a multimodal item. This ensures that if a "

1015 - "request has a mixed prompt (like text tokens TTTT followed by "

1016 - "image tokens IIIIIIIIII) where only some image tokens can be "

1017 - "scheduled (like TTTTIIIII, leaving IIIII), it will be scheduled "

1018 - "as TTTT in one step and IIIIIIIIII in the next.")

1019 -

1034 return parser

1035

1036 @classmethod

@@ -1370,7 +1384,7 @@ class EngineArgs:

1384 recommend_to_remove=False)

1385 return False

1386

1373 - if self.preemption_mode != EngineArgs.preemption_mode:

1387 + if self.preemption_mode != SchedulerConfig.preemption_mode:

1388 _raise_or_fallback(feature_name="--preemption-mode",

1389 recommend_to_remove=True)

1390 return False

@@ -1381,17 +1395,17 @@ class EngineArgs:

1395 recommend_to_remove=True)

1396 return False

1397

1384 - if self.scheduling_policy != EngineArgs.scheduling_policy:

1398 + if self.scheduling_policy != SchedulerConfig.policy:

1399 _raise_or_fallback(feature_name="--scheduling-policy",

1400 recommend_to_remove=False)

1401 return False

1402

1389 - if self.num_scheduler_steps != EngineArgs.num_scheduler_steps:

1403 + if self.num_scheduler_steps != SchedulerConfig.num_scheduler_steps:

1404 _raise_or_fallback(feature_name="--num-scheduler-steps",

1405 recommend_to_remove=True)

1406 return False

1407

1394 - if self.scheduler_delay_factor != EngineArgs.scheduler_delay_factor:

1408 + if self.scheduler_delay_factor != SchedulerConfig.delay_factor:

1409 _raise_or_fallback(feature_name="--scheduler-delay-factor",

1410 recommend_to_remove=True)

1411 return False

@@ -1475,9 +1489,9 @@ class EngineArgs:

1489

1490 # No Concurrent Partial Prefills so far.

1491 if (self.max_num_partial_prefills

1478 - != EngineArgs.max_num_partial_prefills

1492 + != SchedulerConfig.max_num_partial_prefills

1493 or self.max_long_partial_prefills

1480 - != EngineArgs.max_long_partial_prefills):

1494 + != SchedulerConfig.max_long_partial_prefills):

1495 _raise_or_fallback(feature_name="Concurrent Partial Prefill",

1496 recommend_to_remove=False)

1497 return False

vllm/entrypoints/openai/cli_args.py

+12 -12

@@ -11,7 +11,7 @@ import ssl

11 from collections.abc import Sequence

12 from typing import Optional, Union, get_args

14 -from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str

14 +from vllm.engine.arg_utils import AsyncEngineArgs, optional_str

15 from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,

16 validate_chat_template)

17 from vllm.entrypoints.openai.serving_models import (LoRAModulePath,

@@ -79,7 +79,7 @@ class PromptAdapterParserAction(argparse.Action):

80 def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

81 parser.add_argument("--host",

82 - type=nullable_str,

82 + type=optional_str,

83 default=None,

84 help="Host name.")

85 parser.add_argument("--port", type=int, default=8000, help="Port number.")

@@ -108,13 +108,13 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

108 default=["*"],

109 help="Allowed headers.")

110 parser.add_argument("--api-key",

111 - type=nullable_str,

111 + type=optional_str,

112 default=None,

113 help="If provided, the server will require this key "

114 "to be presented in the header.")

115 parser.add_argument(

116 "--lora-modules",

117 - type=nullable_str,

117 + type=optional_str,

118 default=None,

119 nargs='+',

120 action=LoRAParserAction,

@@ -126,14 +126,14 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

126 "\"base_model_name\": \"id\"}``")

127 parser.add_argument(

128 "--prompt-adapters",

129 - type=nullable_str,

129 + type=optional_str,

130 default=None,

131 nargs='+',

132 action=PromptAdapterParserAction,

133 help="Prompt adapter configurations in the format name=path. "

134 "Multiple adapters can be specified.")

135 parser.add_argument("--chat-template",

136 - type=nullable_str,

136 + type=optional_str,

137 default=None,

138 help="The file path to the chat template, "

139 "or the template in single-line form "

@@ -151,20 +151,20 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

151 'similar to OpenAI schema. '

152 'Example: ``[{"type": "text", "text": "Hello world!"}]``')

153 parser.add_argument("--response-role",

154 - type=nullable_str,

154 + type=optional_str,

155 default="assistant",

156 help="The role name to return if "

157 "``request.add_generation_prompt=true``.")

158 parser.add_argument("--ssl-keyfile",

159 - type=nullable_str,

159 + type=optional_str,

160 default=None,

161 help="The file path to the SSL key file.")

162 parser.add_argument("--ssl-certfile",

163 - type=nullable_str,

163 + type=optional_str,

164 default=None,

165 help="The file path to the SSL cert file.")

166 parser.add_argument("--ssl-ca-certs",

167 - type=nullable_str,

167 + type=optional_str,

168 default=None,

169 help="The CA certificates file.")

170 parser.add_argument(

@@ -180,13 +180,13 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

180 )

181 parser.add_argument(

182 "--root-path",

183 - type=nullable_str,

183 + type=optional_str,

184 default=None,

185 help="FastAPI root_path when app is behind a path based routing proxy."

186 )

187 parser.add_argument(

188 "--middleware",

189 - type=nullable_str,

189 + type=optional_str,

190 action="append",

191 default=[],

192 help="Additional ASGI middleware to apply to the app. "

vllm/entrypoints/openai/run_batch.py

+2 -2

@@ -12,7 +12,7 @@ import torch

12 from prometheus_client import start_http_server

13 from tqdm import tqdm

15 -from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str

15 +from vllm.engine.arg_utils import AsyncEngineArgs, optional_str

16 from vllm.engine.async_llm_engine import AsyncLLMEngine

17 from vllm.entrypoints.logger import RequestLogger, logger

18 # yapf: disable

@@ -61,7 +61,7 @@ def parse_args():

61 "to the output URL.",

62 )

63 parser.add_argument("--response-role",

64 - type=nullable_str,

64 + type=optional_str,

65 default="assistant",

66 help="The role name to return if "

67 "`request.add_generation_prompt=True`.")