@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 13698db634
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 12:15 UTC

Improve configs - `ModelConfig` (#17130)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Apr 30, 2025 at 03:38 UTC 13698db634b492151c6a5e04769119e3af29b265
36 files changed +492 -650
tests/conftest.py
+1 -1
@@ -738,7 +738,7 @@ class VllmRunner:
738 - `block_size`: Set to `16` instead of `None` to reduce memory usage.
739 - `enable_chunked_prefill`: Set to `False` instead of `None` for
740 test reproducibility.
741 - - `enforce_eager`: Set to `False` instead of `None` to test CUDA graph.
741 + - `enforce_eager`: Set to `False` to test CUDA graph.
742 """
743
744 def __init__(
tests/engine/test_arg_utils.py
+1 -12
@@ -8,7 +8,7 @@ from typing import Literal, Optional
8
9 import pytest
10
11 -from vllm.config import PoolerConfig, config
11 +from vllm.config import config
12 from vllm.engine.arg_utils import (EngineArgs, contains_type, get_kwargs,
13 get_type, is_not_builtin, is_type,
14 literal_to_kwargs, nullable_kvs,
@@ -222,17 +222,6 @@ def test_prefix_cache_default():
222 assert not engine_args.enable_prefix_caching
223
224
225 -def test_valid_pooling_config():
226 - parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
227 - args = parser.parse_args([
228 - '--override-pooler-config',
229 - '{"pooling_type": "MEAN"}',
230 - ])
231 - engine_args = EngineArgs.from_cli_args(args=args)
232 - assert engine_args.override_pooler_config == PoolerConfig(
233 - pooling_type="MEAN", )
234 -
235 -
225 @pytest.mark.parametrize(
226 ("arg"),
227 [
tests/quantization/test_register_quantization_config.py
+2 -2
@@ -14,7 +14,7 @@ import torch.nn.functional as F
14 from vllm.model_executor.layers.linear import LinearBase # noqa: E501
15 from vllm.model_executor.layers.linear import UnquantizedLinearMethod
16 from vllm.model_executor.layers.quantization import (
17 - get_quantization_config, register_quantization_config)
17 + QuantizationMethods, get_quantization_config, register_quantization_config)
18 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
19 QuantizationConfig)
20
@@ -54,7 +54,7 @@ class CustomQuantConfig(QuantizationConfig):
54 """Initialize the quantization config."""
55 self.num_bits = num_bits
56
57 - def get_name(self) -> str:
57 + def get_name(self) -> QuantizationMethods:
58 """Name of the quantization method."""
59 return "custom_quant"
60
tests/test_config.py
+5 -4
@@ -185,7 +185,7 @@ def test_get_pooling_config():
185 revision=None,
186 )
187
188 - pooling_config = model_config._init_pooler_config(None)
188 + pooling_config = model_config._init_pooler_config()
189 assert pooling_config is not None
190
191 assert pooling_config.normalize
@@ -205,11 +205,12 @@ def test_get_pooling_config_from_args():
205 dtype="float16",
206 revision=None)
207
208 - override_config = PoolerConfig(pooling_type='CLS', normalize=True)
208 + override_pooler_config = PoolerConfig(pooling_type='CLS', normalize=True)
209 + model_config.override_pooler_config = override_pooler_config
210
210 - pooling_config = model_config._init_pooler_config(override_config)
211 + pooling_config = model_config._init_pooler_config()
212 assert pooling_config is not None
212 - assert asdict(pooling_config) == asdict(override_config)
213 + assert asdict(pooling_config) == asdict(override_pooler_config)
214
215
216 @pytest.mark.skipif(current_platform.is_rocm(),
vllm/config.py
+258 -255
@@ -16,9 +16,8 @@ from dataclasses import (MISSING, dataclass, field, fields, is_dataclass,
16 replace)
17 from importlib.util import find_spec
18 from pathlib import Path
19 -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Final, Literal,
20 - Optional, Protocol, TypeVar, Union, cast, get_args,
21 - get_origin)
19 +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Literal, Optional,
20 + Protocol, TypeVar, Union, cast, get_args, get_origin)
21
22 import torch
23 from pydantic import BaseModel, Field, PrivateAttr
@@ -211,103 +210,190 @@ def get_field(cls: ConfigType, name: str) -> Field:
210 f"{cls.__name__}.{name} must have a default value or default factory.")
211
212
214 -class ModelConfig:
215 - """Configuration for the model.
213 +TokenizerMode = Literal["auto", "slow", "mistral", "custom"]
214 +ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
215
217 - Args:
218 - model: Name or path of the huggingface model to use.
219 - It is also used as the content for `model_name` tag in metrics
220 - output when `served_model_name` is not specified.
221 - task: The task to use the model for. Each vLLM instance only supports
222 - one task, even if the same model can be used for multiple tasks.
223 - When the model only supports one task, "auto" can be used to select
224 - it; otherwise, you must specify explicitly which task to use.
225 - tokenizer: Name or path of the huggingface tokenizer to use.
226 - tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
227 - available, "slow" will always use the slow tokenizer,
228 - "mistral" will always use the tokenizer from `mistral_common`, and
229 - "custom" will use --tokenizer to select the preregistered tokenizer.
230 - trust_remote_code: Trust remote code (e.g., from HuggingFace) when
231 - downloading the model and tokenizer.
232 - allowed_local_media_path: Allowing API requests to read local images or
233 - videos from directories specified by the server file system.
234 - This is a security risk. Should only be enabled in trusted
235 - environments.
236 - dtype: Data type for model weights and activations. The "auto" option
237 - will use FP16 precision for FP32 and FP16 models, and BF16 precision
238 - for BF16 models.
239 - seed: Random seed for reproducibility.
240 - revision: The specific model version to use. It can be a branch name,
241 - a tag name, or a commit id. If unspecified, will use the default
242 - version.
243 - code_revision: The specific revision to use for the model code on
244 - Hugging Face Hub. It can be a branch name, a tag name, or a
245 - commit id. If unspecified, will use the default version.
246 - tokenizer_revision: The specific tokenizer version to use. It can be a
247 - branch name, a tag name, or a commit id. If unspecified, will use
248 - the default version.
249 - max_model_len: Maximum length of a sequence (including prompt and
250 - output). If None, will be derived from the model.
251 - spec_target_max_model_len: Specify the the maximum length for spec
252 - decoding draft models.
253 - quantization: Quantization method that was used to quantize the model
254 - weights. If None, we assume the model weights are not quantized.
255 - enforce_eager: Whether to enforce eager execution. If True, we will
256 - disable CUDA graph and always execute the model in eager mode.
257 - If False, we will use CUDA graph and eager execution in hybrid.
258 - If None, the user did not specify, so default to False.
259 - max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
260 - When a sequence has context length larger than this, we fall back
261 - to eager mode. Additionally for encoder-decoder models, if the
262 - sequence length of the encoder input is larger than this, we fall
263 - back to the eager mode.
264 - max_logprobs: Maximum number of log probabilities. Defaults to 20.
265 - disable_sliding_window: Whether to disable sliding window. If True,
266 - we will disable the sliding window functionality of the model.
267 - If the model does not support sliding window, this argument is
268 - ignored.
269 - skip_tokenizer_init: If true, skip initialization of tokenizer and
270 - detokenizer.
271 - served_model_name: The model name used in metrics tag `model_name`,
272 - matches the model name exposed via the APIs. If multiple model
273 - names provided, the first name will be used. If not specified,
274 - the model name will be the same as `model`.
275 - limit_mm_per_prompt: Maximum number of data items per modality
276 - per prompt. Only applicable for multimodal models.
277 - mm_processor_kwargs: Overrides for the multi-modal processor obtained
278 - from `AutoProcessor.from_pretrained`.
279 - disable_mm_preprocessor_cache: If True, disable caching of the
280 - processed multi-modal inputs.
281 - use_async_output_proc: Whether to use async output processor.
282 - Defaults to True.
283 - config_format: The config format which shall be loaded.
284 - Defaults to 'auto' which defaults to 'hf'.
285 - hf_token: The token to use as HTTP bearer authorization for remote files
286 - . If `True`, will use the token generated when running
287 - `huggingface-cli login` (stored in `~/.huggingface`).
288 - hf_overrides: If a dictionary, contains arguments to be forwarded to the
289 - HuggingFace config. If a callable, it is called to update the
290 - HuggingFace config.
291 - override_neuron_config: Initialize non default neuron config or
292 - override default neuron config that are specific to Neuron devices,
293 - this argument will be used to configure the neuron config that
294 - can not be gathered from the vllm arguments.
295 - override_pooler_config: Initialize non default pooling config or
296 - override default pooling config for the pooling model.
297 - logits_processor_pattern: Optional regex pattern specifying valid
298 - logits processor qualified names that can be passed with the
299 - `logits_processors` extra completion argument. Defaults to None,
300 - which allows no processors.
301 - generation_config: Configuration parameter file for generation.
302 - model_impl: Which implementation of the model to use:
303 - "auto" will try to use the vLLM implementation if it exists and
304 - fall back to the Transformers implementation if no vLLM
305 - implementation is available.
306 - "vllm" will use the vLLM model implementation.
307 - "transformers" will use the Transformers model implementation.
308 - override_generation_config: Override the generation config with the
309 - given config.
310 - """
216 +
217 +@config
218 +@dataclass
219 +class ModelConfig:
220 + """Configuration for the model."""
221 +
222 + model: str = "facebook/opt-125m"
223 + """Name or path of the Hugging Face model to use. It is also used as the
224 + content for `model_name` tag in metrics output when `served_model_name` is
225 + not specified."""
226 + task: Literal[TaskOption, Literal["draft"]] = "auto"
227 + """The task to use the model for. Each vLLM instance only supports one
228 + task, even if the same model can be used for multiple tasks. When the model
229 + only supports one task, "auto" can be used to select it; otherwise, you
230 + must specify explicitly which task to use."""
231 + tokenizer: str = None # type: ignore
232 + """Name or path of the Hugging Face tokenizer to use. If unspecified, model
233 + name or path will be used."""
234 + tokenizer_mode: TokenizerMode = "auto"
235 + """Tokenizer mode:\n
236 + - "auto" will use the fast tokenizer if available.\n
237 + - "slow" will always use the slow tokenizer.\n
238 + - "mistral" will always use the tokenizer from `mistral_common`.\n
239 + - "custom" will use --tokenizer to select the preregistered tokenizer."""
240 + trust_remote_code: bool = False
241 + """Trust remote code (e.g., from HuggingFace) when downloading the model
242 + and tokenizer."""
243 + dtype: Union[ModelDType, torch.dtype] = "auto"
244 + """Data type for model weights and activations:\n
245 + - "auto" will use FP16 precision for FP32 and FP16 models, and BF16
246 + precision for BF16 models.\n
247 + - "half" for FP16. Recommended for AWQ quantization.\n
248 + - "float16" is the same as "half".\n
249 + - "bfloat16" for a balance between precision and range.\n
250 + - "float" is shorthand for FP32 precision.\n
251 + - "float32" for FP32 precision."""
252 + seed: Optional[int] = None
253 + """Random seed for reproducibility."""
254 + hf_config_path: Optional[str] = None
255 + """Name or path of the Hugging Face config to use. If unspecified, model
256 + name or path will be used."""
257 + allowed_local_media_path: str = ""
258 + """Allowing API requests to read local images or videos from directories
259 + specified by the server file system. This is a security risk. Should only
260 + be enabled in trusted environments."""
261 + revision: Optional[str] = None
262 + """The specific model version to use. It can be a branch name, a tag name,
263 + or a commit id. If unspecified, will use the default version."""
264 + code_revision: Optional[str] = None
265 + """The specific revision to use for the model code on the Hugging Face Hub.
266 + It can be a branch name, a tag name, or a commit id. If unspecified, will
267 + use the default version."""
268 + rope_scaling: dict[str, Any] = field(default_factory=dict)
269 + """RoPE scaling configuration in JSON format. For example,
270 + `{"rope_type":"dynamic","factor":2.0}`."""
271 + rope_theta: Optional[float] = None
272 + """RoPE theta. Use with `rope_scaling`. In some cases, changing the RoPE
273 + theta improves the performance of the scaled model."""
274 + tokenizer_revision: Optional[str] = None
275 + """The specific revision to use for the tokenizer on the Hugging Face Hub.
276 + It can be a branch name, a tag name, or a commit id. If unspecified, will
277 + use the default version."""
278 + max_model_len: int = None # type: ignore
279 + """Model context length (prompt and output). If unspecified, will be
280 + automatically derived from the model config.
281 +
282 + When passing via `--max-model-len`, supports k/m/g/K/M/G in human-readable
283 + format. Examples:\n
284 + - 1k -> 1000\n
285 + - 1K -> 1024\n
286 + - 25.6k -> 25,600"""
287 + spec_target_max_model_len: Optional[int] = None
288 + """Specify the the maximum length for spec decoding draft models."""
289 + quantization: Optional[QuantizationMethods] = None
290 + """Method used to quantize the weights. If `None`, we first check the
291 + `quantization_config` attribute in the model config file. If that is
292 + `None`, we assume the model weights are not quantized and use `dtype` to
293 + determine the data type of the weights."""
294 + enforce_eager: bool = False
295 + """Whether to always use eager-mode PyTorch. If True, we will disable CUDA
296 + graph and always execute the model in eager mode. If False, we will use
297 + CUDA graph and eager execution in hybrid for maximal performance and
298 + flexibility."""
299 + max_seq_len_to_capture: int = 8192
300 + """Maximum sequence len covered by CUDA graphs. When a sequence has context
301 + length larger than this, we fall back to eager mode. Additionally for
302 + encoder-decoder models, if the sequence length of the encoder input is
303 + larger than this, we fall back to the eager mode."""
304 + max_logprobs: int = 20
305 + """Maximum number of log probabilities to return when `logprobs` is
306 + specified in `SamplingParams`. The default value comes the default for the
307 + OpenAI Chat Completions API."""
308 + disable_sliding_window: bool = False
309 + """Whether to disable sliding window. If True, we will disable the sliding
310 + window functionality of the model, capping to sliding window size. If the
311 + model does not support sliding window, this argument is ignored."""
312 + disable_cascade_attn: bool = False
313 + """Disable cascade attention for V1. While cascade attention does not
314 + change the mathematical correctness, disabling it could be useful for
315 + preventing potential numerical issues. Note that even if this is set to
316 + False, cascade attention will be only used when the heuristic tells that
317 + it's beneficial."""
318 + skip_tokenizer_init: bool = False
319 + """Skip initialization of tokenizer and detokenizer. Expects valid
320 + `prompt_token_ids` and `None` for prompt from the input. The generated
321 + output will contain token ids."""
322 + served_model_name: Optional[Union[str, list[str]]] = None
323 + """The model name(s) used in the API. If multiple names are provided, the
324 + server will respond to any of the provided names. The model name in the
325 + model field of a response will be the first name in this list. If not
326 + specified, the model name will be the same as the `--model` argument. Noted
327 + that this name(s) will also be used in `model_name` tag content of
328 + prometheus metrics, if multiple names provided, metrics tag will take the
329 + first one."""
330 + limit_mm_per_prompt: dict[str, int] = field(default_factory=dict)
331 + """Maximum number of data items per modality per prompt. Only applicable
332 + for multimodal models."""
333 + use_async_output_proc: bool = True
334 + """Whether to use async output processor."""
335 + config_format: Union[str, ConfigFormat] = ConfigFormat.AUTO.value
336 + """The format of the model config to load:\n
337 + - "auto" will try to load the config in hf format if available else it
338 + will try to load in mistral format.\n
339 + - "hf" will load the config in hf format.\n
340 + - "mistral" will load the config in mistral format."""
341 + hf_token: Optional[Union[bool, str]] = None
342 + """The token to use as HTTP bearer authorization for remote files . If
343 + `True`, will use the token generated when running `huggingface-cli login`
344 + (stored in `~/.huggingface`)."""
345 + hf_overrides: HfOverrides = field(default_factory=dict)
346 + """If a dictionary, contains arguments to be forwarded to the Hugging Face
347 + config. If a callable, it is called to update the HuggingFace config. When
348 + specified via CLI, the argument must be a valid JSON string."""
349 + mm_processor_kwargs: Optional[dict[str, Any]] = None
350 + """Arguments to be forwarded to the model's processor for multi-modal data,
351 + e.g., image processor. Overrides for the multi-modal processor obtained
352 + from `AutoProcessor.from_pretrained`. The available overrides depend on the
353 + model that is being run. For example, for Phi-3-Vision: `{"num_crops": 4}`.
354 + When specified via CLI, the argument must be a valid JSON string."""
355 + disable_mm_preprocessor_cache: bool = False
356 + """If `True`, disable caching of the multi-modal preprocessor/mapper (not
357 + recommended)."""
358 + override_neuron_config: dict[str, Any] = field(default_factory=dict)
359 + """Initialize non-default neuron config or override default neuron config
360 + that are specific to Neuron devices, this argument will be used to
361 + configure the neuron config that can not be gathered from the vllm
362 + arguments. e.g. `{"cast_logits_dtype": "bloat16"}`. When specified via CLI,
363 + the argument must be a valid JSON string."""
364 + pooler_config: Optional["PoolerConfig"] = field(init=False)
365 + """Pooler config which controls the behaviour of output pooling in pooling
366 + models."""
367 + override_pooler_config: Optional[Union[dict, "PoolerConfig"]] = None
368 + """Initialize non-default pooling config or override default pooling config
369 + for the pooling model. e.g. `{"pooling_type": "mean", "normalize": false}`.
370 + When specified via CLI, the argument must be a valid JSON string."""
371 + logits_processor_pattern: Optional[str] = None
372 + """Optional regex pattern specifying valid logits processor qualified names
373 + that can be passed with the `logits_processors` extra completion argument.
374 + Defaults to `None`, which allows no processors."""
375 + generation_config: str = "auto"
376 + """The folder path to the generation config. Defaults to `"auto"`, the
377 + generation config will be loaded from model path. If set to `"vllm"`, no
378 + generation config is loaded, vLLM defaults will be used. If set to a folder
379 + path, the generation config will be loaded from the specified folder path.
380 + If `max_new_tokens` is specified in generation config, then it sets a
381 + server-wide limit on the number of output tokens for all requests."""
382 + override_generation_config: dict[str, Any] = field(default_factory=dict)
383 + """Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
384 + used with `--generation-config auto`, the override parameters will be
385 + merged with the default config from the model. If used with
386 + `--generation-config vllm`, only the override parameters are used.
387 + When specified via CLI, the argument must be a valid JSON string."""
388 + enable_sleep_mode: bool = False
389 + """Enable sleep mode for the engine (only cuda platform is supported)."""
390 + model_impl: Union[str, ModelImpl] = ModelImpl.AUTO.value
391 + """Which implementation of the model to use:\n
392 + - "auto" will try to use the vLLM implementation, if it exists, and fall
393 + back to the Transformers implementation if no vLLM implementation is
394 + available.\n
395 + - "vllm" will use the vLLM model implementation.\n
396 + - "transformers" will use the Transformers model implementation."""
397
398 def compute_hash(self) -> str:
399 """
@@ -342,92 +428,43 @@ class ModelConfig:
428 assert_hashable(str_factors)
429 return hashlib.sha256(str(factors).encode()).hexdigest()
430
345 - def __init__(
346 - self,
347 - model: str,
348 - task: Literal[TaskOption, Literal["draft"]],
349 - tokenizer: str,
350 - tokenizer_mode: str,
351 - trust_remote_code: bool,
352 - dtype: Union[str, torch.dtype],
353 - seed: int,
354 - hf_config_path: Optional[str] = None,
355 - allowed_local_media_path: str = "",
356 - revision: Optional[str] = None,
357 - code_revision: Optional[str] = None,
358 - rope_scaling: Optional[dict[str, Any]] = None,
359 - rope_theta: Optional[float] = None,
360 - tokenizer_revision: Optional[str] = None,
361 - max_model_len: Optional[int] = None,
362 - spec_target_max_model_len: Optional[int] = None,
363 - quantization: Optional[str] = None,
364 - enforce_eager: Optional[bool] = None,
365 - max_seq_len_to_capture: Optional[int] = None,
366 - max_logprobs: int = 20,
367 - disable_sliding_window: bool = False,
368 - disable_cascade_attn: bool = False,
369 - skip_tokenizer_init: bool = False,
370 - served_model_name: Optional[Union[str, list[str]]] = None,
371 - limit_mm_per_prompt: Optional[dict[str, int]] = None,
372 - mm_processor_kwargs: Optional[dict[str, Any]] = None,
373 - disable_mm_preprocessor_cache: bool = False,
374 - use_async_output_proc: bool = True,
375 - config_format: ConfigFormat = ConfigFormat.AUTO,
376 - hf_token: Optional[Union[bool, str]] = None,
377 - hf_overrides: Optional[HfOverrides] = None,
378 - override_neuron_config: Optional[dict[str, Any]] = None,
379 - override_pooler_config: Optional["PoolerConfig"] = None,
380 - logits_processor_pattern: Optional[str] = None,
381 - generation_config: str = "auto",
382 - enable_sleep_mode: bool = False,
383 - override_generation_config: Optional[dict[str, Any]] = None,
384 - model_impl: Union[str, ModelImpl] = ModelImpl.AUTO,
385 - ) -> None:
386 - self.model = maybe_model_redirect(model)
387 - self.tokenizer = maybe_model_redirect(tokenizer)
388 -
389 - self.hf_config_path = hf_config_path
390 - if isinstance(hf_config_path, str):
391 - self.hf_config_path = maybe_model_redirect(hf_config_path)
392 -
393 - self.tokenizer_mode = tokenizer_mode
394 - self.trust_remote_code = trust_remote_code
395 - self.allowed_local_media_path = allowed_local_media_path
396 - self.seed = seed
397 - self.revision = revision
398 - self.code_revision = code_revision
399 - self.rope_scaling = rope_scaling
400 - self.rope_theta = rope_theta
401 - self.model_impl = model_impl
402 -
403 - if hf_overrides is None:
404 - hf_overrides = {}
405 -
406 - if callable(hf_overrides):
431 + def __post_init__(self) -> None:
432 + self.model = maybe_model_redirect(self.model)
433 + # The tokenizer is consistent with the model by default.
434 + if self.tokenizer is None:
435 + self.tokenizer = self.model
436 + if self.tokenizer_revision is None:
437 + self.tokenizer_revision = self.revision
438 + self.tokenizer = maybe_model_redirect(self.tokenizer)
439 +
440 + if isinstance(self.hf_config_path, str):
441 + self.hf_config_path = maybe_model_redirect(self.hf_config_path)
442 +
443 + if callable(self.hf_overrides):
444 hf_overrides_kw = {}
408 - hf_overrides_fn = hf_overrides
445 + hf_overrides_fn = self.hf_overrides
446 else:
410 - hf_overrides_kw = hf_overrides
447 + hf_overrides_kw = self.hf_overrides
448 hf_overrides_fn = None
449
413 - if rope_scaling is not None:
414 - hf_override: dict[str, Any] = {"rope_scaling": rope_scaling}
450 + if self.rope_scaling:
451 + hf_override: dict[str, Any] = {"rope_scaling": self.rope_scaling}
452 hf_overrides_kw.update(hf_override)
416 - hf_overrides_str = json.dumps(hf_overrides)
453 + hf_overrides_str = json.dumps(hf_overrides_kw)
454 msg = (
455 "`--rope-scaling` will be removed in a future release. "
456 f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
457 warnings.warn(DeprecationWarning(msg), stacklevel=2)
421 - if rope_theta is not None:
422 - hf_override = {"rope_theta": rope_theta}
458 + if self.rope_theta is not None:
459 + hf_override = {"rope_theta": self.rope_theta}
460 hf_overrides_kw.update(hf_override)
424 - hf_overrides_str = json.dumps(hf_overrides)
461 + hf_overrides_str = json.dumps(hf_overrides_kw)
462 msg = (
463 "`--rope-theta` will be removed in a future release. "
464 f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
465 warnings.warn(DeprecationWarning(msg), stacklevel=2)
466
430 - self.maybe_pull_model_tokenizer_for_s3(model, tokenizer)
467 + self.maybe_pull_model_tokenizer_for_s3(self.model, self.tokenizer)
468
469 if (backend := envs.VLLM_ATTENTION_BACKEND
470 ) and backend == "FLASHINFER" and find_spec("flashinfer") is None:
@@ -437,20 +474,6 @@ class ModelConfig:
474 "https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile " # noqa: E501
475 "for instructions on how to install it.")
476
440 - # The tokenizer version is consistent with the model version by default.
441 - if tokenizer_revision is None:
442 - self.tokenizer_revision = revision
443 - else:
444 - self.tokenizer_revision = tokenizer_revision
445 - self.quantization = quantization
446 - self.enforce_eager = enforce_eager
447 - self.max_seq_len_to_capture = max_seq_len_to_capture
448 - self.max_logprobs = max_logprobs
449 - self.disable_sliding_window = disable_sliding_window
450 - self.disable_cascade_attn = disable_cascade_attn
451 - self.skip_tokenizer_init = skip_tokenizer_init
452 - self.enable_sleep_mode = enable_sleep_mode
453 -
477 from vllm.platforms import current_platform
478
479 if (self.enable_sleep_mode
@@ -458,9 +481,12 @@ class ModelConfig:
481 raise ValueError(
482 "Sleep mode is not supported on current platform.")
483
484 + if isinstance(self.config_format, str):
485 + self.config_format = ConfigFormat(self.config_format)
486 +
487 hf_config = get_config(self.hf_config_path or self.model,
462 - trust_remote_code, revision, code_revision,
463 - config_format)
488 + self.trust_remote_code, self.revision,
489 + self.code_revision, self.config_format)
490
491 if hf_overrides_kw:
492 logger.info("Overriding HF config with %s", hf_overrides_kw)
@@ -476,13 +502,8 @@ class ModelConfig:
502 "attention_chunk_size", None)
503 self.encoder_config = self._get_encoder_config()
504 self.hf_image_processor_config = get_hf_image_processor_config(
479 - self.model, hf_token=hf_token, revision=revision)
480 - self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
481 - self.use_async_output_proc = use_async_output_proc
482 -
483 - # Set enforce_eager to False if the value is unset.
484 - if self.enforce_eager is None:
485 - self.enforce_eager = False
505 + self.model, hf_token=self.hf_token, revision=self.revision)
506 + self.dtype = _get_and_verify_dtype(self.hf_config, self.dtype)
507
508 interleaved_attn_models = ["gemma2", "gemma3_text", "cohere2"]
509 sliding_window = getattr(self.hf_text_config, "sliding_window", None)
@@ -515,18 +536,14 @@ class ModelConfig:
536
537 self.max_model_len = _get_and_verify_max_len(
538 hf_config=self.hf_text_config,
518 - max_model_len=max_model_len,
539 + max_model_len=self.max_model_len,
540 disable_sliding_window=self.disable_sliding_window,
541 sliding_window_len=self.get_hf_config_sliding_window(),
521 - spec_target_max_model_len=spec_target_max_model_len,
542 + spec_target_max_model_len=self.spec_target_max_model_len,
543 encoder_config=self.encoder_config)
523 - self.served_model_name = get_served_model_name(model,
524 - served_model_name)
525 - self.multimodal_config = self._init_multimodal_config(
526 - limit_mm_per_prompt=limit_mm_per_prompt,
527 - mm_processor_kwargs=mm_processor_kwargs,
528 - disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,
529 - )
544 + self.served_model_name = get_served_model_name(self.model,
545 + self.served_model_name)
546 + self.multimodal_config = self._init_multimodal_config()
547 if not self.skip_tokenizer_init:
548 self._verify_tokenizer_mode()
549
@@ -535,24 +552,19 @@ class ModelConfig:
552 self.has_noops = self._init_has_noops()
553 self.has_inner_state = self._init_has_inner_state()
554
538 - if current_platform.is_neuron():
539 - self.override_neuron_config = override_neuron_config
540 - else:
541 - self.override_neuron_config = None
555 + if (not current_platform.is_neuron() and self.override_neuron_config):
556 + raise ValueError(
557 + "`override_neuron_config` is only supported on Neuron.")
558
543 - supported_tasks, task = self._resolve_task(task)
559 + supported_tasks, task = self._resolve_task(self.task)
560 self.supported_tasks = supported_tasks
545 - self.task: Final = task
561 + self.task = task
562 if self.task in ("draft", "generate"):
563 self.truncation_side = "left"
564 else:
565 self.truncation_side = "right"
566
551 - self.pooler_config = self._init_pooler_config(override_pooler_config)
552 - self.logits_processor_pattern = logits_processor_pattern
553 -
554 - self.generation_config = generation_config
555 - self.override_generation_config = override_generation_config or {}
567 + self.pooler_config = self._init_pooler_config()
568
569 self._verify_quantization()
570 self._verify_cuda_graph()
@@ -591,26 +603,21 @@ class ModelConfig:
603 model, ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
604 self.tokenizer = s3_tokenizer.dir
605
594 - def _init_multimodal_config(
595 - self,
596 - limit_mm_per_prompt: Optional[dict[str, int]],
597 - mm_processor_kwargs: Optional[dict[str, Any]],
598 - disable_mm_preprocessor_cache: bool,
599 - ) -> Optional["MultiModalConfig"]:
606 + def _init_multimodal_config(self) -> Optional["MultiModalConfig"]:
607 if self.registry.is_multimodal_model(self.architectures):
608 return MultiModalConfig(
602 - limit_per_prompt=limit_mm_per_prompt or {},
603 - mm_processor_kwargs=mm_processor_kwargs or {},
604 - disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,
605 - )
609 + limit_per_prompt=self.limit_mm_per_prompt,
610 + mm_processor_kwargs=self.mm_processor_kwargs,
611 + disable_mm_preprocessor_cache=self.
612 + disable_mm_preprocessor_cache)
613
607 - if limit_mm_per_prompt:
614 + if self.limit_mm_per_prompt:
615 raise ValueError("`limit_mm_per_prompt` is only supported for "
616 "multimodal models.")
610 - if mm_processor_kwargs:
617 + if self.mm_processor_kwargs:
618 raise ValueError("`mm_processor_kwargs` is only supported for "
619 "multimodal models.")
613 - if disable_mm_preprocessor_cache:
620 + if self.disable_mm_preprocessor_cache:
621 raise ValueError("`disable_mm_preprocessor_cache` is only "
622 "supported for multimodal models.")
623
@@ -620,31 +627,32 @@ class ModelConfig:
627 return get_sentence_transformer_tokenizer_config(
628 self.model, self.revision)
629
623 - def _init_pooler_config(
624 - self,
625 - override_pooler_config: Optional["PoolerConfig"],
626 - ) -> Optional["PoolerConfig"]:
630 + def _init_pooler_config(self) -> Optional["PoolerConfig"]:
631
632 if self.runner_type == "pooling":
629 - user_config = override_pooler_config or PoolerConfig()
633 + if isinstance(self.override_pooler_config, dict):
634 + self.override_pooler_config = PoolerConfig(
635 + **self.override_pooler_config)
636 +
637 + pooler_config = self.override_pooler_config or PoolerConfig()
638
639 base_config = get_pooling_config(self.model, self.revision)
640 if base_config is not None:
641 # Only set values that are not overridden by the user
642 for k, v in base_config.items():
635 - if getattr(user_config, k) is None:
636 - setattr(user_config, k, v)
643 + if getattr(pooler_config, k) is None:
644 + setattr(pooler_config, k, v)
645
646 if self.is_matryoshka:
639 - if user_config.normalize is None:
640 - user_config.normalize = True
641 - elif not user_config.normalize:
647 + if pooler_config.normalize is None:
648 + pooler_config.normalize = True
649 + elif not pooler_config.normalize:
650 raise ValueError(
651 "`normalize` must be enabled (set to True) "
652 "for models that are compatible with "
653 "Matryoshka Representation.")
654
647 - return user_config
655 + return pooler_config
656
657 return None
658
@@ -662,11 +670,11 @@ class ModelConfig:
670 return self.registry.model_has_inner_state(self.architectures)
671
672 def _verify_tokenizer_mode(self) -> None:
665 - tokenizer_mode = self.tokenizer_mode.lower()
666 - if tokenizer_mode not in ["auto", "slow", "mistral", "custom"]:
673 + tokenizer_mode = cast(TokenizerMode, self.tokenizer_mode.lower())
674 + if tokenizer_mode not in get_args(TokenizerMode):
675 raise ValueError(
676 f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
669 - "either 'auto', 'slow', 'mistral' or 'custom'.")
677 + f"one of {get_args(TokenizerMode)}.")
678 self.tokenizer_mode = tokenizer_mode
679
680 def _get_preferred_task(
@@ -781,7 +789,8 @@ class ModelConfig:
789 "quark", "nvfp4", "bitblas", "gptq_bitblas"
790 ]
791 if self.quantization is not None:
784 - self.quantization = self.quantization.lower()
792 + self.quantization = cast(QuantizationMethods,
793 + self.quantization.lower())
794
795 # Parse quantization method from the HF model config, if available.
796 quant_cfg = self._parse_quant_hf_config()
@@ -857,8 +866,6 @@ class ModelConfig:
866 "non-quantized models.", self.quantization)
867
868 def _verify_cuda_graph(self) -> None:
860 - if self.max_seq_len_to_capture is None:
861 - self.max_seq_len_to_capture = self.max_model_len
869 self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
870 self.max_model_len)
871 ROCM_UNSUPPORTED_MODELS = ['mllama']
@@ -1294,7 +1301,7 @@ class ModelConfig:
1301
1302 @property
1303 def runner_type(self) -> RunnerType:
1297 - return _TASK_RUNNER[self.task]
1304 + return _TASK_RUNNER[cast(_ResolvedTask, self.task)]
1305
1306 @property
1307 def is_v1_compatible(self) -> bool:
@@ -2201,7 +2208,7 @@ class SpeculativeConfig:
2208 according to the log probability settings in SamplingParams."""
2209
2210 # Draft model configuration
2204 - quantization: Optional[str] = None
2211 + quantization: Optional[QuantizationMethods] = None
2212 """Quantization method that was used to quantize the draft model weights.
2213 If `None`, we assume the model weights are not quantized. Note that it only
2214 takes effect when using the draft model-based speculative method."""
@@ -2386,7 +2393,6 @@ class SpeculativeConfig:
2393 code_revision=self.code_revision,
2394 tokenizer_revision=self.target_model_config.
2395 tokenizer_revision,
2389 - max_model_len=None,
2396 spec_target_max_model_len=self.target_model_config.
2397 max_model_len,
2398 quantization=self.quantization,
@@ -2793,30 +2799,31 @@ class PromptAdapterConfig:
2799 class MultiModalConfig:
2800 """Controls the behavior of multimodal models."""
2801
2796 - limit_per_prompt: dict[str, int] = field(default_factory=dict)
2802 + limit_per_prompt: dict[str, int] = get_field(ModelConfig,
2803 + "limit_mm_per_prompt")
2804 """
2805 The maximum number of input items allowed per prompt for each modality.
2806 This should be a JSON string that will be parsed into a dictionary.
2807 Defaults to 1 (V0) or 999 (V1) for each modality.
2808
2809 For example, to allow up to 16 images and 2 videos per prompt:
2803 - :code:`{"images": 16, "videos": 2}`
2810 + `{"images": 16, "videos": 2}`
2811 """
2812
2813 mm_processor_kwargs: Optional[dict[str, object]] = None
2814 """
2815 Overrides for the multi-modal processor obtained from
2809 - :meth:`transformers.AutoProcessor.from_pretrained`.
2816 + `transformers.AutoProcessor.from_pretrained`.
2817
2818 The available overrides depend on the model that is being run.
2819
2820 For example, for Phi-3-Vision:
2814 - :code:`{"num_crops": 4}`.
2821 + `{"num_crops": 4}`.
2822 """
2823
2824 disable_mm_preprocessor_cache: bool = False
2825 """
2819 - If :code:`True`, disable caching of the processed multi-modal inputs.
2826 + If `True`, disable caching of the processed multi-modal inputs.
2827 """
2828
2829 def compute_hash(self) -> str:
@@ -2907,10 +2914,6 @@ class PoolerConfig:
2914 usedforsecurity=False).hexdigest()
2915 return hash_str
2916
2910 - @staticmethod
2911 - def from_json(json_str: str) -> "PoolerConfig":
2912 - return PoolerConfig(**json.loads(json_str))
2913 -
2917
2918 _STR_DTYPE_TO_TORCH_DTYPE = {
2919 "half": torch.float16,
vllm/engine/arg_utils.py
+137 -314
@@ -20,15 +20,16 @@ from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
20 DeviceConfig, DistributedExecutorBackend,
21 GuidedDecodingBackend, GuidedDecodingBackendV1,
22 HfOverrides, KVTransferConfig, LoadConfig, LoadFormat,
23 - LoRAConfig, ModelConfig, ModelImpl, MultiModalConfig,
24 - ObservabilityConfig, ParallelConfig, PoolerConfig,
25 - PrefixCachingHashAlgo, PromptAdapterConfig,
26 - SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
27 - TaskOption, TokenizerPoolConfig, VllmConfig,
28 - get_attr_docs, get_field)
23 + LoRAConfig, ModelConfig, ModelDType, ModelImpl,
24 + MultiModalConfig, ObservabilityConfig, ParallelConfig,
25 + PoolerConfig, PrefixCachingHashAlgo,
26 + PromptAdapterConfig, SchedulerConfig, SchedulerPolicy,
27 + SpeculativeConfig, TaskOption, TokenizerMode,
28 + TokenizerPoolConfig, VllmConfig, get_attr_docs,
29 + get_field)
30 from vllm.executor.executor_base import ExecutorBase
31 from vllm.logger import init_logger
31 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
32 +from vllm.model_executor.layers.quantization import QuantizationMethods
33 from vllm.plugins import load_general_plugins
34 from vllm.reasoning import ReasoningParserManager
35 from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3
@@ -183,6 +184,9 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
184 kwargs[name]["nargs"] = "+"
185 elif contains_type(type_hints, int):
186 kwargs[name]["type"] = int
187 + # Special case for large integers
188 + if name in {"max_model_len"}:
189 + kwargs[name]["type"] = human_readable_int
190 elif contains_type(type_hints, float):
191 kwargs[name]["type"] = float
192 elif contains_type(type_hints, dict):
@@ -212,22 +216,23 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
216 @dataclass
217 class EngineArgs:
218 """Arguments for vLLM engine."""
215 - model: str = 'facebook/opt-125m'
216 - served_model_name: Optional[Union[str, List[str]]] = None
217 - tokenizer: Optional[str] = None
218 - hf_config_path: Optional[str] = None
219 - task: TaskOption = "auto"
220 - skip_tokenizer_init: bool = False
221 - tokenizer_mode: str = 'auto'
222 - trust_remote_code: bool = False
223 - allowed_local_media_path: str = ""
219 + model: str = ModelConfig.model
220 + served_model_name: Optional[Union[
221 + str, List[str]]] = ModelConfig.served_model_name
222 + tokenizer: Optional[str] = ModelConfig.tokenizer
223 + hf_config_path: Optional[str] = ModelConfig.hf_config_path
224 + task: TaskOption = ModelConfig.task
225 + skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
226 + tokenizer_mode: TokenizerMode = ModelConfig.tokenizer_mode
227 + trust_remote_code: bool = ModelConfig.trust_remote_code
228 + allowed_local_media_path: str = ModelConfig.allowed_local_media_path
229 download_dir: Optional[str] = LoadConfig.download_dir
230 load_format: str = LoadConfig.load_format
226 - config_format: ConfigFormat = ConfigFormat.AUTO
227 - dtype: str = 'auto'
231 + config_format: str = ModelConfig.config_format
232 + dtype: ModelDType = ModelConfig.dtype
233 kv_cache_dtype: CacheDType = CacheConfig.cache_dtype
229 - seed: Optional[int] = None
230 - max_model_len: Optional[int] = None
234 + seed: Optional[int] = ModelConfig.seed
235 + max_model_len: Optional[int] = ModelConfig.max_model_len
236 # Note: Specifying a custom executor backend by passing a class
237 # is intended for expert use only. The API may change without
238 # notice.
@@ -245,8 +250,8 @@ class EngineArgs:
250 enable_prefix_caching: Optional[bool] = CacheConfig.enable_prefix_caching
251 prefix_caching_hash_algo: PrefixCachingHashAlgo = \
252 CacheConfig.prefix_caching_hash_algo
248 - disable_sliding_window: bool = False
249 - disable_cascade_attn: bool = False
253 + disable_sliding_window: bool = ModelConfig.disable_sliding_window
254 + disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
255 use_v2_block_manager: bool = True
256 swap_space: float = CacheConfig.swap_space
257 cpu_offload_gb: float = CacheConfig.cpu_offload_gb
@@ -258,18 +263,19 @@ class EngineArgs:
263 long_prefill_token_threshold: int = \
264 SchedulerConfig.long_prefill_token_threshold
265 max_num_seqs: Optional[int] = SchedulerConfig.max_num_seqs
261 - max_logprobs: int = 20 # Default value for OpenAI Chat Completions API
266 + max_logprobs: int = ModelConfig.max_logprobs
267 disable_log_stats: bool = False
263 - revision: Optional[str] = None
264 - code_revision: Optional[str] = None
265 - rope_scaling: Optional[Dict[str, Any]] = None
266 - rope_theta: Optional[float] = None
267 - hf_token: Optional[Union[bool, str]] = None
268 - hf_overrides: Optional[HfOverrides] = None
269 - tokenizer_revision: Optional[str] = None
270 - quantization: Optional[str] = None
271 - enforce_eager: Optional[bool] = None
272 - max_seq_len_to_capture: int = 8192
268 + revision: Optional[str] = ModelConfig.revision
269 + code_revision: Optional[str] = ModelConfig.code_revision
270 + rope_scaling: dict[str, Any] = get_field(ModelConfig, "rope_scaling")
271 + rope_theta: Optional[float] = ModelConfig.rope_theta
272 + hf_token: Optional[Union[bool, str]] = ModelConfig.hf_token
273 + hf_overrides: Optional[HfOverrides] = \
274 + get_field(ModelConfig, "hf_overrides")
275 + tokenizer_revision: Optional[str] = ModelConfig.tokenizer_revision
276 + quantization: Optional[QuantizationMethods] = ModelConfig.quantization
277 + enforce_eager: bool = ModelConfig.enforce_eager
278 + max_seq_len_to_capture: int = ModelConfig.max_seq_len_to_capture
279 disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
280 # The following three fields are deprecated and will be removed in a future
281 # release. Setting them will have no effect. Please remove them from your
@@ -280,8 +286,10 @@ class EngineArgs:
286 get_field(TokenizerPoolConfig, "extra_config")
287 limit_mm_per_prompt: dict[str, int] = \
288 get_field(MultiModalConfig, "limit_per_prompt")
283 - mm_processor_kwargs: Optional[Dict[str, Any]] = None
284 - disable_mm_preprocessor_cache: bool = False
289 + mm_processor_kwargs: Optional[Dict[str, Any]] = \
290 + MultiModalConfig.mm_processor_kwargs
291 + disable_mm_preprocessor_cache: bool = \
292 + MultiModalConfig.disable_mm_preprocessor_cache
293 # LoRA fields
294 enable_lora: bool = False
295 enable_lora_bias: bool = LoRAConfig.bias_enabled
@@ -323,7 +331,8 @@ class EngineArgs:
331 DecodingConfig.disable_any_whitespace
332 guided_decoding_disable_additional_properties: bool = \
333 DecodingConfig.disable_additional_properties
326 - logits_processor_pattern: Optional[str] = None
334 + logits_processor_pattern: Optional[
335 + str] = ModelConfig.logits_processor_pattern
336
337 speculative_config: Optional[Dict[str, Any]] = None
338
@@ -331,22 +340,25 @@ class EngineArgs:
340 show_hidden_metrics_for_version: Optional[str] = None
341 otlp_traces_endpoint: Optional[str] = None
342 collect_detailed_traces: Optional[str] = None
334 - disable_async_output_proc: bool = False
343 + disable_async_output_proc: bool = not ModelConfig.use_async_output_proc
344 scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
345 scheduler_cls: Union[str, Type[object]] = SchedulerConfig.scheduler_cls
346
338 - override_neuron_config: Optional[Dict[str, Any]] = None
339 - override_pooler_config: Optional[PoolerConfig] = None
347 + override_neuron_config: dict[str, Any] = \
348 + get_field(ModelConfig, "override_neuron_config")
349 + override_pooler_config: Optional[Union[dict, PoolerConfig]] = \
350 + ModelConfig.override_pooler_config
351 compilation_config: Optional[CompilationConfig] = None
352 worker_cls: str = ParallelConfig.worker_cls
353 worker_extension_cls: str = ParallelConfig.worker_extension_cls
354
355 kv_transfer_config: Optional[KVTransferConfig] = None
356
346 - generation_config: Optional[str] = "auto"
347 - override_generation_config: Optional[Dict[str, Any]] = None
348 - enable_sleep_mode: bool = False
349 - model_impl: str = "auto"
357 + generation_config: str = ModelConfig.generation_config
358 + enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
359 + override_generation_config: dict[str, Any] = \
360 + get_field(ModelConfig, "override_generation_config")
361 + model_impl: str = ModelConfig.model_impl
362
363 calculate_kv_scales: bool = CacheConfig.calculate_kv_scales
364
@@ -356,9 +368,6 @@ class EngineArgs:
368 use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load
369
370 def __post_init__(self):
359 - if not self.tokenizer:
360 - self.tokenizer = self.model
361 -
371 # support `EngineArgs(compilation_config={...})`
372 # without having to manually construct a
373 # CompilationConfig object
@@ -375,80 +384,87 @@ class EngineArgs:
384 """Shared CLI arguments for vLLM engine."""
385
386 # Model arguments
378 - parser.add_argument(
379 - '--model',
380 - type=str,
381 - default=EngineArgs.model,
382 - help='Name or path of the huggingface model to use.')
383 - parser.add_argument(
384 - '--task',
385 - default=EngineArgs.task,
386 - choices=get_args(TaskOption),
387 - help='The task to use the model for. Each vLLM instance only '
388 - 'supports one task, even if the same model can be used for '
389 - 'multiple tasks. When the model only supports one task, ``"auto"`` '
390 - 'can be used to select it; otherwise, you must specify explicitly '
391 - 'which task to use.')
392 - parser.add_argument(
393 - '--tokenizer',
394 - type=optional_type(str),
395 - default=EngineArgs.tokenizer,
396 - help='Name or path of the huggingface tokenizer to use. '
397 - 'If unspecified, model name or path will be used.')
398 - parser.add_argument(
399 - "--hf-config-path",
400 - type=optional_type(str),
401 - default=EngineArgs.hf_config_path,
402 - help='Name or path of the huggingface config to use. '
403 - 'If unspecified, model name or path will be used.')
404 - parser.add_argument(
405 - '--skip-tokenizer-init',
406 - action='store_true',
407 - help='Skip initialization of tokenizer and detokenizer. '
408 - 'Expects valid prompt_token_ids and None for prompt from '
409 - 'the input. The generated output will contain token ids.')
410 - parser.add_argument(
411 - '--revision',
412 - type=optional_type(str),
413 - default=None,
414 - help='The specific model version to use. It can be a branch '
415 - 'name, a tag name, or a commit id. If unspecified, will use '
416 - 'the default version.')
417 - parser.add_argument(
418 - '--code-revision',
419 - type=optional_type(str),
420 - default=None,
421 - help='The specific revision to use for the model code on '
422 - 'Hugging Face Hub. It can be a branch name, a tag name, or a '
423 - 'commit id. If unspecified, will use the default version.')
424 - parser.add_argument(
425 - '--tokenizer-revision',
426 - type=optional_type(str),
427 - default=None,
428 - help='Revision of the huggingface tokenizer to use. '
429 - 'It can be a branch name, a tag name, or a commit id. '
430 - 'If unspecified, will use the default version.')
431 - parser.add_argument(
432 - '--tokenizer-mode',
433 - type=str,
434 - default=EngineArgs.tokenizer_mode,
435 - choices=['auto', 'slow', 'mistral', 'custom'],
436 - help='The tokenizer mode.\n\n* "auto" will use the '
437 - 'fast tokenizer if available.\n* "slow" will '
438 - 'always use the slow tokenizer. \n* '
439 - '"mistral" will always use the `mistral_common` tokenizer. \n* '
440 - '"custom" will use --tokenizer to select the '
441 - 'preregistered tokenizer.')
442 - parser.add_argument('--trust-remote-code',
443 - action='store_true',
444 - help='Trust remote code from huggingface.')
445 - parser.add_argument(
446 - '--allowed-local-media-path',
447 - type=str,
448 - help="Allowing API requests to read local images or videos "
449 - "from directories specified by the server file system. "
450 - "This is a security risk. "
451 - "Should only be enabled in trusted environments.")
387 + model_kwargs = get_kwargs(ModelConfig)
388 + model_group = parser.add_argument_group(
389 + title="ModelConfig",
390 + description=ModelConfig.__doc__,
391 + )
392 + model_group.add_argument("--model", **model_kwargs["model"])
393 + model_group.add_argument("--task", **model_kwargs["task"])
394 + model_group.add_argument("--tokenizer", **model_kwargs["tokenizer"])
395 + model_group.add_argument("--tokenizer-mode",
396 + **model_kwargs["tokenizer_mode"])
397 + model_group.add_argument("--trust-remote-code",
398 + **model_kwargs["trust_remote_code"])
399 + model_group.add_argument("--dtype", **model_kwargs["dtype"])
400 + model_group.add_argument("--seed", **model_kwargs["seed"])
401 + model_group.add_argument("--hf-config-path",
402 + **model_kwargs["hf_config_path"])
403 + model_group.add_argument("--allowed-local-media-path",
404 + **model_kwargs["allowed_local_media_path"])
405 + model_group.add_argument("--revision", **model_kwargs["revision"])
406 + model_group.add_argument("--code-revision",
407 + **model_kwargs["code_revision"])
408 + model_group.add_argument("--rope-scaling",
409 + **model_kwargs["rope_scaling"])
410 + model_group.add_argument("--rope-theta", **model_kwargs["rope_theta"])
411 + model_group.add_argument("--tokenizer-revision",
412 + **model_kwargs["tokenizer_revision"])
413 + model_group.add_argument("--max-model-len",
414 + **model_kwargs["max_model_len"])
415 + model_group.add_argument("--quantization", "-q",
416 + **model_kwargs["quantization"])
417 + model_group.add_argument("--enforce-eager",
418 + **model_kwargs["enforce_eager"])
419 + model_group.add_argument("--max-seq-len-to-capture",
420 + **model_kwargs["max_seq_len_to_capture"])
421 + model_group.add_argument("--max-logprobs",
422 + **model_kwargs["max_logprobs"])
423 + model_group.add_argument("--disable-sliding-window",
424 + **model_kwargs["disable_sliding_window"])
425 + model_group.add_argument("--disable-cascade-attn",
426 + **model_kwargs["disable_cascade_attn"])
427 + model_group.add_argument("--skip-tokenizer-init",
428 + **model_kwargs["skip_tokenizer_init"])
429 + model_group.add_argument("--served-model-name",
430 + **model_kwargs["served_model_name"])
431 + # This one is a special case because it is the
432 + # opposite of ModelConfig.use_async_output_proc
433 + model_group.add_argument(
434 + "--disable-async-output-proc",
435 + action="store_true",
436 + default=EngineArgs.disable_async_output_proc,
437 + help="Disable async output processing. This may result in "
438 + "lower performance.")
439 + model_group.add_argument("--config-format",
440 + choices=[f.value for f in ConfigFormat],
441 + **model_kwargs["config_format"])
442 + # This one is a special case because it can bool
443 + # or str. TODO: Handle this in get_kwargs
444 + model_group.add_argument("--hf-token",
445 + type=str,
446 + nargs="?",
447 + const=True,
448 + default=model_kwargs["hf_token"]["default"],
449 + help=model_kwargs["hf_token"]["help"])
450 + model_group.add_argument("--hf-overrides",
451 + **model_kwargs["hf_overrides"])
452 + model_group.add_argument("--override-neuron-config",
453 + **model_kwargs["override_neuron_config"])
454 + model_group.add_argument("--override-pooler-config",
455 + **model_kwargs["override_pooler_config"])
456 + model_group.add_argument("--logits-processor-pattern",
457 + **model_kwargs["logits_processor_pattern"])
458 + model_group.add_argument("--generation-config",
459 + **model_kwargs["generation_config"])
460 + model_group.add_argument("--override-generation-config",
461 + **model_kwargs["override_generation_config"])
462 + model_group.add_argument("--enable-sleep-mode",
463 + **model_kwargs["enable_sleep_mode"])
464 + model_group.add_argument("--model-impl",
465 + choices=[f.value for f in ModelImpl],
466 + **model_kwargs["model_impl"])
467 +
468 # Model loading arguments
469 load_kwargs = get_kwargs(LoadConfig)
470 load_group = parser.add_argument_group(
@@ -465,38 +481,6 @@ class EngineArgs:
481 load_group.add_argument('--use-tqdm-on-load',
482 **load_kwargs["use_tqdm_on_load"])
483
468 - parser.add_argument(
469 - '--config-format',
470 - default=EngineArgs.config_format,
471 - choices=[f.value for f in ConfigFormat],
472 - help='The format of the model config to load.\n\n'
473 - '* "auto" will try to load the config in hf format '
474 - 'if available else it will try to load in mistral format ')
475 - parser.add_argument(
476 - '--dtype',
477 - type=str,
478 - default=EngineArgs.dtype,
479 - choices=[
480 - 'auto', 'half', 'float16', 'bfloat16', 'float', 'float32'
481 - ],
482 - help='Data type for model weights and activations.\n\n'
483 - '* "auto" will use FP16 precision for FP32 and FP16 models, and '
484 - 'BF16 precision for BF16 models.\n'
485 - '* "half" for FP16. Recommended for AWQ quantization.\n'
486 - '* "float16" is the same as "half".\n'
487 - '* "bfloat16" for a balance between precision and range.\n'
488 - '* "float" is shorthand for FP32 precision.\n'
489 - '* "float32" for FP32 precision.')
490 - parser.add_argument('--max-model-len',
491 - type=human_readable_int,
492 - default=EngineArgs.max_model_len,
493 - help='Model context length. If unspecified, will '
494 - 'be automatically derived from the model config. '
495 - 'Supports k/m/g/K/M/G in human-readable format.\n'
496 - 'Examples:\n'
497 - '- 1k → 1000\n'
498 - '- 1K → 1024\n')
499 -
484 # Guided decoding arguments
485 guided_decoding_kwargs = get_kwargs(DecodingConfig)
486 guided_decoding_group = parser.add_argument_group(
@@ -520,26 +504,6 @@ class EngineArgs:
504 choices=list(ReasoningParserManager.reasoning_parsers),
505 **guided_decoding_kwargs["reasoning_backend"])
506
523 - parser.add_argument(
524 - '--logits-processor-pattern',
525 - type=optional_type(str),
526 - default=None,
527 - help='Optional regex pattern specifying valid logits processor '
528 - 'qualified names that can be passed with the `logits_processors` '
529 - 'extra completion argument. Defaults to None, which allows no '
530 - 'processors.')
531 - parser.add_argument(
532 - '--model-impl',
533 - type=str,
534 - default=EngineArgs.model_impl,
535 - choices=[f.value for f in ModelImpl],
536 - help='Which implementation of the model to use.\n\n'
537 - '* "auto" will try to use the vLLM implementation if it exists '
538 - 'and fall back to the Transformers implementation if no vLLM '
539 - 'implementation is available.\n'
540 - '* "vllm" will use the vLLM model implementation.\n'
541 - '* "transformers" will use the Transformers model '
542 - 'implementation.\n')
507 # Parallel arguments
508 parallel_kwargs = get_kwargs(ParallelConfig)
509 parallel_group = parser.add_argument_group(
@@ -592,10 +556,6 @@ class EngineArgs:
556 cache_group.add_argument('--calculate-kv-scales',
557 **cache_kwargs["calculate_kv_scales"])
558
595 - parser.add_argument('--disable-sliding-window',
596 - action='store_true',
597 - help='Disables sliding window, '
598 - 'capping to sliding window size.')
559 parser.add_argument('--use-v2-block-manager',
560 action='store_true',
561 default=True,
@@ -605,73 +565,9 @@ class EngineArgs:
565 'Setting this flag to True or False'
566 ' has no effect on vLLM behavior.')
567
608 - parser.add_argument('--seed',
609 - type=int,
610 - default=EngineArgs.seed,
611 - help='Random seed for operations.')
612 - parser.add_argument(
613 - '--max-logprobs',
614 - type=int,
615 - default=EngineArgs.max_logprobs,
616 - help=('Max number of log probs to return logprobs is specified in'
617 - ' SamplingParams.'))
568 parser.add_argument('--disable-log-stats',
569 action='store_true',
570 help='Disable logging statistics.')
621 - # Quantization settings.
622 - parser.add_argument('--quantization',
623 - '-q',
624 - type=optional_type(str),
625 - choices=[*QUANTIZATION_METHODS, None],
626 - default=EngineArgs.quantization,
627 - help='Method used to quantize the weights. If '
628 - 'None, we first check the `quantization_config` '
629 - 'attribute in the model config file. If that is '
630 - 'None, we assume the model weights are not '
631 - 'quantized and use `dtype` to determine the data '
632 - 'type of the weights.')
633 - parser.add_argument(
634 - '--rope-scaling',
635 - default=None,
636 - type=json.loads,
637 - help='RoPE scaling configuration in JSON format. '
638 - 'For example, ``{"rope_type":"dynamic","factor":2.0}``')
639 - parser.add_argument('--rope-theta',
640 - default=None,
641 - type=float,
642 - help='RoPE theta. Use with `rope_scaling`. In '
643 - 'some cases, changing the RoPE theta improves the '
644 - 'performance of the scaled model.')
645 - parser.add_argument(
646 - '--hf-token',
647 - type=str,
648 - nargs='?',
649 - const=True,
650 - default=None,
651 - help='The token to use as HTTP bearer authorization'
652 - ' for remote files. If `True`, will use the token '
653 - 'generated when running `huggingface-cli login` '
654 - '(stored in `~/.huggingface`).')
655 - parser.add_argument('--hf-overrides',
656 - type=json.loads,
657 - default=EngineArgs.hf_overrides,
658 - help='Extra arguments for the HuggingFace config. '
659 - 'This should be a JSON string that will be '
660 - 'parsed into a dictionary.')
661 - parser.add_argument('--enforce-eager',
662 - action='store_true',
663 - help='Always use eager-mode PyTorch. If False, '
664 - 'will use eager mode and CUDA graph in hybrid '
665 - 'for maximal performance and flexibility.')
666 - parser.add_argument('--max-seq-len-to-capture',
667 - type=int,
668 - default=EngineArgs.max_seq_len_to_capture,
669 - help='Maximum sequence length covered by CUDA '
670 - 'graphs. When a sequence has context length '
671 - 'larger than this, we fall back to eager mode. '
672 - 'Additionally for encoder-decoder models, if the '
673 - 'sequence length of the encoder input is larger '
674 - 'than this, we fall back to the eager mode.')
571
572 # Tokenizer arguments
573 tokenizer_kwargs = get_kwargs(TokenizerPoolConfig)
@@ -775,20 +671,6 @@ class EngineArgs:
671 "Default to `original/**/*` to avoid repeated loading of llama's "
672 "checkpoints.")
673
778 - parser.add_argument(
779 - "--served-model-name",
780 - nargs="+",
781 - type=str,
782 - default=None,
783 - help="The model name(s) used in the API. If multiple "
784 - "names are provided, the server will respond to any "
785 - "of the provided names. The model name in the model "
786 - "field of a response will be the first name in this "
787 - "list. If not specified, the model name will be the "
788 - "same as the ``--model`` argument. Noted that this name(s) "
789 - "will also be used in `model_name` tag content of "
790 - "prometheus metrics, if multiple names provided, metrics "
791 - "tag will take the first one.")
674 parser.add_argument('--qlora-adapter-name-or-path',
675 type=str,
676 default=None,
@@ -822,13 +704,6 @@ class EngineArgs:
704 "modules. This involves use of possibly costly and or blocking "
705 "operations and hence might have a performance impact.")
706
825 - parser.add_argument(
826 - '--disable-async-output-proc',
827 - action='store_true',
828 - default=EngineArgs.disable_async_output_proc,
829 - help="Disable async output processing. This may result in "
830 - "lower performance.")
831 -
707 # Scheduler arguments
708 scheduler_kwargs = get_kwargs(SchedulerConfig)
709 scheduler_group = parser.add_argument_group(
@@ -871,19 +746,6 @@ class EngineArgs:
746 parser.add_argument('--scheduler-cls',
747 **scheduler_kwargs["scheduler_cls"])
748
874 - parser.add_argument(
875 - '--override-neuron-config',
876 - type=json.loads,
877 - default=None,
878 - help="Override or set neuron device configuration. "
879 - "e.g. ``{\"cast_logits_dtype\": \"bloat16\"}``.")
880 - parser.add_argument(
881 - '--override-pooler-config',
882 - type=PoolerConfig.from_json,
883 - default=None,
884 - help="Override or set the pooling method for pooling models. "
885 - "e.g. ``{\"pooling_type\": \"mean\", \"normalize\": false}``.")
886 -
749 parser.add_argument('--compilation-config',
750 '-O',
751 type=CompilationConfig.from_cli,
@@ -920,34 +782,6 @@ class EngineArgs:
782 help='The worker extension class on top of the worker cls, '
783 'it is useful if you just want to add new functions to the worker '
784 'class without changing the existing functions.')
923 - parser.add_argument(
924 - "--generation-config",
925 - type=optional_type(str),
926 - default="auto",
927 - help="The folder path to the generation config. "
928 - "Defaults to 'auto', the generation config will be loaded from "
929 - "model path. If set to 'vllm', no generation config is loaded, "
930 - "vLLM defaults will be used. If set to a folder path, the "
931 - "generation config will be loaded from the specified folder path. "
932 - "If `max_new_tokens` is specified in generation config, then "
933 - "it sets a server-wide limit on the number of output tokens "
934 - "for all requests.")
935 -
936 - parser.add_argument(
937 - "--override-generation-config",
938 - type=json.loads,
939 - default=None,
940 - help="Overrides or sets generation config in JSON format. "
941 - "e.g. ``{\"temperature\": 0.5}``. If used with "
942 - "--generation-config=auto, the override parameters will be merged "
943 - "with the default config from the model. If generation-config is "
944 - "None, only the override parameters are used.")
945 -
946 - parser.add_argument("--enable-sleep-mode",
947 - action="store_true",
948 - default=False,
949 - help="Enable sleep mode for the engine. "
950 - "(only cuda platform is supported)")
785
786 parser.add_argument(
787 "--additional-config",
@@ -966,16 +800,6 @@ class EngineArgs:
800 "If enabled, the model will be able to generate reasoning content."
801 )
802
969 - parser.add_argument(
970 - "--disable-cascade-attn",
971 - action="store_true",
972 - default=False,
973 - help="Disable cascade attention for V1. While cascade attention "
974 - "does not change the mathematical correctness, disabling it "
975 - "could be useful for preventing potential numerical issues. "
976 - "Note that even if this is set to False, cascade attention will be "
977 - "only used when the heuristic tells that it's beneficial.")
978 -
803 return parser
804
805 @classmethod
@@ -1002,8 +826,7 @@ class EngineArgs:
826 model=self.model,
827 hf_config_path=self.hf_config_path,
828 task=self.task,
1005 - # We know this is not None because we set it in __post_init__
1006 - tokenizer=cast(str, self.tokenizer),
829 + tokenizer=self.tokenizer,
830 tokenizer_mode=self.tokenizer_mode,
831 trust_remote_code=self.trust_remote_code,
832 allowed_local_media_path=self.allowed_local_media_path,
vllm/entrypoints/llm.py
+7 -11
@@ -13,7 +13,7 @@ from typing_extensions import TypeVar, deprecated
13
14 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
15 BeamSearchSequence, get_beam_search_score)
16 -from vllm.config import CompilationConfig
16 +from vllm.config import CompilationConfig, ModelDType, TokenizerMode
17 from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
18 TaskOption)
19 from vllm.engine.llm_engine import LLMEngine
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
32 from vllm.lora.request import LoRARequest
33 from vllm.model_executor.guided_decoding.guided_fields import (
34 GuidedDecodingRequest, LLMGuidedOptions)
35 +from vllm.model_executor.layers.quantization import QuantizationMethods
36 from vllm.outputs import (ClassificationRequestOutput, EmbeddingRequestOutput,
37 PoolingRequestOutput, RequestOutput,
38 ScoringRequestOutput)
@@ -163,20 +164,20 @@ class LLM:
164 self,
165 model: str,
166 tokenizer: Optional[str] = None,
166 - tokenizer_mode: str = "auto",
167 + tokenizer_mode: TokenizerMode = "auto",
168 skip_tokenizer_init: bool = False,
169 trust_remote_code: bool = False,
170 allowed_local_media_path: str = "",
171 tensor_parallel_size: int = 1,
171 - dtype: str = "auto",
172 - quantization: Optional[str] = None,
172 + dtype: ModelDType = "auto",
173 + quantization: Optional[QuantizationMethods] = None,
174 revision: Optional[str] = None,
175 tokenizer_revision: Optional[str] = None,
176 seed: Optional[int] = None,
177 gpu_memory_utilization: float = 0.9,
178 swap_space: float = 4,
179 cpu_offload_gb: float = 0,
179 - enforce_eager: Optional[bool] = None,
180 + enforce_eager: bool = False,
181 max_seq_len_to_capture: int = 8192,
182 disable_custom_all_reduce: bool = False,
183 disable_async_output_proc: bool = False,
@@ -189,12 +190,7 @@ class LLM:
190 compilation_config: Optional[Union[int, dict[str, Any]]] = None,
191 **kwargs,
192 ) -> None:
192 - '''
193 - LLM constructor.
194 -
195 - Note: if enforce_eager is unset (enforce_eager is None)
196 - it defaults to False.
197 - '''
193 + """LLM constructor."""
194
195 if "disable_log_stats" not in kwargs:
196 kwargs["disable_log_stats"] = True
vllm/model_executor/layers/quantization/aqlm.py
+2 -1
@@ -12,6 +12,7 @@ from torch.nn.parameter import Parameter
12
13 from vllm import _custom_ops as ops
14 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
15 +from vllm.model_executor.layers.quantization import QuantizationMethods
16 from vllm.model_executor.layers.quantization.base_config import (
17 QuantizationConfig)
18 from vllm.model_executor.utils import set_weight_attrs
@@ -186,7 +187,7 @@ class AQLMConfig(QuantizationConfig):
187 f"out_group_size={self.out_group_size})")
188
189 @classmethod
189 - def get_name(cls) -> str:
190 + def get_name(cls) -> QuantizationMethods:
191 return "aqlm"
192
193 @classmethod
vllm/model_executor/layers/quantization/awq.py
+2 -1
@@ -7,6 +7,7 @@ import torch
7 from vllm import _custom_ops as ops
8 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 UnquantizedLinearMethod)
10 +from vllm.model_executor.layers.quantization import QuantizationMethods
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.model_executor.parameter import (GroupQuantScaleParameter,
@@ -44,7 +45,7 @@ class AWQConfig(QuantizationConfig):
45 f"zero_point={self.zero_point}, "
46 f"modules_to_not_convert={self.modules_to_not_convert})")
47
47 - def get_name(self) -> str:
48 + def get_name(self) -> QuantizationMethods:
49 return "awq"
50
51 def get_supported_act_dtypes(self) -> List[torch.dtype]:
vllm/model_executor/layers/quantization/awq_marlin.py
+4 -3
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.layer import (
13 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
14 UnquantizedLinearMethod,
15 set_weight_attrs)
16 +from vllm.model_executor.layers.quantization import QuantizationMethods
17 from vllm.model_executor.layers.quantization.awq import (AWQConfig,
18 is_layer_skipped_awq)
19 from vllm.model_executor.layers.quantization.base_config import (
@@ -73,7 +74,7 @@ class AWQMarlinConfig(QuantizationConfig):
74 f"modules_to_not_convert={self.modules_to_not_convert})")
75
76 @classmethod
76 - def get_name(cls) -> str:
77 + def get_name(cls) -> QuantizationMethods:
78 return "awq_marlin"
79
80 @classmethod
@@ -101,8 +102,8 @@ class AWQMarlinConfig(QuantizationConfig):
102 modules_to_not_convert, config)
103
104 @classmethod
104 - def override_quantization_method(cls, hf_quant_cfg,
105 - user_quant) -> Optional[str]:
105 + def override_quantization_method(
106 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
107 can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
108 is_valid_user_quant = (user_quant is None or user_quant == "marlin"
109 or user_quant == "awq_marlin")
vllm/model_executor/layers/quantization/base_config.py
+9 -4
@@ -2,11 +2,16 @@
2
3 import inspect
4 from abc import ABC, abstractmethod
5 -from typing import Any, Dict, List, Optional, Type
5 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
6
7 import torch
8 from torch import nn
9
10 +if TYPE_CHECKING:
11 + from vllm.model_executor.layers.quantization import QuantizationMethods
12 +else:
13 + QuantizationMethods = str
14 +
15
16 class QuantizeMethodBase(ABC):
17 """Base class for different quantized methods."""
@@ -66,7 +71,7 @@ class QuantizationConfig(ABC):
71 self.packed_modules_mapping: Dict[str, List[str]] = dict()
72
73 @abstractmethod
69 - def get_name(self) -> str:
74 + def get_name(self) -> QuantizationMethods:
75 """Name of the quantization method."""
76 raise NotImplementedError
77
@@ -99,8 +104,8 @@ class QuantizationConfig(ABC):
104 raise NotImplementedError
105
106 @classmethod
102 - def override_quantization_method(cls, hf_quant_cfg,
103 - user_quant) -> Optional[str]:
107 + def override_quantization_method(
108 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
109 """
110 Detects if this quantization method can support a given checkpoint
111 format by overriding the user specified quantization method --
vllm/model_executor/layers/quantization/bitblas.py
+4 -3
@@ -5,6 +5,7 @@ import torch
5
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 +from vllm.model_executor.layers.quantization import QuantizationMethods
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
@@ -100,7 +101,7 @@ class BitBLASConfig(QuantizationConfig):
101 f"quant_method={self.quant_method})")
102
103 @classmethod
103 - def get_name(cls) -> str:
104 + def get_name(cls) -> QuantizationMethods:
105 return "bitblas"
106
107 @classmethod
@@ -139,8 +140,8 @@ class BitBLASConfig(QuantizationConfig):
140 lm_head_quantized)
141
142 @classmethod
142 - def override_quantization_method(cls, hf_quant_cfg,
143 - user_quant) -> Optional[str]:
143 + def override_quantization_method(
144 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
145 # compat: autogptq >=0.8.0 use checkpoint_format: str
146 # compat: autogptq <=0.7.1 is_bitblas_format: bool
147 is_bitblas_format = (hf_quant_cfg.get("checkpoint_format") == "bitblas"
vllm/model_executor/layers/quantization/bitsandbytes.py
+2 -1
@@ -7,6 +7,7 @@ import torch
7 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
8 UnquantizedLinearMethod,
9 set_weight_attrs)
10 +from vllm.model_executor.layers.quantization import QuantizationMethods
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.utils import direct_register_custom_op
@@ -56,7 +57,7 @@ class BitsAndBytesConfig(QuantizationConfig):
57 f"llm_int8_skip_modules={self.llm_int8_skip_modules})")
58
59 @classmethod
59 - def get_name(self) -> str:
60 + def get_name(self) -> QuantizationMethods:
61 return "bitsandbytes"
62
63 @classmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2 -1
@@ -16,6 +16,7 @@ from vllm.logger import init_logger
16 from vllm.model_executor.layers.fused_moe import FusedMoE
17 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
18 UnquantizedLinearMethod)
19 +from vllm.model_executor.layers.quantization import QuantizationMethods
20 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
21 QuantizationConfig, QuantizeMethodBase)
22 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
@@ -71,7 +72,7 @@ class CompressedTensorsConfig(QuantizationConfig):
72 def get_min_capability(cls) -> int:
73 return 70
74
74 - def get_name(self) -> str:
75 + def get_name(self) -> QuantizationMethods:
76 return "compressed-tensors"
77
78 def get_quant_method(
vllm/model_executor/layers/quantization/deepspeedfp.py
+3 -2
@@ -7,6 +7,7 @@ import torch.nn as nn
7 import torch.nn.functional as F
8
9 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 +from vllm.model_executor.layers.quantization import QuantizationMethods
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.model_executor.utils import set_weight_attrs
@@ -41,8 +42,8 @@ class DeepSpeedFPConfig(QuantizationConfig):
42 f"group_size={self.group_size}")
43
44 @classmethod
44 - def get_name(cls) -> str:
45 - return "DeepSpeedFP"
45 + def get_name(cls) -> QuantizationMethods:
46 + return "deepspeedfp"
47
48 @classmethod
49 def from_config(cls, config: Dict[str, Any]) -> "DeepSpeedFPConfig":
vllm/model_executor/layers/quantization/experts_int8.py
+2 -1
@@ -8,6 +8,7 @@ from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
8 from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
9 from vllm.model_executor.layers.linear import (LinearBase,
10 UnquantizedLinearMethod)
11 +from vllm.model_executor.layers.quantization import QuantizationMethods
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.utils import set_weight_attrs
@@ -20,7 +21,7 @@ class ExpertsInt8Config(QuantizationConfig):
21 super().__init__()
22
23 @classmethod
23 - def get_name(cls) -> str:
24 + def get_name(cls) -> QuantizationMethods:
25 return "experts_int8"
26
27 @classmethod
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+2 -1
@@ -9,6 +9,7 @@ from torch.nn.parameter import Parameter
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 UnquantizedLinearMethod)
12 +from vllm.model_executor.layers.quantization import QuantizationMethods
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig, QuantizeMethodBase)
15 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
@@ -38,7 +39,7 @@ class FBGEMMFp8Config(QuantizationConfig):
39 self.fp8_linear = Fp8LinearOp()
40
41 @classmethod
41 - def get_name(cls) -> str:
42 + def get_name(cls) -> QuantizationMethods:
43 return "fbgemm_fp8"
44
45 @classmethod
vllm/model_executor/layers/quantization/fp8.py
+2 -1
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
16 FusedMoeWeightScaleSupported)
17 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
18 UnquantizedLinearMethod)
19 +from vllm.model_executor.layers.quantization import QuantizationMethods
20 from vllm.model_executor.layers.quantization.base_config import (
21 QuantizationConfig, QuantizeMethodBase)
22 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
@@ -83,7 +84,7 @@ class Fp8Config(QuantizationConfig):
84 self.weight_block_size = weight_block_size
85
86 @classmethod
86 - def get_name(cls) -> str:
87 + def get_name(cls) -> QuantizationMethods:
88 return "fp8"
89
90 @classmethod
vllm/model_executor/layers/quantization/gguf.py
+2 -1
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.activation import SiluAndMul
13 from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
14 FusedMoEMethodBase)
15 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
16 +from vllm.model_executor.layers.quantization import QuantizationMethods
17 from vllm.model_executor.layers.quantization.base_config import (
18 QuantizationConfig, QuantizeMethodBase)
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -31,7 +32,7 @@ class GGUFConfig(QuantizationConfig):
32 def __repr__(self) -> str:
33 return ("GGUFConfig()")
34
34 - def get_name(self) -> str:
35 + def get_name(self) -> QuantizationMethods:
36 return "gguf"
37
38 def get_supported_act_dtypes(self) -> List[torch.dtype]:
vllm/model_executor/layers/quantization/gptq.py
+2 -1
@@ -10,6 +10,7 @@ from torch.nn.parameter import Parameter
10
11 from vllm import _custom_ops as ops
12 from vllm.model_executor.layers.linear import LinearMethodBase
13 +from vllm.model_executor.layers.quantization import QuantizationMethods
14 from vllm.model_executor.layers.quantization.base_config import (
15 QuantizationConfig)
16 from vllm.model_executor.layers.quantization.utils.gptq_utils import (
@@ -79,7 +80,7 @@ class GPTQConfig(QuantizationConfig):
80 f"dynamic={self.dynamic}")
81
82 @classmethod
82 - def get_name(cls) -> str:
83 + def get_name(cls) -> QuantizationMethods:
84 return "gptq"
85
86 @classmethod
vllm/model_executor/layers/quantization/gptq_bitblas.py
+4 -3
@@ -7,6 +7,7 @@ from torch.nn.parameter import Parameter
7 from vllm.logger import init_logger
8 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 set_weight_attrs)
10 +from vllm.model_executor.layers.quantization import QuantizationMethods
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
@@ -123,7 +124,7 @@ class GPTQBitBLASConfig(QuantizationConfig):
124 f"quant_method={self.quant_method})")
125
126 @classmethod
126 - def get_name(cls) -> str:
127 + def get_name(cls) -> QuantizationMethods:
128 return "gptq_bitblas"
129
130 @classmethod
@@ -151,8 +152,8 @@ class GPTQBitBLASConfig(QuantizationConfig):
152 lm_head_quantized)
153
154 @classmethod
154 - def override_quantization_method(cls, hf_quant_cfg,
155 - user_quant) -> Optional[str]:
155 + def override_quantization_method(
156 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
157 can_convert = cls.is_gptq_bitblas_compatible(hf_quant_cfg)
158
159 is_valid_user_quant = (user_quant is None or user_quant == "bitblas"
vllm/model_executor/layers/quantization/gptq_marlin.py
+4 -3
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe.layer import (
11 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
13 set_weight_attrs)
14 +from vllm.model_executor.layers.quantization import QuantizationMethods
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig, QuantizeMethodBase)
17 from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
@@ -100,7 +101,7 @@ class GPTQMarlinConfig(QuantizationConfig):
101 f"dynamic={self.dynamic}")
102
103 @classmethod
103 - def get_name(cls) -> str:
104 + def get_name(cls) -> QuantizationMethods:
105 return "gptq_marlin"
106
107 @classmethod
@@ -130,8 +131,8 @@ class GPTQMarlinConfig(QuantizationConfig):
131 lm_head_quantized, dynamic, config)
132
133 @classmethod
133 - def override_quantization_method(cls, hf_quant_cfg,
134 - user_quant) -> Optional[str]:
134 + def override_quantization_method(
135 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
136 can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
137
138 is_valid_user_quant = (user_quant is None or user_quant == "marlin"
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+4 -3
@@ -8,6 +8,7 @@ from torch.nn.parameter import Parameter
8 from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 +from vllm.model_executor.layers.quantization import QuantizationMethods
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.parameter import (BasevLLMParameter,
@@ -85,7 +86,7 @@ class GPTQMarlin24Config(QuantizationConfig):
86 self.quant_type, self.group_size)
87
88 @classmethod
88 - def get_name(cls) -> str:
89 + def get_name(cls) -> QuantizationMethods:
90 return "gptq_marlin_24"
91
92 @classmethod
@@ -108,8 +109,8 @@ class GPTQMarlin24Config(QuantizationConfig):
109 return cls(weight_bits, group_size)
110
111 @classmethod
111 - def override_quantization_method(cls, hf_quant_cfg,
112 - user_quant) -> Optional[str]:
112 + def override_quantization_method(
113 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
114 is_marlin_24_format = (
115 hf_quant_cfg.get("checkpoint_format") == "marlin_24")
116
vllm/model_executor/layers/quantization/hqq_marlin.py
+2 -1
@@ -8,6 +8,7 @@ from vllm import _custom_ops as ops
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 UnquantizedLinearMethod)
11 +from vllm.model_executor.layers.quantization import QuantizationMethods
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -50,7 +51,7 @@ class HQQMarlinConfig(QuantizationConfig):
51 f"group_size={self.group_size})")
52
53 @classmethod
53 - def get_name(cls) -> str:
54 + def get_name(cls) -> QuantizationMethods:
55 return "hqq"
56
57 @classmethod
vllm/model_executor/layers/quantization/ipex_quant.py
+4 -3
@@ -6,6 +6,7 @@ import torch
6
7 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
8 UnquantizedLinearMethod)
9 +from vllm.model_executor.layers.quantization import QuantizationMethods
10 from vllm.model_executor.layers.quantization.awq import (AWQLinearMethod,
11 is_layer_skipped_awq)
12 from vllm.model_executor.layers.quantization.base_config import (
@@ -58,7 +59,7 @@ class IPEXConfig(QuantizationConfig):
59 f"group_size={self.group_size})")
60
61 @classmethod
61 - def get_name(cls) -> str:
62 + def get_name(cls) -> QuantizationMethods:
63 return "ipex"
64
65 @classmethod
@@ -97,8 +98,8 @@ class IPEXConfig(QuantizationConfig):
98 lm_head_quantized)
99
100 @classmethod
100 - def override_quantization_method(cls, hf_quant_cfg,
101 - user_quant) -> Optional[str]:
101 + def override_quantization_method(
102 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
103 if not current_platform.is_cpu() and not current_platform.is_xpu():
104 return None
105
vllm/model_executor/layers/quantization/marlin.py
+4 -3
@@ -8,6 +8,7 @@ from torch.nn.parameter import Parameter
8 from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 +from vllm.model_executor.layers.quantization import QuantizationMethods
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -63,7 +64,7 @@ class MarlinConfig(QuantizationConfig):
64 f"lm_head_quantized={self.lm_head_quantized})")
65
66 @classmethod
66 - def get_name(cls) -> str:
67 + def get_name(cls) -> QuantizationMethods:
68 return "marlin"
69
70 @classmethod
@@ -87,8 +88,8 @@ class MarlinConfig(QuantizationConfig):
88 return cls(group_size, lm_head_quantized)
89
90 @classmethod
90 - def override_quantization_method(cls, hf_quant_cfg,
91 - user_quant) -> Optional[str]:
91 + def override_quantization_method(
92 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
93 # compat: autogptq >=0.8.0 use checkpoint_format: str
94 # compat: autogptq <=0.7.1 is_marlin_format: bool
95 is_marlin_format = (hf_quant_cfg.get("checkpoint_format") == "marlin"
vllm/model_executor/layers/quantization/modelopt.py
+4 -3
@@ -11,6 +11,7 @@ from vllm._custom_ops import (cutlass_scaled_fp4_mm,
11 from vllm.logger import init_logger
12 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
13 UnquantizedLinearMethod)
14 +from vllm.model_executor.layers.quantization import QuantizationMethods
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig, QuantizeMethodBase)
17 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
@@ -42,7 +43,7 @@ class ModelOptFp8Config(QuantizationConfig):
43 " the format is experimental and could change.")
44
45 @classmethod
45 - def get_name(cls) -> str:
46 + def get_name(cls) -> QuantizationMethods:
47 return "modelopt"
48
49 @classmethod
@@ -184,8 +185,8 @@ class ModelOptNvFp4Config(QuantizationConfig):
185 self.exclude_modules = exclude_modules
186
187 @classmethod
187 - def get_name(cls) -> str:
188 - return "modelopt_nvfp4"
188 + def get_name(cls) -> QuantizationMethods:
189 + return "nvfp4"
190
191 @classmethod
192 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
vllm/model_executor/layers/quantization/moe_wna16.py
+4 -3
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.fused_moe.layer import (
9 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
10 from vllm.model_executor.layers.linear import (LinearBase,
11 UnquantizedLinearMethod)
12 +from vllm.model_executor.layers.quantization import QuantizationMethods
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig, QuantizeMethodBase)
15 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -64,7 +65,7 @@ class MoeWNA16Config(QuantizationConfig):
65 self.modules_to_not_convert = modules_to_not_convert
66
67 @classmethod
67 - def get_name(cls) -> str:
68 + def get_name(cls) -> QuantizationMethods:
69 return "moe_wna16"
70
71 @classmethod
@@ -100,8 +101,8 @@ class MoeWNA16Config(QuantizationConfig):
101 lm_head_quantized, modules_to_not_convert, config)
102
103 @classmethod
103 - def override_quantization_method(cls, hf_quant_cfg,
104 - user_quant) -> Optional[str]:
104 + def override_quantization_method(
105 + cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
106 can_convert = cls.is_moe_wna16_compatible(hf_quant_cfg)
107 if can_convert and user_quant == "moe_wna16":
108 return cls.get_name()
vllm/model_executor/layers/quantization/neuron_quant.py
+2 -1
@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional
6
7 from torch.nn import Module
8
9 +from vllm.model_executor.layers.quantization import QuantizationMethods
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12
@@ -30,7 +31,7 @@ class NeuronQuantConfig(QuantizationConfig):
31 self.dequant_dtype = dequant_dtype
32 self.quantize_method = quantize_method
33
33 - def get_name(self) -> str:
34 + def get_name(self) -> QuantizationMethods:
35 return "neuron_quant"
36
37 def get_supported_act_dtypes(self) -> List[str]:
vllm/model_executor/layers/quantization/ptpc_fp8.py
+2 -1
@@ -9,6 +9,7 @@ from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.linear import (LinearBase,
11 UnquantizedLinearMethod)
12 +from vllm.model_executor.layers.quantization import QuantizationMethods
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizeMethodBase)
15 from vllm.model_executor.layers.quantization.fp8 import (Fp8Config,
@@ -50,7 +51,7 @@ class PTPCFp8Config(Fp8Config):
51 ignored_layers=ignored_layers)
52
53 @classmethod
53 - def get_name(cls) -> str:
54 + def get_name(cls) -> QuantizationMethods:
55 return "ptpc_fp8"
56
57 @classmethod
vllm/model_executor/layers/quantization/qqq.py
+2 -1
@@ -8,6 +8,7 @@ from torch.nn.parameter import Parameter
8 from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 +from vllm.model_executor.layers.quantization import QuantizationMethods
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.parameter import (BasevLLMParameter,
@@ -84,7 +85,7 @@ class QQQConfig(QuantizationConfig):
85 self.weight_bits, self.group_size)
86
87 @classmethod
87 - def get_name(cls) -> str:
88 + def get_name(cls) -> QuantizationMethods:
89 return "qqq"
90
91 @classmethod
vllm/model_executor/layers/quantization/quark/quark.py
+2 -1
@@ -8,6 +8,7 @@ import torch
8 from vllm.model_executor.layers.fused_moe import FusedMoE
9 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 UnquantizedLinearMethod)
11 +from vllm.model_executor.layers.quantization import QuantizationMethods
12 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
13 QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
@@ -47,7 +48,7 @@ class QuarkConfig(QuantizationConfig):
48 def get_min_capability(cls) -> int:
49 return 70
50
50 - def get_name(self) -> str:
51 + def get_name(self) -> QuantizationMethods:
52 return "quark"
53
54 def get_quant_method(self, layer: torch.nn.Module,
vllm/model_executor/layers/quantization/torchao.py
+2 -1
@@ -6,6 +6,7 @@ import torch.nn.functional as F
6 from torch.nn.parameter import Parameter
7
8 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 +from vllm.model_executor.layers.quantization import QuantizationMethods
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 from vllm.model_executor.utils import set_weight_attrs
@@ -20,7 +21,7 @@ class TorchAOConfig(QuantizationConfig):
21 def __repr__(self) -> str:
22 return f"TorchAOConfig({self.torchao_config})"
23
23 - def get_name(self) -> str:
24 + def get_name(self) -> QuantizationMethods:
25 return "torchao"
26
27 def get_supported_act_dtypes(self) -> List[torch.dtype]:
vllm/model_executor/layers/quantization/tpu_int8.py
+2 -1
@@ -7,6 +7,7 @@ from torch.nn import Module
7 from torch.nn.parameter import Parameter
8
9 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 +from vllm.model_executor.layers.quantization import QuantizationMethods
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.model_executor.parameter import ModelWeightParameter
@@ -27,7 +28,7 @@ class Int8TpuConfig(QuantizationConfig):
28 f"Unsupported activation scheme {activation_scheme}")
29 self.activation_scheme = activation_scheme
30
30 - def get_name(self) -> str:
31 + def get_name(self) -> QuantizationMethods:
32 return "tpu_int8"
33
34 def get_supported_act_dtypes(self) -> List[torch.dtype]:
vllm/model_executor/layers/rotary_embedding.py
+1 -1
@@ -1496,7 +1496,7 @@ def get_rope(
1496 if key in _ROPE_DICT:
1497 return _ROPE_DICT[key]
1498
1499 - if rope_scaling is None:
1499 + if not rope_scaling:
1500 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
1501 is_neox_style, dtype)
1502 else:
vllm/model_executor/model_loader/neuron.py
-1
@@ -180,7 +180,6 @@ def _get_neuron_config_after_override(default_neuron_config,
180 NeuronConfig, QuantizationConfig,
181 SparseAttnConfig)
182
183 - overridden_neuron_config = overridden_neuron_config or {}
183 sparse_attn = overridden_neuron_config.pop("sparse_attn", {})
184 if sparse_attn:
185 overridden_neuron_config["sparse_attn"] = SparseAttnConfig(