A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Move `ModelConfig` from `config/__init__.py` to `config/model.py` (#25252)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Sep 19, 2025 at 17:22 UTC
aed16879a9191a58adc5b8ac3973454dddefe018
13 files changed
+2160
-2149
tests/conftest.py
+2
-1
@@ -39,7 +39,8 @@ from vllm import LLM, SamplingParams
39
from vllm.assets.audio import AudioAsset
40
from vllm.assets.image import ImageAsset
41
from vllm.assets.video import VideoAsset
42
-from vllm.config import ConvertOption, RunnerOption, _get_and_verify_dtype
42
+from vllm.config.model import (ConvertOption, RunnerOption,
43
+ _get_and_verify_dtype)
44
from vllm.connections import global_http_connection
45
from vllm.distributed import (cleanup_dist_env_and_memory,
46
init_distributed_environment,
tests/distributed/test_pipeline_parallel.py
+1
-1
@@ -14,7 +14,7 @@ from typing import Literal, NamedTuple, Optional
14
15
import pytest
16
17
-from vllm.config import _FLOAT16_NOT_SUPPORTED_MODELS, RunnerOption
17
+from vllm.config.model import _FLOAT16_NOT_SUPPORTED_MODELS, RunnerOption
18
from vllm.logger import init_logger
19
from vllm.transformers_utils.config import get_config
20
tests/models/test_initialization.py
+2
-3
@@ -7,7 +7,6 @@ from unittest.mock import patch
7
import pytest
8
9
from vllm import LLM
10
-from vllm.config import ModelImpl
10
from vllm.engine.llm_engine import LLMEngine as V0LLMEngine
11
from vllm.utils import GiB_bytes
12
from vllm.v1.core.kv_cache_utils import get_kv_cache_configs
@@ -111,8 +110,8 @@ def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
110
# these tests seem to produce leftover memory
111
gpu_memory_utilization=0.80,
112
load_format="dummy",
114
- model_impl=ModelImpl.TRANSFORMERS
115
- if model_arch in _TRANSFORMERS_BACKEND_MODELS else ModelImpl.VLLM,
113
+ model_impl="transformers"
114
+ if model_arch in _TRANSFORMERS_BACKEND_MODELS else "vllm",
115
hf_overrides=hf_overrides_fn,
116
max_num_seqs=model_info.max_num_seqs)
117
tests/v1/sample/test_logprobs.py
+4
-5
@@ -3,6 +3,7 @@
3
4
import itertools
5
from collections.abc import Generator
6
+from typing import get_args
7
8
import pytest
9
import torch
@@ -464,7 +465,7 @@ def test_all_logprobs(example_prompts, monkeypatch: pytest.MonkeyPatch):
465
assert len(prompt_logprob) == vocab_size
466
467
467
-@pytest.mark.parametrize("logprobs_mode", list(LogprobsMode))
468
+@pytest.mark.parametrize("logprobs_mode", get_args(LogprobsMode))
469
def test_logprobs_mode(logprobs_mode: LogprobsMode,
470
monkeypatch: pytest.MonkeyPatch):
471
"""Test with LLM engine with different logprobs_mode.
@@ -493,14 +494,12 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode,
494
for logprobs in output.logprobs:
495
for token_id in logprobs:
496
logprob = logprobs[token_id]
496
- if logprobs_mode in (LogprobsMode.RAW_LOGPROBS,
497
- LogprobsMode.PROCESSED_LOGPROBS):
497
+ if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
498
assert logprob.logprob <= 0
499
if logprob.logprob > 0:
500
positive_values = positive_values + 1
501
total_token_with_logprobs = total_token_with_logprobs + 1
502
assert total_token_with_logprobs >= len(results[0].outputs)
503
- if logprobs_mode in (LogprobsMode.RAW_LOGITS,
504
- LogprobsMode.PROCESSED_LOGITS):
503
+ if logprobs_mode in ("raw_logits", "processed_logits"):
504
assert positive_values > 0
505
del llm
vllm/config/__init__.py
+14
-2094
@@ -4,27 +4,22 @@
4
# ruff: noqa: F401
5
import ast
6
import copy
7
-import enum
7
import hashlib
8
import inspect
9
import json
10
import os
11
import textwrap
13
-import warnings
12
from contextlib import contextmanager
15
-from dataclasses import InitVar, field, fields, is_dataclass, replace
13
+from dataclasses import field, fields, is_dataclass, replace
14
from functools import cached_property, lru_cache
17
-from importlib.util import find_spec
18
-from typing import (TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol,
19
- TypeVar, Union, cast, get_args)
15
+from typing import (TYPE_CHECKING, Any, Literal, Optional, Protocol, TypeVar,
16
+ Union, cast)
17
18
import regex as re
19
import torch
23
-from pydantic import (ConfigDict, SkipValidation, field_validator,
24
- model_validator)
20
+from pydantic import ConfigDict, SkipValidation
21
from pydantic.dataclasses import dataclass
26
-from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
27
-from typing_extensions import assert_never, runtime_checkable
22
+from typing_extensions import runtime_checkable
23
24
import vllm.envs as envs
25
from vllm import version
@@ -36,45 +31,31 @@ from vllm.config.kv_events import KVEventsConfig
31
from vllm.config.kv_transfer import KVTransferConfig
32
from vllm.config.load import LoadConfig
33
from vllm.config.lora import LoRAConfig
34
+from vllm.config.model import (ConvertOption, HfOverrides, LogprobsMode,
35
+ ModelConfig, ModelDType, ModelImpl,
36
+ RunnerOption, TaskOption, TokenizerMode,
37
+ iter_architecture_defaults,
38
+ try_match_architecture_defaults)
39
from vllm.config.multimodal import (MMCacheType, MMEncoderTPMode,
40
MultiModalConfig)
41
from vllm.config.parallel import (DistributedExecutorBackend, EPLBConfig,
42
ParallelConfig)
43
from vllm.config.pooler import PoolerConfig
44
-from vllm.config.scheduler import SchedulerConfig, SchedulerPolicy
44
+from vllm.config.scheduler import RunnerType, SchedulerConfig, SchedulerPolicy
45
from vllm.config.speculative import SpeculativeConfig
46
from vllm.config.structured_outputs import StructuredOutputsConfig
47
-from vllm.config.utils import ConfigType, config
47
+from vllm.config.utils import ConfigType, config, get_attr_docs, is_init_field
48
from vllm.logger import init_logger
49
-from vllm.model_executor.layers.quantization import QuantizationMethods
49
from vllm.multimodal import MULTIMODAL_REGISTRY
51
-from vllm.platforms import current_platform
52
-from vllm.transformers_utils.config import (
53
- ConfigFormat, get_config, get_hf_image_processor_config,
54
- get_hf_text_config, get_pooling_config,
55
- get_sentence_transformer_tokenizer_config, is_encoder_decoder,
56
- is_interleaved, maybe_override_with_speculators_target_model,
57
- try_get_generation_config, try_get_safetensors_metadata,
58
- try_get_tokenizer_config, uses_mrope)
59
-from vllm.transformers_utils.runai_utils import (ObjectStorageModel,
60
- is_runai_obj_uri)
61
-from vllm.transformers_utils.utils import maybe_model_redirect
62
-from vllm.utils import (DEFAULT_MAX_NUM_BATCHED_TOKENS,
63
- STR_DUAL_CHUNK_FLASH_ATTN_VAL, LayerBlockType,
64
- LazyLoader, common_broadcastable_dtype, random_uuid)
50
+from vllm.transformers_utils.runai_utils import is_runai_obj_uri
51
+from vllm.utils import random_uuid
52
53
if TYPE_CHECKING:
54
from _typeshed import DataclassInstance
55
from transformers.configuration_utils import PretrainedConfig
56
70
- import vllm.model_executor.layers.quantization as me_quant
71
- import vllm.model_executor.models as me_models
72
- from vllm.model_executor.layers.quantization import QuantizationMethods
57
from vllm.model_executor.layers.quantization.base_config import (
58
QuantizationConfig)
75
- from vllm.v1.sample.logits_processor import LogitsProcessor
76
-
77
- HfOverrides = Union[dict, Callable[[type], type]]
59
else:
60
DataclassInstance = Any
61
PretrainedConfig = Any
@@ -82,83 +63,10 @@ else:
63
QuantizationMethods = Any
64
BaseModelLoader = Any
65
LogitsProcessor = Any
85
- HfOverrides = Union[dict[str, Any], Callable[[type], type]]
86
-
87
- me_quant = LazyLoader("model_executor", globals(),
88
- "vllm.model_executor.layers.quantization")
89
- me_models = LazyLoader("model_executor", globals(),
90
- "vllm.model_executor.models")
66
67
logger = init_logger(__name__)
68
DataclassInstanceT = TypeVar("DataclassInstanceT", bound=DataclassInstance)
69
95
-TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",
96
- "score", "reward", "transcription", "draft"]
97
-
98
-_ResolvedTask = Literal["generate", "transcription", "encode", "embed",
99
- "classify", "reward", "draft"]
100
-
101
-RunnerOption = Literal["auto", "generate", "pooling", "draft"]
102
-
103
-RunnerType = Literal["generate", "pooling", "draft"]
104
-
105
-ConvertOption = Literal["auto", "none", "embed", "classify", "reward"]
106
-
107
-ConvertType = Literal["none", "embed", "classify", "reward"]
108
-
109
-_RUNNER_TASKS: dict[RunnerType, list[TaskOption]] = {
110
- "generate": ["generate", "transcription"],
111
- "pooling": ["embedding", "embed", "classify", "score", "reward"],
112
- "draft": ["draft"],
113
-}
114
-
115
-_RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
116
- "generate": [],
117
- "pooling": ["embed", "classify", "reward"],
118
- "draft": [],
119
-}
120
-
121
-# Some model suffixes are based on auto classes from Transformers:
122
-# https://huggingface.co/docs/transformers/en/model_doc/auto
123
-# NOTE: Items higher on this list priority over lower ones
124
-_SUFFIX_TO_DEFAULTS: list[tuple[str, tuple[RunnerType, ConvertType]]] = [
125
- ("ForCausalLM", ("generate", "none")),
126
- ("ForConditionalGeneration", ("generate", "none")),
127
- ("ChatModel", ("generate", "none")),
128
- ("LMHeadModel", ("generate", "none")),
129
- ("ForTextEncoding", ("pooling", "embed")),
130
- ("EmbeddingModel", ("pooling", "embed")),
131
- ("ForSequenceClassification", ("pooling", "classify")),
132
- ("ForAudioClassification", ("pooling", "classify")),
133
- ("ForImageClassification", ("pooling", "classify")),
134
- ("ForVideoClassification", ("pooling", "classify")),
135
- ("ClassificationModel", ("pooling", "classify")),
136
- ("ForRewardModeling", ("pooling", "reward")),
137
- ("RewardModel", ("pooling", "reward")),
138
- # Let other `*Model`s take priority
139
- ("Model", ("pooling", "embed")),
140
-]
141
-
142
-
143
-def iter_architecture_defaults():
144
- yield from _SUFFIX_TO_DEFAULTS
145
-
146
-
147
-def try_match_architecture_defaults(
148
- architecture: str,
149
- *,
150
- runner_type: Optional[RunnerType] = None,
151
- convert_type: Optional[ConvertType] = None,
152
-) -> Optional[tuple[str, tuple[RunnerType, ConvertType]]]:
153
- for suffix, (default_runner_type,
154
- default_convert_type) in iter_architecture_defaults():
155
- if ((runner_type is None or runner_type == default_runner_type) and
156
- (convert_type is None or convert_type == default_convert_type)
157
- and architecture.endswith(suffix)):
158
- return suffix, (default_runner_type, default_convert_type)
159
-
160
- return None
161
-
70
71
@runtime_checkable
72
class SupportsHash(Protocol):
@@ -173,1608 +81,6 @@ class SupportsMetricsInfo(Protocol):
81
...
82
83
176
-class ModelImpl(str, enum.Enum):
177
- AUTO = "auto"
178
- VLLM = "vllm"
179
- TRANSFORMERS = "transformers"
180
- TERRATORCH = "terratorch"
181
-
182
-
183
-def get_attr_docs(cls: type[Any]) -> dict[str, str]:
184
- """
185
- Get any docstrings placed after attribute assignments in a class body.
186
-
187
- https://davidism.com/mit-license/
188
- """
189
-
190
- def pairwise(iterable):
191
- """
192
- Manually implement https://docs.python.org/3/library/itertools.html#itertools.pairwise
193
-
194
- Can be removed when Python 3.9 support is dropped.
195
- """
196
- iterator = iter(iterable)
197
- a = next(iterator, None)
198
-
199
- for b in iterator:
200
- yield a, b
201
- a = b
202
-
203
- try:
204
- cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
205
- except (OSError, KeyError, TypeError):
206
- # HACK: Python 3.13+ workaround - set missing __firstlineno__
207
- # Workaround can be removed after we upgrade to pydantic==2.12.0
208
- with open(inspect.getfile(cls)) as f:
209
- for i, line in enumerate(f):
210
- if f"class {cls.__name__}" in line and ":" in line:
211
- cls.__firstlineno__ = i + 1
212
- break
213
- cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
214
-
215
- if not isinstance(cls_node, ast.ClassDef):
216
- raise TypeError("Given object was not a class.")
217
-
218
- out = {}
219
-
220
- # Consider each pair of nodes.
221
- for a, b in pairwise(cls_node.body):
222
- # Must be an assignment then a constant string.
223
- if (not isinstance(a, (ast.Assign, ast.AnnAssign))
224
- or not isinstance(b, ast.Expr)
225
- or not isinstance(b.value, ast.Constant)
226
- or not isinstance(b.value.value, str)):
227
- continue
228
-
229
- doc = inspect.cleandoc(b.value.value)
230
-
231
- # An assignment can have multiple targets (a = b = v), but an
232
- # annotated assignment only has one target.
233
- targets = a.targets if isinstance(a, ast.Assign) else [a.target]
234
-
235
- for target in targets:
236
- # Must be assigning to a plain name.
237
- if not isinstance(target, ast.Name):
238
- continue
239
-
240
- out[target.id] = doc
241
-
242
- return out
243
-
244
-
245
-def is_init_field(cls: ConfigType, name: str) -> bool:
246
- return next(f for f in fields(cls) if f.name == name).init
247
-
248
-
249
-TokenizerMode = Literal["auto", "slow", "mistral", "custom"]
250
-ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
251
-
252
-
253
-class LogprobsMode(enum.Enum):
254
- RAW_LOGITS = "raw_logits"
255
- RAW_LOGPROBS = "raw_logprobs"
256
- PROCESSED_LOGITS = "processed_logits"
257
- PROCESSED_LOGPROBS = "processed_logprobs"
258
-
259
-
260
-@config
261
-@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
262
-class ModelConfig:
263
- """Configuration for the model."""
264
-
265
- model: str = "Qwen/Qwen3-0.6B"
266
- """Name or path of the Hugging Face model to use. It is also used as the
267
- content for `model_name` tag in metrics output when `served_model_name` is
268
- not specified."""
269
- runner: RunnerOption = "auto"
270
- """The type of model runner to use. Each vLLM instance only supports one
271
- model runner, even if the same model can be used for multiple types."""
272
- convert: ConvertOption = "auto"
273
- """Convert the model using adapters defined in
274
- [vllm.model_executor.models.adapters][]. The most common use case is to
275
- adapt a text generation model to be used for pooling tasks."""
276
- task: Optional[TaskOption] = None
277
- """[DEPRECATED] The task to use the model for. If the model supports more
278
- than one model runner, this is used to select which model runner to run.
279
-
280
- Note that the model may support other tasks using the same model runner.
281
- """
282
- tokenizer: SkipValidation[str] = None # type: ignore
283
- """Name or path of the Hugging Face tokenizer to use. If unspecified, model
284
- name or path will be used."""
285
- tokenizer_mode: TokenizerMode = "auto"
286
- """Tokenizer mode:\n
287
- - "auto" will use the fast tokenizer if available.\n
288
- - "slow" will always use the slow tokenizer.\n
289
- - "mistral" will always use the tokenizer from `mistral_common`.\n
290
- - "custom" will use --tokenizer to select the preregistered tokenizer."""
291
- trust_remote_code: bool = False
292
- """Trust remote code (e.g., from HuggingFace) when downloading the model
293
- and tokenizer."""
294
- dtype: Union[ModelDType, torch.dtype] = "auto"
295
- """Data type for model weights and activations:\n
296
- - "auto" will use FP16 precision for FP32 and FP16 models, and BF16
297
- precision for BF16 models.\n
298
- - "half" for FP16. Recommended for AWQ quantization.\n
299
- - "float16" is the same as "half".\n
300
- - "bfloat16" for a balance between precision and range.\n
301
- - "float" is shorthand for FP32 precision.\n
302
- - "float32" for FP32 precision."""
303
- seed: Optional[int] = None
304
- """Random seed for reproducibility. Initialized to None in V0, but
305
- initialized to 0 in V1."""
306
- hf_config_path: Optional[str] = None
307
- """Name or path of the Hugging Face config to use. If unspecified, model
308
- name or path will be used."""
309
- allowed_local_media_path: str = ""
310
- """Allowing API requests to read local images or videos from directories
311
- specified by the server file system. This is a security risk. Should only
312
- be enabled in trusted environments."""
313
- revision: Optional[str] = None
314
- """The specific model version to use. It can be a branch name, a tag name,
315
- or a commit id. If unspecified, will use the default version."""
316
- code_revision: Optional[str] = None
317
- """The specific revision to use for the model code on the Hugging Face Hub.
318
- It can be a branch name, a tag name, or a commit id. If unspecified, will
319
- use the default version."""
320
- rope_scaling: dict[str, Any] = field(default_factory=dict)
321
- """RoPE scaling configuration. For example,
322
- `{"rope_type":"dynamic","factor":2.0}`."""
323
- rope_theta: Optional[float] = None
324
- """RoPE theta. Use with `rope_scaling`. In some cases, changing the RoPE
325
- theta improves the performance of the scaled model."""
326
- tokenizer_revision: Optional[str] = None
327
- """The specific revision to use for the tokenizer on the Hugging Face Hub.
328
- It can be a branch name, a tag name, or a commit id. If unspecified, will
329
- use the default version."""
330
- max_model_len: SkipValidation[int] = None # type: ignore
331
- """Model context length (prompt and output). If unspecified, will be
332
- automatically derived from the model config.
333
-
334
- When passing via `--max-model-len`, supports k/m/g/K/M/G in human-readable
335
- format. Examples:\n
336
- - 1k -> 1000\n
337
- - 1K -> 1024\n
338
- - 25.6k -> 25,600"""
339
- spec_target_max_model_len: Optional[int] = None
340
- """Specify the maximum length for spec decoding draft models."""
341
- quantization: SkipValidation[Optional[QuantizationMethods]] = None
342
- """Method used to quantize the weights. If `None`, we first check the
343
- `quantization_config` attribute in the model config file. If that is
344
- `None`, we assume the model weights are not quantized and use `dtype` to
345
- determine the data type of the weights."""
346
- enforce_eager: bool = False
347
- """Whether to always use eager-mode PyTorch. If True, we will disable CUDA
348
- graph and always execute the model in eager mode. If False, we will use
349
- CUDA graph and eager execution in hybrid for maximal performance and
350
- flexibility."""
351
- max_seq_len_to_capture: int = 8192
352
- """Maximum sequence len covered by CUDA graphs. When a sequence has context
353
- length larger than this, we fall back to eager mode. Additionally for
354
- encoder-decoder models, if the sequence length of the encoder input is
355
- larger than this, we fall back to the eager mode."""
356
- max_logprobs: int = 20
357
- """Maximum number of log probabilities to return when `logprobs` is
358
- specified in `SamplingParams`. The default value comes the default for the
359
- OpenAI Chat Completions API. -1 means no cap, i.e. all (output_length *
360
- vocab_size) logprobs are allowed to be returned and it may cause OOM."""
361
- logprobs_mode: LogprobsMode = LogprobsMode.RAW_LOGPROBS
362
- """Indicates the content returned in the logprobs and prompt_logprobs.
363
- Supported mode:
364
- 1) raw_logprobs, 2) processed_logprobs, 3) raw_logits, 4) processed_logits.
365
- Raw means the values before applying any logit processors, like bad words.
366
- Processed means the values after applying all processors, including
367
- temperature and top_k/top_p.
368
- """
369
- disable_sliding_window: bool = False
370
- """Whether to disable sliding window. If True, we will disable the sliding
371
- window functionality of the model, capping to sliding window size. If the
372
- model does not support sliding window, this argument is ignored."""
373
- disable_cascade_attn: bool = False
374
- """Disable cascade attention for V1. While cascade attention does not
375
- change the mathematical correctness, disabling it could be useful for
376
- preventing potential numerical issues. Note that even if this is set to
377
- False, cascade attention will be only used when the heuristic tells that
378
- it's beneficial."""
379
- skip_tokenizer_init: bool = False
380
- """Skip initialization of tokenizer and detokenizer. Expects valid
381
- `prompt_token_ids` and `None` for prompt from the input. The generated
382
- output will contain token ids."""
383
- enable_prompt_embeds: bool = False
384
- """If `True`, enables passing text embeddings as inputs via the
385
- `prompt_embeds` key. Note that enabling this will double the time required
386
- for graph compilation."""
387
- served_model_name: Optional[Union[str, list[str]]] = None
388
- """The model name(s) used in the API. If multiple names are provided, the
389
- server will respond to any of the provided names. The model name in the
390
- model field of a response will be the first name in this list. If not
391
- specified, the model name will be the same as the `--model` argument. Noted
392
- that this name(s) will also be used in `model_name` tag content of
393
- prometheus metrics, if multiple names provided, metrics tag will take the
394
- first one."""
395
- use_async_output_proc: bool = True
396
- """Whether to use async output processor."""
397
- config_format: Union[str, ConfigFormat] = "auto"
398
- """The format of the model config to load:\n
399
- - "auto" will try to load the config in hf format if available else it
400
- will try to load in mistral format.\n
401
- - "hf" will load the config in hf format.\n
402
- - "mistral" will load the config in mistral format."""
403
- hf_token: Optional[Union[bool, str]] = None
404
- """The token to use as HTTP bearer authorization for remote files . If
405
- `True`, will use the token generated when running `huggingface-cli login`
406
- (stored in `~/.huggingface`)."""
407
- hf_overrides: HfOverrides = field(default_factory=dict)
408
- """If a dictionary, contains arguments to be forwarded to the Hugging Face
409
- config. If a callable, it is called to update the HuggingFace config."""
410
- logits_processor_pattern: Optional[str] = None
411
- """Optional regex pattern specifying valid logits processor qualified names
412
- that can be passed with the `logits_processors` extra completion argument.
413
- Defaults to `None`, which allows no processors."""
414
- generation_config: str = "auto"
415
- """The folder path to the generation config. Defaults to `"auto"`, the
416
- generation config will be loaded from model path. If set to `"vllm"`, no
417
- generation config is loaded, vLLM defaults will be used. If set to a folder
418
- path, the generation config will be loaded from the specified folder path.
419
- If `max_new_tokens` is specified in generation config, then it sets a
420
- server-wide limit on the number of output tokens for all requests."""
421
- override_generation_config: dict[str, Any] = field(default_factory=dict)
422
- """Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
423
- used with `--generation-config auto`, the override parameters will be
424
- merged with the default config from the model. If used with
425
- `--generation-config vllm`, only the override parameters are used."""
426
- enable_sleep_mode: bool = False
427
- """Enable sleep mode for the engine (only cuda platform is supported)."""
428
- model_impl: Union[str, ModelImpl] = ModelImpl.AUTO.value
429
- """Which implementation of the model to use:\n
430
- - "auto" will try to use the vLLM implementation, if it exists, and fall
431
- back to the Transformers implementation if no vLLM implementation is
432
- available.\n
433
- - "vllm" will use the vLLM model implementation.\n
434
- - "transformers" will use the Transformers model implementation.\n
435
- - "terratorch" will use the TerraTorch model implementation.
436
- """
437
- override_attention_dtype: Optional[str] = None
438
- """Override dtype for attention"""
439
- logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None
440
- """One or more logits processors' fully-qualified class names or class
441
- definitions"""
442
- io_processor_plugin: Optional[str] = None
443
- """IOProcessor plugin name to load at model startup"""
444
-
445
- # Pooler config
446
- pooler_config: Optional[PoolerConfig] = None
447
- """Pooler config which controls the behaviour of output pooling in pooling
448
- models."""
449
- override_pooler_config: Optional[Union[dict, PoolerConfig]] = None
450
- """[DEPRECATED] Use `pooler_config` instead. This field will be removed in
451
- v0.12.0 or v1.0.0, whichever is sooner."""
452
-
453
- # Multimodal config and init vars
454
- multimodal_config: Optional[MultiModalConfig] = None
455
- """Configuration for multimodal model. If `None`, this will be inferred
456
- from the architecture of `self.model`."""
457
- limit_mm_per_prompt: InitVar[Optional[dict[str, int]]] = None
458
- media_io_kwargs: InitVar[Optional[dict[str, dict[str, Any]]]] = None
459
- mm_processor_kwargs: InitVar[Optional[dict[str, Any]]] = None
460
- mm_processor_cache_gb: InitVar[Optional[float]] = None
461
- mm_processor_cache_type: InitVar[Optional[MMCacheType]] = None
462
- mm_shm_cache_max_object_size_mb: InitVar[Optional[int]] = None
463
- mm_encoder_tp_mode: InitVar[Optional[MMEncoderTPMode]] = None
464
- interleave_mm_strings: InitVar[Optional[bool]] = None
465
- skip_mm_profiling: InitVar[Optional[bool]] = None
466
-
467
- def compute_hash(self) -> str:
468
- """
469
- WARNING: Whenever a new field is added to this config,
470
- ensure that it is included in the factors list if
471
- it affects the computation graph.
472
-
473
- Provide a hash that uniquely identifies all the configs
474
- that affect the structure of the computation
475
- graph from input ids/embeddings to the final hidden states,
476
- excluding anything before input ids/embeddings and after
477
- the final hidden states.
478
- """
479
- factors: list[Any] = []
480
- factors.append(self.model)
481
- factors.append(self.dtype)
482
- factors.append(self.quantization)
483
- factors.append(self.revision)
484
- factors.append(self.code_revision)
485
- factors.append(self.max_model_len)
486
- factors.append(self.max_logprobs)
487
- factors.append(self.disable_sliding_window)
488
- factors.append(self.trust_remote_code)
489
- factors.append(self.generation_config)
490
- factors.append(self.model_impl)
491
- factors.append(self.override_generation_config)
492
- factors.append(self.rope_scaling)
493
- factors.append(self.rope_theta)
494
- # hf_config can control how the model looks!
495
- factors.append(self.hf_config.to_json_string())
496
- str_factors = str(factors)
497
- assert_hashable(str_factors)
498
- return hashlib.sha256(str(factors).encode()).hexdigest()
499
-
500
- def __post_init__(
501
- self,
502
- # Multimodal config init vars
503
- limit_mm_per_prompt: Optional[dict[str, int]],
504
- media_io_kwargs: Optional[dict[str, dict[str, Any]]],
505
- mm_processor_kwargs: Optional[dict[str, Any]],
506
- mm_processor_cache_gb: Optional[float],
507
- mm_processor_cache_type: Optional[MMCacheType],
508
- mm_shm_cache_max_object_size_mb: Optional[int],
509
- mm_encoder_tp_mode: Optional[MMEncoderTPMode],
510
- interleave_mm_strings: Optional[bool],
511
- skip_mm_profiling: Optional[bool]) -> None:
512
- # Set the default seed to 0 in V1.
513
- # NOTE(woosuk): In V0, we set the default seed to None because the
514
- # driver worker shares the same process as the user process, and thus
515
- # setting a seed affects the user process as well.
516
- # In V1, we use separate processes for workers (unless
517
- # VLLM_ENABLE_V1_MULTIPROCESSING=0), so setting a seed here
518
- # doesn't affect the user process. However, without a consistent seed,
519
- # different tensor parallel workers would sample different tokens,
520
- # leading to inconsistent results.
521
- if envs.VLLM_USE_V1 and self.seed is None:
522
- self.seed = 0
523
- if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
524
- logger.warning(
525
- "The global random seed is set to %d. Since "
526
- "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may "
527
- "affect the random state of the Python process that "
528
- "launched vLLM.", self.seed)
529
-
530
- # Keep set served_model_name before maybe_model_redirect(self.model)
531
- self.served_model_name = get_served_model_name(self.model,
532
- self.served_model_name)
533
- self.model = maybe_model_redirect(self.model)
534
- # The tokenizer is consistent with the model by default.
535
- if self.tokenizer is None:
536
- self.tokenizer = self.model
537
- if self.tokenizer_revision is None:
538
- self.tokenizer_revision = self.revision
539
- self.tokenizer = maybe_model_redirect(self.tokenizer)
540
-
541
- if isinstance(self.hf_config_path, str):
542
- self.hf_config_path = maybe_model_redirect(self.hf_config_path)
543
-
544
- if callable(self.hf_overrides):
545
- hf_overrides_kw = {}
546
- hf_overrides_fn = self.hf_overrides
547
- else:
548
- hf_overrides_kw = self.hf_overrides
549
- hf_overrides_fn = None
550
-
551
- if self.rope_scaling:
552
- hf_override: dict[str, Any] = {"rope_scaling": self.rope_scaling}
553
- hf_overrides_kw.update(hf_override)
554
- hf_overrides_str = json.dumps(hf_overrides_kw)
555
- msg = (
556
- "`--rope-scaling` will be removed in a future release. "
557
- f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
558
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
559
- if self.rope_theta is not None:
560
- hf_override = {"rope_theta": self.rope_theta}
561
- hf_overrides_kw.update(hf_override)
562
- hf_overrides_str = json.dumps(hf_overrides_kw)
563
- msg = (
564
- "`--rope-theta` will be removed in a future release. "
565
- f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
566
- warnings.warn(DeprecationWarning(msg), stacklevel=2)
567
-
568
- self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer)
569
-
570
- if self.runner != "draft":
571
- # If we're not running the draft model, check for speculators config
572
- # If speculators config, set model / tokenizer to be target model
573
- self.model, self.tokenizer = maybe_override_with_speculators_target_model( # noqa: E501
574
- model=self.model,
575
- tokenizer=self.tokenizer,
576
- revision=self.revision,
577
- trust_remote_code=self.trust_remote_code)
578
-
579
- if (backend := envs.VLLM_ATTENTION_BACKEND
580
- ) and backend == "FLASHINFER" and find_spec("flashinfer") is None:
581
- raise ValueError(
582
- "VLLM_ATTENTION_BACKEND is set to FLASHINFER, but flashinfer "
583
- "module was not found. See "
584
- "https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile " # noqa: E501
585
- "for instructions on how to install it.")
586
-
587
- from vllm.platforms import current_platform
588
-
589
- if (self.override_attention_dtype is not None
590
- and not current_platform.is_rocm()):
591
- warnings.warn(
592
- "override-attention-dtype is set but not using ROCm platform",
593
- stacklevel=2)
594
-
595
- if (self.enable_sleep_mode
596
- and not current_platform.is_sleep_mode_available()):
597
- raise ValueError(
598
- "Sleep mode is not supported on current platform.")
599
-
600
- hf_config = get_config(self.hf_config_path or self.model,
601
- self.trust_remote_code,
602
- self.revision,
603
- self.code_revision,
604
- self.config_format,
605
- hf_overrides_kw=hf_overrides_kw,
606
- hf_overrides_fn=hf_overrides_fn)
607
-
608
- self.hf_config = hf_config
609
- self.hf_text_config = get_hf_text_config(self.hf_config)
610
- self.attention_chunk_size = getattr(self.hf_text_config,
611
- "attention_chunk_size", None)
612
- self.encoder_config = self._get_encoder_config()
613
- self.hf_image_processor_config = get_hf_image_processor_config(
614
- self.model, hf_token=self.hf_token, revision=self.revision)
615
-
616
- architectures = self.architectures
617
- registry = self.registry
618
- is_generative_model = registry.is_text_generation_model(
619
- architectures, self)
620
- is_pooling_model = registry.is_pooling_model(architectures, self)
621
-
622
- def _task_to_convert(task: TaskOption) -> ConvertType:
623
- if task == "embedding" or task == "embed":
624
- return "embed"
625
- if task == "classify":
626
- return "classify"
627
- if task == "reward":
628
- return "reward"
629
- if task == "score":
630
- new_task = self._get_default_pooling_task(architectures)
631
- return "classify" if new_task == "classify" else "embed"
632
-
633
- return "none"
634
-
635
- if self.task is not None:
636
- runner: RunnerOption = "auto"
637
- convert: ConvertOption = "auto"
638
- msg_prefix = ("The 'task' option has been deprecated and will be "
639
- "removed in v0.13.0 or v1.0, whichever comes first.")
640
- msg_hint = "Please remove this option."
641
-
642
- is_generative_task = self.task in _RUNNER_TASKS["generate"]
643
- is_pooling_task = self.task in _RUNNER_TASKS["pooling"]
644
-
645
- if is_generative_model and is_pooling_model:
646
- if is_generative_task:
647
- runner = "generate"
648
- convert = "auto"
649
- msg_hint = ("Please replace this option with `--runner "
650
- "generate` to continue using this model "
651
- "as a generative model.")
652
- elif is_pooling_task:
653
- runner = "pooling"
654
- convert = "auto"
655
- msg_hint = ("Please replace this option with `--runner "
656
- "pooling` to continue using this model "
657
- "as a pooling model.")
658
- else: # task == "auto"
659
- pass
660
- elif is_generative_model or is_pooling_model:
661
- if is_generative_task:
662
- runner = "generate"
663
- convert = "auto"
664
- msg_hint = "Please remove this option"
665
- elif is_pooling_task:
666
- runner = "pooling"
667
- convert = _task_to_convert(self.task)
668
- msg_hint = ("Please replace this option with `--convert "
669
- f"{convert}` to continue using this model "
670
- "as a pooling model.")
671
- else: # task == "auto"
672
- pass
673
- else:
674
- raise AssertionError("The model should be a generative or "
675
- "pooling model when task is set to "
676
- f"{self.task!r}.")
677
-
678
- self.runner = runner
679
- self.convert = convert
680
-
681
- msg = f"{msg_prefix} {msg_hint}"
682
- warnings.warn(msg, DeprecationWarning, stacklevel=2)
683
-
684
- self.runner_type = self._get_runner_type(architectures, self.runner)
685
- self.convert_type = self._get_convert_type(architectures,
686
- self.runner_type,
687
- self.convert)
688
-
689
- if self.runner_type == "generate" and not is_generative_model:
690
- generate_converts = _RUNNER_CONVERTS["generate"]
691
- if self.convert_type not in generate_converts:
692
- # Currently we don't have any converters for generative models
693
- raise ValueError(
694
- "This model does not support `--runner generate`.")
695
- if self.runner_type == "pooling" and not is_pooling_model:
696
- pooling_converts = _RUNNER_CONVERTS["pooling"]
697
- if self.convert_type not in pooling_converts:
698
- convert_option = "<" + "|".join(pooling_converts) + ">"
699
- raise ValueError(
700
- "This model does not support `--runner pooling`. "
701
- f"You can pass `--convert {convert_option} to adapt "
702
- "it into a pooling model.")
703
-
704
- self.supported_tasks = self._get_supported_tasks(
705
- architectures, self.runner_type, self.convert_type)
706
-
707
- # Note: Initialize these attributes early because transformers fallback
708
- # may fail to load dynamic modules in child processes
709
- model_info, arch = registry.inspect_model_cls(architectures, self)
710
- self._model_info = model_info
711
- self._architecture = arch
712
- logger.info("Resolved architecture: %s", arch)
713
-
714
- # Init pooler config if needed
715
- if self.runner_type == "pooling":
716
- if self.override_pooler_config is not None:
717
- logger.warning_once(
718
- "`override_pooler_config` is deprecated and will be "
719
- "removed in v0.12.0 or v1.0.0, whichever is sooner. "
720
- "Please use `pooler_config` instead.")
721
-
722
- if isinstance(self.override_pooler_config, dict):
723
- self.pooler_config = PoolerConfig(
724
- **self.override_pooler_config)
725
- else:
726
- self.pooler_config = self.override_pooler_config
727
-
728
- if self.pooler_config is None:
729
- self.pooler_config = PoolerConfig()
730
-
731
- base_config = get_pooling_config(self.model, self.revision)
732
- if base_config is not None:
733
- # Only set values that are not overridden by the user
734
- for k, v in base_config.items():
735
- if getattr(self.pooler_config, k) is None:
736
- setattr(self.pooler_config, k, v)
737
-
738
- default_pooling_type = self._model_info.default_pooling_type
739
- if self.pooler_config.pooling_type is None:
740
- self.pooler_config.pooling_type = default_pooling_type
741
-
742
- self.dtype: torch.dtype = _get_and_verify_dtype(
743
- self.model,
744
- self.hf_config,
745
- self.dtype,
746
- is_pooling_model=self.runner_type == "pooling",
747
- revision=self.revision,
748
- )
749
-
750
- # Interleaved attention is not supported by some backends in V0
751
- if (not self.disable_sliding_window
752
- and is_interleaved(self.hf_text_config)
753
- and not envs.VLLM_USE_V1
754
- and (backend := envs.VLLM_ATTENTION_BACKEND)
755
- in ("XFORMERS", "FLASHINFER")):
756
- logger.warning_once(
757
- "%s has interleaved attention, which is currently not "
758
- "supported by the %s backend. Disabling sliding window and "
759
- "capping the max length to the sliding window size (%d).",
760
- self.hf_text_config.model_type,
761
- backend,
762
- self.hf_text_config.sliding_window,
763
- )
764
- self.disable_sliding_window = True
765
-
766
- self.original_max_model_len = self.max_model_len
767
- self.max_model_len = self.get_and_verify_max_len(self.max_model_len)
768
- # Init multimodal config if needed
769
- if self._model_info.supports_multimodal:
770
- if (mm_encoder_tp_mode == "data" and
771
- not self._model_info.supports_multimodal_encoder_tp_data):
772
- logger.warning_once(
773
- "This model does not support `--mm-encoder-tp-mode data`. "
774
- "Falling back to `--mm-encoder-tp-mode weights`.")
775
- mm_encoder_tp_mode = "weights"
776
-
777
- mm_config_kwargs = dict(
778
- limit_per_prompt=limit_mm_per_prompt,
779
- media_io_kwargs=media_io_kwargs,
780
- mm_processor_kwargs=mm_processor_kwargs,
781
- mm_processor_cache_gb=mm_processor_cache_gb,
782
- mm_processor_cache_type=mm_processor_cache_type,
783
- mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
784
- mm_encoder_tp_mode=mm_encoder_tp_mode,
785
- interleave_mm_strings=interleave_mm_strings,
786
- skip_mm_profiling=skip_mm_profiling,
787
- )
788
-
789
- mm_config_kwargs = {
790
- k: v
791
- for k, v in mm_config_kwargs.items() if v is not None
792
- }
793
-
794
- self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
795
-
796
- if self.disable_sliding_window:
797
- # Set after get_and_verify_max_len to ensure that max_model_len
798
- # can be correctly capped to sliding window size
799
- self.hf_text_config.sliding_window = None
800
-
801
- if not self.skip_tokenizer_init:
802
- self._verify_tokenizer_mode()
803
-
804
- # Avoid running try_verify_and_update_config multiple times
805
- self.config_updated = False
806
-
807
- self._verify_quantization()
808
- self._verify_cuda_graph()
809
- self._verify_bnb_config()
810
-
811
- @field_validator("quantization", mode="before")
812
- @classmethod
813
- def validate_quantization_before(cls, value: Any) -> Any:
814
- if isinstance(value, str):
815
- return value.lower()
816
- return value
817
-
818
- @model_validator(mode="after")
819
- def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
820
- if not isinstance(self.tokenizer, str):
821
- raise ValueError("tokenizer must be a string after __post_init__.")
822
- if not isinstance(self.max_model_len, int):
823
- raise ValueError(
824
- "max_model_len must be an integer after __post_init__.")
825
- return self
826
-
827
- def _get_transformers_backend_cls(self) -> str:
828
- """Determine which Transformers backend class will be used if
829
- `model_impl` is set to `transformers` or `auto`."""
830
- if getattr(self, "runner_type", self.runner) == "pooling":
831
- return "TransformersModel"
832
- if self.hf_config != self.hf_text_config:
833
- # If 'hf_text_config' is the same as 'hf_config'. If not, it is
834
- # probably a composite config, i.e. multimodal
835
- return "TransformersForMultimodalLM"
836
- return "TransformersForCausalLM"
837
-
838
- def using_transformers_backend(self) -> bool:
839
- """Check if the model is using the Transformers backend class."""
840
- return self.architecture == self._get_transformers_backend_cls()
841
-
842
- @property
843
- def registry(self):
844
- return me_models.ModelRegistry
845
-
846
- @property
847
- def architectures(self) -> list[str]:
848
- return getattr(self.hf_config, "architectures", [])
849
-
850
- @property
851
- def architecture(self) -> str:
852
- """The architecture vllm actually used."""
853
- return self._architecture
854
-
855
- def maybe_pull_model_tokenizer_for_runai(self, model: str,
856
- tokenizer: str) -> None:
857
- """Pull model/tokenizer from Object Storage to temporary
858
- directory when needed.
859
-
860
- Args:
861
- model: Model name or path
862
- tokenizer: Tokenizer name or path
863
- """
864
- if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)):
865
- return
866
-
867
- if is_runai_obj_uri(model):
868
- object_storage_model = ObjectStorageModel()
869
- object_storage_model.pull_files(
870
- model, allow_pattern=["*.model", "*.py", "*.json"])
871
- self.model_weights = model
872
- self.model = object_storage_model.dir
873
-
874
- # If tokenizer is same as model, download to same directory
875
- if model == tokenizer:
876
- object_storage_model.pull_files(model,
877
- ignore_pattern=[
878
- "*.pt", "*.safetensors",
879
- "*.bin", "*.tensors",
880
- "*.pth"
881
- ])
882
- self.tokenizer = object_storage_model.dir
883
- return
884
-
885
- # Only download tokenizer if needed and not already handled
886
- if is_runai_obj_uri(tokenizer):
887
- object_storage_tokenizer = ObjectStorageModel()
888
- object_storage_tokenizer.pull_files(model,
889
- ignore_pattern=[
890
- "*.pt", "*.safetensors",
891
- "*.bin", "*.tensors",
892
- "*.pth"
893
- ])
894
- self.tokenizer = object_storage_tokenizer.dir
895
-
896
- def _get_encoder_config(self):
897
- return get_sentence_transformer_tokenizer_config(
898
- self.model, self.revision)
899
-
900
- def _verify_tokenizer_mode(self) -> None:
901
- tokenizer_mode = cast(TokenizerMode, self.tokenizer_mode.lower())
902
- if tokenizer_mode not in get_args(TokenizerMode):
903
- raise ValueError(
904
- f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
905
- f"one of {get_args(TokenizerMode)}.")
906
- self.tokenizer_mode = tokenizer_mode
907
-
908
- def _get_default_runner_type(
909
- self,
910
- architectures: list[str],
911
- ) -> RunnerType:
912
- registry = self.registry
913
-
914
- # Some Sentence Transformers models use *ForCausalLM archs
915
- if get_pooling_config(self.model, self.revision):
916
- return "pooling"
917
-
918
- for arch in architectures:
919
- if arch in registry.get_supported_archs():
920
- if registry.is_pooling_model(architectures, self):
921
- return "pooling"
922
- if registry.is_text_generation_model(architectures, self):
923
- return "generate"
924
-
925
- match = try_match_architecture_defaults(arch)
926
- if match:
927
- _, (runner_type, _) = match
928
- return runner_type
929
-
930
- return "generate"
931
-
932
- def _get_runner_type(
933
- self,
934
- architectures: list[str],
935
- runner: RunnerOption,
936
- ) -> RunnerType:
937
- if runner != "auto":
938
- return runner
939
-
940
- runner_type = self._get_default_runner_type(architectures)
941
-
942
- # Don't log the most common case
943
- if runner_type != "generate":
944
- logger.info(
945
- "Resolved `--runner auto` to `--runner %s`. "
946
- "Pass the value explicitly to silence this message.",
947
- runner_type)
948
-
949
- return runner_type
950
-
951
- def _get_default_convert_type(
952
- self,
953
- architectures: list[str],
954
- runner_type: RunnerType,
955
- ) -> ConvertType:
956
- registry = self.registry
957
-
958
- for arch in architectures:
959
- if arch in registry.get_supported_archs():
960
- if (runner_type == "generate"
961
- and registry.is_text_generation_model(
962
- architectures, self)):
963
- return "none"
964
- if (runner_type == "pooling"
965
- and registry.is_pooling_model(architectures, self)):
966
- return "none"
967
-
968
- match = try_match_architecture_defaults(arch,
969
- runner_type=runner_type)
970
- if match:
971
- _, (_, convert_type) = match
972
- return convert_type
973
-
974
- # This is to handle Sentence Transformers models that use *ForCausalLM
975
- # and also multi-modal pooling models which are not defined as
976
- # Sentence Transformers models
977
- if runner_type == "pooling":
978
- return "embed"
979
-
980
- return "none"
981
-
982
- def _get_convert_type(
983
- self,
984
- architectures: list[str],
985
- runner_type: RunnerType,
986
- convert: ConvertOption,
987
- ) -> ConvertType:
988
- if convert != "auto":
989
- return convert
990
-
991
- convert_type = self._get_default_convert_type(architectures,
992
- runner_type)
993
-
994
- # Don't log the most common case
995
- if convert_type != "none":
996
- logger.info(
997
- "Resolved `--convert auto` to `--convert %s`. "
998
- "Pass the value explicitly to silence this message.",
999
- convert_type)
1000
-
1001
- return convert_type
1002
-
1003
- def _get_supported_generation_tasks(
1004
- self,
1005
- architectures: list[str],
1006
- convert_type: ConvertType,
1007
- ) -> list[_ResolvedTask]:
1008
- registry = self.registry
1009
-
1010
- if registry.is_transcription_only_model(architectures, self):
1011
- return ["transcription"]
1012
-
1013
- # TODO: Use get_supported_generation_tasks once V0 is removed
1014
- supported_tasks = list[_ResolvedTask]()
1015
- if (registry.is_text_generation_model(architectures, self)
1016
- or convert_type in _RUNNER_CONVERTS["generate"]):
1017
- supported_tasks.append("generate")
1018
-
1019
- if registry.is_transcription_model(architectures, self):
1020
- supported_tasks.append("transcription")
1021
-
1022
- return supported_tasks
1023
-
1024
- def _get_default_pooling_task(
1025
- self,
1026
- architectures: list[str],
1027
- ) -> Literal["embed", "classify", "reward"]:
1028
- if self.registry.is_cross_encoder_model(architectures, self):
1029
- return "classify"
1030
-
1031
- for arch in architectures:
1032
- match = try_match_architecture_defaults(arch,
1033
- runner_type="pooling")
1034
- if match:
1035
- _, (_, convert_type) = match
1036
- assert convert_type != "none"
1037
- return convert_type
1038
-
1039
- return "embed"
1040
-
1041
- def _get_supported_pooling_tasks(
1042
- self,
1043
- architectures: list[str],
1044
- convert_type: ConvertType,
1045
- ) -> list[_ResolvedTask]:
1046
- registry = self.registry
1047
-
1048
- # TODO: Use get_supported_pooling_tasks once V0 is removed
1049
- supported_tasks = list[_ResolvedTask]()
1050
- if (registry.is_pooling_model(architectures, self)
1051
- or convert_type in _RUNNER_CONVERTS["pooling"]):
1052
- supported_tasks.append("encode")
1053
-
1054
- extra_task = (self._get_default_pooling_task(architectures)
1055
- if convert_type == "none" else convert_type)
1056
- supported_tasks.append(extra_task)
1057
-
1058
- return supported_tasks
1059
-
1060
- def _get_supported_tasks(
1061
- self,
1062
- architectures: list[str],
1063
- runner_type: RunnerType,
1064
- convert_type: ConvertType,
1065
- ) -> list[_ResolvedTask]:
1066
- if runner_type == "generate":
1067
- return self._get_supported_generation_tasks(
1068
- architectures, convert_type)
1069
- if runner_type == "pooling":
1070
- return self._get_supported_pooling_tasks(architectures,
1071
- convert_type)
1072
- if runner_type == "draft":
1073
- return ["draft"]
1074
-
1075
- assert_never(runner_type)
1076
-
1077
- def _parse_quant_hf_config(self, hf_config: PretrainedConfig):
1078
- quant_cfg = getattr(hf_config, "quantization_config", None)
1079
- if quant_cfg is None:
1080
- # compressed-tensors uses a "compression_config" key
1081
- quant_cfg = getattr(hf_config, "compression_config", None)
1082
-
1083
- else:
1084
- # Set quant_method for ModelOpt models.
1085
- producer_name = quant_cfg.get("producer", {}).get("name")
1086
- if producer_name == "modelopt":
1087
- quant_algo = quant_cfg.get("quantization",
1088
- {}).get("quant_algo")
1089
- if quant_algo == "FP8":
1090
- quant_cfg["quant_method"] = "modelopt"
1091
- elif quant_algo == "NVFP4":
1092
- quant_cfg["quant_method"] = "modelopt_fp4"
1093
- elif quant_algo is not None:
1094
- raise ValueError(
1095
- f"Unknown ModelOpt quant algo: {quant_algo}")
1096
-
1097
- return quant_cfg
1098
-
1099
- def _verify_quantization(self) -> None:
1100
- supported_quantization = me_quant.QUANTIZATION_METHODS
1101
- if self.quantization is not None:
1102
- self.quantization = cast(me_quant.QuantizationMethods,
1103
- self.quantization)
1104
-
1105
- # Parse quantization method from the HF model config, if available.
1106
- quant_cfg = self._parse_quant_hf_config(self.hf_config)
1107
- if quant_cfg is None and (text_config := getattr(
1108
- self.hf_config, "text_config", None)):
1109
- # Check the text config as well for multi-modal models.
1110
- quant_cfg = self._parse_quant_hf_config(text_config)
1111
-
1112
- if quant_cfg is not None:
1113
- # Use the community standard 'quant_method'
1114
- quant_method = quant_cfg.get("quant_method", "").lower()
1115
-
1116
- # Normalize library names
1117
- quant_method = quant_method.replace("compressed_tensors",
1118
- "compressed-tensors")
1119
-
1120
- quant_cfg["quant_method"] = quant_method
1121
-
1122
- # Quantization methods which are overrides (i.e. they have a
1123
- # `override_quantization_method` method) must be checked in order
1124
- # of preference (this is particularly important for GPTQ).
1125
- overrides = [
1126
- "bitblas",
1127
- "gptq_marlin_24",
1128
- "gptq_marlin",
1129
- "gptq_bitblas",
1130
- "awq_marlin",
1131
- "ipex",
1132
- "moe_wna16",
1133
- "modelopt",
1134
- "modelopt_fp4",
1135
- "petit_nvfp4",
1136
- ]
1137
- quantization_methods = [
1138
- q for q in supported_quantization if q not in overrides
1139
- ]
1140
- # Any custom overrides will be in quantization_methods so we place
1141
- # them at the start of the list so custom overrides have preference
1142
- # over the built-in ones.
1143
- quantization_methods = quantization_methods + overrides
1144
-
1145
- # Detect which checkpoint is it
1146
- for name in quantization_methods:
1147
- method = me_quant.get_quantization_config(name)
1148
- quantization_override = method.override_quantization_method(
1149
- quant_cfg, self.quantization)
1150
- if quantization_override is not None:
1151
- # Raise error if the override is not custom (custom would
1152
- # be in QUANTIZATION_METHODS but not QuantizationMethods)
1153
- # and hasn't been added to the overrides list.
1154
- if (name in get_args(me_quant.QuantizationMethods)
1155
- and name not in overrides):
1156
- raise ValueError(
1157
- f"Quantization method {name} is an override but "
1158
- "is has not been added to the `overrides` list "
1159
- "above. This is necessary to ensure that the "
1160
- "overrides are checked in order of preference.")
1161
- quant_method = quantization_override
1162
- self.quantization = quantization_override
1163
- break
1164
-
1165
- # Verify quantization configurations.
1166
- if self.quantization is None:
1167
- self.quantization = quant_method
1168
- elif self.quantization != quant_method:
1169
- raise ValueError(
1170
- "Quantization method specified in the model config "
1171
- f"({quant_method}) does not match the quantization "
1172
- f"method specified in the `quantization` argument "
1173
- f"({self.quantization}).")
1174
-
1175
- if self.quantization is not None:
1176
- if self.quantization not in supported_quantization:
1177
- raise ValueError(
1178
- f"Unknown quantization method: {self.quantization}. Must "
1179
- f"be one of {supported_quantization}.")
1180
- from vllm.platforms import current_platform
1181
- current_platform.verify_quantization(self.quantization)
1182
-
1183
- def _verify_cuda_graph(self) -> None:
1184
- # The `max_seq_len_to_capture` was incorrectly
1185
- # based on the encoder's input length (448)
1186
- # but not the decoder's larger input length (1500).
1187
- # This change ensures the CUDA Graph captures the correct,
1188
- # larger sequence length, allowing it to work as intended.
1189
- effective_max_seq_len = self.max_model_len
1190
- if self.is_encoder_decoder:
1191
- effective_max_seq_len = max(
1192
- effective_max_seq_len,
1193
- getattr(self.hf_config, "max_source_positions", 0))
1194
- self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
1195
- effective_max_seq_len)
1196
- # CUDAGraph capture not supported for encoder-decoder models on ROCm
1197
- unsupported_rocm = self.is_encoder_decoder
1198
-
1199
- if (unsupported_rocm and not self.enforce_eager
1200
- and current_platform.is_rocm()):
1201
- logger.warning(
1202
- "CUDA graph is not supported for %s on ROCm yet, fallback "
1203
- "to eager mode.", self.hf_config.model_type)
1204
- self.enforce_eager = True
1205
-
1206
- def _verify_bnb_config(self) -> None:
1207
- """
1208
- The current version of bitsandbytes (0.46.1) with 8-bit models does not
1209
- yet support CUDA graph.
1210
- # TODO Remove this when bitsandbytes supports.
1211
- """
1212
- is_bitsandbytes = self.quantization == "bitsandbytes"
1213
- has_quantization_config = (getattr(self.hf_config,
1214
- "quantization_config", None)
1215
- is not None)
1216
- is_8bit = (self.hf_config.quantization_config.get(
1217
- "load_in_8bit", False) if has_quantization_config else False)
1218
- if all([
1219
- is_bitsandbytes,
1220
- has_quantization_config,
1221
- is_8bit,
1222
- not self.enforce_eager,
1223
- ]):
1224
- logger.warning(
1225
- "CUDA graph is not supported on BitsAndBytes 8bit yet, "
1226
- "fallback to the eager mode.")
1227
-
1228
- self.enforce_eager = True
1229
-
1230
- def _verify_with_expert_parallelism(self) -> None:
1231
- num_expert_names = [
1232
- "moe_num_experts", # Dbrx
1233
- "num_experts", # Jamba
1234
- "n_routed_experts", # DeepSeek
1235
- "num_local_experts", # Mixtral
1236
- ]
1237
- num_experts = 0
1238
- for name in num_expert_names:
1239
- num_experts = getattr(self.hf_text_config, name, 0)
1240
- if num_experts > 0:
1241
- break
1242
- if num_experts < 1:
1243
- raise ValueError(
1244
- "Number of experts in the model must be greater than 0 "
1245
- "when expert parallelism is enabled.")
1246
-
1247
- def verify_dual_chunk_attention_config(
1248
- self,
1249
- load_config: "LoadConfig",
1250
- ) -> None:
1251
- if hasattr(self.hf_config, "dual_chunk_attention_config"):
1252
- # Try loading the sparse attention config
1253
- from vllm.model_executor.model_loader.weight_utils import (
1254
- get_sparse_attention_config)
1255
- sparse_attn_config = get_sparse_attention_config(self, load_config)
1256
- if sparse_attn_config:
1257
- self.hf_config.dual_chunk_attention_config[
1258
- "sparse_attention_config"] = sparse_attn_config
1259
- if "sparse_attention_enabled" not in \
1260
- self.hf_config.dual_chunk_attention_config:
1261
- self.hf_config.dual_chunk_attention_config[
1262
- "sparse_attention_enabled"] = True
1263
-
1264
- if envs.VLLM_ATTENTION_BACKEND != STR_DUAL_CHUNK_FLASH_ATTN_VAL:
1265
- raise ValueError("please set VLLM_ATTENTION_BACKEND to "
1266
- f"{STR_DUAL_CHUNK_FLASH_ATTN_VAL}")
1267
-
1268
- def verify_async_output_proc(self, parallel_config, speculative_config,
1269
- device_config) -> None:
1270
- if not self.use_async_output_proc:
1271
- # Nothing to check
1272
- return
1273
-
1274
- if parallel_config.pipeline_parallel_size > 1:
1275
- self.use_async_output_proc = False
1276
- return
1277
-
1278
- # Reminder: Please update docs/features/compatibility_matrix.md
1279
- # If the feature combo become valid
1280
- from vllm.platforms import current_platform
1281
- if not current_platform.is_async_output_supported(self.enforce_eager):
1282
- self.use_async_output_proc = False
1283
- return
1284
-
1285
- if envs.VLLM_USE_RAY_SPMD_WORKER:
1286
- self.use_async_output_proc = False
1287
- return
1288
-
1289
- # Async postprocessor is not necessary for pooling models
1290
- # since there is no token generation
1291
- if self.runner_type == "pooling":
1292
- self.use_async_output_proc = False
1293
-
1294
- # Reminder: Please update docs/features/compatibility_matrix.md
1295
- # If the feature combo become valid
1296
- if speculative_config:
1297
- self.use_async_output_proc = False
1298
-
1299
- def verify_with_parallel_config(
1300
- self,
1301
- parallel_config: "ParallelConfig",
1302
- ) -> None:
1303
-
1304
- if parallel_config.distributed_executor_backend == "external_launcher":
1305
- assert self.seed is not None, (
1306
- "Seed must be set when using external launcher backend to "
1307
- "make sure sampling results are the same across workers.")
1308
-
1309
- total_num_attention_heads = getattr(self.hf_text_config,
1310
- "num_attention_heads", 0)
1311
- tensor_parallel_size = parallel_config.tensor_parallel_size
1312
- if total_num_attention_heads % tensor_parallel_size != 0:
1313
- raise ValueError(
1314
- f"Total number of attention heads ({total_num_attention_heads})"
1315
- " must be divisible by tensor parallel size "
1316
- f"({tensor_parallel_size}).")
1317
-
1318
- if parallel_config.enable_expert_parallel:
1319
- self._verify_with_expert_parallelism()
1320
-
1321
- pipeline_parallel_size = parallel_config.pipeline_parallel_size
1322
- if pipeline_parallel_size > 1:
1323
- if not self.registry.is_pp_supported_model(self.architectures,
1324
- self):
1325
- raise NotImplementedError(
1326
- "Pipeline parallelism is not supported for this model. "
1327
- "Supported models implement the `SupportsPP` interface.")
1328
-
1329
- if self.use_async_output_proc:
1330
- self.use_async_output_proc = False
1331
-
1332
- def get_sliding_window(self) -> Optional[int]:
1333
- """Get the sliding window size from the HF text config if present."""
1334
- return getattr(self.hf_text_config, "sliding_window", None)
1335
-
1336
- def get_vocab_size(self) -> int:
1337
- return getattr(self.hf_text_config, "vocab_size", 0)
1338
-
1339
- def get_hidden_size(self) -> int:
1340
- return getattr(self.hf_text_config, "hidden_size", 0)
1341
-
1342
- @property
1343
- def is_deepseek_mla(self) -> bool:
1344
- if not hasattr(self.hf_text_config, "model_type"):
1345
- return False
1346
- elif self.hf_text_config.model_type in \
1347
- ('deepseek_v2', 'deepseek_v3', 'deepseek_mtp', 'kimi_k2'):
1348
- return self.hf_text_config.kv_lora_rank is not None
1349
- elif self.hf_text_config.model_type == 'eagle':
1350
- # if the model is an EAGLE module, check for the
1351
- # underlying architecture
1352
- return self.hf_text_config.model.model_type in \
1353
- ('deepseek_v2', 'deepseek_v3') \
1354
- and self.hf_text_config.kv_lora_rank is not None
1355
- return False
1356
-
1357
- def get_head_size(self) -> int:
1358
- # TODO remove hard code
1359
- if self.is_deepseek_mla:
1360
- qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
1361
- 0)
1362
- if self.use_mla:
1363
- return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
1364
- else:
1365
- qk_nope_head_dim = getattr(self.hf_text_config,
1366
- "qk_nope_head_dim", 0)
1367
- if qk_rope_head_dim and qk_nope_head_dim:
1368
- return qk_rope_head_dim + qk_nope_head_dim
1369
-
1370
- if hasattr(self.hf_text_config,
1371
- "model_type") and (self.hf_text_config.model_type
1372
- == "zamba2"):
1373
- return self.hf_text_config.attention_head_dim
1374
-
1375
- if self.is_attention_free:
1376
- return 0
1377
-
1378
- # NOTE: Some configs may set head_dim=None in the config
1379
- if getattr(self.hf_text_config, "head_dim", None) is not None:
1380
- return self.hf_text_config.head_dim
1381
-
1382
- # NOTE: Some models (such as PLaMo2.1) use `hidden_size_per_head`
1383
- if getattr(self.hf_text_config, "hidden_size_per_head",
1384
- None) is not None:
1385
- return self.hf_text_config.hidden_size_per_head
1386
-
1387
- # FIXME(woosuk): This may not be true for all models.
1388
- return (self.hf_text_config.hidden_size //
1389
- self.hf_text_config.num_attention_heads)
1390
-
1391
- def get_total_num_kv_heads(self) -> int:
1392
- """Returns the total number of KV heads."""
1393
- # For GPTBigCode & Falcon:
1394
- # NOTE: for falcon, when new_decoder_architecture is True, the
1395
- # multi_query flag is ignored and we use n_head_kv for the number of
1396
- # KV heads.
1397
- falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
1398
- new_decoder_arch_falcon = (
1399
- self.hf_config.model_type in falcon_model_types
1400
- and getattr(self.hf_config, "new_decoder_architecture", False))
1401
- if not new_decoder_arch_falcon and getattr(self.hf_text_config,
1402
- "multi_query", False):
1403
- # Multi-query attention, only one KV head.
1404
- # Currently, tensor parallelism is not supported in this case.
1405
- return 1
1406
-
1407
- # For DBRX and MPT
1408
- if self.hf_config.model_type == "mpt":
1409
- if "kv_n_heads" in self.hf_config.attn_config:
1410
- return self.hf_config.attn_config["kv_n_heads"]
1411
- return self.hf_config.num_attention_heads
1412
- if self.hf_config.model_type == "dbrx":
1413
- return getattr(self.hf_config.attn_config, "kv_n_heads",
1414
- self.hf_config.num_attention_heads)
1415
-
1416
- if self.hf_config.model_type == "nemotron-nas":
1417
- for block in self.hf_config.block_configs:
1418
- if not block.attention.no_op:
1419
- return self.hf_config.num_attention_heads \
1420
- // block.attention.n_heads_in_group
1421
-
1422
- raise RuntimeError("Couldn't determine number of kv heads")
1423
-
1424
- if self.is_attention_free:
1425
- return 0
1426
-
1427
- attributes = [
1428
- # For Falcon:
1429
- "n_head_kv",
1430
- "num_kv_heads",
1431
- # For LLaMA-2:
1432
- "num_key_value_heads",
1433
- # For ChatGLM:
1434
- "multi_query_group_num",
1435
- ]
1436
- for attr in attributes:
1437
- num_kv_heads = getattr(self.hf_text_config, attr, None)
1438
- if num_kv_heads is not None:
1439
- return num_kv_heads
1440
-
1441
- # For non-grouped-query attention models, the number of KV heads is
1442
- # equal to the number of attention heads.
1443
- return self.hf_text_config.num_attention_heads
1444
-
1445
- def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
1446
- """Returns the number of KV heads per GPU."""
1447
- if self.use_mla:
1448
- # When using MLA during decode it becomes MQA
1449
- return 1
1450
-
1451
- total_num_kv_heads = self.get_total_num_kv_heads()
1452
- # If tensor parallelism is used, we divide the number of KV heads by
1453
- # the tensor parallel size. We will replicate the KV heads in the
1454
- # case where the number of KV heads is smaller than the tensor
1455
- # parallel size so each GPU has at least one KV head.
1456
- return max(1,
1457
- total_num_kv_heads // parallel_config.tensor_parallel_size)
1458
-
1459
- def get_num_attention_heads(self,
1460
- parallel_config: "ParallelConfig") -> int:
1461
- num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
1462
- return num_heads // parallel_config.tensor_parallel_size
1463
-
1464
- def get_layers_start_end_indices(
1465
- self, parallel_config: "ParallelConfig") -> tuple[int, int]:
1466
- from vllm.distributed.utils import get_pp_indices
1467
- if (self.hf_text_config.model_type == "deepseek_mtp"
1468
- or self.hf_config.model_type == "mimo_mtp"
1469
- or self.hf_config.model_type == "glm4_moe_mtp"
1470
- or self.hf_config.model_type == "ernie_mtp"
1471
- or self.hf_config.model_type == "qwen3_next_mtp"):
1472
- total_num_hidden_layers = getattr(self.hf_text_config,
1473
- "num_nextn_predict_layers", 0)
1474
- else:
1475
- total_num_hidden_layers = getattr(self.hf_text_config,
1476
- "num_hidden_layers", 0)
1477
- # the layout order is: DP x PP x TP
1478
- pp_rank = (parallel_config.rank // parallel_config.tensor_parallel_size
1479
- ) % parallel_config.pipeline_parallel_size
1480
- pp_size = parallel_config.pipeline_parallel_size
1481
- start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
1482
- return start, end
1483
-
1484
- def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
1485
- start, end = self.get_layers_start_end_indices(parallel_config)
1486
- return end - start
1487
-
1488
- def get_num_layers_by_block_type(
1489
- self,
1490
- parallel_config: "ParallelConfig",
1491
- block_type: LayerBlockType = LayerBlockType.attention,
1492
- ) -> int:
1493
- # This function relies on 'layers_block_type' in hf_config,
1494
- # for w/o this attribute, we will need to have workarounds like so
1495
- attn_block_type = block_type == LayerBlockType.attention
1496
- is_transformer = not self.is_hybrid and \
1497
- not self.has_noops and \
1498
- not self.is_attention_free
1499
- start, end = self.get_layers_start_end_indices(parallel_config)
1500
-
1501
- if is_transformer:
1502
- # Handle the basic case first
1503
- return end - start if attn_block_type else 0
1504
- elif self.is_attention_free:
1505
- # Attention free
1506
- # Note that this code assumes there
1507
- # is only one type of attention-free block type.
1508
- return 0 if attn_block_type else end - start
1509
- elif self.has_noops:
1510
- block_configs = self.hf_config.block_configs
1511
- return sum(not bc.attention.no_op
1512
- for bc in block_configs[start:end])
1513
- else:
1514
- # Hybrid model Jamba
1515
- layers_block_type_value = getattr(self.hf_text_config,
1516
- "layers_block_type", None)
1517
- if layers_block_type_value is not None:
1518
- if hasattr(self.hf_text_config,
1519
- "model_type") and (self.hf_text_config.model_type
1520
- == "zamba2"):
1521
- if attn_block_type:
1522
- return sum(t == "hybrid"
1523
- for t in layers_block_type_value[start:end])
1524
- else:
1525
- return self.get_num_layers(parallel_config)
1526
- return sum(t == block_type.value
1527
- for t in layers_block_type_value[start:end])
1528
-
1529
- # Hybrid model Minimax
1530
- attn_type_list = getattr(self.hf_config, "attn_type_list", None)
1531
- if attn_type_list:
1532
- return sum(t == 1 for t in attn_type_list[start:end])
1533
-
1534
- # Hybrid model Qwen3Next
1535
- layer_types_value = getattr(self.hf_config, "layer_types", None)
1536
- if layer_types_value is not None:
1537
- if getattr(block_type, "value", block_type) == "attention":
1538
- return sum(t == "full_attention"
1539
- for t in layer_types_value[start:end])
1540
- elif getattr(block_type, "value",
1541
- block_type) == "linear_attention":
1542
- return sum(t == "linear_attention"
1543
- for t in layer_types_value[start:end])
1544
- else:
1545
- return sum(t == getattr(block_type, "value", block_type)
1546
- for t in layer_types_value[start:end])
1547
-
1548
- if (layers_block_type_value is None and attn_type_list is None
1549
- and layer_types_value is None):
1550
- raise ValueError(
1551
- "The model is an hybrid without a"
1552
- "layers_block_type or an attn_type_list, or a layer_types "
1553
- "in the hf_config, cannot determine the num of "
1554
- f"{block_type.value} layers")
1555
-
1556
- def get_mamba_chunk_size(self) -> Optional[int]:
1557
- """
1558
- Returns the mamba chunk size if it exists
1559
- """
1560
- # used by e.g. Bamba, FalconH1, Granite, PLaMo2
1561
- chunk_size = getattr(self.hf_text_config, "mamba_chunk_size", None)
1562
- if chunk_size is None:
1563
- # used by e.g. Mamba2, NemotronH, Zamba
1564
- chunk_size = getattr(self.hf_text_config, "chunk_size", None)
1565
- return chunk_size
1566
-
1567
- def get_multimodal_config(self) -> "MultiModalConfig":
1568
- """
1569
- Get the multimodal configuration of the model.
1570
-
1571
- Raises:
1572
- ValueError: If the model is not multimodal.
1573
- """
1574
- if self.multimodal_config is None:
1575
- raise ValueError("The model is not multimodal.")
1576
-
1577
- return self.multimodal_config
1578
-
1579
- def try_get_generation_config(self) -> dict[str, Any]:
1580
- """
1581
- This method attempts to retrieve the non-default values of the
1582
- generation config for this model.
1583
-
1584
- The generation config can contain information about special tokens, as
1585
- well as sampling parameters. Which is why this method exists separately
1586
- to `get_diff_sampling_param`.
1587
-
1588
- Returns:
1589
- A dictionary containing the non-default generation config.
1590
- """
1591
- if self.generation_config in {"auto", "vllm"}:
1592
- config = try_get_generation_config(
1593
- self.hf_config_path or self.model,
1594
- trust_remote_code=self.trust_remote_code,
1595
- revision=self.revision,
1596
- )
1597
- else:
1598
- config = try_get_generation_config(
1599
- self.generation_config,
1600
- trust_remote_code=self.trust_remote_code,
1601
- )
1602
-
1603
- if config is None:
1604
- return {}
1605
-
1606
- return config.to_diff_dict()
1607
-
1608
- def get_diff_sampling_param(self) -> dict[str, Any]:
1609
- """
1610
- This method returns a dictionary containing the non-default sampling
1611
- parameters with `override_generation_config` applied.
1612
-
1613
- The default sampling parameters are:
1614
-
1615
- - vLLM's neutral defaults if `self.generation_config="vllm"`
1616
- - the model's defaults if `self.generation_config="auto"`
1617
- - as defined in `generation_config.json` if
1618
- `self.generation_config="path/to/generation_config/dir"`
1619
-
1620
- Returns:
1621
- A dictionary containing the non-default sampling parameters.
1622
- """
1623
- if self.generation_config == "vllm":
1624
- config = {}
1625
- else:
1626
- config = self.try_get_generation_config()
1627
-
1628
- # Overriding with given generation config
1629
- config.update(self.override_generation_config)
1630
-
1631
- available_params = [
1632
- "repetition_penalty",
1633
- "temperature",
1634
- "top_k",
1635
- "top_p",
1636
- "min_p",
1637
- "max_new_tokens",
1638
- ]
1639
- if any(p in config for p in available_params):
1640
- diff_sampling_param = {
1641
- p: config.get(p)
1642
- for p in available_params if config.get(p) is not None
1643
- }
1644
- # Huggingface definition of max_new_tokens is equivalent
1645
- # to vLLM's max_tokens
1646
- if "max_new_tokens" in diff_sampling_param:
1647
- diff_sampling_param["max_tokens"] = diff_sampling_param.pop(
1648
- "max_new_tokens")
1649
- else:
1650
- diff_sampling_param = {}
1651
-
1652
- if diff_sampling_param:
1653
- logger.warning_once(
1654
- "Default sampling parameters have been overridden by the "
1655
- "model's Hugging Face generation config recommended from the "
1656
- "model creator. If this is not intended, please relaunch "
1657
- "vLLM instance with `--generation-config vllm`.")
1658
- return diff_sampling_param
1659
-
1660
- @property
1661
- def is_encoder_decoder(self) -> bool:
1662
- """Extract the HF encoder/decoder model flag."""
1663
- return is_encoder_decoder(self.hf_config)
1664
-
1665
- @property
1666
- def uses_mrope(self) -> bool:
1667
- return uses_mrope(self.hf_config)
1668
-
1669
- @property
1670
- def is_multimodal_model(self) -> bool:
1671
- return self.multimodal_config is not None
1672
-
1673
- @property
1674
- def is_multimodal_raw_input_only_model(self) -> bool:
1675
- return self._model_info.supports_multimodal_raw_input_only
1676
-
1677
- @property
1678
- def is_cross_encoder(self) -> bool:
1679
- return (self._model_info.supports_cross_encoding
1680
- or self.convert_type == "classify")
1681
-
1682
- @property
1683
- def is_pp_supported(self) -> bool:
1684
- return self._model_info.supports_pp
1685
-
1686
- @property
1687
- def is_attention_free(self) -> bool:
1688
- return self._model_info.is_attention_free
1689
-
1690
- @property
1691
- def is_hybrid(self) -> bool:
1692
- return self._model_info.is_hybrid
1693
-
1694
- @property
1695
- def has_noops(self) -> bool:
1696
- return self._model_info.has_noops
1697
-
1698
- @property
1699
- def has_inner_state(self):
1700
- return self._model_info.has_inner_state
1701
-
1702
- @property
1703
- def is_v1_compatible(self) -> bool:
1704
- return not self._model_info.supports_v0_only
1705
-
1706
- @property
1707
- def use_mla(self) -> bool:
1708
- return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE
1709
-
1710
- @property
1711
- def is_matryoshka(self) -> bool:
1712
- return (bool(getattr(self.hf_config, "matryoshka_dimensions", None))
1713
- or getattr(self.hf_config, "is_matryoshka", False))
1714
-
1715
- @property
1716
- def matryoshka_dimensions(self):
1717
- return getattr(self.hf_config, "matryoshka_dimensions", None)
1718
-
1719
- @property
1720
- def use_pad_token(self) -> bool:
1721
- # cross_encoder models defaults to using pad_token.
1722
- # `llm as reranker` models defaults to not using pad_token.
1723
- return getattr(self.hf_config, "use_pad_token", True)
1724
-
1725
- @property
1726
- def head_dtype(self) -> torch.dtype:
1727
- """
1728
- "head" refers to the last Linear layer(s) of an LLM,
1729
- such as the lm_head in a generation model,
1730
- or the score or classifier in a classification model.
1731
-
1732
- `head_dtype` currently only supports pooling models.\n
1733
- - The pooling model defaults to using fp32 head,
1734
- you can use --hf-overrides '{"head_dtype": "model"}' to disable it.
1735
- """
1736
-
1737
- head_dtype = _get_head_dtype(config=self.hf_config,
1738
- dtype=self.dtype,
1739
- runner_type=self.runner_type)
1740
-
1741
- if self.runner_type != "pooling" and head_dtype != self.dtype:
1742
- logger.warning_once(
1743
- "`head_dtype` currently only supports pooling models."
1744
- "fallback to model dtype [%s].", self.dtype)
1745
- return self.dtype
1746
-
1747
- if head_dtype not in current_platform.supported_dtypes:
1748
- logger.warning_once(
1749
- "The current platform does not support [%s] head dtype, "
1750
- "fallback to model dtype [%s].", head_dtype, self.dtype)
1751
- return self.dtype
1752
-
1753
- logger.debug_once("head dtype: %s", head_dtype)
1754
- return head_dtype
1755
-
1756
- def get_and_verify_max_len(self, max_model_len: int):
1757
- # Consider max_model_len in tokenizer_config only when
1758
- # pooling models use absolute position_embedding.
1759
- tokenizer_config = None
1760
- if (self.runner_type == "pooling" and getattr(
1761
- self.hf_config, "position_embedding_type", "") == "absolute"):
1762
- tokenizer_config = try_get_tokenizer_config(
1763
- self.tokenizer,
1764
- trust_remote_code=self.trust_remote_code,
1765
- revision=self.tokenizer_revision)
1766
- max_model_len = _get_and_verify_max_len(
1767
- hf_config=self.hf_text_config,
1768
- tokenizer_config=tokenizer_config,
1769
- max_model_len=max_model_len,
1770
- disable_sliding_window=self.disable_sliding_window,
1771
- sliding_window=self.get_sliding_window(),
1772
- spec_target_max_model_len=self.spec_target_max_model_len,
1773
- encoder_config=self.encoder_config)
1774
- logger.info("Using max model len %s", max_model_len)
1775
- return max_model_len
1776
-
1777
-
84
Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]
85
86
@@ -1838,365 +144,6 @@ class DeviceConfig:
144
self.device = torch.device(self.device_type)
145
146
1841
-_STR_DTYPE_TO_TORCH_DTYPE = {
1842
- "half": torch.float16,
1843
- "float16": torch.float16,
1844
- "float": torch.float32,
1845
- "float32": torch.float32,
1846
- "bfloat16": torch.bfloat16,
1847
-}
1848
-
1849
-# model_type -> reason
1850
-_FLOAT16_NOT_SUPPORTED_MODELS = {
1851
- "gemma2": "Numerical instability. Please use bfloat16 or float32 instead.",
1852
- "gemma3": "Numerical instability. Please use bfloat16 or float32 instead.",
1853
- "gemma3_text":
1854
- "Numerical instability. Please use bfloat16 or float32 instead.",
1855
- "plamo2": "Numerical instability. Please use bfloat16 or float32 instead.",
1856
- "glm4": "Numerical instability. Please use bfloat16 or float32 instead.",
1857
-}
1858
-
1859
-
1860
-def _is_valid_dtype(model_type: str, dtype: torch.dtype):
1861
- if model_type in _FLOAT16_NOT_SUPPORTED_MODELS and dtype == torch.float16: # noqa: E501, SIM103
1862
- return False
1863
-
1864
- return True
1865
-
1866
-
1867
-def _check_valid_dtype(model_type: str, dtype: torch.dtype):
1868
- if model_type in _FLOAT16_NOT_SUPPORTED_MODELS and dtype == torch.float16:
1869
- reason = _FLOAT16_NOT_SUPPORTED_MODELS[model_type]
1870
- raise ValueError(f"The model type {model_type!r} "
1871
- f"does not support float16. Reason: {reason}")
1872
-
1873
- return True
1874
-
1875
-
1876
-def _find_dtype(
1877
- model_id: str,
1878
- config: PretrainedConfig,
1879
- *,
1880
- revision: Optional[str],
1881
-):
1882
- # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
1883
- # because config.torch_dtype can be None.
1884
- config_dtype = getattr(config, "torch_dtype", None)
1885
-
1886
- # Fallbacks for multi-modal models if the root config
1887
- # does not define torch_dtype
1888
- if config_dtype is None:
1889
- config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
1890
- if config_dtype is None and hasattr(config, "vision_config"):
1891
- config_dtype = getattr(config.vision_config, "torch_dtype", None)
1892
- if config_dtype is None and hasattr(config, "encoder_config"):
1893
- config_dtype = getattr(config.encoder_config, "torch_dtype", None)
1894
-
1895
- # Try to read the dtype of the weights if they are in safetensors format
1896
- if config_dtype is None:
1897
- repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
1898
-
1899
- if repo_mt and (files_mt := repo_mt.files_metadata):
1900
- param_dtypes: set[torch.dtype] = {
1901
- _SAFETENSORS_TO_TORCH_DTYPE[dtype_str]
1902
- for file_mt in files_mt.values()
1903
- for dtype_str in file_mt.parameter_count
1904
- if dtype_str in _SAFETENSORS_TO_TORCH_DTYPE
1905
- }
1906
-
1907
- if param_dtypes:
1908
- return common_broadcastable_dtype(param_dtypes)
1909
-
1910
- if config_dtype is None:
1911
- config_dtype = torch.float32
1912
-
1913
- return config_dtype
1914
-
1915
-
1916
-def _resolve_auto_dtype(
1917
- model_type: str,
1918
- config_dtype: torch.dtype,
1919
- *,
1920
- is_pooling_model: bool,
1921
-):
1922
- from vllm.platforms import current_platform
1923
-
1924
- supported_dtypes = [
1925
- dtype for dtype in current_platform.supported_dtypes
1926
- if _is_valid_dtype(model_type, dtype)
1927
- ]
1928
-
1929
- if is_pooling_model and torch.float16 in supported_dtypes:
1930
- preferred_dtype = torch.float16
1931
- else:
1932
- preferred_dtype = supported_dtypes[0]
1933
-
1934
- # Downcast for float32 models
1935
- if config_dtype == torch.float32:
1936
- config_dtype = preferred_dtype
1937
-
1938
- if config_dtype in supported_dtypes:
1939
- return config_dtype
1940
-
1941
- # Ensure device compatibility
1942
- device_name = current_platform.get_device_name()
1943
- device_capability = current_platform.get_device_capability()
1944
-
1945
- if device_capability is None:
1946
- device_str = f"{device_name!r}"
1947
- else:
1948
- version_str = device_capability.as_version_str()
1949
- device_str = f"{device_name!r} (with compute capability {version_str})"
1950
-
1951
- logger.warning(
1952
- "Your device %s doesn't support %s. "
1953
- "Falling back to %s for compatibility.",
1954
- device_str,
1955
- config_dtype,
1956
- preferred_dtype,
1957
- )
1958
-
1959
- return preferred_dtype
1960
-
1961
-
1962
-def _get_and_verify_dtype(
1963
- model_id: str,
1964
- config: PretrainedConfig,
1965
- dtype: Union[str, torch.dtype],
1966
- *,
1967
- is_pooling_model: bool,
1968
- revision: Optional[str] = None,
1969
-) -> torch.dtype:
1970
- config_dtype = _find_dtype(model_id, config, revision=revision)
1971
- model_type = config.model_type
1972
-
1973
- if isinstance(dtype, str):
1974
- dtype = dtype.lower()
1975
- if dtype == "auto":
1976
- # Set default dtype from model config
1977
- torch_dtype = _resolve_auto_dtype(
1978
- model_type,
1979
- config_dtype,
1980
- is_pooling_model=is_pooling_model,
1981
- )
1982
- else:
1983
- if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
1984
- raise ValueError(f"Unknown dtype: {dtype!r}")
1985
- torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
1986
- elif isinstance(dtype, torch.dtype):
1987
- torch_dtype = dtype
1988
- else:
1989
- raise ValueError(f"Unknown dtype: {dtype}")
1990
-
1991
- _check_valid_dtype(model_type, torch_dtype)
1992
-
1993
- if torch_dtype != config_dtype:
1994
- if torch_dtype == torch.float32:
1995
- # Upcasting to float32 is allowed.
1996
- logger.info("Upcasting %s to %s.", config_dtype, torch_dtype)
1997
- elif config_dtype == torch.float32:
1998
- # Downcasting from float32 to float16 or bfloat16 is allowed.
1999
- logger.info("Downcasting %s to %s.", config_dtype, torch_dtype)
2000
- else:
2001
- # Casting between float16 and bfloat16 is allowed with a warning.
2002
- logger.warning("Casting %s to %s.", config_dtype, torch_dtype)
2003
-
2004
- return torch_dtype
2005
-
2006
-
2007
-def _get_head_dtype(config: PretrainedConfig, dtype: torch.dtype,
2008
- runner_type: str) -> torch.dtype:
2009
- head_dtype: Optional[Union[str,
2010
- torch.dtype]] = getattr(config, "head_dtype",
2011
- None)
2012
-
2013
- if head_dtype == "model":
2014
- return dtype
2015
- elif isinstance(head_dtype, str):
2016
- head_dtype = head_dtype.lower()
2017
- if head_dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
2018
- raise ValueError(f"Unknown dtype: {head_dtype!r}")
2019
- return _STR_DTYPE_TO_TORCH_DTYPE[head_dtype]
2020
- elif isinstance(head_dtype, torch.dtype):
2021
- return head_dtype
2022
- elif head_dtype is None:
2023
- if torch.float32 not in current_platform.supported_dtypes:
2024
- return dtype
2025
- if runner_type == "pooling":
2026
- return torch.float32
2027
- return dtype
2028
- else:
2029
- raise ValueError(f"Unknown dtype: {head_dtype}")
2030
-
2031
-
2032
-def _get_and_verify_max_len(
2033
- hf_config: PretrainedConfig,
2034
- tokenizer_config: Optional[dict],
2035
- max_model_len: Optional[int],
2036
- disable_sliding_window: bool,
2037
- sliding_window: Optional[int],
2038
- spec_target_max_model_len: Optional[int] = None,
2039
- encoder_config: Optional[Any] = None,
2040
-) -> int:
2041
- """Get and verify the model's maximum length."""
2042
- derived_max_model_len = float("inf")
2043
- possible_keys = [
2044
- # OPT
2045
- "max_position_embeddings",
2046
- # GPT-2
2047
- "n_positions",
2048
- # MPT
2049
- "max_seq_len",
2050
- # ChatGLM2
2051
- "seq_length",
2052
- # Command-R
2053
- "model_max_length",
2054
- # Whisper
2055
- "max_target_positions",
2056
- # Others
2057
- "max_sequence_length",
2058
- "max_seq_length",
2059
- "seq_len",
2060
- ]
2061
- # Choose the smallest "max_length" from the possible keys
2062
- max_len_key = None
2063
- for key in possible_keys:
2064
- max_len = getattr(hf_config, key, None)
2065
- if max_len is not None:
2066
- max_len_key = key if max_len < derived_max_model_len \
2067
- else max_len_key
2068
- derived_max_model_len = min(derived_max_model_len, max_len)
2069
- # For Command-R / Cohere, Cohere2 / Aya Vision models
2070
- if tmp_max_len := getattr(hf_config, "model_max_length", None):
2071
- max_len_key = "model_max_length"
2072
- derived_max_model_len = tmp_max_len
2073
-
2074
- # If sliding window is manually disabled, max_length should be less
2075
- # than the sliding window length in the model config.
2076
- if (disable_sliding_window and sliding_window is not None
2077
- and sliding_window < derived_max_model_len):
2078
- max_len_key = "sliding_window"
2079
- derived_max_model_len = sliding_window
2080
-
2081
- # Consider model_max_length in tokenizer_config
2082
- if tokenizer_config:
2083
- tokenizer_model_max_length = tokenizer_config.get(
2084
- "model_max_length", derived_max_model_len)
2085
- derived_max_model_len = min(derived_max_model_len,
2086
- tokenizer_model_max_length)
2087
-
2088
- # If none of the keys were found in the config, use a default and
2089
- # log a warning.
2090
- if derived_max_model_len == float("inf"):
2091
- if max_model_len is not None:
2092
- # If max_model_len is specified, we use it.
2093
- return max_model_len
2094
-
2095
- if spec_target_max_model_len is not None:
2096
- # If this is a speculative draft model, we use the max model len
2097
- # from the target model.
2098
- return spec_target_max_model_len
2099
-
2100
- default_max_len = 2048
2101
- logger.warning(
2102
- "The model's config.json does not contain any of the following "
2103
- "keys to determine the original maximum length of the model: "
2104
- "%s. Assuming the model's maximum length is %d.", possible_keys,
2105
- default_max_len)
2106
- derived_max_model_len = default_max_len
2107
-
2108
- rope_scaling = getattr(hf_config, "rope_scaling", None)
2109
- # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE
2110
- # scaling, so we skip applying the scaling factor again.
2111
- if rope_scaling is not None and "gemma3" not in hf_config.model_type:
2112
- # No need to consider "type" key because of patch_rope_scaling when
2113
- # loading HF config
2114
- rope_type = rope_scaling["rope_type"]
2115
-
2116
- if rope_type not in ("su", "longrope", "llama3"):
2117
- if disable_sliding_window:
2118
- # TODO(robertgshaw): Find a model that supports rope_scaling
2119
- # with sliding window to see if this case should be allowed.
2120
- raise NotImplementedError(
2121
- "Disabling sliding window is not supported for models "
2122
- "with rope_scaling. Please raise an issue so we can "
2123
- "investigate.")
2124
-
2125
- # NOTE: rope_type == "default" does not define factor
2126
- # https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
2127
- scaling_factor = rope_scaling.get("factor", 1.0)
2128
-
2129
- if rope_type == "yarn":
2130
- derived_max_model_len = rope_scaling[
2131
- "original_max_position_embeddings"]
2132
- derived_max_model_len *= scaling_factor
2133
-
2134
- if encoder_config and "max_seq_length" in encoder_config:
2135
- derived_max_model_len = encoder_config["max_seq_length"]
2136
-
2137
- # If the user specified a max length, make sure it is smaller than the
2138
- # derived length from the HF model config.
2139
- if max_model_len is None:
2140
- max_model_len = int(derived_max_model_len)
2141
- if current_platform.is_tpu():
2142
- logger.warning(
2143
- "--max-model-len is not specified, "
2144
- "it's currently using model's default length %s, "
2145
- "which might be too large."
2146
- "Please input with --max-model-len based on your "
2147
- "request input length and output length, to avoid "
2148
- "unnecessary degradation.", max_model_len)
2149
- elif max_model_len > derived_max_model_len:
2150
- # Some models might have a separate key for specifying model_max_length
2151
- # that will be bigger than derived_max_model_len. We compare user input
2152
- # with model_max_length and allow this override when it's smaller.
2153
- model_max_length = getattr(hf_config, "model_max_length", None)
2154
- if model_max_length is not None and max_model_len <= model_max_length:
2155
- if disable_sliding_window:
2156
- # TODO(robertgshaw): Find a model that has model_max_length
2157
- # with sliding window to see if this case should be allowed.
2158
- raise NotImplementedError(
2159
- "Disabling sliding window is not supported for models "
2160
- "model_max_length in the config. Please raise an issue "
2161
- "so we can investigate.")
2162
- else:
2163
- msg = (
2164
- f"User-specified max_model_len ({max_model_len}) is greater "
2165
- f"than the derived max_model_len ({max_len_key}="
2166
- f"{derived_max_model_len} or model_max_length="
2167
- f"{model_max_length} in model's config.json).")
2168
- warning = (
2169
- "VLLM_ALLOW_LONG_MAX_MODEL_LEN must be used with extreme "
2170
- "caution. If the model uses relative position encoding (RoPE), "
2171
- "positions exceeding derived_max_model_len lead to nan. If the "
2172
- "model uses absolute position encoding, positions exceeding "
2173
- "derived_max_model_len will cause a CUDA array out-of-bounds "
2174
- "error.")
2175
- if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN:
2176
- logger.warning_once("%s %s", msg, warning)
2177
- else:
2178
- raise ValueError(
2179
- f"{msg} To allow overriding this maximum, set "
2180
- f"the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. {warning}")
2181
- return int(max_model_len)
2182
-
2183
-
2184
-def get_served_model_name(model: str,
2185
- served_model_name: Optional[Union[str, list[str]]]):
2186
- """
2187
- If the input is a non-empty list, the first model_name in
2188
- `served_model_name` is taken.
2189
- If the input is a non-empty string, it is used directly.
2190
- For cases where the input is either an empty string or an
2191
- empty list, the fallback is to use `self.model`.
2192
- """
2193
- if not served_model_name:
2194
- return model
2195
- if isinstance(served_model_name, list):
2196
- return served_model_name[0]
2197
- return served_model_name
2198
-
2199
-
147
DetailedTraceModules = Literal["model", "worker", "all"]
148
149
@@ -3012,33 +959,6 @@ def get_current_model_prefix() -> str:
959
return _current_prefix
960
961
3015
-def contains_object_print(text):
3016
- """
3017
- Check if the text looks like a printed Python object, e.g.
3018
- contains any substring matching the pattern: "at 0xFFFFFFF>"
3019
- We match against 0x followed by 2-16 hex chars (there's
3020
- a max of 16 on a 64-bit system).
3021
-
3022
- Args:
3023
- text (str): The text to check
3024
-
3025
- Returns:
3026
- result (bool): `True` if a match is found, `False` otherwise.
3027
- """
3028
- pattern = r'at 0x[a-fA-F0-9]{2,16}>'
3029
- match = re.search(pattern, text)
3030
- return match is not None
3031
-
3032
-
3033
-def assert_hashable(text):
3034
- if not contains_object_print(text):
3035
- return True
3036
- raise AssertionError(
3037
- f"vLLM tried to hash some configs that may have Python objects ids "
3038
- f"in them. This is a bug, please file an issue. "
3039
- f"Text being hashed: {text}")
3040
-
3041
-
962
T = TypeVar("T")
963
964
vllm/config/model.py
new
+2006
@@ -0,0 +1,2006 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import hashlib
5
+import json
6
+import warnings
7
+from dataclasses import InitVar, field
8
+from importlib.util import find_spec
9
+from typing import (TYPE_CHECKING, Any, Callable, Literal, Optional, Union,
10
+ cast, get_args)
11
+
12
+import torch
13
+from pydantic import (ConfigDict, SkipValidation, field_validator,
14
+ model_validator)
15
+from pydantic.dataclasses import dataclass
16
+from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
17
+from typing_extensions import assert_never
18
+
19
+import vllm.envs as envs
20
+from vllm.config.multimodal import (MMCacheType, MMEncoderTPMode,
21
+ MultiModalConfig)
22
+from vllm.config.pooler import PoolerConfig
23
+from vllm.config.utils import assert_hashable, config
24
+from vllm.logger import init_logger
25
+from vllm.platforms import current_platform
26
+from vllm.transformers_utils.config import (
27
+ ConfigFormat, get_config, get_hf_image_processor_config,
28
+ get_hf_text_config, get_pooling_config,
29
+ get_sentence_transformer_tokenizer_config, is_encoder_decoder,
30
+ is_interleaved, maybe_override_with_speculators_target_model,
31
+ try_get_generation_config, try_get_safetensors_metadata,
32
+ try_get_tokenizer_config, uses_mrope)
33
+from vllm.transformers_utils.runai_utils import (ObjectStorageModel,
34
+ is_runai_obj_uri)
35
+from vllm.transformers_utils.utils import maybe_model_redirect
36
+from vllm.utils import (STR_DUAL_CHUNK_FLASH_ATTN_VAL, LayerBlockType,
37
+ LazyLoader, common_broadcastable_dtype)
38
+
39
+if TYPE_CHECKING:
40
+ from transformers import PretrainedConfig
41
+
42
+ import vllm.model_executor.layers.quantization as me_quant
43
+ import vllm.model_executor.models as me_models
44
+ from vllm.config.load import LoadConfig
45
+ from vllm.config.parallel import ParallelConfig
46
+ from vllm.config.scheduler import RunnerType
47
+ from vllm.model_executor.layers.quantization import QuantizationMethods
48
+ from vllm.v1.sample.logits_processor import LogitsProcessor
49
+else:
50
+ PretrainedConfig = Any
51
+
52
+ me_quant = LazyLoader("model_executor", globals(),
53
+ "vllm.model_executor.layers.quantization")
54
+ me_models = LazyLoader("model_executor", globals(),
55
+ "vllm.model_executor.models")
56
+ LoadConfig = Any
57
+ ParallelConfig = Any
58
+ RunnerType = Any
59
+ QuantizationMethods = Any
60
+ LogitsProcessor = Any
61
+
62
+logger = init_logger(__name__)
63
+
64
+RunnerOption = Literal["auto", "generate", "pooling", "draft"]
65
+ConvertType = Literal["none", "embed", "classify", "reward"]
66
+ConvertOption = Literal["auto", ConvertType]
67
+TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",
68
+ "score", "reward", "transcription", "draft"]
69
+_ResolvedTask = Literal["generate", "transcription", "encode", "embed",
70
+ "classify", "reward", "draft"]
71
+TokenizerMode = Literal["auto", "slow", "mistral", "custom"]
72
+ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
73
+LogprobsMode = Literal["raw_logits", "raw_logprobs", "processed_logits",
74
+ "processed_logprobs"]
75
+HfOverrides = Union[dict[str, Any], Callable[[type], type]]
76
+ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"]
77
+
78
+_RUNNER_TASKS: dict[RunnerType, list[TaskOption]] = {
79
+ "generate": ["generate", "transcription"],
80
+ "pooling": ["embedding", "embed", "classify", "score", "reward"],
81
+ "draft": ["draft"],
82
+}
83
+
84
+_RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
85
+ "generate": [],
86
+ "pooling": ["embed", "classify", "reward"],
87
+ "draft": [],
88
+}
89
+
90
+
91
+@config
92
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
93
+class ModelConfig:
94
+ """Configuration for the model."""
95
+
96
+ model: str = "Qwen/Qwen3-0.6B"
97
+ """Name or path of the Hugging Face model to use. It is also used as the
98
+ content for `model_name` tag in metrics output when `served_model_name` is
99
+ not specified."""
100
+ runner: RunnerOption = "auto"
101
+ """The type of model runner to use. Each vLLM instance only supports one
102
+ model runner, even if the same model can be used for multiple types."""
103
+ convert: ConvertOption = "auto"
104
+ """Convert the model using adapters defined in
105
+ [vllm.model_executor.models.adapters][]. The most common use case is to
106
+ adapt a text generation model to be used for pooling tasks."""
107
+ task: Optional[TaskOption] = None
108
+ """[DEPRECATED] The task to use the model for. If the model supports more
109
+ than one model runner, this is used to select which model runner to run.
110
+
111
+ Note that the model may support other tasks using the same model runner.
112
+ """
113
+ tokenizer: SkipValidation[str] = None # type: ignore
114
+ """Name or path of the Hugging Face tokenizer to use. If unspecified, model
115
+ name or path will be used."""
116
+ tokenizer_mode: TokenizerMode = "auto"
117
+ """Tokenizer mode:\n
118
+ - "auto" will use the fast tokenizer if available.\n
119
+ - "slow" will always use the slow tokenizer.\n
120
+ - "mistral" will always use the tokenizer from `mistral_common`.\n
121
+ - "custom" will use --tokenizer to select the preregistered tokenizer."""
122
+ trust_remote_code: bool = False
123
+ """Trust remote code (e.g., from HuggingFace) when downloading the model
124
+ and tokenizer."""
125
+ dtype: Union[ModelDType, torch.dtype] = "auto"
126
+ """Data type for model weights and activations:\n
127
+ - "auto" will use FP16 precision for FP32 and FP16 models, and BF16
128
+ precision for BF16 models.\n
129
+ - "half" for FP16. Recommended for AWQ quantization.\n
130
+ - "float16" is the same as "half".\n
131
+ - "bfloat16" for a balance between precision and range.\n
132
+ - "float" is shorthand for FP32 precision.\n
133
+ - "float32" for FP32 precision."""
134
+ seed: Optional[int] = None
135
+ """Random seed for reproducibility. Initialized to None in V0, but
136
+ initialized to 0 in V1."""
137
+ hf_config_path: Optional[str] = None
138
+ """Name or path of the Hugging Face config to use. If unspecified, model
139
+ name or path will be used."""
140
+ allowed_local_media_path: str = ""
141
+ """Allowing API requests to read local images or videos from directories
142
+ specified by the server file system. This is a security risk. Should only
143
+ be enabled in trusted environments."""
144
+ revision: Optional[str] = None
145
+ """The specific model version to use. It can be a branch name, a tag name,
146
+ or a commit id. If unspecified, will use the default version."""
147
+ code_revision: Optional[str] = None
148
+ """The specific revision to use for the model code on the Hugging Face Hub.
149
+ It can be a branch name, a tag name, or a commit id. If unspecified, will
150
+ use the default version."""
151
+ rope_scaling: dict[str, Any] = field(default_factory=dict)
152
+ """RoPE scaling configuration. For example,
153
+ `{"rope_type":"dynamic","factor":2.0}`."""
154
+ rope_theta: Optional[float] = None
155
+ """RoPE theta. Use with `rope_scaling`. In some cases, changing the RoPE
156
+ theta improves the performance of the scaled model."""
157
+ tokenizer_revision: Optional[str] = None
158
+ """The specific revision to use for the tokenizer on the Hugging Face Hub.
159
+ It can be a branch name, a tag name, or a commit id. If unspecified, will
160
+ use the default version."""
161
+ max_model_len: SkipValidation[int] = None # type: ignore
162
+ """Model context length (prompt and output). If unspecified, will be
163
+ automatically derived from the model config.
164
+
165
+ When passing via `--max-model-len`, supports k/m/g/K/M/G in human-readable
166
+ format. Examples:\n
167
+ - 1k -> 1000\n
168
+ - 1K -> 1024\n
169
+ - 25.6k -> 25,600"""
170
+ spec_target_max_model_len: Optional[int] = None
171
+ """Specify the maximum length for spec decoding draft models."""
172
+ quantization: SkipValidation[Optional[QuantizationMethods]] = None
173
+ """Method used to quantize the weights. If `None`, we first check the
174
+ `quantization_config` attribute in the model config file. If that is
175
+ `None`, we assume the model weights are not quantized and use `dtype` to
176
+ determine the data type of the weights."""
177
+ enforce_eager: bool = False
178
+ """Whether to always use eager-mode PyTorch. If True, we will disable CUDA
179
+ graph and always execute the model in eager mode. If False, we will use
180
+ CUDA graph and eager execution in hybrid for maximal performance and
181
+ flexibility."""
182
+ max_seq_len_to_capture: int = 8192
183
+ """Maximum sequence len covered by CUDA graphs. When a sequence has context
184
+ length larger than this, we fall back to eager mode. Additionally for
185
+ encoder-decoder models, if the sequence length of the encoder input is
186
+ larger than this, we fall back to the eager mode."""
187
+ max_logprobs: int = 20
188
+ """Maximum number of log probabilities to return when `logprobs` is
189
+ specified in `SamplingParams`. The default value comes the default for the
190
+ OpenAI Chat Completions API. -1 means no cap, i.e. all (output_length *
191
+ vocab_size) logprobs are allowed to be returned and it may cause OOM."""
192
+ logprobs_mode: LogprobsMode = "raw_logprobs"
193
+ """Indicates the content returned in the logprobs and prompt_logprobs.
194
+ Supported mode:
195
+ 1) raw_logprobs, 2) processed_logprobs, 3) raw_logits, 4) processed_logits.
196
+ Raw means the values before applying any logit processors, like bad words.
197
+ Processed means the values after applying all processors, including
198
+ temperature and top_k/top_p.
199
+ """
200
+ disable_sliding_window: bool = False
201
+ """Whether to disable sliding window. If True, we will disable the sliding
202
+ window functionality of the model, capping to sliding window size. If the
203
+ model does not support sliding window, this argument is ignored."""
204
+ disable_cascade_attn: bool = False
205
+ """Disable cascade attention for V1. While cascade attention does not
206
+ change the mathematical correctness, disabling it could be useful for
207
+ preventing potential numerical issues. Note that even if this is set to
208
+ False, cascade attention will be only used when the heuristic tells that
209
+ it's beneficial."""
210
+ skip_tokenizer_init: bool = False
211
+ """Skip initialization of tokenizer and detokenizer. Expects valid
212
+ `prompt_token_ids` and `None` for prompt from the input. The generated
213
+ output will contain token ids."""
214
+ enable_prompt_embeds: bool = False
215
+ """If `True`, enables passing text embeddings as inputs via the
216
+ `prompt_embeds` key. Note that enabling this will double the time required
217
+ for graph compilation."""
218
+ served_model_name: Optional[Union[str, list[str]]] = None
219
+ """The model name(s) used in the API. If multiple names are provided, the
220
+ server will respond to any of the provided names. The model name in the
221
+ model field of a response will be the first name in this list. If not
222
+ specified, the model name will be the same as the `--model` argument. Noted
223
+ that this name(s) will also be used in `model_name` tag content of
224
+ prometheus metrics, if multiple names provided, metrics tag will take the
225
+ first one."""
226
+ use_async_output_proc: bool = True
227
+ """Whether to use async output processor."""
228
+ config_format: Union[str, ConfigFormat] = "auto"
229
+ """The format of the model config to load:\n
230
+ - "auto" will try to load the config in hf format if available else it
231
+ will try to load in mistral format.\n
232
+ - "hf" will load the config in hf format.\n
233
+ - "mistral" will load the config in mistral format."""
234
+ hf_token: Optional[Union[bool, str]] = None
235
+ """The token to use as HTTP bearer authorization for remote files . If
236
+ `True`, will use the token generated when running `huggingface-cli login`
237
+ (stored in `~/.huggingface`)."""
238
+ hf_overrides: HfOverrides = field(default_factory=dict)
239
+ """If a dictionary, contains arguments to be forwarded to the Hugging Face
240
+ config. If a callable, it is called to update the HuggingFace config."""
241
+ logits_processor_pattern: Optional[str] = None
242
+ """Optional regex pattern specifying valid logits processor qualified names
243
+ that can be passed with the `logits_processors` extra completion argument.
244
+ Defaults to `None`, which allows no processors."""
245
+ generation_config: str = "auto"
246
+ """The folder path to the generation config. Defaults to `"auto"`, the
247
+ generation config will be loaded from model path. If set to `"vllm"`, no
248
+ generation config is loaded, vLLM defaults will be used. If set to a folder
249
+ path, the generation config will be loaded from the specified folder path.
250
+ If `max_new_tokens` is specified in generation config, then it sets a
251
+ server-wide limit on the number of output tokens for all requests."""
252
+ override_generation_config: dict[str, Any] = field(default_factory=dict)
253
+ """Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
254
+ used with `--generation-config auto`, the override parameters will be
255
+ merged with the default config from the model. If used with
256
+ `--generation-config vllm`, only the override parameters are used."""
257
+ enable_sleep_mode: bool = False
258
+ """Enable sleep mode for the engine (only cuda platform is supported)."""
259
+ model_impl: Union[str, ModelImpl] = "auto"
260
+ """Which implementation of the model to use:\n
261
+ - "auto" will try to use the vLLM implementation, if it exists, and fall
262
+ back to the Transformers implementation if no vLLM implementation is
263
+ available.\n
264
+ - "vllm" will use the vLLM model implementation.\n
265
+ - "transformers" will use the Transformers model implementation.\n
266
+ - "terratorch" will use the TerraTorch model implementation.
267
+ """
268
+ override_attention_dtype: Optional[str] = None
269
+ """Override dtype for attention"""
270
+ logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None
271
+ """One or more logits processors' fully-qualified class names or class
272
+ definitions"""
273
+ io_processor_plugin: Optional[str] = None
274
+ """IOProcessor plugin name to load at model startup"""
275
+
276
+ # Pooler config
277
+ pooler_config: Optional[PoolerConfig] = None
278
+ """Pooler config which controls the behaviour of output pooling in pooling
279
+ models."""
280
+ override_pooler_config: Optional[Union[dict, PoolerConfig]] = None
281
+ """[DEPRECATED] Use `pooler_config` instead. This field will be removed in
282
+ v0.12.0 or v1.0.0, whichever is sooner."""
283
+
284
+ # Multimodal config and init vars
285
+ multimodal_config: Optional[MultiModalConfig] = None
286
+ """Configuration for multimodal model. If `None`, this will be inferred
287
+ from the architecture of `self.model`."""
288
+ limit_mm_per_prompt: InitVar[Optional[dict[str, int]]] = None
289
+ media_io_kwargs: InitVar[Optional[dict[str, dict[str, Any]]]] = None
290
+ mm_processor_kwargs: InitVar[Optional[dict[str, Any]]] = None
291
+ mm_processor_cache_gb: InitVar[Optional[float]] = None
292
+ mm_processor_cache_type: InitVar[Optional[MMCacheType]] = None
293
+ mm_shm_cache_max_object_size_mb: InitVar[Optional[int]] = None
294
+ mm_encoder_tp_mode: InitVar[Optional[MMEncoderTPMode]] = None
295
+ interleave_mm_strings: InitVar[Optional[bool]] = None
296
+ skip_mm_profiling: InitVar[Optional[bool]] = None
297
+
298
+ def compute_hash(self) -> str:
299
+ """
300
+ WARNING: Whenever a new field is added to this config,
301
+ ensure that it is included in the factors list if
302
+ it affects the computation graph.
303
+
304
+ Provide a hash that uniquely identifies all the configs
305
+ that affect the structure of the computation
306
+ graph from input ids/embeddings to the final hidden states,
307
+ excluding anything before input ids/embeddings and after
308
+ the final hidden states.
309
+ """
310
+ factors: list[Any] = []
311
+ factors.append(self.model)
312
+ factors.append(self.dtype)
313
+ factors.append(self.quantization)
314
+ factors.append(self.revision)
315
+ factors.append(self.code_revision)
316
+ factors.append(self.max_model_len)
317
+ factors.append(self.max_logprobs)
318
+ factors.append(self.disable_sliding_window)
319
+ factors.append(self.trust_remote_code)
320
+ factors.append(self.generation_config)
321
+ factors.append(self.model_impl)
322
+ factors.append(self.override_generation_config)
323
+ factors.append(self.rope_scaling)
324
+ factors.append(self.rope_theta)
325
+ # hf_config can control how the model looks!
326
+ factors.append(self.hf_config.to_json_string())
327
+ str_factors = str(factors)
328
+ assert_hashable(str_factors)
329
+ return hashlib.sha256(str(factors).encode()).hexdigest()
330
+
331
+ def __post_init__(
332
+ self,
333
+ # Multimodal config init vars
334
+ limit_mm_per_prompt: Optional[dict[str, int]],
335
+ media_io_kwargs: Optional[dict[str, dict[str, Any]]],
336
+ mm_processor_kwargs: Optional[dict[str, Any]],
337
+ mm_processor_cache_gb: Optional[float],
338
+ mm_processor_cache_type: Optional[MMCacheType],
339
+ mm_shm_cache_max_object_size_mb: Optional[int],
340
+ mm_encoder_tp_mode: Optional[MMEncoderTPMode],
341
+ interleave_mm_strings: Optional[bool],
342
+ skip_mm_profiling: Optional[bool]) -> None:
343
+ # Set the default seed to 0 in V1.
344
+ # NOTE(woosuk): In V0, we set the default seed to None because the
345
+ # driver worker shares the same process as the user process, and thus
346
+ # setting a seed affects the user process as well.
347
+ # In V1, we use separate processes for workers (unless
348
+ # VLLM_ENABLE_V1_MULTIPROCESSING=0), so setting a seed here
349
+ # doesn't affect the user process. However, without a consistent seed,
350
+ # different tensor parallel workers would sample different tokens,
351
+ # leading to inconsistent results.
352
+ if envs.VLLM_USE_V1 and self.seed is None:
353
+ self.seed = 0
354
+ if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
355
+ logger.warning(
356
+ "The global random seed is set to %d. Since "
357
+ "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may "
358
+ "affect the random state of the Python process that "
359
+ "launched vLLM.", self.seed)
360
+
361
+ # Keep set served_model_name before maybe_model_redirect(self.model)
362
+ self.served_model_name = get_served_model_name(self.model,
363
+ self.served_model_name)
364
+ self.model = maybe_model_redirect(self.model)
365
+ # The tokenizer is consistent with the model by default.
366
+ if self.tokenizer is None:
367
+ self.tokenizer = self.model
368
+ if self.tokenizer_revision is None:
369
+ self.tokenizer_revision = self.revision
370
+ self.tokenizer = maybe_model_redirect(self.tokenizer)
371
+
372
+ if isinstance(self.hf_config_path, str):
373
+ self.hf_config_path = maybe_model_redirect(self.hf_config_path)
374
+
375
+ if callable(self.hf_overrides):
376
+ hf_overrides_kw = {}
377
+ hf_overrides_fn = self.hf_overrides
378
+ else:
379
+ hf_overrides_kw = self.hf_overrides
380
+ hf_overrides_fn = None
381
+
382
+ if self.rope_scaling:
383
+ hf_override: dict[str, Any] = {"rope_scaling": self.rope_scaling}
384
+ hf_overrides_kw.update(hf_override)
385
+ hf_overrides_str = json.dumps(hf_overrides_kw)
386
+ msg = (
387
+ "`--rope-scaling` will be removed in a future release. "
388
+ f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
389
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
390
+ if self.rope_theta is not None:
391
+ hf_override = {"rope_theta": self.rope_theta}
392
+ hf_overrides_kw.update(hf_override)
393
+ hf_overrides_str = json.dumps(hf_overrides_kw)
394
+ msg = (
395
+ "`--rope-theta` will be removed in a future release. "
396
+ f"'Please instead use `--hf-overrides '{hf_overrides_str}'`")
397
+ warnings.warn(DeprecationWarning(msg), stacklevel=2)
398
+
399
+ self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer)
400
+
401
+ if self.runner != "draft":
402
+ # If we're not running the draft model, check for speculators config
403
+ # If speculators config, set model / tokenizer to be target model
404
+ self.model, self.tokenizer = maybe_override_with_speculators_target_model( # noqa: E501
405
+ model=self.model,
406
+ tokenizer=self.tokenizer,
407
+ revision=self.revision,
408
+ trust_remote_code=self.trust_remote_code)
409
+
410
+ if (backend := envs.VLLM_ATTENTION_BACKEND
411
+ ) and backend == "FLASHINFER" and find_spec("flashinfer") is None:
412
+ raise ValueError(
413
+ "VLLM_ATTENTION_BACKEND is set to FLASHINFER, but flashinfer "
414
+ "module was not found. See "
415
+ "https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile " # noqa: E501
416
+ "for instructions on how to install it.")
417
+
418
+ from vllm.platforms import current_platform
419
+
420
+ if (self.override_attention_dtype is not None
421
+ and not current_platform.is_rocm()):
422
+ warnings.warn(
423
+ "override-attention-dtype is set but not using ROCm platform",
424
+ stacklevel=2)
425
+
426
+ if (self.enable_sleep_mode
427
+ and not current_platform.is_sleep_mode_available()):
428
+ raise ValueError(
429
+ "Sleep mode is not supported on current platform.")
430
+
431
+ hf_config = get_config(self.hf_config_path or self.model,
432
+ self.trust_remote_code,
433
+ self.revision,
434
+ self.code_revision,
435
+ self.config_format,
436
+ hf_overrides_kw=hf_overrides_kw,
437
+ hf_overrides_fn=hf_overrides_fn)
438
+
439
+ self.hf_config = hf_config
440
+ self.hf_text_config = get_hf_text_config(self.hf_config)
441
+ self.attention_chunk_size = getattr(self.hf_text_config,
442
+ "attention_chunk_size", None)
443
+ self.encoder_config = self._get_encoder_config()
444
+ self.hf_image_processor_config = get_hf_image_processor_config(
445
+ self.model, hf_token=self.hf_token, revision=self.revision)
446
+
447
+ architectures = self.architectures
448
+ registry = self.registry
449
+ is_generative_model = registry.is_text_generation_model(
450
+ architectures, self)
451
+ is_pooling_model = registry.is_pooling_model(architectures, self)
452
+
453
+ def _task_to_convert(task: TaskOption) -> ConvertType:
454
+ if task == "embedding" or task == "embed":
455
+ return "embed"
456
+ if task == "classify":
457
+ return "classify"
458
+ if task == "reward":
459
+ return "reward"
460
+ if task == "score":
461
+ new_task = self._get_default_pooling_task(architectures)
462
+ return "classify" if new_task == "classify" else "embed"
463
+
464
+ return "none"
465
+
466
+ if self.task is not None:
467
+ runner: RunnerOption = "auto"
468
+ convert: ConvertOption = "auto"
469
+ msg_prefix = ("The 'task' option has been deprecated and will be "
470
+ "removed in v0.13.0 or v1.0, whichever comes first.")
471
+ msg_hint = "Please remove this option."
472
+
473
+ is_generative_task = self.task in _RUNNER_TASKS["generate"]
474
+ is_pooling_task = self.task in _RUNNER_TASKS["pooling"]
475
+
476
+ if is_generative_model and is_pooling_model:
477
+ if is_generative_task:
478
+ runner = "generate"
479
+ convert = "auto"
480
+ msg_hint = ("Please replace this option with `--runner "
481
+ "generate` to continue using this model "
482
+ "as a generative model.")
483
+ elif is_pooling_task:
484
+ runner = "pooling"
485
+ convert = "auto"
486
+ msg_hint = ("Please replace this option with `--runner "
487
+ "pooling` to continue using this model "
488
+ "as a pooling model.")
489
+ else: # task == "auto"
490
+ pass
491
+ elif is_generative_model or is_pooling_model:
492
+ if is_generative_task:
493
+ runner = "generate"
494
+ convert = "auto"
495
+ msg_hint = "Please remove this option"
496
+ elif is_pooling_task:
497
+ runner = "pooling"
498
+ convert = _task_to_convert(self.task)
499
+ msg_hint = ("Please replace this option with `--convert "
500
+ f"{convert}` to continue using this model "
501
+ "as a pooling model.")
502
+ else: # task == "auto"
503
+ pass
504
+ else:
505
+ raise AssertionError("The model should be a generative or "
506
+ "pooling model when task is set to "
507
+ f"{self.task!r}.")
508
+
509
+ self.runner = runner
510
+ self.convert = convert
511
+
512
+ msg = f"{msg_prefix} {msg_hint}"
513
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
514
+
515
+ self.runner_type = self._get_runner_type(architectures, self.runner)
516
+ self.convert_type = self._get_convert_type(architectures,
517
+ self.runner_type,
518
+ self.convert)
519
+
520
+ if self.runner_type == "generate" and not is_generative_model:
521
+ generate_converts = _RUNNER_CONVERTS["generate"]
522
+ if self.convert_type not in generate_converts:
523
+ # Currently we don't have any converters for generative models
524
+ raise ValueError(
525
+ "This model does not support `--runner generate`.")
526
+ if self.runner_type == "pooling" and not is_pooling_model:
527
+ pooling_converts = _RUNNER_CONVERTS["pooling"]
528
+ if self.convert_type not in pooling_converts:
529
+ convert_option = "<" + "|".join(pooling_converts) + ">"
530
+ raise ValueError(
531
+ "This model does not support `--runner pooling`. "
532
+ f"You can pass `--convert {convert_option} to adapt "
533
+ "it into a pooling model.")
534
+
535
+ self.supported_tasks = self._get_supported_tasks(
536
+ architectures, self.runner_type, self.convert_type)
537
+
538
+ # Note: Initialize these attributes early because transformers fallback
539
+ # may fail to load dynamic modules in child processes
540
+ model_info, arch = registry.inspect_model_cls(architectures, self)
541
+ self._model_info = model_info
542
+ self._architecture = arch
543
+ logger.info("Resolved architecture: %s", arch)
544
+
545
+ # Init pooler config if needed
546
+ if self.runner_type == "pooling":
547
+ if self.override_pooler_config is not None:
548
+ logger.warning_once(
549
+ "`override_pooler_config` is deprecated and will be "
550
+ "removed in v0.12.0 or v1.0.0, whichever is sooner. "
551
+ "Please use `pooler_config` instead.")
552
+
553
+ if isinstance(self.override_pooler_config, dict):
554
+ self.pooler_config = PoolerConfig(
555
+ **self.override_pooler_config)
556
+ else:
557
+ self.pooler_config = self.override_pooler_config
558
+
559
+ if self.pooler_config is None:
560
+ self.pooler_config = PoolerConfig()
561
+
562
+ base_config = get_pooling_config(self.model, self.revision)
563
+ if base_config is not None:
564
+ # Only set values that are not overridden by the user
565
+ for k, v in base_config.items():
566
+ if getattr(self.pooler_config, k) is None:
567
+ setattr(self.pooler_config, k, v)
568
+
569
+ default_pooling_type = self._model_info.default_pooling_type
570
+ if self.pooler_config.pooling_type is None:
571
+ self.pooler_config.pooling_type = default_pooling_type
572
+
573
+ self.dtype: torch.dtype = _get_and_verify_dtype(
574
+ self.model,
575
+ self.hf_config,
576
+ self.dtype,
577
+ is_pooling_model=self.runner_type == "pooling",
578
+ revision=self.revision,
579
+ )
580
+
581
+ # Interleaved attention is not supported by some backends in V0
582
+ if (not self.disable_sliding_window
583
+ and is_interleaved(self.hf_text_config)
584
+ and not envs.VLLM_USE_V1
585
+ and (backend := envs.VLLM_ATTENTION_BACKEND)
586
+ in ("XFORMERS", "FLASHINFER")):
587
+ logger.warning_once(
588
+ "%s has interleaved attention, which is currently not "
589
+ "supported by the %s backend. Disabling sliding window and "
590
+ "capping the max length to the sliding window size (%d).",
591
+ self.hf_text_config.model_type,
592
+ backend,
593
+ self.hf_text_config.sliding_window,
594
+ )
595
+ self.disable_sliding_window = True
596
+
597
+ self.original_max_model_len = self.max_model_len
598
+ self.max_model_len = self.get_and_verify_max_len(self.max_model_len)
599
+ # Init multimodal config if needed
600
+ if self._model_info.supports_multimodal:
601
+ if (mm_encoder_tp_mode == "data" and
602
+ not self._model_info.supports_multimodal_encoder_tp_data):
603
+ logger.warning_once(
604
+ "This model does not support `--mm-encoder-tp-mode data`. "
605
+ "Falling back to `--mm-encoder-tp-mode weights`.")
606
+ mm_encoder_tp_mode = "weights"
607
+
608
+ mm_config_kwargs = dict(
609
+ limit_per_prompt=limit_mm_per_prompt,
610
+ media_io_kwargs=media_io_kwargs,
611
+ mm_processor_kwargs=mm_processor_kwargs,
612
+ mm_processor_cache_gb=mm_processor_cache_gb,
613
+ mm_processor_cache_type=mm_processor_cache_type,
614
+ mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
615
+ mm_encoder_tp_mode=mm_encoder_tp_mode,
616
+ interleave_mm_strings=interleave_mm_strings,
617
+ skip_mm_profiling=skip_mm_profiling,
618
+ )
619
+
620
+ mm_config_kwargs = {
621
+ k: v
622
+ for k, v in mm_config_kwargs.items() if v is not None
623
+ }
624
+
625
+ self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
626
+
627
+ if self.disable_sliding_window:
628
+ # Set after get_and_verify_max_len to ensure that max_model_len
629
+ # can be correctly capped to sliding window size
630
+ self.hf_text_config.sliding_window = None
631
+
632
+ if not self.skip_tokenizer_init:
633
+ self._verify_tokenizer_mode()
634
+
635
+ # Avoid running try_verify_and_update_config multiple times
636
+ self.config_updated = False
637
+
638
+ self._verify_quantization()
639
+ self._verify_cuda_graph()
640
+ self._verify_bnb_config()
641
+
642
+ @field_validator("quantization", mode="before")
643
+ @classmethod
644
+ def validate_quantization_before(cls, value: Any) -> Any:
645
+ if isinstance(value, str):
646
+ return value.lower()
647
+ return value
648
+
649
+ @model_validator(mode="after")
650
+ def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
651
+ if not isinstance(self.tokenizer, str):
652
+ raise ValueError("tokenizer must be a string after __post_init__.")
653
+ if not isinstance(self.max_model_len, int):
654
+ raise ValueError(
655
+ "max_model_len must be an integer after __post_init__.")
656
+ return self
657
+
658
+ def _get_transformers_backend_cls(self) -> str:
659
+ """Determine which Transformers backend class will be used if
660
+ `model_impl` is set to `transformers` or `auto`."""
661
+ if getattr(self, "runner_type", self.runner) == "pooling":
662
+ return "TransformersModel"
663
+ if self.hf_config != self.hf_text_config:
664
+ # If 'hf_text_config' is the same as 'hf_config'. If not, it is
665
+ # probably a composite config, i.e. multimodal
666
+ return "TransformersForMultimodalLM"
667
+ return "TransformersForCausalLM"
668
+
669
+ def using_transformers_backend(self) -> bool:
670
+ """Check if the model is using the Transformers backend class."""
671
+ return self.architecture == self._get_transformers_backend_cls()
672
+
673
+ @property
674
+ def registry(self):
675
+ return me_models.ModelRegistry
676
+
677
+ @property
678
+ def architectures(self) -> list[str]:
679
+ return getattr(self.hf_config, "architectures", [])
680
+
681
+ @property
682
+ def architecture(self) -> str:
683
+ """The architecture vllm actually used."""
684
+ return self._architecture
685
+
686
+ def maybe_pull_model_tokenizer_for_runai(self, model: str,
687
+ tokenizer: str) -> None:
688
+ """Pull model/tokenizer from Object Storage to temporary
689
+ directory when needed.
690
+
691
+ Args:
692
+ model: Model name or path
693
+ tokenizer: Tokenizer name or path
694
+ """
695
+ if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)):
696
+ return
697
+
698
+ if is_runai_obj_uri(model):
699
+ object_storage_model = ObjectStorageModel()
700
+ object_storage_model.pull_files(
701
+ model, allow_pattern=["*.model", "*.py", "*.json"])
702
+ self.model_weights = model
703
+ self.model = object_storage_model.dir
704
+
705
+ # If tokenizer is same as model, download to same directory
706
+ if model == tokenizer:
707
+ object_storage_model.pull_files(model,
708
+ ignore_pattern=[
709
+ "*.pt", "*.safetensors",
710
+ "*.bin", "*.tensors",
711
+ "*.pth"
712
+ ])
713
+ self.tokenizer = object_storage_model.dir
714
+ return
715
+
716
+ # Only download tokenizer if needed and not already handled
717
+ if is_runai_obj_uri(tokenizer):
718
+ object_storage_tokenizer = ObjectStorageModel()
719
+ object_storage_tokenizer.pull_files(model,
720
+ ignore_pattern=[
721
+ "*.pt", "*.safetensors",
722
+ "*.bin", "*.tensors",
723
+ "*.pth"
724
+ ])
725
+ self.tokenizer = object_storage_tokenizer.dir
726
+
727
+ def _get_encoder_config(self):
728
+ return get_sentence_transformer_tokenizer_config(
729
+ self.model, self.revision)
730
+
731
+ def _verify_tokenizer_mode(self) -> None:
732
+ tokenizer_mode = cast(TokenizerMode, self.tokenizer_mode.lower())
733
+ if tokenizer_mode not in get_args(TokenizerMode):
734
+ raise ValueError(
735
+ f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
736
+ f"one of {get_args(TokenizerMode)}.")
737
+ self.tokenizer_mode = tokenizer_mode
738
+
739
+ def _get_default_runner_type(
740
+ self,
741
+ architectures: list[str],
742
+ ) -> RunnerType:
743
+ registry = self.registry
744
+
745
+ # Some Sentence Transformers models use *ForCausalLM archs
746
+ if get_pooling_config(self.model, self.revision):
747
+ return "pooling"
748
+
749
+ for arch in architectures:
750
+ if arch in registry.get_supported_archs():
751
+ if registry.is_pooling_model(architectures, self):
752
+ return "pooling"
753
+ if registry.is_text_generation_model(architectures, self):
754
+ return "generate"
755
+
756
+ match = try_match_architecture_defaults(arch)
757
+ if match:
758
+ _, (runner_type, _) = match
759
+ return runner_type
760
+
761
+ return "generate"
762
+
763
+ def _get_runner_type(
764
+ self,
765
+ architectures: list[str],
766
+ runner: RunnerOption,
767
+ ) -> RunnerType:
768
+ if runner != "auto":
769
+ return runner
770
+
771
+ runner_type = self._get_default_runner_type(architectures)
772
+
773
+ # Don't log the most common case
774
+ if runner_type != "generate":
775
+ logger.info(
776
+ "Resolved `--runner auto` to `--runner %s`. "
777
+ "Pass the value explicitly to silence this message.",
778
+ runner_type)
779
+
780
+ return runner_type
781
+
782
+ def _get_default_convert_type(
783
+ self,
784
+ architectures: list[str],
785
+ runner_type: RunnerType,
786
+ ) -> ConvertType:
787
+ registry = self.registry
788
+
789
+ for arch in architectures:
790
+ if arch in registry.get_supported_archs():
791
+ if (runner_type == "generate"
792
+ and registry.is_text_generation_model(
793
+ architectures, self)):
794
+ return "none"
795
+ if (runner_type == "pooling"
796
+ and registry.is_pooling_model(architectures, self)):
797
+ return "none"
798
+
799
+ match = try_match_architecture_defaults(arch,
800
+ runner_type=runner_type)
801
+ if match:
802
+ _, (_, convert_type) = match
803
+ return convert_type
804
+
805
+ # This is to handle Sentence Transformers models that use *ForCausalLM
806
+ # and also multi-modal pooling models which are not defined as
807
+ # Sentence Transformers models
808
+ if runner_type == "pooling":
809
+ return "embed"
810
+
811
+ return "none"
812
+
813
+ def _get_convert_type(
814
+ self,
815
+ architectures: list[str],
816
+ runner_type: RunnerType,
817
+ convert: ConvertOption,
818
+ ) -> ConvertType:
819
+ if convert != "auto":
820
+ return convert
821
+
822
+ convert_type = self._get_default_convert_type(architectures,
823
+ runner_type)
824
+
825
+ # Don't log the most common case
826
+ if convert_type != "none":
827
+ logger.info(
828
+ "Resolved `--convert auto` to `--convert %s`. "
829
+ "Pass the value explicitly to silence this message.",
830
+ convert_type)
831
+
832
+ return convert_type
833
+
834
+ def _get_supported_generation_tasks(
835
+ self,
836
+ architectures: list[str],
837
+ convert_type: ConvertType,
838
+ ) -> list[_ResolvedTask]:
839
+ registry = self.registry
840
+
841
+ if registry.is_transcription_only_model(architectures, self):
842
+ return ["transcription"]
843
+
844
+ # TODO: Use get_supported_generation_tasks once V0 is removed
845
+ supported_tasks = list[_ResolvedTask]()
846
+ if (registry.is_text_generation_model(architectures, self)
847
+ or convert_type in _RUNNER_CONVERTS["generate"]):
848
+ supported_tasks.append("generate")
849
+
850
+ if registry.is_transcription_model(architectures, self):
851
+ supported_tasks.append("transcription")
852
+
853
+ return supported_tasks
854
+
855
+ def _get_default_pooling_task(
856
+ self,
857
+ architectures: list[str],
858
+ ) -> Literal["embed", "classify", "reward"]:
859
+ if self.registry.is_cross_encoder_model(architectures, self):
860
+ return "classify"
861
+
862
+ for arch in architectures:
863
+ match = try_match_architecture_defaults(arch,
864
+ runner_type="pooling")
865
+ if match:
866
+ _, (_, convert_type) = match
867
+ assert convert_type != "none"
868
+ return convert_type
869
+
870
+ return "embed"
871
+
872
+ def _get_supported_pooling_tasks(
873
+ self,
874
+ architectures: list[str],
875
+ convert_type: ConvertType,
876
+ ) -> list[_ResolvedTask]:
877
+ registry = self.registry
878
+
879
+ # TODO: Use get_supported_pooling_tasks once V0 is removed
880
+ supported_tasks = list[_ResolvedTask]()
881
+ if (registry.is_pooling_model(architectures, self)
882
+ or convert_type in _RUNNER_CONVERTS["pooling"]):
883
+ supported_tasks.append("encode")
884
+
885
+ extra_task = (self._get_default_pooling_task(architectures)
886
+ if convert_type == "none" else convert_type)
887
+ supported_tasks.append(extra_task)
888
+
889
+ return supported_tasks
890
+
891
+ def _get_supported_tasks(
892
+ self,
893
+ architectures: list[str],
894
+ runner_type: RunnerType,
895
+ convert_type: ConvertType,
896
+ ) -> list[_ResolvedTask]:
897
+ if runner_type == "generate":
898
+ return self._get_supported_generation_tasks(
899
+ architectures, convert_type)
900
+ if runner_type == "pooling":
901
+ return self._get_supported_pooling_tasks(architectures,
902
+ convert_type)
903
+ if runner_type == "draft":
904
+ return ["draft"]
905
+
906
+ assert_never(runner_type)
907
+
908
+ def _parse_quant_hf_config(self, hf_config: PretrainedConfig):
909
+ quant_cfg = getattr(hf_config, "quantization_config", None)
910
+ if quant_cfg is None:
911
+ # compressed-tensors uses a "compression_config" key
912
+ quant_cfg = getattr(hf_config, "compression_config", None)
913
+
914
+ else:
915
+ # Set quant_method for ModelOpt models.
916
+ producer_name = quant_cfg.get("producer", {}).get("name")
917
+ if producer_name == "modelopt":
918
+ quant_algo = quant_cfg.get("quantization",
919
+ {}).get("quant_algo")
920
+ if quant_algo == "FP8":
921
+ quant_cfg["quant_method"] = "modelopt"
922
+ elif quant_algo == "NVFP4":
923
+ quant_cfg["quant_method"] = "modelopt_fp4"
924
+ elif quant_algo is not None:
925
+ raise ValueError(
926
+ f"Unknown ModelOpt quant algo: {quant_algo}")
927
+
928
+ return quant_cfg
929
+
930
+ def _verify_quantization(self) -> None:
931
+ supported_quantization = me_quant.QUANTIZATION_METHODS
932
+ if self.quantization is not None:
933
+ self.quantization = cast(me_quant.QuantizationMethods,
934
+ self.quantization)
935
+
936
+ # Parse quantization method from the HF model config, if available.
937
+ quant_cfg = self._parse_quant_hf_config(self.hf_config)
938
+ if quant_cfg is None and (text_config := getattr(
939
+ self.hf_config, "text_config", None)):
940
+ # Check the text config as well for multi-modal models.
941
+ quant_cfg = self._parse_quant_hf_config(text_config)
942
+
943
+ if quant_cfg is not None:
944
+ # Use the community standard 'quant_method'
945
+ quant_method = quant_cfg.get("quant_method", "").lower()
946
+
947
+ # Normalize library names
948
+ quant_method = quant_method.replace("compressed_tensors",
949
+ "compressed-tensors")
950
+
951
+ quant_cfg["quant_method"] = quant_method
952
+
953
+ # Quantization methods which are overrides (i.e. they have a
954
+ # `override_quantization_method` method) must be checked in order
955
+ # of preference (this is particularly important for GPTQ).
956
+ overrides = [
957
+ "bitblas",
958
+ "gptq_marlin_24",
959
+ "gptq_marlin",
960
+ "gptq_bitblas",
961
+ "awq_marlin",
962
+ "ipex",
963
+ "moe_wna16",
964
+ "modelopt",
965
+ "modelopt_fp4",
966
+ "petit_nvfp4",
967
+ ]
968
+ quantization_methods = [
969
+ q for q in supported_quantization if q not in overrides
970
+ ]
971
+ # Any custom overrides will be in quantization_methods so we place
972
+ # them at the start of the list so custom overrides have preference
973
+ # over the built-in ones.
974
+ quantization_methods = quantization_methods + overrides
975
+
976
+ # Detect which checkpoint is it
977
+ for name in quantization_methods:
978
+ method = me_quant.get_quantization_config(name)
979
+ quantization_override = method.override_quantization_method(
980
+ quant_cfg, self.quantization)
981
+ if quantization_override is not None:
982
+ # Raise error if the override is not custom (custom would
983
+ # be in QUANTIZATION_METHODS but not QuantizationMethods)
984
+ # and hasn't been added to the overrides list.
985
+ if (name in get_args(me_quant.QuantizationMethods)
986
+ and name not in overrides):
987
+ raise ValueError(
988
+ f"Quantization method {name} is an override but "
989
+ "is has not been added to the `overrides` list "
990
+ "above. This is necessary to ensure that the "
991
+ "overrides are checked in order of preference.")
992
+ quant_method = quantization_override
993
+ self.quantization = quantization_override
994
+ break
995
+
996
+ # Verify quantization configurations.
997
+ if self.quantization is None:
998
+ self.quantization = quant_method
999
+ elif self.quantization != quant_method:
1000
+ raise ValueError(
1001
+ "Quantization method specified in the model config "
1002
+ f"({quant_method}) does not match the quantization "
1003
+ f"method specified in the `quantization` argument "
1004
+ f"({self.quantization}).")
1005
+
1006
+ if self.quantization is not None:
1007
+ if self.quantization not in supported_quantization:
1008
+ raise ValueError(
1009
+ f"Unknown quantization method: {self.quantization}. Must "
1010
+ f"be one of {supported_quantization}.")
1011
+ from vllm.platforms import current_platform
1012
+ current_platform.verify_quantization(self.quantization)
1013
+
1014
+ def _verify_cuda_graph(self) -> None:
1015
+ # The `max_seq_len_to_capture` was incorrectly
1016
+ # based on the encoder's input length (448)
1017
+ # but not the decoder's larger input length (1500).
1018
+ # This change ensures the CUDA Graph captures the correct,
1019
+ # larger sequence length, allowing it to work as intended.
1020
+ effective_max_seq_len = self.max_model_len
1021
+ if self.is_encoder_decoder:
1022
+ effective_max_seq_len = max(
1023
+ effective_max_seq_len,
1024
+ getattr(self.hf_config, "max_source_positions", 0))
1025
+ self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
1026
+ effective_max_seq_len)
1027
+ # CUDAGraph capture not supported for encoder-decoder models on ROCm
1028
+ unsupported_rocm = self.is_encoder_decoder
1029
+
1030
+ if (unsupported_rocm and not self.enforce_eager
1031
+ and current_platform.is_rocm()):
1032
+ logger.warning(
1033
+ "CUDA graph is not supported for %s on ROCm yet, fallback "
1034
+ "to eager mode.", self.hf_config.model_type)
1035
+ self.enforce_eager = True
1036
+
1037
+ def _verify_bnb_config(self) -> None:
1038
+ """
1039
+ The current version of bitsandbytes (0.46.1) with 8-bit models does not
1040
+ yet support CUDA graph.
1041
+ # TODO Remove this when bitsandbytes supports.
1042
+ """
1043
+ is_bitsandbytes = self.quantization == "bitsandbytes"
1044
+ has_quantization_config = (getattr(self.hf_config,
1045
+ "quantization_config", None)
1046
+ is not None)
1047
+ is_8bit = (self.hf_config.quantization_config.get(
1048
+ "load_in_8bit", False) if has_quantization_config else False)
1049
+ if all([
1050
+ is_bitsandbytes,
1051
+ has_quantization_config,
1052
+ is_8bit,
1053
+ not self.enforce_eager,
1054
+ ]):
1055
+ logger.warning(
1056
+ "CUDA graph is not supported on BitsAndBytes 8bit yet, "
1057
+ "fallback to the eager mode.")
1058
+
1059
+ self.enforce_eager = True
1060
+
1061
+ def _verify_with_expert_parallelism(self) -> None:
1062
+ num_expert_names = [
1063
+ "moe_num_experts", # Dbrx
1064
+ "num_experts", # Jamba
1065
+ "n_routed_experts", # DeepSeek
1066
+ "num_local_experts", # Mixtral
1067
+ ]
1068
+ num_experts = 0
1069
+ for name in num_expert_names:
1070
+ num_experts = getattr(self.hf_text_config, name, 0)
1071
+ if num_experts > 0:
1072
+ break
1073
+ if num_experts < 1:
1074
+ raise ValueError(
1075
+ "Number of experts in the model must be greater than 0 "
1076
+ "when expert parallelism is enabled.")
1077
+
1078
+ def verify_dual_chunk_attention_config(
1079
+ self,
1080
+ load_config: LoadConfig,
1081
+ ) -> None:
1082
+ if hasattr(self.hf_config, "dual_chunk_attention_config"):
1083
+ # Try loading the sparse attention config
1084
+ from vllm.model_executor.model_loader.weight_utils import (
1085
+ get_sparse_attention_config)
1086
+ sparse_attn_config = get_sparse_attention_config(self, load_config)
1087
+ if sparse_attn_config:
1088
+ self.hf_config.dual_chunk_attention_config[
1089
+ "sparse_attention_config"] = sparse_attn_config
1090
+ if "sparse_attention_enabled" not in \
1091
+ self.hf_config.dual_chunk_attention_config:
1092
+ self.hf_config.dual_chunk_attention_config[
1093
+ "sparse_attention_enabled"] = True
1094
+
1095
+ if envs.VLLM_ATTENTION_BACKEND != STR_DUAL_CHUNK_FLASH_ATTN_VAL:
1096
+ raise ValueError("please set VLLM_ATTENTION_BACKEND to "
1097
+ f"{STR_DUAL_CHUNK_FLASH_ATTN_VAL}")
1098
+
1099
+ def verify_async_output_proc(self, parallel_config, speculative_config,
1100
+ device_config) -> None:
1101
+ if not self.use_async_output_proc:
1102
+ # Nothing to check
1103
+ return
1104
+
1105
+ if parallel_config.pipeline_parallel_size > 1:
1106
+ self.use_async_output_proc = False
1107
+ return
1108
+
1109
+ # Reminder: Please update docs/features/compatibility_matrix.md
1110
+ # If the feature combo become valid
1111
+ from vllm.platforms import current_platform
1112
+ if not current_platform.is_async_output_supported(self.enforce_eager):
1113
+ self.use_async_output_proc = False
1114
+ return
1115
+
1116
+ if envs.VLLM_USE_RAY_SPMD_WORKER:
1117
+ self.use_async_output_proc = False
1118
+ return
1119
+
1120
+ # Async postprocessor is not necessary for pooling models
1121
+ # since there is no token generation
1122
+ if self.runner_type == "pooling":
1123
+ self.use_async_output_proc = False
1124
+
1125
+ # Reminder: Please update docs/features/compatibility_matrix.md
1126
+ # If the feature combo become valid
1127
+ if speculative_config:
1128
+ self.use_async_output_proc = False
1129
+
1130
+ def verify_with_parallel_config(
1131
+ self,
1132
+ parallel_config: ParallelConfig,
1133
+ ) -> None:
1134
+
1135
+ if parallel_config.distributed_executor_backend == "external_launcher":
1136
+ assert self.seed is not None, (
1137
+ "Seed must be set when using external launcher backend to "
1138
+ "make sure sampling results are the same across workers.")
1139
+
1140
+ total_num_attention_heads = getattr(self.hf_text_config,
1141
+ "num_attention_heads", 0)
1142
+ tensor_parallel_size = parallel_config.tensor_parallel_size
1143
+ if total_num_attention_heads % tensor_parallel_size != 0:
1144
+ raise ValueError(
1145
+ f"Total number of attention heads ({total_num_attention_heads})"
1146
+ " must be divisible by tensor parallel size "
1147
+ f"({tensor_parallel_size}).")
1148
+
1149
+ if parallel_config.enable_expert_parallel:
1150
+ self._verify_with_expert_parallelism()
1151
+
1152
+ pipeline_parallel_size = parallel_config.pipeline_parallel_size
1153
+ if pipeline_parallel_size > 1:
1154
+ if not self.registry.is_pp_supported_model(self.architectures,
1155
+ self):
1156
+ raise NotImplementedError(
1157
+ "Pipeline parallelism is not supported for this model. "
1158
+ "Supported models implement the `SupportsPP` interface.")
1159
+
1160
+ if self.use_async_output_proc:
1161
+ self.use_async_output_proc = False
1162
+
1163
+ def get_sliding_window(self) -> Optional[int]:
1164
+ """Get the sliding window size from the HF text config if present."""
1165
+ return getattr(self.hf_text_config, "sliding_window", None)
1166
+
1167
+ def get_vocab_size(self) -> int:
1168
+ return getattr(self.hf_text_config, "vocab_size", 0)
1169
+
1170
+ def get_hidden_size(self) -> int:
1171
+ return getattr(self.hf_text_config, "hidden_size", 0)
1172
+
1173
+ @property
1174
+ def is_deepseek_mla(self) -> bool:
1175
+ if not hasattr(self.hf_text_config, "model_type"):
1176
+ return False
1177
+ elif self.hf_text_config.model_type in \
1178
+ ('deepseek_v2', 'deepseek_v3', 'deepseek_mtp', 'kimi_k2'):
1179
+ return self.hf_text_config.kv_lora_rank is not None
1180
+ elif self.hf_text_config.model_type == 'eagle':
1181
+ # if the model is an EAGLE module, check for the
1182
+ # underlying architecture
1183
+ return self.hf_text_config.model.model_type in \
1184
+ ('deepseek_v2', 'deepseek_v3') \
1185
+ and self.hf_text_config.kv_lora_rank is not None
1186
+ return False
1187
+
1188
+ def get_head_size(self) -> int:
1189
+ # TODO remove hard code
1190
+ if self.is_deepseek_mla:
1191
+ qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
1192
+ 0)
1193
+ if self.use_mla:
1194
+ return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
1195
+ else:
1196
+ qk_nope_head_dim = getattr(self.hf_text_config,
1197
+ "qk_nope_head_dim", 0)
1198
+ if qk_rope_head_dim and qk_nope_head_dim:
1199
+ return qk_rope_head_dim + qk_nope_head_dim
1200
+
1201
+ if hasattr(self.hf_text_config,
1202
+ "model_type") and (self.hf_text_config.model_type
1203
+ == "zamba2"):
1204
+ return self.hf_text_config.attention_head_dim
1205
+
1206
+ if self.is_attention_free:
1207
+ return 0
1208
+
1209
+ # NOTE: Some configs may set head_dim=None in the config
1210
+ if getattr(self.hf_text_config, "head_dim", None) is not None:
1211
+ return self.hf_text_config.head_dim
1212
+
1213
+ # NOTE: Some models (such as PLaMo2.1) use `hidden_size_per_head`
1214
+ if getattr(self.hf_text_config, "hidden_size_per_head",
1215
+ None) is not None:
1216
+ return self.hf_text_config.hidden_size_per_head
1217
+
1218
+ # FIXME(woosuk): This may not be true for all models.
1219
+ return (self.hf_text_config.hidden_size //
1220
+ self.hf_text_config.num_attention_heads)
1221
+
1222
+ def get_total_num_kv_heads(self) -> int:
1223
+ """Returns the total number of KV heads."""
1224
+ # For GPTBigCode & Falcon:
1225
+ # NOTE: for falcon, when new_decoder_architecture is True, the
1226
+ # multi_query flag is ignored and we use n_head_kv for the number of
1227
+ # KV heads.
1228
+ falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
1229
+ new_decoder_arch_falcon = (
1230
+ self.hf_config.model_type in falcon_model_types
1231
+ and getattr(self.hf_config, "new_decoder_architecture", False))
1232
+ if not new_decoder_arch_falcon and getattr(self.hf_text_config,
1233
+ "multi_query", False):
1234
+ # Multi-query attention, only one KV head.
1235
+ # Currently, tensor parallelism is not supported in this case.
1236
+ return 1
1237
+
1238
+ # For DBRX and MPT
1239
+ if self.hf_config.model_type == "mpt":
1240
+ if "kv_n_heads" in self.hf_config.attn_config:
1241
+ return self.hf_config.attn_config["kv_n_heads"]
1242
+ return self.hf_config.num_attention_heads
1243
+ if self.hf_config.model_type == "dbrx":
1244
+ return getattr(self.hf_config.attn_config, "kv_n_heads",
1245
+ self.hf_config.num_attention_heads)
1246
+
1247
+ if self.hf_config.model_type == "nemotron-nas":
1248
+ for block in self.hf_config.block_configs:
1249
+ if not block.attention.no_op:
1250
+ return self.hf_config.num_attention_heads \
1251
+ // block.attention.n_heads_in_group
1252
+
1253
+ raise RuntimeError("Couldn't determine number of kv heads")
1254
+
1255
+ if self.is_attention_free:
1256
+ return 0
1257
+
1258
+ attributes = [
1259
+ # For Falcon:
1260
+ "n_head_kv",
1261
+ "num_kv_heads",
1262
+ # For LLaMA-2:
1263
+ "num_key_value_heads",
1264
+ # For ChatGLM:
1265
+ "multi_query_group_num",
1266
+ ]
1267
+ for attr in attributes:
1268
+ num_kv_heads = getattr(self.hf_text_config, attr, None)
1269
+ if num_kv_heads is not None:
1270
+ return num_kv_heads
1271
+
1272
+ # For non-grouped-query attention models, the number of KV heads is
1273
+ # equal to the number of attention heads.
1274
+ return self.hf_text_config.num_attention_heads
1275
+
1276
+ def get_num_kv_heads(self, parallel_config: ParallelConfig) -> int:
1277
+ """Returns the number of KV heads per GPU."""
1278
+ if self.use_mla:
1279
+ # When using MLA during decode it becomes MQA
1280
+ return 1
1281
+
1282
+ total_num_kv_heads = self.get_total_num_kv_heads()
1283
+ # If tensor parallelism is used, we divide the number of KV heads by
1284
+ # the tensor parallel size. We will replicate the KV heads in the
1285
+ # case where the number of KV heads is smaller than the tensor
1286
+ # parallel size so each GPU has at least one KV head.
1287
+ return max(1,
1288
+ total_num_kv_heads // parallel_config.tensor_parallel_size)
1289
+
1290
+ def get_num_attention_heads(self, parallel_config: ParallelConfig) -> int:
1291
+ num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
1292
+ return num_heads // parallel_config.tensor_parallel_size
1293
+
1294
+ def get_layers_start_end_indices(
1295
+ self, parallel_config: ParallelConfig) -> tuple[int, int]:
1296
+ from vllm.distributed.utils import get_pp_indices
1297
+ if (self.hf_text_config.model_type == "deepseek_mtp"
1298
+ or self.hf_config.model_type == "mimo_mtp"
1299
+ or self.hf_config.model_type == "glm4_moe_mtp"
1300
+ or self.hf_config.model_type == "ernie_mtp"
1301
+ or self.hf_config.model_type == "qwen3_next_mtp"):
1302
+ total_num_hidden_layers = getattr(self.hf_text_config,
1303
+ "num_nextn_predict_layers", 0)
1304
+ else:
1305
+ total_num_hidden_layers = getattr(self.hf_text_config,
1306
+ "num_hidden_layers", 0)
1307
+ # the layout order is: DP x PP x TP
1308
+ pp_rank = (parallel_config.rank // parallel_config.tensor_parallel_size
1309
+ ) % parallel_config.pipeline_parallel_size
1310
+ pp_size = parallel_config.pipeline_parallel_size
1311
+ start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
1312
+ return start, end
1313
+
1314
+ def get_num_layers(self, parallel_config: ParallelConfig) -> int:
1315
+ start, end = self.get_layers_start_end_indices(parallel_config)
1316
+ return end - start
1317
+
1318
+ def get_num_layers_by_block_type(
1319
+ self,
1320
+ parallel_config: ParallelConfig,
1321
+ block_type: LayerBlockType = LayerBlockType.attention,
1322
+ ) -> int:
1323
+ # This function relies on 'layers_block_type' in hf_config,
1324
+ # for w/o this attribute, we will need to have workarounds like so
1325
+ attn_block_type = block_type == LayerBlockType.attention
1326
+ is_transformer = not self.is_hybrid and \
1327
+ not self.has_noops and \
1328
+ not self.is_attention_free
1329
+ start, end = self.get_layers_start_end_indices(parallel_config)
1330
+
1331
+ if is_transformer:
1332
+ # Handle the basic case first
1333
+ return end - start if attn_block_type else 0
1334
+ elif self.is_attention_free:
1335
+ # Attention free
1336
+ # Note that this code assumes there
1337
+ # is only one type of attention-free block type.
1338
+ return 0 if attn_block_type else end - start
1339
+ elif self.has_noops:
1340
+ block_configs = self.hf_config.block_configs
1341
+ return sum(not bc.attention.no_op
1342
+ for bc in block_configs[start:end])
1343
+ else:
1344
+ # Hybrid model Jamba
1345
+ layers_block_type_value = getattr(self.hf_text_config,
1346
+ "layers_block_type", None)
1347
+ if layers_block_type_value is not None:
1348
+ if hasattr(self.hf_text_config,
1349
+ "model_type") and (self.hf_text_config.model_type
1350
+ == "zamba2"):
1351
+ if attn_block_type:
1352
+ return sum(t == "hybrid"
1353
+ for t in layers_block_type_value[start:end])
1354
+ else:
1355
+ return self.get_num_layers(parallel_config)
1356
+ return sum(t == block_type.value
1357
+ for t in layers_block_type_value[start:end])
1358
+
1359
+ # Hybrid model Minimax
1360
+ attn_type_list = getattr(self.hf_config, "attn_type_list", None)
1361
+ if attn_type_list:
1362
+ return sum(t == 1 for t in attn_type_list[start:end])
1363
+
1364
+ # Hybrid model Qwen3Next
1365
+ layer_types_value = getattr(self.hf_config, "layer_types", None)
1366
+ if layer_types_value is not None:
1367
+ if getattr(block_type, "value", block_type) == "attention":
1368
+ return sum(t == "full_attention"
1369
+ for t in layer_types_value[start:end])
1370
+ elif getattr(block_type, "value",
1371
+ block_type) == "linear_attention":
1372
+ return sum(t == "linear_attention"
1373
+ for t in layer_types_value[start:end])
1374
+ else:
1375
+ return sum(t == getattr(block_type, "value", block_type)
1376
+ for t in layer_types_value[start:end])
1377
+
1378
+ if (layers_block_type_value is None and attn_type_list is None
1379
+ and layer_types_value is None):
1380
+ raise ValueError(
1381
+ "The model is an hybrid without a"
1382
+ "layers_block_type or an attn_type_list, or a layer_types "
1383
+ "in the hf_config, cannot determine the num of "
1384
+ f"{block_type.value} layers")
1385
+
1386
+ def get_mamba_chunk_size(self) -> Optional[int]:
1387
+ """
1388
+ Returns the mamba chunk size if it exists
1389
+ """
1390
+ # used by e.g. Bamba, FalconH1, Granite, PLaMo2
1391
+ chunk_size = getattr(self.hf_text_config, "mamba_chunk_size", None)
1392
+ if chunk_size is None:
1393
+ # used by e.g. Mamba2, NemotronH, Zamba
1394
+ chunk_size = getattr(self.hf_text_config, "chunk_size", None)
1395
+ return chunk_size
1396
+
1397
+ def get_multimodal_config(self) -> MultiModalConfig:
1398
+ """
1399
+ Get the multimodal configuration of the model.
1400
+
1401
+ Raises:
1402
+ ValueError: If the model is not multimodal.
1403
+ """
1404
+ if self.multimodal_config is None:
1405
+ raise ValueError("The model is not multimodal.")
1406
+
1407
+ return self.multimodal_config
1408
+
1409
+ def try_get_generation_config(self) -> dict[str, Any]:
1410
+ """
1411
+ This method attempts to retrieve the non-default values of the
1412
+ generation config for this model.
1413
+
1414
+ The generation config can contain information about special tokens, as
1415
+ well as sampling parameters. Which is why this method exists separately
1416
+ to `get_diff_sampling_param`.
1417
+
1418
+ Returns:
1419
+ A dictionary containing the non-default generation config.
1420
+ """
1421
+ if self.generation_config in {"auto", "vllm"}:
1422
+ config = try_get_generation_config(
1423
+ self.hf_config_path or self.model,
1424
+ trust_remote_code=self.trust_remote_code,
1425
+ revision=self.revision,
1426
+ )
1427
+ else:
1428
+ config = try_get_generation_config(
1429
+ self.generation_config,
1430
+ trust_remote_code=self.trust_remote_code,
1431
+ )
1432
+
1433
+ if config is None:
1434
+ return {}
1435
+
1436
+ return config.to_diff_dict()
1437
+
1438
+ def get_diff_sampling_param(self) -> dict[str, Any]:
1439
+ """
1440
+ This method returns a dictionary containing the non-default sampling
1441
+ parameters with `override_generation_config` applied.
1442
+
1443
+ The default sampling parameters are:
1444
+
1445
+ - vLLM's neutral defaults if `self.generation_config="vllm"`
1446
+ - the model's defaults if `self.generation_config="auto"`
1447
+ - as defined in `generation_config.json` if
1448
+ `self.generation_config="path/to/generation_config/dir"`
1449
+
1450
+ Returns:
1451
+ A dictionary containing the non-default sampling parameters.
1452
+ """
1453
+ if self.generation_config == "vllm":
1454
+ config = {}
1455
+ else:
1456
+ config = self.try_get_generation_config()
1457
+
1458
+ # Overriding with given generation config
1459
+ config.update(self.override_generation_config)
1460
+
1461
+ available_params = [
1462
+ "repetition_penalty",
1463
+ "temperature",
1464
+ "top_k",
1465
+ "top_p",
1466
+ "min_p",
1467
+ "max_new_tokens",
1468
+ ]
1469
+ if any(p in config for p in available_params):
1470
+ diff_sampling_param = {
1471
+ p: config.get(p)
1472
+ for p in available_params if config.get(p) is not None
1473
+ }
1474
+ # Huggingface definition of max_new_tokens is equivalent
1475
+ # to vLLM's max_tokens
1476
+ if "max_new_tokens" in diff_sampling_param:
1477
+ diff_sampling_param["max_tokens"] = diff_sampling_param.pop(
1478
+ "max_new_tokens")
1479
+ else:
1480
+ diff_sampling_param = {}
1481
+
1482
+ if diff_sampling_param:
1483
+ logger.warning_once(
1484
+ "Default sampling parameters have been overridden by the "
1485
+ "model's Hugging Face generation config recommended from the "
1486
+ "model creator. If this is not intended, please relaunch "
1487
+ "vLLM instance with `--generation-config vllm`.")
1488
+ return diff_sampling_param
1489
+
1490
+ @property
1491
+ def is_encoder_decoder(self) -> bool:
1492
+ """Extract the HF encoder/decoder model flag."""
1493
+ return is_encoder_decoder(self.hf_config)
1494
+
1495
+ @property
1496
+ def uses_mrope(self) -> bool:
1497
+ return uses_mrope(self.hf_config)
1498
+
1499
+ @property
1500
+ def is_multimodal_model(self) -> bool:
1501
+ return self.multimodal_config is not None
1502
+
1503
+ @property
1504
+ def is_multimodal_raw_input_only_model(self) -> bool:
1505
+ return self._model_info.supports_multimodal_raw_input_only
1506
+
1507
+ @property
1508
+ def is_cross_encoder(self) -> bool:
1509
+ return (self._model_info.supports_cross_encoding
1510
+ or self.convert_type == "classify")
1511
+
1512
+ @property
1513
+ def is_pp_supported(self) -> bool:
1514
+ return self._model_info.supports_pp
1515
+
1516
+ @property
1517
+ def is_attention_free(self) -> bool:
1518
+ return self._model_info.is_attention_free
1519
+
1520
+ @property
1521
+ def is_hybrid(self) -> bool:
1522
+ return self._model_info.is_hybrid
1523
+
1524
+ @property
1525
+ def has_noops(self) -> bool:
1526
+ return self._model_info.has_noops
1527
+
1528
+ @property
1529
+ def has_inner_state(self):
1530
+ return self._model_info.has_inner_state
1531
+
1532
+ @property
1533
+ def is_v1_compatible(self) -> bool:
1534
+ return not self._model_info.supports_v0_only
1535
+
1536
+ @property
1537
+ def use_mla(self) -> bool:
1538
+ return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE
1539
+
1540
+ @property
1541
+ def is_matryoshka(self) -> bool:
1542
+ return (bool(getattr(self.hf_config, "matryoshka_dimensions", None))
1543
+ or getattr(self.hf_config, "is_matryoshka", False))
1544
+
1545
+ @property
1546
+ def matryoshka_dimensions(self):
1547
+ return getattr(self.hf_config, "matryoshka_dimensions", None)
1548
+
1549
+ @property
1550
+ def use_pad_token(self) -> bool:
1551
+ # cross_encoder models defaults to using pad_token.
1552
+ # `llm as reranker` models defaults to not using pad_token.
1553
+ return getattr(self.hf_config, "use_pad_token", True)
1554
+
1555
+ @property
1556
+ def head_dtype(self) -> torch.dtype:
1557
+ """
1558
+ "head" refers to the last Linear layer(s) of an LLM,
1559
+ such as the lm_head in a generation model,
1560
+ or the score or classifier in a classification model.
1561
+
1562
+ `head_dtype` currently only supports pooling models.\n
1563
+ - The pooling model defaults to using fp32 head,
1564
+ you can use --hf-overrides '{"head_dtype": "model"}' to disable it.
1565
+ """
1566
+
1567
+ head_dtype = _get_head_dtype(config=self.hf_config,
1568
+ dtype=self.dtype,
1569
+ runner_type=self.runner_type)
1570
+
1571
+ if self.runner_type != "pooling" and head_dtype != self.dtype:
1572
+ logger.warning_once(
1573
+ "`head_dtype` currently only supports pooling models."
1574
+ "fallback to model dtype [%s].", self.dtype)
1575
+ return self.dtype
1576
+
1577
+ if head_dtype not in current_platform.supported_dtypes:
1578
+ logger.warning_once(
1579
+ "The current platform does not support [%s] head dtype, "
1580
+ "fallback to model dtype [%s].", head_dtype, self.dtype)
1581
+ return self.dtype
1582
+
1583
+ logger.debug_once("head dtype: %s", head_dtype)
1584
+ return head_dtype
1585
+
1586
+ def get_and_verify_max_len(self, max_model_len: int):
1587
+ # Consider max_model_len in tokenizer_config only when
1588
+ # pooling models use absolute position_embedding.
1589
+ tokenizer_config = None
1590
+ if (self.runner_type == "pooling" and getattr(
1591
+ self.hf_config, "position_embedding_type", "") == "absolute"):
1592
+ tokenizer_config = try_get_tokenizer_config(
1593
+ self.tokenizer,
1594
+ trust_remote_code=self.trust_remote_code,
1595
+ revision=self.tokenizer_revision)
1596
+ max_model_len = _get_and_verify_max_len(
1597
+ hf_config=self.hf_text_config,
1598
+ tokenizer_config=tokenizer_config,
1599
+ max_model_len=max_model_len,
1600
+ disable_sliding_window=self.disable_sliding_window,
1601
+ sliding_window=self.get_sliding_window(),
1602
+ spec_target_max_model_len=self.spec_target_max_model_len,
1603
+ encoder_config=self.encoder_config)
1604
+ logger.info("Using max model len %s", max_model_len)
1605
+ return max_model_len
1606
+
1607
+
1608
+def get_served_model_name(model: str,
1609
+ served_model_name: Optional[Union[str, list[str]]]):
1610
+ """
1611
+ If the input is a non-empty list, the first model_name in
1612
+ `served_model_name` is taken.
1613
+ If the input is a non-empty string, it is used directly.
1614
+ For cases where the input is either an empty string or an
1615
+ empty list, the fallback is to use `self.model`.
1616
+ """
1617
+ if not served_model_name:
1618
+ return model
1619
+ if isinstance(served_model_name, list):
1620
+ return served_model_name[0]
1621
+ return served_model_name
1622
+
1623
+
1624
+# Some model suffixes are based on auto classes from Transformers:
1625
+# https://huggingface.co/docs/transformers/en/model_doc/auto
1626
+# NOTE: Items higher on this list priority over lower ones
1627
+_SUFFIX_TO_DEFAULTS: list[tuple[str, tuple[RunnerType, ConvertType]]] = [
1628
+ ("ForCausalLM", ("generate", "none")),
1629
+ ("ForConditionalGeneration", ("generate", "none")),
1630
+ ("ChatModel", ("generate", "none")),
1631
+ ("LMHeadModel", ("generate", "none")),
1632
+ ("ForTextEncoding", ("pooling", "embed")),
1633
+ ("EmbeddingModel", ("pooling", "embed")),
1634
+ ("ForSequenceClassification", ("pooling", "classify")),
1635
+ ("ForAudioClassification", ("pooling", "classify")),
1636
+ ("ForImageClassification", ("pooling", "classify")),
1637
+ ("ForVideoClassification", ("pooling", "classify")),
1638
+ ("ClassificationModel", ("pooling", "classify")),
1639
+ ("ForRewardModeling", ("pooling", "reward")),
1640
+ ("RewardModel", ("pooling", "reward")),
1641
+ # Let other `*Model`s take priority
1642
+ ("Model", ("pooling", "embed")),
1643
+]
1644
+
1645
+
1646
+def iter_architecture_defaults():
1647
+ yield from _SUFFIX_TO_DEFAULTS
1648
+
1649
+
1650
+def try_match_architecture_defaults(
1651
+ architecture: str,
1652
+ *,
1653
+ runner_type: Optional[RunnerType] = None,
1654
+ convert_type: Optional[ConvertType] = None,
1655
+) -> Optional[tuple[str, tuple[RunnerType, ConvertType]]]:
1656
+ for suffix, (default_runner_type,
1657
+ default_convert_type) in iter_architecture_defaults():
1658
+ if ((runner_type is None or runner_type == default_runner_type) and
1659
+ (convert_type is None or convert_type == default_convert_type)
1660
+ and architecture.endswith(suffix)):
1661
+ return suffix, (default_runner_type, default_convert_type)
1662
+
1663
+ return None
1664
+
1665
+
1666
+_STR_DTYPE_TO_TORCH_DTYPE = {
1667
+ "half": torch.float16,
1668
+ "float16": torch.float16,
1669
+ "float": torch.float32,
1670
+ "float32": torch.float32,
1671
+ "bfloat16": torch.bfloat16,
1672
+}
1673
+
1674
+# model_type -> reason
1675
+_FLOAT16_NOT_SUPPORTED_MODELS = {
1676
+ "gemma2": "Numerical instability. Please use bfloat16 or float32 instead.",
1677
+ "gemma3": "Numerical instability. Please use bfloat16 or float32 instead.",
1678
+ "gemma3_text":
1679
+ "Numerical instability. Please use bfloat16 or float32 instead.",
1680
+ "plamo2": "Numerical instability. Please use bfloat16 or float32 instead.",
1681
+ "glm4": "Numerical instability. Please use bfloat16 or float32 instead.",
1682
+}
1683
+
1684
+
1685
+def _is_valid_dtype(model_type: str, dtype: torch.dtype):
1686
+ if model_type in _FLOAT16_NOT_SUPPORTED_MODELS and dtype == torch.float16: # noqa: E501, SIM103
1687
+ return False
1688
+
1689
+ return True
1690
+
1691
+
1692
+def _check_valid_dtype(model_type: str, dtype: torch.dtype):
1693
+ if model_type in _FLOAT16_NOT_SUPPORTED_MODELS and dtype == torch.float16:
1694
+ reason = _FLOAT16_NOT_SUPPORTED_MODELS[model_type]
1695
+ raise ValueError(f"The model type {model_type!r} "
1696
+ f"does not support float16. Reason: {reason}")
1697
+
1698
+ return True
1699
+
1700
+
1701
+def _find_dtype(
1702
+ model_id: str,
1703
+ config: PretrainedConfig,
1704
+ *,
1705
+ revision: Optional[str],
1706
+):
1707
+ # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
1708
+ # because config.torch_dtype can be None.
1709
+ config_dtype = getattr(config, "torch_dtype", None)
1710
+
1711
+ # Fallbacks for multi-modal models if the root config
1712
+ # does not define torch_dtype
1713
+ if config_dtype is None:
1714
+ config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
1715
+ if config_dtype is None and hasattr(config, "vision_config"):
1716
+ config_dtype = getattr(config.vision_config, "torch_dtype", None)
1717
+ if config_dtype is None and hasattr(config, "encoder_config"):
1718
+ config_dtype = getattr(config.encoder_config, "torch_dtype", None)
1719
+
1720
+ # Try to read the dtype of the weights if they are in safetensors format
1721
+ if config_dtype is None:
1722
+ repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
1723
+
1724
+ if repo_mt and (files_mt := repo_mt.files_metadata):
1725
+ param_dtypes: set[torch.dtype] = {
1726
+ _SAFETENSORS_TO_TORCH_DTYPE[dtype_str]
1727
+ for file_mt in files_mt.values()
1728
+ for dtype_str in file_mt.parameter_count
1729
+ if dtype_str in _SAFETENSORS_TO_TORCH_DTYPE
1730
+ }
1731
+
1732
+ if param_dtypes:
1733
+ return common_broadcastable_dtype(param_dtypes)
1734
+
1735
+ if config_dtype is None:
1736
+ config_dtype = torch.float32
1737
+
1738
+ return config_dtype
1739
+
1740
+
1741
+def _resolve_auto_dtype(
1742
+ model_type: str,
1743
+ config_dtype: torch.dtype,
1744
+ *,
1745
+ is_pooling_model: bool,
1746
+):
1747
+ from vllm.platforms import current_platform
1748
+
1749
+ supported_dtypes = [
1750
+ dtype for dtype in current_platform.supported_dtypes
1751
+ if _is_valid_dtype(model_type, dtype)
1752
+ ]
1753
+
1754
+ if is_pooling_model and torch.float16 in supported_dtypes:
1755
+ preferred_dtype = torch.float16
1756
+ else:
1757
+ preferred_dtype = supported_dtypes[0]
1758
+
1759
+ # Downcast for float32 models
1760
+ if config_dtype == torch.float32:
1761
+ config_dtype = preferred_dtype
1762
+
1763
+ if config_dtype in supported_dtypes:
1764
+ return config_dtype
1765
+
1766
+ # Ensure device compatibility
1767
+ device_name = current_platform.get_device_name()
1768
+ device_capability = current_platform.get_device_capability()
1769
+
1770
+ if device_capability is None:
1771
+ device_str = f"{device_name!r}"
1772
+ else:
1773
+ version_str = device_capability.as_version_str()
1774
+ device_str = f"{device_name!r} (with compute capability {version_str})"
1775
+
1776
+ logger.warning(
1777
+ "Your device %s doesn't support %s. "
1778
+ "Falling back to %s for compatibility.",
1779
+ device_str,
1780
+ config_dtype,
1781
+ preferred_dtype,
1782
+ )
1783
+
1784
+ return preferred_dtype
1785
+
1786
+
1787
+def _get_and_verify_dtype(
1788
+ model_id: str,
1789
+ config: PretrainedConfig,
1790
+ dtype: Union[str, torch.dtype],
1791
+ *,
1792
+ is_pooling_model: bool,
1793
+ revision: Optional[str] = None,
1794
+) -> torch.dtype:
1795
+ config_dtype = _find_dtype(model_id, config, revision=revision)
1796
+ model_type = config.model_type
1797
+
1798
+ if isinstance(dtype, str):
1799
+ dtype = dtype.lower()
1800
+ if dtype == "auto":
1801
+ # Set default dtype from model config
1802
+ torch_dtype = _resolve_auto_dtype(
1803
+ model_type,
1804
+ config_dtype,
1805
+ is_pooling_model=is_pooling_model,
1806
+ )
1807
+ else:
1808
+ if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
1809
+ raise ValueError(f"Unknown dtype: {dtype!r}")
1810
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
1811
+ elif isinstance(dtype, torch.dtype):
1812
+ torch_dtype = dtype
1813
+ else:
1814
+ raise ValueError(f"Unknown dtype: {dtype}")
1815
+
1816
+ _check_valid_dtype(model_type, torch_dtype)
1817
+
1818
+ if torch_dtype != config_dtype:
1819
+ if torch_dtype == torch.float32:
1820
+ # Upcasting to float32 is allowed.
1821
+ logger.info("Upcasting %s to %s.", config_dtype, torch_dtype)
1822
+ elif config_dtype == torch.float32:
1823
+ # Downcasting from float32 to float16 or bfloat16 is allowed.
1824
+ logger.info("Downcasting %s to %s.", config_dtype, torch_dtype)
1825
+ else:
1826
+ # Casting between float16 and bfloat16 is allowed with a warning.
1827
+ logger.warning("Casting %s to %s.", config_dtype, torch_dtype)
1828
+
1829
+ return torch_dtype
1830
+
1831
+
1832
+def _get_head_dtype(config: PretrainedConfig, dtype: torch.dtype,
1833
+ runner_type: str) -> torch.dtype:
1834
+ head_dtype: Optional[Union[str,
1835
+ torch.dtype]] = getattr(config, "head_dtype",
1836
+ None)
1837
+
1838
+ if head_dtype == "model":
1839
+ return dtype
1840
+ elif isinstance(head_dtype, str):
1841
+ head_dtype = head_dtype.lower()
1842
+ if head_dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
1843
+ raise ValueError(f"Unknown dtype: {head_dtype!r}")
1844
+ return _STR_DTYPE_TO_TORCH_DTYPE[head_dtype]
1845
+ elif isinstance(head_dtype, torch.dtype):
1846
+ return head_dtype
1847
+ elif head_dtype is None:
1848
+ if torch.float32 not in current_platform.supported_dtypes:
1849
+ return dtype
1850
+ if runner_type == "pooling":
1851
+ return torch.float32
1852
+ return dtype
1853
+ else:
1854
+ raise ValueError(f"Unknown dtype: {head_dtype}")
1855
+
1856
+
1857
+def _get_and_verify_max_len(
1858
+ hf_config: PretrainedConfig,
1859
+ tokenizer_config: Optional[dict],
1860
+ max_model_len: Optional[int],
1861
+ disable_sliding_window: bool,
1862
+ sliding_window: Optional[int],
1863
+ spec_target_max_model_len: Optional[int] = None,
1864
+ encoder_config: Optional[Any] = None,
1865
+) -> int:
1866
+ """Get and verify the model's maximum length."""
1867
+ derived_max_model_len = float("inf")
1868
+ possible_keys = [
1869
+ # OPT
1870
+ "max_position_embeddings",
1871
+ # GPT-2
1872
+ "n_positions",
1873
+ # MPT
1874
+ "max_seq_len",
1875
+ # ChatGLM2
1876
+ "seq_length",
1877
+ # Command-R
1878
+ "model_max_length",
1879
+ # Whisper
1880
+ "max_target_positions",
1881
+ # Others
1882
+ "max_sequence_length",
1883
+ "max_seq_length",
1884
+ "seq_len",
1885
+ ]
1886
+ # Choose the smallest "max_length" from the possible keys
1887
+ max_len_key = None
1888
+ for key in possible_keys:
1889
+ max_len = getattr(hf_config, key, None)
1890
+ if max_len is not None:
1891
+ max_len_key = key if max_len < derived_max_model_len \
1892
+ else max_len_key
1893
+ derived_max_model_len = min(derived_max_model_len, max_len)
1894
+ # For Command-R / Cohere, Cohere2 / Aya Vision models
1895
+ if tmp_max_len := getattr(hf_config, "model_max_length", None):
1896
+ max_len_key = "model_max_length"
1897
+ derived_max_model_len = tmp_max_len
1898
+
1899
+ # If sliding window is manually disabled, max_length should be less
1900
+ # than the sliding window length in the model config.
1901
+ if (disable_sliding_window and sliding_window is not None
1902
+ and sliding_window < derived_max_model_len):
1903
+ max_len_key = "sliding_window"
1904
+ derived_max_model_len = sliding_window
1905
+
1906
+ # Consider model_max_length in tokenizer_config
1907
+ if tokenizer_config:
1908
+ tokenizer_model_max_length = tokenizer_config.get(
1909
+ "model_max_length", derived_max_model_len)
1910
+ derived_max_model_len = min(derived_max_model_len,
1911
+ tokenizer_model_max_length)
1912
+
1913
+ # If none of the keys were found in the config, use a default and
1914
+ # log a warning.
1915
+ if derived_max_model_len == float("inf"):
1916
+ if max_model_len is not None:
1917
+ # If max_model_len is specified, we use it.
1918
+ return max_model_len
1919
+
1920
+ if spec_target_max_model_len is not None:
1921
+ # If this is a speculative draft model, we use the max model len
1922
+ # from the target model.
1923
+ return spec_target_max_model_len
1924
+
1925
+ default_max_len = 2048
1926
+ logger.warning(
1927
+ "The model's config.json does not contain any of the following "
1928
+ "keys to determine the original maximum length of the model: "
1929
+ "%s. Assuming the model's maximum length is %d.", possible_keys,
1930
+ default_max_len)
1931
+ derived_max_model_len = default_max_len
1932
+
1933
+ rope_scaling = getattr(hf_config, "rope_scaling", None)
1934
+ # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE
1935
+ # scaling, so we skip applying the scaling factor again.
1936
+ if rope_scaling is not None and "gemma3" not in hf_config.model_type:
1937
+ # No need to consider "type" key because of patch_rope_scaling when
1938
+ # loading HF config
1939
+ rope_type = rope_scaling["rope_type"]
1940
+
1941
+ if rope_type not in ("su", "longrope", "llama3"):
1942
+ if disable_sliding_window:
1943
+ # TODO(robertgshaw): Find a model that supports rope_scaling
1944
+ # with sliding window to see if this case should be allowed.
1945
+ raise NotImplementedError(
1946
+ "Disabling sliding window is not supported for models "
1947
+ "with rope_scaling. Please raise an issue so we can "
1948
+ "investigate.")
1949
+
1950
+ # NOTE: rope_type == "default" does not define factor
1951
+ # https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py
1952
+ scaling_factor = rope_scaling.get("factor", 1.0)
1953
+
1954
+ if rope_type == "yarn":
1955
+ derived_max_model_len = rope_scaling[
1956
+ "original_max_position_embeddings"]
1957
+ derived_max_model_len *= scaling_factor
1958
+
1959
+ if encoder_config and "max_seq_length" in encoder_config:
1960
+ derived_max_model_len = encoder_config["max_seq_length"]
1961
+
1962
+ # If the user specified a max length, make sure it is smaller than the
1963
+ # derived length from the HF model config.
1964
+ if max_model_len is None:
1965
+ max_model_len = int(derived_max_model_len)
1966
+ if current_platform.is_tpu():
1967
+ logger.warning(
1968
+ "--max-model-len is not specified, "
1969
+ "it's currently using model's default length %s, "
1970
+ "which might be too large."
1971
+ "Please input with --max-model-len based on your "
1972
+ "request input length and output length, to avoid "
1973
+ "unnecessary degradation.", max_model_len)
1974
+ elif max_model_len > derived_max_model_len:
1975
+ # Some models might have a separate key for specifying model_max_length
1976
+ # that will be bigger than derived_max_model_len. We compare user input
1977
+ # with model_max_length and allow this override when it's smaller.
1978
+ model_max_length = getattr(hf_config, "model_max_length", None)
1979
+ if model_max_length is not None and max_model_len <= model_max_length:
1980
+ if disable_sliding_window:
1981
+ # TODO(robertgshaw): Find a model that has model_max_length
1982
+ # with sliding window to see if this case should be allowed.
1983
+ raise NotImplementedError(
1984
+ "Disabling sliding window is not supported for models "
1985
+ "model_max_length in the config. Please raise an issue "
1986
+ "so we can investigate.")
1987
+ else:
1988
+ msg = (
1989
+ f"User-specified max_model_len ({max_model_len}) is greater "
1990
+ f"than the derived max_model_len ({max_len_key}="
1991
+ f"{derived_max_model_len} or model_max_length="
1992
+ f"{model_max_length} in model's config.json).")
1993
+ warning = (
1994
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN must be used with extreme "
1995
+ "caution. If the model uses relative position encoding (RoPE), "
1996
+ "positions exceeding derived_max_model_len lead to nan. If the "
1997
+ "model uses absolute position encoding, positions exceeding "
1998
+ "derived_max_model_len will cause a CUDA array out-of-bounds "
1999
+ "error.")
2000
+ if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN:
2001
+ logger.warning_once("%s %s", msg, warning)
2002
+ else:
2003
+ raise ValueError(
2004
+ f"{msg} To allow overriding this maximum, set "
2005
+ f"the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. {warning}")
2006
+ return int(max_model_len)
vllm/config/scheduler.py
+2
-6
@@ -3,7 +3,7 @@
3
4
import hashlib
5
from dataclasses import field
6
-from typing import TYPE_CHECKING, Any, Literal, Optional, Union
6
+from typing import Any, Literal, Optional, Union
7
8
from pydantic import SkipValidation, model_validator
9
from pydantic.dataclasses import dataclass
@@ -15,13 +15,9 @@ from vllm.utils import (DEFAULT_MAX_NUM_BATCHED_TOKENS,
15
MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
16
POOLING_MODEL_MAX_NUM_BATCHED_TOKENS)
17
18
-if TYPE_CHECKING:
19
- from vllm.config import RunnerType
20
-else:
21
- RunnerType = Any
22
-
18
logger = init_logger(__name__)
19
20
+RunnerType = Literal["generate", "pooling", "draft"]
21
PreemptionMode = Literal["swap", "recompute"]
22
SchedulerPolicy = Literal["fcfs", "priority"]
23
vllm/config/utils.py
+99
-1
@@ -1,8 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import ast
5
+import inspect
6
+import textwrap
7
from dataclasses import MISSING, Field, field, fields, is_dataclass
5
-from typing import TYPE_CHECKING, TypeVar
8
+from typing import TYPE_CHECKING, Any, TypeVar
9
+
10
+import regex as re
11
12
if TYPE_CHECKING:
13
from _typeshed import DataclassInstance
@@ -45,3 +50,96 @@ def get_field(cls: ConfigType, name: str) -> Field:
50
return field(default=default)
51
raise ValueError(
52
f"{cls.__name__}.{name} must have a default value or default factory.")
53
+
54
+
55
+def contains_object_print(text: str) -> bool:
56
+ """
57
+ Check if the text looks like a printed Python object, e.g.
58
+ contains any substring matching the pattern: "at 0xFFFFFFF>"
59
+ We match against 0x followed by 2-16 hex chars (there's
60
+ a max of 16 on a 64-bit system).
61
+
62
+ Args:
63
+ text (str): The text to check
64
+
65
+ Returns:
66
+ result (bool): `True` if a match is found, `False` otherwise.
67
+ """
68
+ pattern = r'at 0x[a-fA-F0-9]{2,16}>'
69
+ match = re.search(pattern, text)
70
+ return match is not None
71
+
72
+
73
+def assert_hashable(text: str) -> bool:
74
+ if not contains_object_print(text):
75
+ return True
76
+ raise AssertionError(
77
+ f"vLLM tried to hash some configs that may have Python objects ids "
78
+ f"in them. This is a bug, please file an issue. "
79
+ f"Text being hashed: {text}")
80
+
81
+
82
+def get_attr_docs(cls: type[Any]) -> dict[str, str]:
83
+ """
84
+ Get any docstrings placed after attribute assignments in a class body.
85
+
86
+ https://davidism.com/mit-license/
87
+ """
88
+
89
+ def pairwise(iterable):
90
+ """
91
+ Manually implement https://docs.python.org/3/library/itertools.html#itertools.pairwise
92
+
93
+ Can be removed when Python 3.9 support is dropped.
94
+ """
95
+ iterator = iter(iterable)
96
+ a = next(iterator, None)
97
+
98
+ for b in iterator:
99
+ yield a, b
100
+ a = b
101
+
102
+ try:
103
+ cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
104
+ except (OSError, KeyError, TypeError):
105
+ # HACK: Python 3.13+ workaround - set missing __firstlineno__
106
+ # Workaround can be removed after we upgrade to pydantic==2.12.0
107
+ with open(inspect.getfile(cls)) as f:
108
+ for i, line in enumerate(f):
109
+ if f"class {cls.__name__}" in line and ":" in line:
110
+ cls.__firstlineno__ = i + 1
111
+ break
112
+ cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
113
+
114
+ if not isinstance(cls_node, ast.ClassDef):
115
+ raise TypeError("Given object was not a class.")
116
+
117
+ out = {}
118
+
119
+ # Consider each pair of nodes.
120
+ for a, b in pairwise(cls_node.body):
121
+ # Must be an assignment then a constant string.
122
+ if (not isinstance(a, (ast.Assign, ast.AnnAssign))
123
+ or not isinstance(b, ast.Expr)
124
+ or not isinstance(b.value, ast.Constant)
125
+ or not isinstance(b.value.value, str)):
126
+ continue
127
+
128
+ doc = inspect.cleandoc(b.value.value)
129
+
130
+ # An assignment can have multiple targets (a = b = v), but an
131
+ # annotated assignment only has one target.
132
+ targets = a.targets if isinstance(a, ast.Assign) else [a.target]
133
+
134
+ for target in targets:
135
+ # Must be assigning to a plain name.
136
+ if not isinstance(target, ast.Name):
137
+ continue
138
+
139
+ out[target.id] = doc
140
+
141
+ return out
142
+
143
+
144
+def is_init_field(cls: ConfigType, name: str) -> bool:
145
+ return next(f for f in fields(cls) if f.name == name).init
vllm/engine/arg_utils.py
+6
-9
@@ -27,11 +27,11 @@ from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
27
EPLBConfig, HfOverrides, KVEventsConfig,
28
KVTransferConfig, LoadConfig, LogprobsMode,
29
LoRAConfig, MambaDType, MMEncoderTPMode, ModelConfig,
30
- ModelDType, ModelImpl, ObservabilityConfig,
31
- ParallelConfig, PoolerConfig, PrefixCachingHashAlgo,
32
- RunnerOption, SchedulerConfig, SchedulerPolicy,
33
- SpeculativeConfig, StructuredOutputsConfig,
34
- TaskOption, TokenizerMode, VllmConfig, get_attr_docs)
30
+ ModelDType, ObservabilityConfig, ParallelConfig,
31
+ PoolerConfig, PrefixCachingHashAlgo, RunnerOption,
32
+ SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
33
+ StructuredOutputsConfig, TaskOption, TokenizerMode,
34
+ VllmConfig, get_attr_docs)
35
from vllm.config.multimodal import MMCacheType, MultiModalConfig
36
from vllm.config.parallel import ExpertPlacementStrategy
37
from vllm.config.utils import get_field
@@ -548,7 +548,6 @@ class EngineArgs:
548
model_group.add_argument("--max-logprobs",
549
**model_kwargs["max_logprobs"])
550
model_group.add_argument("--logprobs-mode",
551
- choices=[f.value for f in LogprobsMode],
551
**model_kwargs["logprobs_mode"])
552
model_group.add_argument("--disable-sliding-window",
553
**model_kwargs["disable_sliding_window"])
@@ -593,9 +592,7 @@ class EngineArgs:
592
**model_kwargs["override_generation_config"])
593
model_group.add_argument("--enable-sleep-mode",
594
**model_kwargs["enable_sleep_mode"])
596
- model_group.add_argument("--model-impl",
597
- choices=[f.value for f in ModelImpl],
598
- **model_kwargs["model_impl"])
595
+ model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
596
model_group.add_argument("--override-attention-dtype",
597
**model_kwargs["override_attention_dtype"])
598
model_group.add_argument("--logits-processors",
vllm/model_executor/model_loader/utils.py
+3
-4
@@ -13,8 +13,7 @@ from torch import nn
13
from typing_extensions import assert_never
14
15
from vllm.attention import Attention
16
-from vllm.config import (ModelConfig, ModelImpl, VllmConfig,
17
- set_current_vllm_config)
16
+from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.linear import QKVCrossParallelLinear
19
from vllm.model_executor.layers.quantization.base_config import (
@@ -176,8 +175,8 @@ def get_model_architecture(
175
)
176
177
if arch == model_config._get_transformers_backend_cls():
179
- assert model_config.model_impl != ModelImpl.VLLM
180
- if model_config.model_impl == ModelImpl.AUTO:
178
+ assert model_config.model_impl != "vllm"
179
+ if model_config.model_impl == "auto":
180
logger.warning_once(
181
"%s has no vLLM implementation, falling back to Transformers "
182
"implementation. Some features may not be supported and "
vllm/model_executor/models/registry.py
+11
-11
@@ -19,7 +19,7 @@ from typing import Callable, Optional, TypeVar, Union
19
import torch.nn as nn
20
import transformers
21
22
-from vllm.config import (ModelConfig, ModelImpl, iter_architecture_defaults,
22
+from vllm.config import (ModelConfig, iter_architecture_defaults,
23
try_match_architecture_defaults)
24
from vllm.logger import init_logger
25
from vllm.transformers_utils.dynamic_module import (
@@ -587,7 +587,7 @@ class _ModelRegistry:
587
if model_module is not None:
588
break
589
else:
590
- if model_config.model_impl != ModelImpl.TRANSFORMERS:
590
+ if model_config.model_impl != "transformers":
591
return None
592
593
raise ValueError(
@@ -598,7 +598,7 @@ class _ModelRegistry:
598
"'auto_map' (relevant if the model is custom).")
599
600
if not model_module.is_backend_compatible():
601
- if model_config.model_impl != ModelImpl.TRANSFORMERS:
601
+ if model_config.model_impl != "transformers":
602
return None
603
604
raise ValueError(
@@ -644,20 +644,20 @@ class _ModelRegistry:
644
raise ValueError("No model architectures are specified")
645
646
# Require transformers impl
647
- if model_config.model_impl == ModelImpl.TRANSFORMERS:
647
+ if model_config.model_impl == "transformers":
648
arch = self._try_resolve_transformers(architectures[0],
649
model_config)
650
if arch is not None:
651
model_info = self._try_inspect_model_cls(arch)
652
if model_info is not None:
653
return (model_info, arch)
654
- elif model_config.model_impl == ModelImpl.TERRATORCH:
654
+ elif model_config.model_impl == "terratorch":
655
model_info = self._try_inspect_model_cls("Terratorch")
656
return (model_info, "Terratorch")
657
658
# Fallback to transformers impl (after resolving convert_type)
659
if (all(arch not in self.models for arch in architectures)
660
- and model_config.model_impl == ModelImpl.AUTO
660
+ and model_config.model_impl == "auto"
661
and getattr(model_config, "convert_type", "none") == "none"):
662
arch = self._try_resolve_transformers(architectures[0],
663
model_config)
@@ -674,7 +674,7 @@ class _ModelRegistry:
674
675
# Fallback to transformers impl (before resolving runner_type)
676
if (all(arch not in self.models for arch in architectures)
677
- and model_config.model_impl == ModelImpl.AUTO):
677
+ and model_config.model_impl == "auto"):
678
arch = self._try_resolve_transformers(architectures[0],
679
model_config)
680
if arch is not None:
@@ -695,14 +695,14 @@ class _ModelRegistry:
695
raise ValueError("No model architectures are specified")
696
697
# Require transformers impl
698
- if model_config.model_impl == ModelImpl.TRANSFORMERS:
698
+ if model_config.model_impl == "transformers":
699
arch = self._try_resolve_transformers(architectures[0],
700
model_config)
701
if arch is not None:
702
model_cls = self._try_load_model_cls(arch)
703
if model_cls is not None:
704
return (model_cls, arch)
705
- elif model_config.model_impl == ModelImpl.TERRATORCH:
705
+ elif model_config.model_impl == "terratorch":
706
arch = "Terratorch"
707
model_cls = self._try_load_model_cls(arch)
708
if model_cls is not None:
@@ -710,7 +710,7 @@ class _ModelRegistry:
710
711
# Fallback to transformers impl (after resolving convert_type)
712
if (all(arch not in self.models for arch in architectures)
713
- and model_config.model_impl == ModelImpl.AUTO
713
+ and model_config.model_impl == "auto"
714
and getattr(model_config, "convert_type", "none") == "none"):
715
arch = self._try_resolve_transformers(architectures[0],
716
model_config)
@@ -727,7 +727,7 @@ class _ModelRegistry:
727
728
# Fallback to transformers impl (before resolving runner_type)
729
if (all(arch not in self.models for arch in architectures)
730
- and model_config.model_impl == ModelImpl.AUTO):
730
+ and model_config.model_impl == "auto"):
731
arch = self._try_resolve_transformers(architectures[0],
732
model_config)
733
if arch is not None:
vllm/v1/sample/ops/topk_topp_sampler.py
+5
-8
@@ -29,15 +29,12 @@ class TopKTopPSampler(nn.Module):
29
Implementations may update the logits tensor in-place.
30
"""
31
32
- def __init__(
33
- self,
34
- logprobs_mode: LogprobsMode = LogprobsMode.RAW_LOGPROBS) -> None:
32
+ def __init__(self, logprobs_mode: LogprobsMode = "raw_logprobs") -> None:
33
super().__init__()
34
self.logprobs_mode = logprobs_mode
35
# flashinfer optimization does not apply if intermediate
36
# logprobs/logits after top_k/top_p need to be returned
39
- if logprobs_mode not in (LogprobsMode.PROCESSED_LOGITS,
40
- LogprobsMode.PROCESSED_LOGPROBS
37
+ if logprobs_mode not in ("processed_logits", "processed_logprobs"
38
) and current_platform.is_cuda():
39
if is_flashinfer_available:
40
flashinfer_version = flashinfer.__version__
@@ -90,9 +87,9 @@ class TopKTopPSampler(nn.Module):
87
"""
88
logits = self.apply_top_k_top_p(logits, k, p)
89
logits_to_return = None
93
- if self.logprobs_mode == LogprobsMode.PROCESSED_LOGITS:
90
+ if self.logprobs_mode == "processed_logits":
91
logits_to_return = logits
95
- elif self.logprobs_mode == LogprobsMode.PROCESSED_LOGPROBS:
92
+ elif self.logprobs_mode == "processed_logprobs":
93
logits_to_return = logits.log_softmax(dim=-1, dtype=torch.float32)
94
probs = logits.softmax(dim=-1, dtype=torch.float32)
95
return random_sample(probs, generators), logits_to_return
@@ -115,7 +112,7 @@ class TopKTopPSampler(nn.Module):
112
"PyTorch-native implementation.")
113
return self.forward_native(logits, generators, k, p)
114
assert self.logprobs_mode not in (
118
- LogprobsMode.PROCESSED_LOGITS, LogprobsMode.PROCESSED_LOGPROBS
115
+ "processed_logits", "processed_logprobs"
116
), "FlashInfer does not support returning logits/logprobs"
117
# flashinfer sampling functions expect contiguous logits.
118
# In flex_attn/triton_attn fp32 inference, logits can be non-contiguous
vllm/v1/sample/sampler.py
+5
-6
@@ -60,8 +60,7 @@ class Sampler(nn.Module):
60
9. Return the final `SamplerOutput`.
61
"""
62
63
- def __init__(self,
64
- logprobs_mode: LogprobsMode = LogprobsMode.RAW_LOGPROBS):
63
+ def __init__(self, logprobs_mode: LogprobsMode = "raw_logprobs"):
64
super().__init__()
65
self.topk_topp_sampler = TopKTopPSampler(logprobs_mode)
66
self.pin_memory = is_pin_memory_available()
@@ -78,9 +77,9 @@ class Sampler(nn.Module):
77
# is used for sampling (after penalties and temperature scaling).
78
num_logprobs = sampling_metadata.max_num_logprobs
79
if num_logprobs is not None:
81
- if self.logprobs_mode == LogprobsMode.RAW_LOGPROBS:
80
+ if self.logprobs_mode == "raw_logprobs":
81
raw_logprobs = self.compute_logprobs(logits)
83
- elif self.logprobs_mode == LogprobsMode.RAW_LOGITS:
82
+ elif self.logprobs_mode == "raw_logits":
83
raw_logprobs = logits.clone()
84
85
# Use float32 for the logits.
@@ -156,9 +155,9 @@ class Sampler(nn.Module):
155
if sampling_metadata.all_greedy:
156
processed_logprobs = None
157
if sampling_metadata.max_num_logprobs is not None:
159
- if self.logprobs_mode == LogprobsMode.PROCESSED_LOGITS:
158
+ if self.logprobs_mode == "processed_logits":
159
processed_logprobs = logits
161
- elif self.logprobs_mode == LogprobsMode.PROCESSED_LOGPROBS:
160
+ elif self.logprobs_mode == "processed_logprobs":
161
processed_logprobs = self.compute_logprobs(logits)
162
return greedy_sampled, processed_logprobs
163