@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 61aedb5ffe

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Move`VllmConfig` from `config/init.py` to `config/vllm.py` (#25271)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Sep 30, 2025 at 03:49 UTC 61aedb5ffe056f83b1edab15610a644d32f40071

36 files changed +964 -905

vllm/attention/layer.py

+1 -2

@@ -20,8 +20,7 @@ from vllm.forward_context import ForwardContext, get_forward_context

20 from vllm.logger import init_logger

21 from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase

22 from vllm.model_executor.layers.linear import UnquantizedLinearMethod

23 -from vllm.model_executor.layers.quantization.base_config import (

24 - QuantizationConfig)

23 +from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8

25 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod

26 from vllm.model_executor.layers.quantization.utils.quant_utils import (

vllm/attention/layers/chunked_local_attention.py

+2 -1

@@ -9,7 +9,8 @@ from vllm import envs

9 from vllm.attention.backends.abstract import (AttentionBackend,

10 AttentionMetadata)

11 from vllm.attention.selector import get_attn_backend

12 -from vllm.config import CacheConfig, QuantizationConfig

12 +from vllm.config import CacheConfig

13 +from vllm.model_executor.layers.quantization import QuantizationConfig

14 from vllm.v1.attention.backends.utils import (

15 CommonAttentionMetadata, make_local_attention_virtual_batches,

16 subclass_attention_backend)

vllm/config/__init__.py

+79 -826

@@ -1,29 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -# ruff: noqa: F401

5 -import ast

6 -import copy

7 -import hashlib

8 -import inspect

9 -import json

10 -import os

11 -import textwrap

12 -from contextlib import contextmanager

13 -from dataclasses import field, fields, is_dataclass, replace

14 -from functools import cached_property, lru_cache

15 -from pathlib import Path

16 -from typing import (TYPE_CHECKING, Any, Literal, Optional, Protocol, TypeVar,

17 - Union, cast)

18 -

19 -import regex as re

20 -import torch

21 -from pydantic import ConfigDict, SkipValidation

22 -from pydantic.dataclasses import dataclass

23 -from typing_extensions import runtime_checkable

24 -

25 -import vllm.envs as envs

26 -from vllm import version

4 from vllm.config.cache import (BlockSize, CacheConfig, CacheDType, MambaDType,

5 PrefixCachingHashAlgo)

6 from vllm.config.compilation import (CompilationConfig, CompilationLevel,

@@ -48,806 +25,82 @@ from vllm.config.scheduler import RunnerType, SchedulerConfig, SchedulerPolicy

25 from vllm.config.speculative import SpeculativeConfig

26 from vllm.config.speech_to_text import SpeechToTextConfig

27 from vllm.config.structured_outputs import StructuredOutputsConfig

51 -from vllm.config.utils import ConfigType, config, get_attr_docs, is_init_field

52 -from vllm.logger import init_logger

53 -from vllm.multimodal import MULTIMODAL_REGISTRY

54 -from vllm.transformers_utils.runai_utils import is_runai_obj_uri

55 -from vllm.utils import random_uuid

56 -

57 -if TYPE_CHECKING:

58 - from _typeshed import DataclassInstance

59 - from transformers.configuration_utils import PretrainedConfig

60 -

61 - from vllm.model_executor.layers.quantization.base_config import (

62 - QuantizationConfig)

63 -else:

64 - DataclassInstance = Any

65 - PretrainedConfig = Any

66 - QuantizationConfig = Any

67 - QuantizationMethods = Any

68 - BaseModelLoader = Any

69 - LogitsProcessor = Any

70 -

71 -logger = init_logger(__name__)

72 -DataclassInstanceT = TypeVar("DataclassInstanceT", bound=DataclassInstance)

73 -

74 -

75 -@runtime_checkable

76 -class SupportsHash(Protocol):

77 -

78 - def compute_hash(self) -> str:

79 - ...

80 -

81 -

82 -class SupportsMetricsInfo(Protocol):

83 -

84 - def metrics_info(self) -> dict[str, str]:

85 - ...

86 -

87 -

88 -@config

89 -@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

90 -class VllmConfig:

91 - """Dataclass which contains all vllm-related configuration. This

92 - simplifies passing around the distinct configurations in the codebase.

93 - """

94 -

95 - # TODO: use default_factory once default constructing ModelConfig doesn't

96 - # try to download a model

97 - model_config: ModelConfig = None # type: ignore

98 - """Model configuration."""

99 - cache_config: CacheConfig = field(default_factory=CacheConfig)

100 - """Cache configuration."""

101 - parallel_config: ParallelConfig = field(default_factory=ParallelConfig)

102 - """Parallel configuration."""

103 - scheduler_config: SchedulerConfig = field(default_factory=SchedulerConfig)

104 - """Scheduler configuration."""

105 - device_config: DeviceConfig = field(default_factory=DeviceConfig)

106 - """Device configuration."""

107 - load_config: LoadConfig = field(default_factory=LoadConfig)

108 - """Load configuration."""

109 - lora_config: Optional[LoRAConfig] = None

110 - """LoRA configuration."""

111 - speculative_config: Optional[SpeculativeConfig] = None

112 - """Speculative decoding configuration."""

113 - structured_outputs_config: StructuredOutputsConfig = field(

114 - default_factory=StructuredOutputsConfig)

115 - """Structured outputs configuration."""

116 - observability_config: Optional[ObservabilityConfig] = None

117 - """Observability configuration."""

118 - quant_config: Optional[QuantizationConfig] = None

119 - """Quantization configuration."""

120 - compilation_config: CompilationConfig = field(

121 - default_factory=CompilationConfig)

122 - """`torch.compile` and cudagraph capture configuration for the model.

123 -

124 - As a shorthand, `-O<n>` can be used to directly specify the compilation

125 - level `n`: `-O3` is equivalent to `-O.level=3` (same as `-O='{"level":3}'`).

126 - Currently, -O <n> and -O=<n> are supported as well but this will likely be

127 - removed in favor of clearer -O<n> syntax in the future.

128 -

129 - NOTE: level 0 is the default level without any optimization. level 1 and 2

130 - are for internal testing only. level 3 is the recommended level for

131 - production, also default in V1.

132 -

133 - You can specify the full compilation config like so:

134 - `{"level": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`

135 - """

136 - kv_transfer_config: Optional[KVTransferConfig] = None

137 - """The configurations for distributed KV cache transfer."""

138 - kv_events_config: Optional[KVEventsConfig] = None

139 - """The configurations for event publishing."""

140 - # some opaque config, only used to provide additional information

141 - # for the hash computation, mainly used for testing, debugging or out of

142 - # tree config registration.

143 - additional_config: Union[dict, SupportsHash] = field(default_factory=dict)

144 - """Additional config for specified platform. Different platforms may

145 - support different configs. Make sure the configs are valid for the platform

146 - you are using. Contents must be hashable."""

147 - instance_id: str = ""

148 - """The ID of the vLLM instance."""

149 -

150 - def compute_hash(self) -> str:

151 - """

152 - WARNING: Whenever a new field is added to this config,

153 - ensure that it is included in the factors list if

154 - it affects the computation graph.

155 -

156 - Provide a hash that uniquely identifies all the configs

157 - that affect the structure of the computation

158 - graph from input ids/embeddings to the final hidden states,

159 - excluding anything before input ids/embeddings and after

160 - the final hidden states.

161 - """

162 - factors: list[Any] = []

163 -

164 - # summarize vllm config

165 - vllm_factors: list[Any] = []

166 - from vllm import __version__

167 - vllm_factors.append(__version__)

168 - vllm_factors.append(envs.VLLM_USE_V1)

169 - if self.model_config:

170 - vllm_factors.append(self.model_config.compute_hash())

171 - else:

172 - vllm_factors.append("None")

173 - if self.cache_config:

174 - vllm_factors.append(self.cache_config.compute_hash())

175 - else:

176 - vllm_factors.append("None")

177 - if self.parallel_config:

178 - vllm_factors.append(self.parallel_config.compute_hash())

179 - else:

180 - vllm_factors.append("None")

181 - if self.scheduler_config:

182 - vllm_factors.append(self.scheduler_config.compute_hash())

183 - else:

184 - vllm_factors.append("None")

185 - if self.device_config:

186 - vllm_factors.append(self.device_config.compute_hash())

187 - else:

188 - vllm_factors.append("None")

189 - if self.load_config:

190 - vllm_factors.append(self.load_config.compute_hash())

191 - else:

192 - vllm_factors.append("None")

193 - if self.lora_config:

194 - vllm_factors.append(self.lora_config.compute_hash())

195 - # LoRA creates static buffers based on max_num_batched_tokens.

196 - # The tensor sizes and strides get captured in the torch.compile

197 - # graph explicitly.

198 - vllm_factors.append(

199 - str(self.scheduler_config.max_num_batched_tokens))

200 - else:

201 - vllm_factors.append("None")

202 - if self.speculative_config:

203 - vllm_factors.append(self.speculative_config.compute_hash())

204 - else:

205 - vllm_factors.append("None")

206 - if self.structured_outputs_config:

207 - vllm_factors.append(self.structured_outputs_config.compute_hash())

208 - else:

209 - vllm_factors.append("None")

210 - if self.observability_config:

211 - vllm_factors.append(self.observability_config.compute_hash())

212 - else:

213 - vllm_factors.append("None")

214 - if self.quant_config:

215 - pass # should be captured by model_config.quantization

216 - if self.compilation_config:

217 - vllm_factors.append(self.compilation_config.compute_hash())

218 - else:

219 - vllm_factors.append("None")

220 - if self.kv_transfer_config:

221 - vllm_factors.append(self.kv_transfer_config.compute_hash())

222 - else:

223 - vllm_factors.append("None")

224 - if self.additional_config:

225 - if isinstance(additional_config := self.additional_config, dict):

226 - additional_config_hash = hashlib.md5(

227 - json.dumps(additional_config, sort_keys=True).encode(),

228 - usedforsecurity=False,

229 - ).hexdigest()

230 - else:

231 - additional_config_hash = additional_config.compute_hash()

232 - vllm_factors.append(additional_config_hash)

233 - else:

234 - vllm_factors.append("None")

235 - factors.append(vllm_factors)

236 -

237 - hash_str = hashlib.md5(str(factors).encode(),

238 - usedforsecurity=False).hexdigest()[:10]

239 - return hash_str

240 -

241 - def pad_for_cudagraph(self, batch_size: int) -> int:

242 - # if batch_size > self.compilation_config.max_capture_size,

243 - # it should raise an IndexError.

244 - # the caller should make sure the batch_size is within the range,

245 - # i.e., batch_size <= self.compilation_config.max_capture_size

246 - return self.compilation_config.bs_to_padded_graph_size[batch_size]

247 -

248 - @staticmethod

249 - def _get_quantization_config(

250 - model_config: ModelConfig,

251 - load_config: LoadConfig) -> Optional[QuantizationConfig]:

252 - """Get the quantization config."""

253 - from vllm.platforms import current_platform

254 - if model_config.quantization is not None:

255 - from vllm.model_executor.model_loader.weight_utils import (

256 - get_quant_config)

257 - quant_config = get_quant_config(model_config, load_config)

258 - capability_tuple = current_platform.get_device_capability()

259 -

260 - if capability_tuple is not None:

261 - capability = capability_tuple.to_int()

262 - if capability < quant_config.get_min_capability():

263 - raise ValueError(

264 - f"The quantization method {model_config.quantization} "

265 - "is not supported for the current GPU. Minimum "

266 - f"capability: {quant_config.get_min_capability()}. "

267 - f"Current capability: {capability}.")

268 - supported_dtypes = quant_config.get_supported_act_dtypes()

269 - if model_config.dtype not in supported_dtypes:

270 - raise ValueError(

271 - f"{model_config.dtype} is not supported for quantization "

272 - f"method {model_config.quantization}. Supported dtypes: "

273 - f"{supported_dtypes}")

274 - quant_config.maybe_update_config(model_config.model)

275 - return quant_config

276 - return None

277 -

278 - @staticmethod

279 - def get_quantization_config(

280 - model_config: ModelConfig,

281 - load_config: LoadConfig) -> Optional[QuantizationConfig]:

282 - import copy

283 -

284 - # For some reason, the _ version of this modifies the model_config

285 - # object, so using deepcopy to avoid this problem.

286 - return VllmConfig._get_quantization_config(copy.deepcopy(model_config),

287 - load_config)

288 -

289 - def with_hf_config(

290 - self,

291 - hf_config: PretrainedConfig,

292 - architectures: Optional[list[str]] = None,

293 - ) -> "VllmConfig":

294 - if architectures is not None:

295 - hf_config = copy.deepcopy(hf_config)

296 - hf_config.architectures = architectures

297 -

298 - model_config = copy.deepcopy(self.model_config)

299 - model_config.hf_config = hf_config

300 -

301 - return replace(self, model_config=model_config)

302 -

303 - def __post_init__(self):

304 - """Verify configs are valid & consistent with each other.

305 - """

306 -

307 - self.try_verify_and_update_config()

308 -

309 - if self.model_config is not None:

310 - self.model_config.verify_with_parallel_config(self.parallel_config)

311 - self.model_config.verify_dual_chunk_attention_config(

312 - self.load_config)

313 -

314 - self.cache_config.verify_with_parallel_config(self.parallel_config)

315 -

316 - if self.lora_config is not None:

317 - self.lora_config.verify_with_cache_config(self.cache_config)

318 - self.lora_config.verify_with_model_config(self.model_config)

319 -

320 - if self.quant_config is None and self.model_config is not None:

321 - self.quant_config = VllmConfig._get_quantization_config(

322 - self.model_config, self.load_config)

323 -

324 - from vllm.platforms import current_platform

325 - if self.model_config is not None and \

326 - self.scheduler_config.chunked_prefill_enabled and \

327 - self.model_config.dtype == torch.float32 and \

328 - current_platform.get_device_capability() == (7, 5):

329 - logger.warning_once(

330 - "Turing devices tensor cores do not support float32 matmul. "

331 - "To workaround this limitation, vLLM will set 'ieee' input "

332 - "precision for chunked prefill triton kernels.")

333 -

334 - # If the user does not explicitly set a compilation level, then

335 - # we use the default level. The default level depends on other

336 - # settings (see the below code).

337 - if self.compilation_config.level is None:

338 - if envs.VLLM_USE_V1:

339 - if (self.model_config is not None

340 - and not self.model_config.enforce_eager):

341 - self.compilation_config.level = CompilationLevel.PIECEWISE

342 - else:

343 - self.compilation_config.level = \

344 - CompilationLevel.NO_COMPILATION

345 -

346 - else:

347 - # NB: Passing both --enforce-eager and a compilation level

348 - # in V0 means the compilation level wins out.

349 - self.compilation_config.level = CompilationLevel.NO_COMPILATION

350 -

351 - # async tp is built on top of sequence parallelism

352 - # and requires it to be enabled.

353 - if self.compilation_config.pass_config.enable_async_tp:

354 - self.compilation_config.pass_config.enable_sequence_parallelism = \

355 - True

356 - if self.compilation_config.pass_config.enable_sequence_parallelism:

357 - self.compilation_config.custom_ops.append("+rms_norm")

358 -

359 - if current_platform.support_static_graph_mode():

360 - # if cudagraph_mode is not explicitly set by users, set default

361 - # value

362 - if self.compilation_config.cudagraph_mode is None:

363 - if envs.VLLM_USE_V1 and self.compilation_config.level \

364 - == CompilationLevel.PIECEWISE:

365 - # default to full and piecewise for most models

366 - self.compilation_config.cudagraph_mode = \

367 - CUDAGraphMode.FULL_AND_PIECEWISE

368 -

369 - # pooling models and encoder-decoder models

370 - # do not support full cudagraphs

371 - if self.model_config is not None and \

372 - (self.model_config.pooler_config is not None

373 - or self.model_config.is_encoder_decoder):

374 - self.compilation_config.cudagraph_mode = \

375 - CUDAGraphMode.PIECEWISE

376 - else:

377 - self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

378 -

379 - # disable cudagraph when enforce eager execution

380 - if self.model_config is not None and \

381 - self.model_config.enforce_eager:

382 - logger.info("Cudagraph is disabled under eager mode")

383 - self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

384 - elif envs.VLLM_USE_V1:

385 - self.compilation_config.cudagraph_num_of_warmups = 1

386 -

387 - self._set_cudagraph_sizes()

388 - else:

389 - self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

390 -

391 - if self.cache_config.kv_sharing_fast_prefill:

392 -

393 - if self.speculative_config is not None and \

394 - self.speculative_config.use_eagle():

395 - raise NotImplementedError(

396 - "Fast prefill optimization for KV sharing is not "

397 - "compatible with EAGLE as EAGLE requires correct logits "

398 - "for all tokens while fast prefill gives incorrect logits "

399 - "for prompt tokens.")

400 -

401 - logger.warning_once(

402 - "--kv-sharing-fast-prefill requires changes on model side for "

403 - "correctness and to realize prefill savings. ")

404 -

405 - disable_chunked_prefill_reasons: list[str] = []

406 -

407 - if self.model_config:

408 - if self.model_config.pooler_config:

409 - pooling_type = self.model_config.pooler_config.pooling_type

410 - if pooling_type is None or pooling_type.lower() != "last":

411 - disable_chunked_prefill_reasons.append(

412 - "Only \"last\" pooling supports chunked "

413 - "prefill and prefix caching; disabling both.")

414 - if not getattr(self.model_config.hf_config, "is_causal", True):

415 - disable_chunked_prefill_reasons.append(

416 - "Only models using causal attention supports chunked "

417 - "prefill and prefix caching; disabling both.")

418 - elif self.model_config.is_encoder_decoder:

419 - self.scheduler_config.max_num_encoder_input_tokens = \

420 - MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.model_config)

421 - logger.debug(

422 - "Encoder-decoder model detected: setting "

423 - "`max_num_encoder_input_tokens` to encoder length (%s)",

424 - self.scheduler_config.max_num_encoder_input_tokens)

425 - self.scheduler_config.disable_chunked_mm_input = True

426 - disable_chunked_prefill_reasons.append(

427 - "Encoder-decoder models do not support chunked prefill nor"

428 - " prefix caching; disabling both.")

429 - if (self.model_config.architecture

430 - == "WhisperForConditionalGeneration"

431 - and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD")

432 - != "spawn"):

433 - logger.warning(

434 - "Whisper is known to have issues with "

435 - "forked workers. If startup is hanging, "

436 - "try setting 'VLLM_WORKER_MULTIPROC_METHOD' "

437 - "to 'spawn'.")

438 -

439 - if disable_chunked_prefill_reasons:

440 - for reason in disable_chunked_prefill_reasons:

441 - logger.info(reason)

442 - self.scheduler_config.chunked_prefill_enabled = False

443 - self.scheduler_config.long_prefill_token_threshold = 0

444 -

445 - if self.cache_config is not None:

446 - self.cache_config.enable_prefix_caching = False

447 -

448 - if (self.kv_events_config is not None

449 - and self.kv_events_config.enable_kv_cache_events

450 - and not self.cache_config.enable_prefix_caching):

451 - logger.warning(

452 - "KV cache events are on, but prefix caching is not enabled."

453 - "Use --enable-prefix-caching to enable.")

454 - if (self.kv_events_config is not None

455 - and self.kv_events_config.publisher != "null"

456 - and not self.kv_events_config.enable_kv_cache_events):

457 - logger.warning("KV cache events are disabled,"

458 - "but the scheduler is configured to publish them."

459 - "Modify KVEventsConfig.enable_kv_cache_events"

460 - "to True to enable.")

461 - current_platform.check_and_update_config(self)

462 -

463 - # Do this after all the updates to compilation_config.level

464 - if envs.VLLM_USE_V1 and \

465 - self.compilation_config.level == CompilationLevel.PIECEWISE:

466 - self.compilation_config.set_splitting_ops_for_v1()

467 -

468 - # final check of cudagraph mode after all possible updates

469 - if envs.VLLM_USE_V1 and current_platform.is_cuda_alike():

470 - if self.compilation_config.cudagraph_mode.has_full_cudagraphs()\

471 - and self.model_config is not None and \

472 - not self.model_config.disable_cascade_attn and\

473 - not self.compilation_config.cudagraph_mode.\

474 - has_piecewise_cudagraphs():

475 - logger.warning_once(

476 - "No piecewise cudagraph for executing cascade attention."

477 - " Will fall back to eager execution if a batch runs "

478 - "into cascade attentions")

479 -

480 - if self.compilation_config.cudagraph_mode\

481 - .requires_piecewise_compilation():

482 - assert self.compilation_config.level == \

483 - CompilationLevel.PIECEWISE, \

484 - "Compilation level should be CompilationLevel.PIECEWISE "\

485 - "when cudagraph_mode piecewise cudagraphs is used, "\

486 - f"cudagraph_mode={self.compilation_config.cudagraph_mode}"

487 -

488 - # final migrate the deprecated flags

489 - self.compilation_config.use_cudagraph = self.compilation_config.\

490 - cudagraph_mode!= CUDAGraphMode.NONE

491 - self.compilation_config.full_cuda_graph = self.compilation_config.\

492 - cudagraph_mode.has_full_cudagraphs()

493 -

494 - if self.parallel_config.enable_dbo:

495 - a2a_backend = envs.VLLM_ALL2ALL_BACKEND

496 - assert a2a_backend in \

497 - ["deepep_low_latency", "deepep_high_throughput"], \

498 - "Microbatching currently only supports the deepep_low_latency and "\

499 - f"deepep_high_throughput all2all backend. {a2a_backend} is not "\

500 - "supported. To fix set the VLLM_ALL2ALL_BACKEND environment "\

501 - "variable to deepep_low_latency or deepep_high_throughput and "\

502 - "install the DeepEP kernels."

503 -

504 - if not self.model_config.disable_cascade_attn:

505 - self.model_config.disable_cascade_attn = True

506 - logger.warning_once(

507 - "Disabling cascade attention when DBO is enabled.")

508 -

509 - if not self.instance_id:

510 - self.instance_id = random_uuid()[:5]

511 -

512 - if (envs.VLLM_USE_V1

513 - and not self.scheduler_config.disable_hybrid_kv_cache_manager):

514 - # logger should only print warning message for hybrid models. As we

515 - # can't know whether the model is hybrid or not now, so we don't log

516 - # warning message here and will log it later.

517 - if not current_platform.support_hybrid_kv_cache():

518 - # Hybrid KV cache manager is not supported on non-GPU platforms.

519 - self.scheduler_config.disable_hybrid_kv_cache_manager = True

520 - if self.kv_transfer_config is not None:

521 - # Hybrid KV cache manager is not compatible with KV transfer.

522 - self.scheduler_config.disable_hybrid_kv_cache_manager = True

523 - if self.kv_events_config is not None:

524 - # Hybrid KV cache manager is not compatible with KV events.

525 - self.scheduler_config.disable_hybrid_kv_cache_manager = True

526 - if self.model_config is not None and \

527 - self.model_config.attention_chunk_size is not None:

528 - if self.speculative_config is not None and \

529 - self.speculative_config.use_eagle():

530 - # Hybrid KV cache manager is not yet supported with chunked

531 - # local attention + eagle.

532 - self.scheduler_config.disable_hybrid_kv_cache_manager = True

533 - elif \

534 - not envs.VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE:

535 - logger.warning(

536 - "There is a latency regression when using chunked local"

537 - " attention with the hybrid KV cache manager. Disabling"

538 - " it, by default. To enable it, set the environment "

539 - "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE=1."

540 - )

541 - # Hybrid KV cache manager is not yet supported with chunked

542 - # local attention.

543 - self.scheduler_config.disable_hybrid_kv_cache_manager = True

544 -

545 - if self.compilation_config.debug_dump_path:

546 - self.compilation_config.debug_dump_path = \

547 - self.compilation_config.debug_dump_path.absolute().expanduser()

548 - if envs.VLLM_DEBUG_DUMP_PATH is not None:

549 - env_path = Path(envs.VLLM_DEBUG_DUMP_PATH).absolute().expanduser()

550 - if self.compilation_config.debug_dump_path:

551 - logger.warning(

552 - "Config-specified debug dump path is overridden"

553 - " by VLLM_DEBUG_DUMP_PATH to %s", env_path)

554 - self.compilation_config.debug_dump_path = env_path

555 -

556 - def update_sizes_for_sequence_parallelism(self,

557 - possible_sizes: list) -> list:

558 - # remove the sizes that not multiple of tp_size when

559 - # enable sequence parallelism

560 - removed_sizes = [

561 - size for size in possible_sizes

562 - if size % self.parallel_config.tensor_parallel_size != 0

563 - ]

564 - if removed_sizes:

565 - logger.warning(

566 - "Batch sizes %s are removed because they are not "

567 - "multiple of tp_size %d when "

568 - "sequence parallelism is enabled", removed_sizes,

569 - self.parallel_config.tensor_parallel_size)

570 -

571 - return [

572 - size for size in possible_sizes

573 - if size % self.parallel_config.tensor_parallel_size == 0

574 - ]

575 -

576 - def _set_cudagraph_sizes(self):

577 - """

578 - vLLM defines the default candidate list of batch sizes for CUDA graph

579 - capture as:

580 -

581 - ```python

582 - max_graph_size = min(max_num_seqs * 2, 512)

583 - # 1, 2, 4, then multiples of 8 up to max_graph_size

584 - cuda_graph_sizes = [1, 2, 4, 8, 16, 24, 32, 40, ..., max_graph_size]

585 -

586 - In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`

587 - will be the final sizes to capture cudagraph (in descending order).

588 -

589 - These sizes are used to capture and reuse CUDA graphs for

590 - performance-critical paths (e.g., decoding). Capturing enables

591 - significantly faster kernel dispatch by avoiding Python overhead. The

592 - list is then filtered based on `max_num_batched_tokens` (e.g., 8192 on

593 - most GPUs), which controls the total allowed number of tokens in a

594 - batch. Since each sequence may have a variable number of tokens, the

595 - maximum usable batch size will depend on actual sequence lengths.

596 -

597 - Example:

598 - With `max_num_batched_tokens = 8192`, and typical sequences

599 - averaging ~32 tokens, most practical batch sizes fall below 256.

600 - However, the system will still allow capture sizes up to 512 if

601 - shape and memory permit.

602 -

603 - Note:

604 - If users explicitly specify cudagraph capture sizes in the

605 - compilation config, those will override this default logic.

606 - At runtime:

607 -

608 - - If batch size <= one of the `cudagraph_capture_sizes`, the closest

609 - padded CUDA graph will be used.

610 - - If batch size > largest `cudagraph_capture_sizes`, cudagraph will

611 - not be used.

612 - """

613 -

614 - # calculate the default `batch_size_capture_list`

615 - batch_size_capture_list = []

616 - if self.model_config is not None and \

617 - not self.model_config.enforce_eager:

618 - cuda_graph_sizes = self.scheduler_config.cuda_graph_sizes

619 - if len(cuda_graph_sizes) == 1:

620 - batch_size_capture_list = [1, 2, 4] + [

621 - i for i in range(8, cuda_graph_sizes[0] + 1, 8)

622 - ]

623 - elif len(cuda_graph_sizes) > 1:

624 - batch_size_capture_list = sorted(cuda_graph_sizes)

625 - else:

626 - raise TypeError(f"Invalid value for {cuda_graph_sizes=}.")

627 - if self.parallel_config.tensor_parallel_size > 1 and \

628 - self.compilation_config.pass_config.enable_sequence_parallelism:

629 - batch_size_capture_list = \

630 - self.update_sizes_for_sequence_parallelism(batch_size_capture_list)

631 - max_num_tokens = self.scheduler_config.max_num_batched_tokens

632 - batch_size_capture_list = [

633 - size for size in batch_size_capture_list

634 - if size <= max_num_tokens

635 - ]

636 -

637 - self.compilation_config.init_with_cudagraph_sizes(

638 - batch_size_capture_list)

639 -

640 - def recalculate_max_model_len(self, max_model_len: int):

641 - # Can only be called in try_verify_and_update_config

642 - model_config = self.model_config

643 - max_model_len = model_config.get_and_verify_max_len(max_model_len)

644 - self.model_config.max_model_len = max_model_len

645 - self.scheduler_config.max_model_len = max_model_len

646 -

647 - def try_verify_and_update_config(self):

648 - if self.model_config is None:

649 - return

650 -

651 - # Avoid running try_verify_and_update_config multiple times

652 - if getattr(self.model_config, "config_updated", False):

653 - return

654 - self.model_config.config_updated = True

655 -

656 - architecture = self.model_config.architecture

657 - if architecture is None:

658 - return

659 -

660 - from vllm.model_executor.models.config import (

661 - MODELS_CONFIG_MAP, HybridAttentionMambaModelConfig)

662 - cls = MODELS_CONFIG_MAP.get(architecture, None)

663 - if cls is not None:

664 - cls.verify_and_update_config(self)

665 -

666 - if self.model_config.is_hybrid:

667 - HybridAttentionMambaModelConfig.verify_and_update_config(self)

668 -

669 - if self.model_config.convert_type == "classify":

670 - # Maybe convert ForCausalLM into ForSequenceClassification model.

671 - from vllm.model_executor.models.adapters import (

672 - SequenceClassificationConfig)

673 - SequenceClassificationConfig.verify_and_update_config(self)

674 -

675 - if hasattr(self.model_config, "model_weights") and is_runai_obj_uri(

676 - self.model_config.model_weights):

677 - if self.load_config.load_format == "auto":

678 - logger.info("Detected Run:ai model config. "

679 - "Overriding `load_format` to 'runai_streamer'")

680 - self.load_config.load_format = "runai_streamer"

681 - elif self.load_config.load_format != "runai_streamer":

682 - raise ValueError(f"To load a model from S3, 'load_format' "

683 - f"must be 'runai_streamer', "

684 - f"but got '{self.load_config.load_format}'. "

685 - f"Model: {self.model_config.model}")

686 -

687 - def compile_debug_dump_path(self) -> Optional[Path]:

688 - """Returns a rank-aware path for dumping

689 - torch.compile debug information.

690 - """

691 - if self.compilation_config.debug_dump_path is None:

692 - return None

693 - tp_rank = self.parallel_config.rank

694 - dp_rank = self.parallel_config.data_parallel_rank

695 - data_parallel_size = self.parallel_config.data_parallel_size

696 - append_path = f"rank_{tp_rank}" if data_parallel_size == 1 \

697 - else f"rank_{tp_rank}_dp_{dp_rank}"

698 - path = self.compilation_config.debug_dump_path / append_path

699 - return path

700 -

701 - def __str__(self):

702 - return (

703 - f"model={self.model_config.model!r}, "

704 - f"speculative_config={self.speculative_config!r}, "

705 - f"tokenizer={self.model_config.tokenizer!r}, "

706 - f"skip_tokenizer_init={self.model_config.skip_tokenizer_init}, "

707 - f"tokenizer_mode={self.model_config.tokenizer_mode}, "

708 - f"revision={self.model_config.revision}, "

709 - f"tokenizer_revision={self.model_config.tokenizer_revision}, "

710 - f"trust_remote_code={self.model_config.trust_remote_code}, "

711 - f"dtype={self.model_config.dtype}, "

712 - f"max_seq_len={self.model_config.max_model_len}, "

713 - f"download_dir={self.load_config.download_dir!r}, "

714 - f"load_format={self.load_config.load_format}, "

715 - f"tensor_parallel_size={self.parallel_config.tensor_parallel_size}, " # noqa

716 - f"pipeline_parallel_size={self.parallel_config.pipeline_parallel_size}, " # noqa

717 - f"data_parallel_size={self.parallel_config.data_parallel_size}, " # noqa

718 - f"disable_custom_all_reduce={self.parallel_config.disable_custom_all_reduce}, " # noqa

719 - f"quantization={self.model_config.quantization}, "

720 - f"enforce_eager={self.model_config.enforce_eager}, "

721 - f"kv_cache_dtype={self.cache_config.cache_dtype}, "

722 - f"device_config={self.device_config.device}, "

723 - f"structured_outputs_config={self.structured_outputs_config!r}, "

724 - f"observability_config={self.observability_config!r}, "

725 - f"seed={self.model_config.seed}, "

726 - f"served_model_name={self.model_config.served_model_name}, "

727 - f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "

728 - f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa

729 - f"pooler_config={self.model_config.pooler_config!r}, "

730 - f"compilation_config={self.compilation_config!r}")

731 -

732 -

733 -_current_vllm_config: Optional[VllmConfig] = None

734 -_current_prefix: Optional[str] = None

735 -

736 -

737 -@contextmanager

738 -def set_current_vllm_config(vllm_config: VllmConfig,

739 - check_compile=False,

740 - prefix: Optional[str] = None):

741 - """

742 - Temporarily set the current vLLM config.

743 - Used during model initialization.

744 - We save the current vLLM config in a global variable,

745 - so that all modules can access it, e.g. custom ops

746 - can access the vLLM config to determine how to dispatch.

747 - """

748 - global _current_vllm_config, _current_prefix

749 - old_vllm_config = _current_vllm_config

750 - old_prefix = _current_prefix

751 - from vllm.compilation.counter import compilation_counter

752 - num_models_seen = compilation_counter.num_models_seen

753 - try:

754 - _current_vllm_config = vllm_config

755 - _current_prefix = prefix

756 - yield

757 - except Exception:

758 - raise

759 - else:

760 - if check_compile:

761 - vllm_config.compilation_config.custom_op_log_check()

762 -

763 - if check_compile and \

764 - vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \

765 - and compilation_counter.num_models_seen == num_models_seen:

766 - # If the model supports compilation,

767 - # compilation_counter.num_models_seen should be increased

768 - # by at least 1.

769 - # If it is not increased, it means the model does not support

770 - # compilation (does not have @support_torch_compile decorator).

771 - logger.warning(

772 - "`torch.compile` is turned on, but the model %s"

773 - " does not support it. Please open an issue on GitHub"

774 - " if you want it to be supported.",

775 - vllm_config.model_config.model)

776 - finally:

777 - _current_vllm_config = old_vllm_config

778 - _current_prefix = old_prefix

779 - # Clear the compilation config cache when context changes

780 - get_cached_compilation_config.cache_clear()

781 -

782 -

783 -@lru_cache(maxsize=1)

784 -def get_cached_compilation_config():

785 - """Cache config to avoid repeated calls to get_current_vllm_config()"""

786 - return get_current_vllm_config().compilation_config

787 -

788 -

789 -def get_current_vllm_config() -> VllmConfig:

790 - if _current_vllm_config is None:

791 - # in ci, usually when we test custom ops/modules directly,

792 - # we don't set the vllm config. In that case, we set a default

793 - # config.

794 - logger.warning("Current vLLM config is not set.")

795 - from vllm.config import VllmConfig

796 - return VllmConfig()

797 - return _current_vllm_config

798 -

799 -

800 -def get_current_model_prefix() -> str:

801 - """

802 - Get the prefix of the model that's currently being initialized.

803 - """

804 - assert _current_prefix is not None, \

805 - "Current model prefix is not set. "

806 - return _current_prefix

807 -

808 -

809 -T = TypeVar("T")

810 -

811 -

812 -def get_layers_from_vllm_config(

813 - vllm_config: VllmConfig,

814 - layer_type: type[T],

815 - layer_names: Optional[list[str]] = None) -> dict[str, T]:

816 - """

817 - Get layers from the vLLM config.

818 -

819 - Args:

820 - vllm_config: The vLLM config.

821 - layer_type: The type of the layer to get.

822 - layer_names: The names of the layers to get. If None, return all layers.

823 - """

824 -

825 - if layer_names is None:

826 - layer_names = list(

827 - vllm_config.compilation_config.static_forward_context.keys())

828 -

829 - forward_context = vllm_config.compilation_config.static_forward_context

830 -

831 - return {

832 - layer_name: forward_context[layer_name]

833 - for layer_name in layer_names

834 - if isinstance(forward_context[layer_name], layer_type)

835 - }

836 -

837 -

838 -def update_config(config: DataclassInstanceT,

839 - overrides: dict[str, Any]) -> DataclassInstanceT:

840 - processed_overrides = {}

841 - for field_name, value in overrides.items():

842 - assert hasattr(

843 - config, field_name), f"{type(config)} has no field `{field_name}`"

844 - current_value = getattr(config, field_name)

845 - if is_dataclass(current_value) and not is_dataclass(value):

846 - assert isinstance(value, dict), (

847 - f"Overrides to {type(config)}.{field_name} must be a dict"

848 - f" or {type(current_value)}, but got {type(value)}")

849 - value = update_config(

850 - current_value, # type: ignore[type-var]

851 - value)

852 - processed_overrides[field_name] = value

853 - return replace(config, **processed_overrides)

28 +from vllm.config.utils import (ConfigType, SupportsMetricsInfo, config,

29 + get_attr_docs, is_init_field, update_config)

30 +from vllm.config.vllm import (VllmConfig, get_cached_compilation_config,

31 + get_current_vllm_config,

32 + get_layers_from_vllm_config,

33 + set_current_vllm_config)

34 +

35 +__all__ = [

36 + # From vllm.config.cache

37 + "BlockSize",

38 + "CacheConfig",

39 + "CacheDType",

40 + "MambaDType",

41 + "PrefixCachingHashAlgo",

42 + # From vllm.config.compilation

43 + "CompilationConfig",

44 + "CompilationLevel",

45 + "CUDAGraphMode",

46 + "PassConfig",

47 + # From vllm.config.device

48 + "Device",

49 + "DeviceConfig",

50 + # From vllm.config.kv_events

51 + "KVEventsConfig",

52 + # From vllm.config.kv_transfer

53 + "KVTransferConfig",

54 + # From vllm.config.load

55 + "LoadConfig",

56 + # From vllm.config.lora

57 + "LoRAConfig",

58 + # From vllm.config.model

59 + "ConvertOption",

60 + "HfOverrides",

61 + "LogprobsMode",

62 + "ModelConfig",

63 + "ModelDType",

64 + "ModelImpl",

65 + "RunnerOption",

66 + "TaskOption",

67 + "TokenizerMode",

68 + "iter_architecture_defaults",

69 + "try_match_architecture_defaults",

70 + # From vllm.config.multimodal

71 + "MMCacheType",

72 + "MMEncoderTPMode",

73 + "MultiModalConfig",

74 + # From vllm.config.observability

75 + "DetailedTraceModules",

76 + "ObservabilityConfig",

77 + # From vllm.config.parallel

78 + "DistributedExecutorBackend",

79 + "EPLBConfig",

80 + "ParallelConfig",

81 + # From vllm.config.pooler

82 + "PoolerConfig",

83 + # From vllm.config.scheduler

84 + "RunnerType",

85 + "SchedulerConfig",

86 + "SchedulerPolicy",

87 + # From vllm.config.speculative

88 + "SpeculativeConfig",

89 + # From vllm.config.speech_to_text

90 + "SpeechToTextConfig",

91 + # From vllm.config.structured_outputs

92 + "StructuredOutputsConfig",

93 + # From vllm.config.utils

94 + "ConfigType",

95 + "SupportsMetricsInfo",

96 + "config",

97 + "get_attr_docs",

98 + "is_init_field",

99 + "update_config",

100 + # From vllm.config.vllm

101 + "VllmConfig",

102 + "get_cached_compilation_config",

103 + "get_current_vllm_config",

104 + "set_current_vllm_config",

105 + "get_layers_from_vllm_config",

106 +]

vllm/config/utils.py

+36 -6

@@ -1,21 +1,21 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

3 +"""Utility functions for vLLM config dataclasses."""

4 import ast

5 import inspect

6 import textwrap

7 -from dataclasses import MISSING, Field, field, fields, is_dataclass

8 -from typing import TYPE_CHECKING, Any, TypeVar

7 +from dataclasses import MISSING, Field, field, fields, is_dataclass, replace

8 +from typing import TYPE_CHECKING, Any, Protocol, TypeVar

10 import regex as re

11 +from typing_extensions import runtime_checkable

13 if TYPE_CHECKING:

14 from _typeshed import DataclassInstance

14 -

15 - ConfigType = type[DataclassInstance]

15 else:

17 - ConfigType = type

16 + DataclassInstance = Any

18 +ConfigType = type[DataclassInstance]

19 ConfigT = TypeVar("ConfigT", bound=ConfigType)

@@ -143,3 +143,33 @@ def get_attr_docs(cls: type[Any]) -> dict[str, str]:

143

144 def is_init_field(cls: ConfigType, name: str) -> bool:

145 return next(f for f in fields(cls) if f.name == name).init

146 +

147 +

148 +@runtime_checkable

149 +class SupportsHash(Protocol):

150 +

151 + def compute_hash(self) -> str:

152 + ...

153 +

154 +

155 +class SupportsMetricsInfo(Protocol):

156 +

157 + def metrics_info(self) -> dict[str, str]:

158 + ...

159 +

160 +

161 +def update_config(config: ConfigT, overrides: dict[str, Any]) -> ConfigT:

162 + processed_overrides = {}

163 + for field_name, value in overrides.items():

164 + assert hasattr(

165 + config, field_name), f"{type(config)} has no field `{field_name}`"

166 + current_value = getattr(config, field_name)

167 + if is_dataclass(current_value) and not is_dataclass(value):

168 + assert isinstance(value, dict), (

169 + f"Overrides to {type(config)}.{field_name} must be a dict"

170 + f" or {type(current_value)}, but got {type(value)}")

171 + value = update_config(

172 + current_value, # type: ignore[type-var]

173 + value)

174 + processed_overrides[field_name] = value

175 + return replace(config, **processed_overrides)

vllm/config/vllm.py new

+789

@@ -0,0 +1,789 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import copy

5 +import hashlib

6 +import json

7 +import os

8 +from contextlib import contextmanager

9 +from dataclasses import field, replace

10 +from functools import lru_cache

11 +from pathlib import Path

12 +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union

13 +

14 +import torch

15 +from pydantic import ConfigDict

16 +from pydantic.dataclasses import dataclass

17 +

18 +import vllm.envs as envs

19 +from vllm.logger import init_logger

20 +from vllm.transformers_utils.runai_utils import is_runai_obj_uri

21 +from vllm.utils import random_uuid

22 +

23 +from .cache import CacheConfig

24 +from .compilation import CompilationConfig, CompilationLevel, CUDAGraphMode

25 +from .device import DeviceConfig

26 +from .kv_events import KVEventsConfig

27 +from .kv_transfer import KVTransferConfig

28 +from .load import LoadConfig

29 +from .lora import LoRAConfig

30 +from .model import ModelConfig

31 +from .observability import ObservabilityConfig

32 +from .parallel import ParallelConfig

33 +from .scheduler import SchedulerConfig

34 +from .speculative import SpeculativeConfig

35 +from .structured_outputs import StructuredOutputsConfig

36 +from .utils import SupportsHash, config

37 +

38 +if TYPE_CHECKING:

39 + from transformers import PretrainedConfig

40 +

41 + from vllm.model_executor.layers.quantization.base_config import (

42 + QuantizationConfig)

43 +else:

44 + PretrainedConfig = Any

45 +

46 + QuantizationConfig = Any

47 +

48 +logger = init_logger(__name__)

49 +

50 +

51 +@config

52 +@dataclass(config=ConfigDict(arbitrary_types_allowed=True))

53 +class VllmConfig:

54 + """Dataclass which contains all vllm-related configuration. This

55 + simplifies passing around the distinct configurations in the codebase.

56 + """

57 +

58 + # TODO: use default_factory once default constructing ModelConfig doesn't

59 + # try to download a model

60 + model_config: ModelConfig = None # type: ignore

61 + """Model configuration."""

62 + cache_config: CacheConfig = field(default_factory=CacheConfig)

63 + """Cache configuration."""

64 + parallel_config: ParallelConfig = field(default_factory=ParallelConfig)

65 + """Parallel configuration."""

66 + scheduler_config: SchedulerConfig = field(default_factory=SchedulerConfig)

67 + """Scheduler configuration."""

68 + device_config: DeviceConfig = field(default_factory=DeviceConfig)

69 + """Device configuration."""

70 + load_config: LoadConfig = field(default_factory=LoadConfig)

71 + """Load configuration."""

72 + lora_config: Optional[LoRAConfig] = None

73 + """LoRA configuration."""

74 + speculative_config: Optional[SpeculativeConfig] = None

75 + """Speculative decoding configuration."""

76 + structured_outputs_config: StructuredOutputsConfig = field(

77 + default_factory=StructuredOutputsConfig)

78 + """Structured outputs configuration."""

79 + observability_config: Optional[ObservabilityConfig] = None

80 + """Observability configuration."""

81 + quant_config: Optional[QuantizationConfig] = None

82 + """Quantization configuration."""

83 + compilation_config: CompilationConfig = field(

84 + default_factory=CompilationConfig)

85 + """`torch.compile` and cudagraph capture configuration for the model.

86 +

87 + As a shorthand, `-O<n>` can be used to directly specify the compilation

88 + level `n`: `-O3` is equivalent to `-O.level=3` (same as `-O='{"level":3}'`).

89 + Currently, -O <n> and -O=<n> are supported as well but this will likely be

90 + removed in favor of clearer -O<n> syntax in the future.

91 +

92 + NOTE: level 0 is the default level without any optimization. level 1 and 2

93 + are for internal testing only. level 3 is the recommended level for

94 + production, also default in V1.

95 +

96 + You can specify the full compilation config like so:

97 + `{"level": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`

98 + """

99 + kv_transfer_config: Optional[KVTransferConfig] = None

100 + """The configurations for distributed KV cache transfer."""

101 + kv_events_config: Optional[KVEventsConfig] = None

102 + """The configurations for event publishing."""

103 + # some opaque config, only used to provide additional information

104 + # for the hash computation, mainly used for testing, debugging or out of

105 + # tree config registration.

106 + additional_config: Union[dict, SupportsHash] = field(default_factory=dict)

107 + """Additional config for specified platform. Different platforms may

108 + support different configs. Make sure the configs are valid for the platform

109 + you are using. Contents must be hashable."""

110 + instance_id: str = ""

111 + """The ID of the vLLM instance."""

112 +

113 + def compute_hash(self) -> str:

114 + """

115 + WARNING: Whenever a new field is added to this config,

116 + ensure that it is included in the factors list if

117 + it affects the computation graph.

118 +

119 + Provide a hash that uniquely identifies all the configs

120 + that affect the structure of the computation

121 + graph from input ids/embeddings to the final hidden states,

122 + excluding anything before input ids/embeddings and after

123 + the final hidden states.

124 + """

125 + factors: list[Any] = []

126 +

127 + # summarize vllm config

128 + vllm_factors: list[Any] = []

129 + from vllm import __version__

130 + vllm_factors.append(__version__)

131 + vllm_factors.append(envs.VLLM_USE_V1)

132 + if self.model_config:

133 + vllm_factors.append(self.model_config.compute_hash())

134 + else:

135 + vllm_factors.append("None")

136 + if self.cache_config:

137 + vllm_factors.append(self.cache_config.compute_hash())

138 + else:

139 + vllm_factors.append("None")

140 + if self.parallel_config:

141 + vllm_factors.append(self.parallel_config.compute_hash())

142 + else:

143 + vllm_factors.append("None")

144 + if self.scheduler_config:

145 + vllm_factors.append(self.scheduler_config.compute_hash())

146 + else:

147 + vllm_factors.append("None")

148 + if self.device_config:

149 + vllm_factors.append(self.device_config.compute_hash())

150 + else:

151 + vllm_factors.append("None")

152 + if self.load_config:

153 + vllm_factors.append(self.load_config.compute_hash())

154 + else:

155 + vllm_factors.append("None")

156 + if self.lora_config:

157 + vllm_factors.append(self.lora_config.compute_hash())

158 + # LoRA creates static buffers based on max_num_batched_tokens.

159 + # The tensor sizes and strides get captured in the torch.compile

160 + # graph explicitly.

161 + vllm_factors.append(

162 + str(self.scheduler_config.max_num_batched_tokens))

163 + else:

164 + vllm_factors.append("None")

165 + if self.speculative_config:

166 + vllm_factors.append(self.speculative_config.compute_hash())

167 + else:

168 + vllm_factors.append("None")

169 + if self.structured_outputs_config:

170 + vllm_factors.append(self.structured_outputs_config.compute_hash())

171 + else:

172 + vllm_factors.append("None")

173 + if self.observability_config:

174 + vllm_factors.append(self.observability_config.compute_hash())

175 + else:

176 + vllm_factors.append("None")

177 + if self.quant_config:

178 + pass # should be captured by model_config.quantization

179 + if self.compilation_config:

180 + vllm_factors.append(self.compilation_config.compute_hash())

181 + else:

182 + vllm_factors.append("None")

183 + if self.kv_transfer_config:

184 + vllm_factors.append(self.kv_transfer_config.compute_hash())

185 + else:

186 + vllm_factors.append("None")

187 + if self.additional_config:

188 + if isinstance(additional_config := self.additional_config, dict):

189 + additional_config_hash = hashlib.md5(

190 + json.dumps(additional_config, sort_keys=True).encode(),

191 + usedforsecurity=False,

192 + ).hexdigest()

193 + else:

194 + additional_config_hash = additional_config.compute_hash()

195 + vllm_factors.append(additional_config_hash)

196 + else:

197 + vllm_factors.append("None")

198 + factors.append(vllm_factors)

199 +

200 + hash_str = hashlib.md5(str(factors).encode(),

201 + usedforsecurity=False).hexdigest()[:10]

202 + return hash_str

203 +

204 + def pad_for_cudagraph(self, batch_size: int) -> int:

205 + # if batch_size > self.compilation_config.max_capture_size,

206 + # it should raise an IndexError.

207 + # the caller should make sure the batch_size is within the range,

208 + # i.e., batch_size <= self.compilation_config.max_capture_size

209 + return self.compilation_config.bs_to_padded_graph_size[batch_size]

210 +

211 + @staticmethod

212 + def _get_quantization_config(

213 + model_config: ModelConfig,

214 + load_config: LoadConfig) -> Optional[QuantizationConfig]:

215 + """Get the quantization config."""

216 + from vllm.platforms import current_platform

217 + if model_config.quantization is not None:

218 + from vllm.model_executor.model_loader.weight_utils import (

219 + get_quant_config)

220 + quant_config = get_quant_config(model_config, load_config)

221 + capability_tuple = current_platform.get_device_capability()

222 +

223 + if capability_tuple is not None:

224 + capability = capability_tuple.to_int()

225 + if capability < quant_config.get_min_capability():

226 + raise ValueError(

227 + f"The quantization method {model_config.quantization} "

228 + "is not supported for the current GPU. Minimum "

229 + f"capability: {quant_config.get_min_capability()}. "

230 + f"Current capability: {capability}.")

231 + supported_dtypes = quant_config.get_supported_act_dtypes()

232 + if model_config.dtype not in supported_dtypes:

233 + raise ValueError(

234 + f"{model_config.dtype} is not supported for quantization "

235 + f"method {model_config.quantization}. Supported dtypes: "

236 + f"{supported_dtypes}")

237 + quant_config.maybe_update_config(model_config.model)

238 + return quant_config

239 + return None

240 +

241 + @staticmethod

242 + def get_quantization_config(

243 + model_config: ModelConfig,

244 + load_config: LoadConfig) -> Optional[QuantizationConfig]:

245 + import copy

246 +

247 + # For some reason, the _ version of this modifies the model_config

248 + # object, so using deepcopy to avoid this problem.

249 + return VllmConfig._get_quantization_config(copy.deepcopy(model_config),

250 + load_config)

251 +

252 + def with_hf_config(

253 + self,

254 + hf_config: PretrainedConfig,

255 + architectures: Optional[list[str]] = None,

256 + ) -> "VllmConfig":

257 + if architectures is not None:

258 + hf_config = copy.deepcopy(hf_config)

259 + hf_config.architectures = architectures

260 +

261 + model_config = copy.deepcopy(self.model_config)

262 + model_config.hf_config = hf_config

263 +

264 + return replace(self, model_config=model_config)

265 +

266 + def __post_init__(self):

267 + """Verify configs are valid & consistent with each other.

268 + """

269 +

270 + self.try_verify_and_update_config()

271 +

272 + if self.model_config is not None:

273 + self.model_config.verify_with_parallel_config(self.parallel_config)

274 + self.model_config.verify_dual_chunk_attention_config(

275 + self.load_config)

276 +

277 + self.cache_config.verify_with_parallel_config(self.parallel_config)

278 +

279 + if self.lora_config is not None:

280 + self.lora_config.verify_with_cache_config(self.cache_config)

281 + self.lora_config.verify_with_model_config(self.model_config)

282 +

283 + if self.quant_config is None and self.model_config is not None:

284 + self.quant_config = VllmConfig._get_quantization_config(

285 + self.model_config, self.load_config)

286 +

287 + from vllm.platforms import current_platform

288 + if self.model_config is not None and \

289 + self.scheduler_config.chunked_prefill_enabled and \

290 + self.model_config.dtype == torch.float32 and \

291 + current_platform.get_device_capability() == (7, 5):

292 + logger.warning_once(

293 + "Turing devices tensor cores do not support float32 matmul. "

294 + "To workaround this limitation, vLLM will set 'ieee' input "

295 + "precision for chunked prefill triton kernels.")

296 +

297 + # If the user does not explicitly set a compilation level, then

298 + # we use the default level. The default level depends on other

299 + # settings (see the below code).

300 + if self.compilation_config.level is None:

301 + if envs.VLLM_USE_V1:

302 + if (self.model_config is not None

303 + and not self.model_config.enforce_eager):

304 + self.compilation_config.level = CompilationLevel.PIECEWISE

305 + else:

306 + self.compilation_config.level = \

307 + CompilationLevel.NO_COMPILATION

308 +

309 + else:

310 + # NB: Passing both --enforce-eager and a compilation level

311 + # in V0 means the compilation level wins out.

312 + self.compilation_config.level = CompilationLevel.NO_COMPILATION

313 +

314 + # async tp is built on top of sequence parallelism

315 + # and requires it to be enabled.

316 + if self.compilation_config.pass_config.enable_async_tp:

317 + self.compilation_config.pass_config.enable_sequence_parallelism = \

318 + True

319 + if self.compilation_config.pass_config.enable_sequence_parallelism:

320 + self.compilation_config.custom_ops.append("+rms_norm")

321 +

322 + if current_platform.support_static_graph_mode():

323 + # if cudagraph_mode is not explicitly set by users, set default

324 + # value

325 + if self.compilation_config.cudagraph_mode is None:

326 + if envs.VLLM_USE_V1 and self.compilation_config.level \

327 + == CompilationLevel.PIECEWISE:

328 + # default to full and piecewise for most models

329 + self.compilation_config.cudagraph_mode = \

330 + CUDAGraphMode.FULL_AND_PIECEWISE

331 +

332 + # pooling models and encoder-decoder models

333 + # do not support full cudagraphs

334 + if self.model_config is not None and \

335 + (self.model_config.pooler_config is not None

336 + or self.model_config.is_encoder_decoder):

337 + self.compilation_config.cudagraph_mode = \

338 + CUDAGraphMode.PIECEWISE

339 + else:

340 + self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

341 +

342 + # disable cudagraph when enforce eager execution

343 + if self.model_config is not None and \

344 + self.model_config.enforce_eager:

345 + logger.info("Cudagraph is disabled under eager mode")

346 + self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

347 + elif envs.VLLM_USE_V1:

348 + self.compilation_config.cudagraph_num_of_warmups = 1

349 +

350 + self._set_cudagraph_sizes()

351 + else:

352 + self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

353 +

354 + if self.cache_config.kv_sharing_fast_prefill:

355 +

356 + if self.speculative_config is not None and \

357 + self.speculative_config.use_eagle():

358 + raise NotImplementedError(

359 + "Fast prefill optimization for KV sharing is not "

360 + "compatible with EAGLE as EAGLE requires correct logits "

361 + "for all tokens while fast prefill gives incorrect logits "

362 + "for prompt tokens.")

363 +

364 + logger.warning_once(

365 + "--kv-sharing-fast-prefill requires changes on model side for "

366 + "correctness and to realize prefill savings. ")

367 +

368 + disable_chunked_prefill_reasons: list[str] = []

369 +

370 + if self.model_config:

371 + if self.model_config.pooler_config:

372 + pooling_type = self.model_config.pooler_config.pooling_type

373 + if pooling_type is None or pooling_type.lower() != "last":

374 + disable_chunked_prefill_reasons.append(

375 + "Only \"last\" pooling supports chunked "

376 + "prefill and prefix caching; disabling both.")

377 + if not getattr(self.model_config.hf_config, "is_causal", True):

378 + disable_chunked_prefill_reasons.append(

379 + "Only models using causal attention supports chunked "

380 + "prefill and prefix caching; disabling both.")

381 + elif self.model_config.is_encoder_decoder:

382 + from vllm.multimodal import MULTIMODAL_REGISTRY

383 + self.scheduler_config.max_num_encoder_input_tokens = \

384 + MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(self.model_config)

385 + logger.debug(

386 + "Encoder-decoder model detected: setting "

387 + "`max_num_encoder_input_tokens` to encoder length (%s)",

388 + self.scheduler_config.max_num_encoder_input_tokens)

389 + self.scheduler_config.disable_chunked_mm_input = True

390 + disable_chunked_prefill_reasons.append(

391 + "Encoder-decoder models do not support chunked prefill nor"

392 + " prefix caching; disabling both.")

393 + if (self.model_config.architecture

394 + == "WhisperForConditionalGeneration"

395 + and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD")

396 + != "spawn"):

397 + logger.warning(

398 + "Whisper is known to have issues with "

399 + "forked workers. If startup is hanging, "

400 + "try setting 'VLLM_WORKER_MULTIPROC_METHOD' "

401 + "to 'spawn'.")

402 +

403 + if disable_chunked_prefill_reasons:

404 + for reason in disable_chunked_prefill_reasons:

405 + logger.info(reason)

406 + self.scheduler_config.chunked_prefill_enabled = False

407 + self.scheduler_config.long_prefill_token_threshold = 0

408 +

409 + if self.cache_config is not None:

410 + self.cache_config.enable_prefix_caching = False

411 +

412 + if (self.kv_events_config is not None

413 + and self.kv_events_config.enable_kv_cache_events

414 + and not self.cache_config.enable_prefix_caching):

415 + logger.warning(

416 + "KV cache events are on, but prefix caching is not enabled."

417 + "Use --enable-prefix-caching to enable.")

418 + if (self.kv_events_config is not None

419 + and self.kv_events_config.publisher != "null"

420 + and not self.kv_events_config.enable_kv_cache_events):

421 + logger.warning("KV cache events are disabled,"

422 + "but the scheduler is configured to publish them."

423 + "Modify KVEventsConfig.enable_kv_cache_events"

424 + "to True to enable.")

425 + current_platform.check_and_update_config(self)

426 +

427 + # Do this after all the updates to compilation_config.level

428 + if envs.VLLM_USE_V1 and \

429 + self.compilation_config.level == CompilationLevel.PIECEWISE:

430 + self.compilation_config.set_splitting_ops_for_v1()

431 +

432 + # final check of cudagraph mode after all possible updates

433 + if envs.VLLM_USE_V1 and current_platform.is_cuda_alike():

434 + if self.compilation_config.cudagraph_mode.has_full_cudagraphs()\

435 + and self.model_config is not None and \

436 + not self.model_config.disable_cascade_attn and\

437 + not self.compilation_config.cudagraph_mode.\

438 + has_piecewise_cudagraphs():

439 + logger.warning_once(

440 + "No piecewise cudagraph for executing cascade attention."

441 + " Will fall back to eager execution if a batch runs "

442 + "into cascade attentions")

443 +

444 + if self.compilation_config.cudagraph_mode\

445 + .requires_piecewise_compilation():

446 + assert self.compilation_config.level == \

447 + CompilationLevel.PIECEWISE, \

448 + "Compilation level should be CompilationLevel.PIECEWISE "\

449 + "when cudagraph_mode piecewise cudagraphs is used, "\

450 + f"cudagraph_mode={self.compilation_config.cudagraph_mode}"

451 +

452 + # final migrate the deprecated flags

453 + self.compilation_config.use_cudagraph = self.compilation_config.\

454 + cudagraph_mode!= CUDAGraphMode.NONE

455 + self.compilation_config.full_cuda_graph = self.compilation_config.\

456 + cudagraph_mode.has_full_cudagraphs()

457 +

458 + if self.parallel_config.enable_dbo:

459 + a2a_backend = envs.VLLM_ALL2ALL_BACKEND

460 + assert a2a_backend in \

461 + ["deepep_low_latency", "deepep_high_throughput"], \

462 + "Microbatching currently only supports the deepep_low_latency and "\

463 + f"deepep_high_throughput all2all backend. {a2a_backend} is not "\

464 + "supported. To fix set the VLLM_ALL2ALL_BACKEND environment "\

465 + "variable to deepep_low_latency or deepep_high_throughput and "\

466 + "install the DeepEP kernels."

467 +

468 + if not self.model_config.disable_cascade_attn:

469 + self.model_config.disable_cascade_attn = True

470 + logger.warning_once(

471 + "Disabling cascade attention when DBO is enabled.")

472 +

473 + if not self.instance_id:

474 + self.instance_id = random_uuid()[:5]

475 +

476 + if (envs.VLLM_USE_V1

477 + and not self.scheduler_config.disable_hybrid_kv_cache_manager):

478 + # logger should only print warning message for hybrid models. As we

479 + # can't know whether the model is hybrid or not now, so we don't log

480 + # warning message here and will log it later.

481 + if not current_platform.support_hybrid_kv_cache():

482 + # Hybrid KV cache manager is not supported on non-GPU platforms.

483 + self.scheduler_config.disable_hybrid_kv_cache_manager = True

484 + if self.kv_transfer_config is not None:

485 + # Hybrid KV cache manager is not compatible with KV transfer.

486 + self.scheduler_config.disable_hybrid_kv_cache_manager = True

487 + if self.kv_events_config is not None:

488 + # Hybrid KV cache manager is not compatible with KV events.

489 + self.scheduler_config.disable_hybrid_kv_cache_manager = True

490 + if self.model_config is not None and \

491 + self.model_config.attention_chunk_size is not None:

492 + if self.speculative_config is not None and \

493 + self.speculative_config.use_eagle():

494 + # Hybrid KV cache manager is not yet supported with chunked

495 + # local attention + eagle.

496 + self.scheduler_config.disable_hybrid_kv_cache_manager = True

497 + elif \

498 + not envs.VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE:

499 + logger.warning(

500 + "There is a latency regression when using chunked local"

501 + " attention with the hybrid KV cache manager. Disabling"

502 + " it, by default. To enable it, set the environment "

503 + "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE=1."

504 + )

505 + # Hybrid KV cache manager is not yet supported with chunked

506 + # local attention.

507 + self.scheduler_config.disable_hybrid_kv_cache_manager = True

508 +

509 + if self.compilation_config.debug_dump_path:

510 + self.compilation_config.debug_dump_path = \

511 + self.compilation_config.debug_dump_path.absolute().expanduser()

512 + if envs.VLLM_DEBUG_DUMP_PATH is not None:

513 + env_path = Path(envs.VLLM_DEBUG_DUMP_PATH).absolute().expanduser()

514 + if self.compilation_config.debug_dump_path:

515 + logger.warning(

516 + "Config-specified debug dump path is overridden"

517 + " by VLLM_DEBUG_DUMP_PATH to %s", env_path)

518 + self.compilation_config.debug_dump_path = env_path

519 +

520 + def update_sizes_for_sequence_parallelism(self,

521 + possible_sizes: list) -> list:

522 + # remove the sizes that not multiple of tp_size when

523 + # enable sequence parallelism

524 + removed_sizes = [

525 + size for size in possible_sizes

526 + if size % self.parallel_config.tensor_parallel_size != 0

527 + ]

528 + if removed_sizes:

529 + logger.warning(

530 + "Batch sizes %s are removed because they are not "

531 + "multiple of tp_size %d when "

532 + "sequence parallelism is enabled", removed_sizes,

533 + self.parallel_config.tensor_parallel_size)

534 +

535 + return [

536 + size for size in possible_sizes

537 + if size % self.parallel_config.tensor_parallel_size == 0

538 + ]

539 +

540 + def _set_cudagraph_sizes(self):

541 + """

542 + vLLM defines the default candidate list of batch sizes for CUDA graph

543 + capture as:

544 +

545 + ```python

546 + max_graph_size = min(max_num_seqs * 2, 512)

547 + # 1, 2, 4, then multiples of 8 up to max_graph_size

548 + cuda_graph_sizes = [1, 2, 4, 8, 16, 24, 32, 40, ..., max_graph_size]

549 +

550 + In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`

551 + will be the final sizes to capture cudagraph (in descending order).

552 +

553 + These sizes are used to capture and reuse CUDA graphs for

554 + performance-critical paths (e.g., decoding). Capturing enables

555 + significantly faster kernel dispatch by avoiding Python overhead. The

556 + list is then filtered based on `max_num_batched_tokens` (e.g., 8192 on

557 + most GPUs), which controls the total allowed number of tokens in a

558 + batch. Since each sequence may have a variable number of tokens, the

559 + maximum usable batch size will depend on actual sequence lengths.

560 +

561 + Example:

562 + With `max_num_batched_tokens = 8192`, and typical sequences

563 + averaging ~32 tokens, most practical batch sizes fall below 256.

564 + However, the system will still allow capture sizes up to 512 if

565 + shape and memory permit.

566 +

567 + Note:

568 + If users explicitly specify cudagraph capture sizes in the

569 + compilation config, those will override this default logic.

570 + At runtime:

571 +

572 + - If batch size <= one of the `cudagraph_capture_sizes`, the closest

573 + padded CUDA graph will be used.

574 + - If batch size > largest `cudagraph_capture_sizes`, cudagraph will

575 + not be used.

576 + """

577 +

578 + # calculate the default `batch_size_capture_list`

579 + batch_size_capture_list = []

580 + if self.model_config is not None and \

581 + not self.model_config.enforce_eager:

582 + cuda_graph_sizes = self.scheduler_config.cuda_graph_sizes

583 + if len(cuda_graph_sizes) == 1:

584 + batch_size_capture_list = [1, 2, 4] + [

585 + i for i in range(8, cuda_graph_sizes[0] + 1, 8)

586 + ]

587 + elif len(cuda_graph_sizes) > 1:

588 + batch_size_capture_list = sorted(cuda_graph_sizes)

589 + else:

590 + raise TypeError(f"Invalid value for {cuda_graph_sizes=}.")

591 + if self.parallel_config.tensor_parallel_size > 1 and \

592 + self.compilation_config.pass_config.enable_sequence_parallelism:

593 + batch_size_capture_list = \

594 + self.update_sizes_for_sequence_parallelism(batch_size_capture_list)

595 + max_num_tokens = self.scheduler_config.max_num_batched_tokens

596 + batch_size_capture_list = [

597 + size for size in batch_size_capture_list

598 + if size <= max_num_tokens

599 + ]

600 +

601 + self.compilation_config.init_with_cudagraph_sizes(

602 + batch_size_capture_list)

603 +

604 + def recalculate_max_model_len(self, max_model_len: int):

605 + # Can only be called in try_verify_and_update_config

606 + model_config = self.model_config

607 + max_model_len = model_config.get_and_verify_max_len(max_model_len)

608 + self.model_config.max_model_len = max_model_len

609 + self.scheduler_config.max_model_len = max_model_len

610 +

611 + def try_verify_and_update_config(self):

612 + if self.model_config is None:

613 + return

614 +

615 + # Avoid running try_verify_and_update_config multiple times

616 + if getattr(self.model_config, "config_updated", False):

617 + return

618 + self.model_config.config_updated = True

619 +

620 + architecture = self.model_config.architecture

621 + if architecture is None:

622 + return

623 +

624 + from vllm.model_executor.models.config import (

625 + MODELS_CONFIG_MAP, HybridAttentionMambaModelConfig)

626 + cls = MODELS_CONFIG_MAP.get(architecture, None)

627 + if cls is not None:

628 + cls.verify_and_update_config(self)

629 +

630 + if self.model_config.is_hybrid:

631 + HybridAttentionMambaModelConfig.verify_and_update_config(self)

632 +

633 + if self.model_config.convert_type == "classify":

634 + # Maybe convert ForCausalLM into ForSequenceClassification model.

635 + from vllm.model_executor.models.adapters import (

636 + SequenceClassificationConfig)

637 + SequenceClassificationConfig.verify_and_update_config(self)

638 +

639 + if hasattr(self.model_config, "model_weights") and is_runai_obj_uri(

640 + self.model_config.model_weights):

641 + if self.load_config.load_format == "auto":

642 + logger.info("Detected Run:ai model config. "

643 + "Overriding `load_format` to 'runai_streamer'")

644 + self.load_config.load_format = "runai_streamer"

645 + elif self.load_config.load_format != "runai_streamer":

646 + raise ValueError(f"To load a model from S3, 'load_format' "

647 + f"must be 'runai_streamer', "

648 + f"but got '{self.load_config.load_format}'. "

649 + f"Model: {self.model_config.model}")

650 +

651 + def compile_debug_dump_path(self) -> Optional[Path]:

652 + """Returns a rank-aware path for dumping

653 + torch.compile debug information.

654 + """

655 + if self.compilation_config.debug_dump_path is None:

656 + return None

657 + tp_rank = self.parallel_config.rank

658 + dp_rank = self.parallel_config.data_parallel_rank

659 + data_parallel_size = self.parallel_config.data_parallel_size

660 + append_path = f"rank_{tp_rank}" if data_parallel_size == 1 \

661 + else f"rank_{tp_rank}_dp_{dp_rank}"

662 + path = self.compilation_config.debug_dump_path / append_path

663 + return path

664 +

665 + def __str__(self):

666 + return (

667 + f"model={self.model_config.model!r}, "

668 + f"speculative_config={self.speculative_config!r}, "

669 + f"tokenizer={self.model_config.tokenizer!r}, "

670 + f"skip_tokenizer_init={self.model_config.skip_tokenizer_init}, "

671 + f"tokenizer_mode={self.model_config.tokenizer_mode}, "

672 + f"revision={self.model_config.revision}, "

673 + f"tokenizer_revision={self.model_config.tokenizer_revision}, "

674 + f"trust_remote_code={self.model_config.trust_remote_code}, "

675 + f"dtype={self.model_config.dtype}, "

676 + f"max_seq_len={self.model_config.max_model_len}, "

677 + f"download_dir={self.load_config.download_dir!r}, "

678 + f"load_format={self.load_config.load_format}, "

679 + f"tensor_parallel_size={self.parallel_config.tensor_parallel_size}, " # noqa

680 + f"pipeline_parallel_size={self.parallel_config.pipeline_parallel_size}, " # noqa

681 + f"data_parallel_size={self.parallel_config.data_parallel_size}, " # noqa

682 + f"disable_custom_all_reduce={self.parallel_config.disable_custom_all_reduce}, " # noqa

683 + f"quantization={self.model_config.quantization}, "

684 + f"enforce_eager={self.model_config.enforce_eager}, "

685 + f"kv_cache_dtype={self.cache_config.cache_dtype}, "

686 + f"device_config={self.device_config.device}, "

687 + f"structured_outputs_config={self.structured_outputs_config!r}, "

688 + f"observability_config={self.observability_config!r}, "

689 + f"seed={self.model_config.seed}, "

690 + f"served_model_name={self.model_config.served_model_name}, "

691 + f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "

692 + f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa

693 + f"pooler_config={self.model_config.pooler_config!r}, "

694 + f"compilation_config={self.compilation_config!r}")

695 +

696 +

697 +_current_vllm_config: Optional[VllmConfig] = None

698 +_current_prefix: Optional[str] = None

699 +

700 +

701 +@contextmanager

702 +def set_current_vllm_config(vllm_config: VllmConfig,

703 + check_compile=False,

704 + prefix: Optional[str] = None):

705 + """

706 + Temporarily set the current vLLM config.

707 + Used during model initialization.

708 + We save the current vLLM config in a global variable,

709 + so that all modules can access it, e.g. custom ops

710 + can access the vLLM config to determine how to dispatch.

711 + """

712 + global _current_vllm_config, _current_prefix

713 + old_vllm_config = _current_vllm_config

714 + old_prefix = _current_prefix

715 + from vllm.compilation.counter import compilation_counter

716 + num_models_seen = compilation_counter.num_models_seen

717 + try:

718 + _current_vllm_config = vllm_config

719 + _current_prefix = prefix

720 + yield

721 + except Exception:

722 + raise

723 + else:

724 + if check_compile:

725 + vllm_config.compilation_config.custom_op_log_check()

726 +

727 + if check_compile and \

728 + vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \

729 + and compilation_counter.num_models_seen == num_models_seen:

730 + # If the model supports compilation,

731 + # compilation_counter.num_models_seen should be increased

732 + # by at least 1.

733 + # If it is not increased, it means the model does not support

734 + # compilation (does not have @support_torch_compile decorator).

735 + logger.warning(

736 + "`torch.compile` is turned on, but the model %s"

737 + " does not support it. Please open an issue on GitHub"

738 + " if you want it to be supported.",

739 + vllm_config.model_config.model)

740 + finally:

741 + _current_vllm_config = old_vllm_config

742 + _current_prefix = old_prefix

743 + # Clear the compilation config cache when context changes

744 + get_cached_compilation_config.cache_clear()

745 +

746 +

747 +@lru_cache(maxsize=1)

748 +def get_cached_compilation_config():

749 + """Cache config to avoid repeated calls to get_current_vllm_config()"""

750 + return get_current_vllm_config().compilation_config

751 +

752 +

753 +def get_current_vllm_config() -> VllmConfig:

754 + if _current_vllm_config is None:

755 + # in ci, usually when we test custom ops/modules directly,

756 + # we don't set the vllm config. In that case, we set a default

757 + # config.

758 + logger.warning("Current vLLM config is not set.")

759 + return VllmConfig()

760 + return _current_vllm_config

761 +

762 +

763 +T = TypeVar("T")

764 +

765 +

766 +def get_layers_from_vllm_config(

767 + vllm_config: VllmConfig,

768 + layer_type: type[T],

769 + layer_names: Optional[list[str]] = None) -> dict[str, T]:

770 + """

771 + Get layers from the vLLM config.

772 +

773 + Args:

774 + vllm_config: The vLLM config.

775 + layer_type: The type of the layer to get.

776 + layer_names: The names of the layers to get. If None, return all layers.

777 + """

778 +

779 + if layer_names is None:

780 + layer_names = list(

781 + vllm_config.compilation_config.static_forward_context.keys())

782 +

783 + forward_context = vllm_config.compilation_config.static_forward_context

784 +

785 + return {

786 + layer_name: forward_context[layer_name]

787 + for layer_name in layer_names

788 + if isinstance(forward_context[layer_name], layer_type)

789 + }

vllm/model_executor/layers/mamba/linear_attn.py

+1 -2

@@ -29,8 +29,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

29 from vllm.model_executor.layers.mamba.abstract import MambaBase

30 from vllm.model_executor.layers.mamba.mamba_utils import (

31 MambaStateDtypeCalculator, MambaStateShapeCalculator)

32 -from vllm.model_executor.layers.quantization.base_config import (

33 - QuantizationConfig)

32 +from vllm.model_executor.layers.quantization import QuantizationConfig

33 from vllm.utils import direct_register_custom_op

34 from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata

vllm/model_executor/layers/quantization/auto_round.py

+2 -3

@@ -9,9 +9,8 @@ import torch

9 from vllm.logger import init_logger

10 from vllm.model_executor.layers.linear import (LinearBase,

11 UnquantizedLinearMethod)

12 -from vllm.model_executor.layers.quantization import QuantizationMethods

13 -from vllm.model_executor.layers.quantization.base_config import (

14 - QuantizationConfig)

12 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

13 + QuantizationMethods)

14 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

15 from vllm.platforms import current_platform

16 from vllm.scalar_type import scalar_types

vllm/model_executor/layers/quantization/bitblas.py

+2 -3

@@ -7,9 +7,8 @@ from packaging import version

8 from vllm.logger import init_logger

9 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

10 -from vllm.model_executor.layers.quantization import QuantizationMethods

11 -from vllm.model_executor.layers.quantization.base_config import (

12 - QuantizationConfig)

10 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

11 + QuantizationMethods)

12 from vllm.model_executor.layers.quantization.utils.bitblas_utils import (

13 BITBLAS_OPTIMIZE_FEATURES, BITBLAS_SUPPORTED_NUM_BITS,

14 BITBLAS_SUPPORTED_SYM, MINIMUM_BITBLAS_VERSION)

vllm/model_executor/layers/quantization/bitsandbytes.py

+2 -3

@@ -13,9 +13,8 @@ from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,

13 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,

14 UnquantizedLinearMethod,

15 set_weight_attrs)

16 -from vllm.model_executor.layers.quantization import QuantizationMethods

17 -from vllm.model_executor.layers.quantization.base_config import (

18 - QuantizationConfig)

16 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

17 + QuantizationMethods)

18 from vllm.platforms import current_platform

19 from vllm.utils import direct_register_custom_op

vllm/model_executor/layers/quantization/deepspeedfp.py

+2 -3

@@ -9,9 +9,8 @@ import torch.nn.functional as F

9 from packaging import version

11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

12 -from vllm.model_executor.layers.quantization import QuantizationMethods

13 -from vllm.model_executor.layers.quantization.base_config import (

14 - QuantizationConfig)

12 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

13 + QuantizationMethods)

14 from vllm.model_executor.utils import set_weight_attrs

vllm/model_executor/layers/quantization/gptq.py

+6 -2

@@ -4,7 +4,7 @@

4 import enum

5 from enum import Enum

6 from fractions import Fraction

7 -from typing import Any, Optional, Union

7 +from typing import TYPE_CHECKING, Any, Optional, Union

9 import torch

10 from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

@@ -13,7 +13,6 @@ from torch.nn.parameter import Parameter

13 from vllm import _custom_ops as ops

14 from vllm.model_executor.layers.fused_moe.layer import FusedMoE

15 from vllm.model_executor.layers.linear import LinearMethodBase

16 -from vllm.model_executor.layers.quantization import QuantizationMethods

16 from vllm.model_executor.layers.quantization.base_config import (

17 QuantizationConfig, QuantizeMethodBase)

18 from vllm.model_executor.layers.quantization.utils.gptq_utils import (

@@ -26,6 +25,11 @@ from vllm.model_executor.parameter import (ChannelQuantScaleParameter,

25 from vllm.transformers_utils.config import get_safetensors_params_metadata

26 from vllm.utils import is_list_of

28 +if TYPE_CHECKING:

29 + from vllm.model_executor.layers.quantization import QuantizationMethods

30 +else:

31 + QuantizationMethods = str

32 +

34 class GPTQConfig(QuantizationConfig):

35 """Config class for GPTQ.

vllm/model_executor/layers/quantization/gptq_bitblas.py

+2 -3

@@ -9,9 +9,8 @@ from torch.nn.parameter import Parameter

9 from vllm.logger import init_logger

10 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,

11 set_weight_attrs)

12 -from vllm.model_executor.layers.quantization import QuantizationMethods

13 -from vllm.model_executor.layers.quantization.base_config import (

14 - QuantizationConfig)

12 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

13 + QuantizationMethods)

14 from vllm.model_executor.layers.quantization.kernels.mixed_precision import (

15 BitBLASLinearKernel, MPLinearLayerConfig)

16 from vllm.model_executor.layers.quantization.utils.bitblas_utils import (

vllm/model_executor/layers/quantization/gptq_marlin.py

+1 -1

@@ -43,7 +43,7 @@ logger = init_logger(__name__)

45 def get_moe_quant_method(

46 - config: QuantizationConfig,

46 + config: "GPTQMarlinConfig",

47 layer: torch.nn.Module,

48 prefix: str,

49 moe_method_cls: type,

vllm/model_executor/layers/quantization/gptq_marlin_24.py

+2 -3

@@ -9,9 +9,8 @@ from torch.nn.parameter import Parameter

9 from vllm import _custom_ops as ops

10 from vllm.logger import init_logger

11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

12 -from vllm.model_executor.layers.quantization import QuantizationMethods

13 -from vllm.model_executor.layers.quantization.base_config import (

14 - QuantizationConfig)

12 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

13 + QuantizationMethods)

14 from vllm.model_executor.parameter import (BasevLLMParameter,

15 ChannelQuantScaleParameter,

16 GroupQuantScaleParameter,

vllm/model_executor/layers/quantization/ipex_quant.py

+2 -3

@@ -14,11 +14,10 @@ from vllm.model_executor.layers.fused_moe import (FusedMoEMethodBase,

14 from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig

15 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,

16 UnquantizedLinearMethod)

17 -from vllm.model_executor.layers.quantization import QuantizationMethods

17 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

18 + QuantizationMethods)

19 from vllm.model_executor.layers.quantization.awq import (AWQLinearMethod,

20 is_layer_skipped_awq)

20 -from vllm.model_executor.layers.quantization.base_config import (

21 - QuantizationConfig)

21 from vllm.model_executor.layers.quantization.fp8 import (Fp8Config,

22 Fp8LinearMethod)

23 from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod

vllm/model_executor/layers/quantization/kernels/mixed_precision/bitblas.py

+1 -2

@@ -7,8 +7,7 @@ import torch

7 from packaging import version

9 from vllm.logger import init_logger

10 -from vllm.model_executor.layers.quantization.base_config import (

11 - QuantizationConfig)

10 +from vllm.model_executor.layers.quantization import QuantizationConfig

11 from vllm.model_executor.layers.quantization.utils import replace_parameter

12 from vllm.model_executor.layers.quantization.utils.bitblas_utils import (

13 BITBLAS_OPTIMIZE_FEATURES, BITBLAS_SUPPORTED_GROUP_SIZES,

vllm/model_executor/layers/quantization/tpu_int8.py

+2 -3

@@ -8,9 +8,8 @@ from torch.nn import Module

8 from torch.nn.parameter import Parameter

10 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

11 -from vllm.model_executor.layers.quantization import QuantizationMethods

12 -from vllm.model_executor.layers.quantization.base_config import (

13 - QuantizationConfig)

11 +from vllm.model_executor.layers.quantization import (QuantizationConfig,

12 + QuantizationMethods)

13 from vllm.model_executor.parameter import ModelWeightParameter

15 ACTIVATION_SCHEMES = ["none", "dynamic"]

vllm/model_executor/layers/quantization/utils/gptq_utils.py

+13 -5

@@ -4,21 +4,27 @@ from collections.abc import Mapping

4 from copy import deepcopy

5 from fractions import Fraction

6 from types import MappingProxyType

7 -from typing import Optional, Union

7 +from typing import TYPE_CHECKING, Optional, Union

9 import regex as re

10 import torch

12 -from vllm.config import QuantizationConfig

12 from vllm.model_executor.layers.linear import (LinearBase,

13 UnquantizedLinearMethod)

14 from vllm.model_executor.layers.vocab_parallel_embedding import (

15 ParallelLMHead, UnquantizedEmbeddingMethod)

17 +if TYPE_CHECKING:

18 + from ..gptq import GPTQConfig

19 + from ..gptq_marlin import GPTQMarlinConfig

20 +else:

21 + GPTQConfig = object

22 + GPTQMarlinConfig = object

23 +

25 # Match dynamic rules with module name (prefix) and override quantize

26 # config if module (prefix) matches a rule

21 -def override_config(config: QuantizationConfig, prefix: str):

27 +def override_config(config: Union[GPTQConfig, GPTQMarlinConfig], prefix: str):

28 weight_bits = get_dynamic_override(config, prefix, "bits",

29 config.weight_bits)

30 if isinstance(weight_bits, int):

@@ -34,6 +40,7 @@ def override_config(config: QuantizationConfig, prefix: str):

41 config.pack_factor = Fraction(32, config.weight_bits) # packed into int32

42 if config.get_name() == "gptq_marlin":

43 + assert isinstance(config, GPTQMarlinConfig)

44 is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)

45 if isinstance(is_sym, bool):

46 config.is_sym = is_sym

@@ -45,6 +52,7 @@ def override_config(config: QuantizationConfig, prefix: str):

52 config.quant_type = config.TYPE_MAP[(config.weight_bits,

53 config.is_sym)]

54 elif config.get_name() == "gptq":

55 + assert isinstance(config, GPTQConfig)

56 if config.weight_bits not in [2, 3, 4, 8]:

57 raise ValueError(

58 "Currently, only 2/3/4/8-bit weight quantization is "

@@ -52,7 +60,7 @@ def override_config(config: QuantizationConfig, prefix: str):

62 def get_dynamic_override(

55 - config: QuantizationConfig,

63 + config: Union[GPTQConfig, GPTQMarlinConfig],

64 layer_name: str,

65 key: Optional[str] = None,

66 default_value: Union[int, bool,

@@ -116,7 +124,7 @@ def is_layer_gptq_quantized(

124

125

126 def get_linear_quant_method(

119 - config: QuantizationConfig,

127 + config: Union[GPTQConfig, GPTQMarlinConfig],

128 layer: torch.nn.Module,

129 prefix: str,

130 linear_method_cls: type,

vllm/model_executor/models/aimv2.py

+1 -2

@@ -17,8 +17,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm

17 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

18 QKVParallelLinear,

19 RowParallelLinear)

20 -from vllm.model_executor.layers.quantization.base_config import (

21 - QuantizationConfig)

20 +from vllm.model_executor.layers.quantization import QuantizationConfig

21 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

22 from vllm.transformers_utils.configs.ovis import AIMv2Config

vllm/model_executor/models/aria.py

+2 -1

@@ -9,13 +9,14 @@ from transformers import AriaConfig, AriaTextConfig, BatchFeature

9 from transformers.models.aria.modeling_aria import AriaCrossAttention

10 from transformers.models.aria.processing_aria import AriaProcessor

12 -from vllm.config import QuantizationConfig, VllmConfig

12 +from vllm.config import VllmConfig

13 from vllm.distributed import get_tensor_model_parallel_rank

14 from vllm.model_executor.layers.activation import get_act_fn

15 from vllm.model_executor.layers.fused_moe import FusedMoE

16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

17 RowParallelLinear)

18 from vllm.model_executor.layers.logits_processor import LogitsProcessor

19 +from vllm.model_executor.layers.quantization import QuantizationConfig

20 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

21 from vllm.model_executor.model_loader.weight_utils import (

22 default_weight_loader, maybe_remap_kv_scale_name)

vllm/model_executor/models/bailing_moe.py

+1 -2

@@ -45,8 +45,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

45 QKVParallelLinear,

46 RowParallelLinear)

47 from vllm.model_executor.layers.logits_processor import LogitsProcessor

48 -from vllm.model_executor.layers.quantization.base_config import (

49 - QuantizationConfig)

48 +from vllm.model_executor.layers.quantization import QuantizationConfig

49 from vllm.model_executor.layers.rotary_embedding import get_rope

50 from vllm.model_executor.layers.vocab_parallel_embedding import (

51 ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/granite.py

+1 -2

@@ -41,8 +41,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

41 QKVParallelLinear,

42 RowParallelLinear)

43 from vllm.model_executor.layers.logits_processor import LogitsProcessor

44 -from vllm.model_executor.layers.quantization.base_config import (

45 - QuantizationConfig)

44 +from vllm.model_executor.layers.quantization import QuantizationConfig

45 from vllm.model_executor.layers.rotary_embedding import get_rope

46 from vllm.model_executor.layers.vocab_parallel_embedding import (

47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/granitemoe.py

+1 -2

@@ -42,8 +42,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

42 ReplicatedLinear,

43 RowParallelLinear)

44 from vllm.model_executor.layers.logits_processor import LogitsProcessor

45 -from vllm.model_executor.layers.quantization.base_config import (

46 - QuantizationConfig)

45 +from vllm.model_executor.layers.quantization import QuantizationConfig

46 from vllm.model_executor.layers.rotary_embedding import get_rope

47 from vllm.model_executor.layers.vocab_parallel_embedding import (

48 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/granitemoeshared.py

+1 -2

@@ -21,8 +21,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm

21 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

22 RowParallelLinear)

23 from vllm.model_executor.layers.logits_processor import LogitsProcessor

24 -from vllm.model_executor.layers.quantization.base_config import (

25 - QuantizationConfig)

24 +from vllm.model_executor.layers.quantization import QuantizationConfig

25 from vllm.model_executor.layers.vocab_parallel_embedding import (

26 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

27 from vllm.sequence import IntermediateTensors

vllm/model_executor/models/hunyuan_v1.py

+1 -2

@@ -47,8 +47,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

47 ReplicatedLinear,

48 RowParallelLinear)

49 from vllm.model_executor.layers.logits_processor import LogitsProcessor

50 -from vllm.model_executor.layers.quantization.base_config import (

51 - QuantizationConfig)

50 +from vllm.model_executor.layers.quantization import QuantizationConfig

51 from vllm.model_executor.layers.rotary_embedding import get_rope

52 from vllm.model_executor.layers.vocab_parallel_embedding import (

53 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/interfaces.py

+1 -2

@@ -16,8 +16,7 @@ from vllm.config import ModelConfig, SpeechToTextConfig

16 from vllm.inputs import TokensPrompt

17 from vllm.inputs.data import PromptType

18 from vllm.logger import init_logger

19 -from vllm.model_executor.layers.quantization.base_config import (

20 - QuantizationConfig)

19 +from vllm.model_executor.layers.quantization import QuantizationConfig

20 from vllm.utils import supports_kw

22 from .interfaces_base import VllmModel, is_pooling_model

vllm/model_executor/models/llama4_eagle.py

+1 -2

@@ -28,8 +28,7 @@ from vllm.distributed.parallel_state import get_pp_group

28 from vllm.logger import init_logger

29 from vllm.model_executor.layers.layernorm import RMSNorm

30 from vllm.model_executor.layers.logits_processor import LogitsProcessor

31 -from vllm.model_executor.layers.quantization.base_config import (

32 - QuantizationConfig)

31 +from vllm.model_executor.layers.quantization import QuantizationConfig

32 from vllm.model_executor.layers.quantization.torchao import TorchAOConfig

33 from vllm.model_executor.layers.vocab_parallel_embedding import (

34 VocabParallelEmbedding)

vllm/model_executor/models/mamba.py

+1 -2

@@ -16,8 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

16 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer

17 from vllm.model_executor.layers.mamba.mamba_utils import (

18 MambaStateDtypeCalculator, MambaStateShapeCalculator)

19 -from vllm.model_executor.layers.quantization.base_config import (

20 - QuantizationConfig)

19 +from vllm.model_executor.layers.quantization import QuantizationConfig

20 from vllm.model_executor.layers.vocab_parallel_embedding import (

21 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

vllm/model_executor/models/mamba2.py

+1 -2

@@ -16,8 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

16 from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2

17 from vllm.model_executor.layers.mamba.mamba_utils import (

18 MambaStateDtypeCalculator, MambaStateShapeCalculator)

19 -from vllm.model_executor.layers.quantization.base_config import (

20 - QuantizationConfig)

19 +from vllm.model_executor.layers.quantization import QuantizationConfig

20 from vllm.model_executor.layers.vocab_parallel_embedding import (

21 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

vllm/model_executor/models/minimax_text_01.py

+1 -2

@@ -33,8 +33,7 @@ from vllm.model_executor.layers.mamba.linear_attn import (

33 MiniMaxText01LinearAttention)

34 from vllm.model_executor.layers.mamba.mamba_utils import (

35 MambaStateDtypeCalculator, MambaStateShapeCalculator)

36 -from vllm.model_executor.layers.quantization.base_config import (

37 - QuantizationConfig)

36 +from vllm.model_executor.layers.quantization import QuantizationConfig

37 from vllm.model_executor.layers.rotary_embedding import get_rope

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/ovis.py

+1 -2

@@ -29,8 +29,7 @@ from transformers import BatchFeature, PretrainedConfig

30 from vllm.config import VllmConfig

31 from vllm.model_executor.layers.linear import ReplicatedLinear

32 -from vllm.model_executor.layers.quantization.base_config import (

33 - QuantizationConfig)

32 +from vllm.model_executor.layers.quantization import QuantizationConfig

33 from vllm.model_executor.models.aimv2 import AIMv2Model

34 from vllm.model_executor.models.siglip import SiglipVisionModel

35 from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,

vllm/model_executor/models/ovis2_5.py

+1 -2

@@ -11,8 +11,7 @@ from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

12 from vllm.config import VllmConfig

13 from vllm.model_executor.layers.linear import ReplicatedLinear

14 -from vllm.model_executor.layers.quantization.base_config import (

15 - QuantizationConfig)

14 +from vllm.model_executor.layers.quantization import QuantizationConfig

15 from vllm.model_executor.models.ovis import (OvisImagePatchInputs,

16 VisualEmbedding)

17 from vllm.model_executor.models.siglip2navit import Siglip2NavitModel

vllm/model_executor/models/phimoe.py

+1 -2

@@ -40,8 +40,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

40 ReplicatedLinear,

41 RowParallelLinear)

42 from vllm.model_executor.layers.logits_processor import LogitsProcessor

43 -from vllm.model_executor.layers.quantization.base_config import (

44 - QuantizationConfig)

43 +from vllm.model_executor.layers.quantization import QuantizationConfig

44 from vllm.model_executor.layers.rotary_embedding import get_rope

45 from vllm.model_executor.layers.vocab_parallel_embedding import (

46 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/siglip2navit.py

+1 -1

@@ -14,13 +14,13 @@ from transformers import Siglip2VisionConfig

14 from transformers.configuration_utils import PretrainedConfig

16 from vllm.attention.layer import check_upstream_fa_availability

17 -from vllm.config import QuantizationConfig

17 from vllm.distributed import divide, get_tensor_model_parallel_world_size

18 from vllm.model_executor.layers.activation import get_act_fn

19 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

20 LinearBase, QKVParallelLinear,

21 ReplicatedLinear,

22 RowParallelLinear)

23 +from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

25 from vllm.platforms import _Backend

vllm/model_executor/models/step3_text.py

+1 -2

@@ -23,8 +23,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

23 ReplicatedLinear,

24 RowParallelLinear)

25 from vllm.model_executor.layers.logits_processor import LogitsProcessor

26 -from vllm.model_executor.layers.quantization.base_config import (

27 - QuantizationConfig)

26 +from vllm.model_executor.layers.quantization import QuantizationConfig

27 from vllm.model_executor.layers.rotary_embedding import get_rope

28 from vllm.model_executor.layers.vocab_parallel_embedding import (

29 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

vllm/model_executor/models/whisper.py

+1 -2

@@ -26,8 +26,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

26 QKVParallelLinear,

27 RowParallelLinear)

28 from vllm.model_executor.layers.logits_processor import LogitsProcessor

29 -from vllm.model_executor.layers.quantization.base_config import (

30 - QuantizationConfig)

29 +from vllm.model_executor.layers.quantization import QuantizationConfig

30 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

31 from vllm.model_executor.model_loader.utils import set_default_torch_dtype

32 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

Move`VllmConfig` from `config/__init__.py` to `config/vllm.py` (#25271)

Move`VllmConfig` from `config/init.py` to `config/vllm.py` (#25271)