A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 01:45 UTC
[Bugfix] mamba-block-size is set for vision language model (#27773)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Oct 29, 2025 at 19:39 UTC
d7fb10c574a3a9cbf596bec086bf02603b71c5c8
2 files changed
+16
-10
vllm/config/cache.py
+1
-9
@@ -5,7 +5,7 @@ import hashlib
5
from dataclasses import field
6
from typing import TYPE_CHECKING, Any, Literal
7
8
-from pydantic import Field, SkipValidation, field_validator, model_validator
8
+from pydantic import Field, SkipValidation, field_validator
9
from pydantic.dataclasses import dataclass
10
11
from vllm.config.utils import config
@@ -185,11 +185,3 @@ class CacheConfig:
185
raise ValueError("Too large swap space. " + msg)
186
elif cpu_memory_usage > 0.4 * total_cpu_memory:
187
logger.warning("Possibly too large swap space. %s", msg)
188
-
189
- @model_validator(mode="after")
190
- def validate_mamba_block_size(self) -> "CacheConfig":
191
- if self.mamba_block_size is not None and not self.enable_prefix_caching:
192
- raise ValueError(
193
- "--mamba-block-size can only be set with --enable-prefix-caching"
194
- )
195
- return self
vllm/config/vllm.py
+15
-1
@@ -17,7 +17,7 @@ from pathlib import Path
17
from typing import TYPE_CHECKING, Any, TypeVar
18
19
import torch
20
-from pydantic import ConfigDict, Field
20
+from pydantic import ConfigDict, Field, model_validator
21
from pydantic.dataclasses import dataclass
22
23
import vllm.envs as envs
@@ -943,6 +943,20 @@ class VllmConfig:
943
f"compilation_config={self.compilation_config!r}"
944
)
945
946
+ @model_validator(mode="after")
947
+ def validate_mamba_block_size(self) -> "VllmConfig":
948
+ if self.model_config is None:
949
+ return self
950
+ mamba_block_size_is_set = (
951
+ self.cache_config.mamba_block_size is not None
952
+ and self.cache_config.mamba_block_size != self.model_config.max_model_len
953
+ )
954
+ if mamba_block_size_is_set and not self.cache_config.enable_prefix_caching:
955
+ raise ValueError(
956
+ "--mamba-block-size can only be set with --enable-prefix-caching"
957
+ )
958
+ return self
959
+
960
961
_current_vllm_config: VllmConfig | None = None
962
_current_prefix: str | None = None