A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 03:45 UTC
Fix mistral sliding window parsing (#33521)
Signed-off-by: Andy Lo <andy@mistral.ai>
Andy Lo committed
Feb 2, 2026 at 05:08 UTC
beb889948276843a8df5e74fc704118a6f54b2b8
2 files changed
+25
-22
vllm/transformers_utils/config.py
-13
@@ -225,19 +225,6 @@ class MistralConfigParser(ConfigParserBase):
225
226
config = adapt_config_dict(config_dict, defaults=hf_config_dict)
227
228
- # Mistral configs may define sliding_window as list[int]. Convert it
229
- # to int and add the layer_types list[str] to make it HF compatible
230
- if (sliding_window := getattr(config, "sliding_window", None)) and isinstance(
231
- sliding_window, list
232
- ):
233
- pattern_repeats = config.num_hidden_layers // len(sliding_window)
234
- layer_types = sliding_window * pattern_repeats
235
- config.layer_types = [
236
- "full_attention" if layer_type is None else "sliding_attention"
237
- for layer_type in layer_types
238
- ]
239
- config.sliding_window = next(filter(None, sliding_window), None)
240
-
228
return config_dict, config
229
230
vllm/transformers_utils/configs/mistral.py
+25
-9
@@ -14,6 +14,7 @@ def adapt_config_dict(
14
defaults: dict[str, Any],
15
) -> PretrainedConfig:
16
config_dict = _remap_general_mistral_args(config_dict)
17
+ config_dict = _remap_mistral_sliding_window(config_dict)
18
19
if bool(config_dict.get("quantization")):
20
config_dict = _remap_mistral_quantization_args(config_dict)
@@ -161,6 +162,29 @@ def _remap_general_mistral_args(config: dict) -> dict:
162
return config
163
164
165
+def _remap_mistral_sliding_window(config: dict) -> dict:
166
+ # Remap sliding_window (list) -> layer_types (list) + sliding window (int)
167
+ # for HF compatibility
168
+ # Mistral configs may define sliding_window as list[int]. Convert it
169
+ # to int and add the layer_types list[str] to make it HF compatible
170
+ if sliding_window := config.get("sliding_window"):
171
+ if isinstance(sliding_window, list):
172
+ pattern_repeats = config["num_hidden_layers"] // len(sliding_window)
173
+ layer_types = sliding_window * pattern_repeats
174
+ config["layer_types"] = [
175
+ "full_attention" if layer_type is None else "sliding_attention"
176
+ for layer_type in layer_types
177
+ ]
178
+ assert len(set(sliding_window) - {None}) <= 1, sliding_window
179
+ config["sliding_window"] = next(filter(None, sliding_window), None)
180
+ elif isinstance(sliding_window, int) and config.get("layer_types") is None:
181
+ config["layer_types"] = ["sliding_attention"] * config["num_hidden_layers"]
182
+ else:
183
+ raise ValueError(f"Unsupported sliding_window type: {sliding_window}")
184
+
185
+ return config
186
+
187
+
188
def _remap_mistral_quantization_args(config: dict) -> dict:
189
if config.get("quantization"):
190
quantization = config.pop("quantization", {})
@@ -195,14 +219,6 @@ def _remap_mistral_audio_args(config: dict) -> dict:
219
else:
220
block_pool_size = 1
221
198
- _maybe_sliding_window = encoder_args.get("ragged_attention", None)
199
- if _maybe_sliding_window is None:
200
- sliding_window = None
201
- elif _maybe_sliding_window.isdigit():
202
- sliding_window = int(_maybe_sliding_window)
203
- else:
204
- raise NotImplementedError(f"Unsupported: {_maybe_sliding_window=}")
205
-
222
architecture = (
223
"VoxtralRealtimeGeneration"
224
if encoder_args.get("causal")
@@ -229,7 +245,7 @@ def _remap_mistral_audio_args(config: dict) -> dict:
245
max_source_positions=encoder_args["max_source_positions"],
246
is_encoder_decoder=False, # Override WhisperConfig default
247
is_causal=encoder_args.get("causal", False),
232
- sliding_window=sliding_window,
248
+ sliding_window=encoder_args.get("sliding_window", None),
249
block_pool_size=block_pool_size,
250
pos_embed=encoder_args.get("pos_embed", "sinusoidal"),
251
# only needed for RoPE