@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c49848396d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Refactor sliding window configuration to Transformers best practice (#21927)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Aug 10, 2025 at 04:50 UTC c49848396d34a1059fbec2a197394484acf5a903

16 files changed +124 -232

docs/contributing/model/basic.md

+1 -1

@@ -117,7 +117,7 @@ For models with interleaving sliding windows (e.g. `google/gemma-2-2b-it` and `m

117

118 To support a model with interleaving sliding windows, we need to take care of the following details:

119

120 -- Make sure the model's `config.json` contains `sliding_window_pattern`. vLLM then sets `self.hf_text_config.interleaved_sliding_window` to the value of `self.hf_text_config.sliding_window` and deletes `sliding_window` from `self.hf_text_config`. The model will then be treated as a full-attention model.

120 +- Make sure the model's `config.json` contains `layer_types`.

121 - In the modeling code, parse the correct sliding window value for every layer, and pass it to the attention layer's `per_layer_sliding_window` argument. For reference, check [this line](https://github.com/vllm-project/vllm/blob/996357e4808ca5eab97d4c97c7d25b3073f46aab/vllm/model_executor/models/llama.py#L171).

122

123 With these two steps, interleave sliding windows should work with the model.

tests/test_config.py

-22

@@ -200,28 +200,6 @@ def test_disable_sliding_window(model_id_expected):

200 assert model_config.max_model_len == expected

201

202

203 -def test_get_sliding_window():

204 - TEST_SLIDING_WINDOW = 4096

205 - # Test that the sliding window is correctly computed.

206 - # For Qwen1.5/Qwen2, get_sliding_window() should be None

207 - # when use_sliding_window is False.

208 - qwen2_model_config = ModelConfig("Qwen/Qwen1.5-7B")

209 -

210 - qwen2_model_config.hf_config.use_sliding_window = False

211 - qwen2_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW

212 - assert qwen2_model_config.get_sliding_window() is None

213 -

214 - qwen2_model_config.hf_config.use_sliding_window = True

215 - assert qwen2_model_config.get_sliding_window() == TEST_SLIDING_WINDOW

216 -

217 - mistral_model_config = ModelConfig("mistralai/Mistral-7B-v0.1")

218 - mistral_model_config.hf_config.sliding_window = None

219 - assert mistral_model_config.get_sliding_window() is None

220 -

221 - mistral_model_config.hf_config.sliding_window = TEST_SLIDING_WINDOW

222 - assert mistral_model_config.get_sliding_window() == TEST_SLIDING_WINDOW

223 -

224 -

203 @pytest.mark.skipif(current_platform.is_rocm(),

204 reason="Xformers backend is not supported on ROCm.")

205 def test_get_pooling_config():

vllm/config/__init__.py

+31 -80

@@ -40,8 +40,9 @@ from vllm.transformers_utils.config import (

40 ConfigFormat, get_config, get_hf_image_processor_config,

41 get_hf_text_config, get_pooling_config,

42 get_sentence_transformer_tokenizer_config, is_encoder_decoder,

43 - maybe_override_with_speculators_target_model, try_get_generation_config,

44 - try_get_safetensors_metadata, try_get_tokenizer_config, uses_mrope)

43 + is_interleaved, maybe_override_with_speculators_target_model,

44 + try_get_generation_config, try_get_safetensors_metadata,

45 + try_get_tokenizer_config, uses_mrope)

46 from vllm.transformers_utils.s3_utils import S3Model

47 from vllm.transformers_utils.utils import is_s3, maybe_model_redirect

48 # yapf conflicts with isort for this block

@@ -714,53 +715,31 @@ class ModelConfig:

715 revision=self.revision,

716 )

717

717 - # Workaround for Gemma 2 which uses interleaved sliding window

718 - # attention, but it's not specified in its config.

719 - # TODO: remove this when Gemma 2 config updated in HuggingFace.

720 - if self.hf_text_config.model_type == "gemma2":

721 - self.hf_text_config.sliding_window_pattern = 2

722 -

723 - # TODO: remove this when Gemma 3n config updated in HuggingFace.

724 - if self.hf_text_config.model_type == "gemma3n_text":

725 - # 4 sliding window attention followed by 1 full attention

726 - self.hf_text_config.sliding_window_pattern = "LLLLG"

727 -

728 - sliding_window = getattr(self.hf_text_config, "sliding_window", None)

729 - sliding_window_pattern = getattr(self.hf_text_config,

730 - "sliding_window_pattern", None)

731 - has_interleaved_attention = sliding_window_pattern is not None or (

732 - isinstance(sliding_window, list))

733 -

734 - if not self.disable_sliding_window and has_interleaved_attention:

735 - if not envs.VLLM_USE_V1 and (backend := envs.VLLM_ATTENTION_BACKEND

736 - ) in ("XFORMERS", "FLASHINFER"):

737 - sliding_window_len_min = get_min_sliding_window(

738 - self.hf_text_config.sliding_window)

739 -

740 - logger.warning_once(

741 - "%s has interleaved attention, which is currently not supported by the %s backend. Disabling sliding window and capping the max length to the sliding window size (%d).", # noqa: E501

742 - self.hf_text_config.model_type,

743 - backend,

744 - sliding_window_len_min,

745 - )

746 - self.disable_sliding_window = True

747 - else:

748 - # for a model with interleaved attention,

749 - # the scheduler and the model treat it as full attention

750 - # (i.e., not dropping any tokens outside the window).

751 - # only the attention layer itself is aware of the sliding

752 - # window, and use the window size to compute the attention.

753 - self.hf_text_config.interleaved_sliding_window = sliding_window

754 -

755 - if hasattr(self.hf_text_config, "sliding_window"):

756 - delattr(self.hf_text_config, "sliding_window")

757 -

758 - sliding_window = None

718 + # Interleaved attention is not supported by some backends in V0

719 + if (not self.disable_sliding_window

720 + and is_interleaved(self.hf_text_config)

721 + and not envs.VLLM_USE_V1

722 + and (backend := envs.VLLM_ATTENTION_BACKEND)

723 + in ("XFORMERS", "FLASHINFER")):

724 + logger.warning_once(

725 + "%s has interleaved attention, which is currently not "

726 + "supported by the %s backend. Disabling sliding window and "

727 + "capping the max length to the sliding window size (%d).",

728 + self.hf_text_config.model_type,

729 + backend,

730 + self.hf_text_config.sliding_window,

731 + )

732 + self.disable_sliding_window = True

733

734 self.original_max_model_len = self.max_model_len

735 self.max_model_len = self.get_and_verify_max_len(self.max_model_len)

736 self.multimodal_config = self._init_multimodal_config()

737

738 + if self.disable_sliding_window:

739 + # Set after get_and_verify_max_len to ensure that max_model_len

740 + # can be correctly capped to sliding window size

741 + self.hf_text_config.sliding_window = None

742 +

743 if not self.skip_tokenizer_init:

744 self._verify_tokenizer_mode()

745

@@ -1322,27 +1301,10 @@ class ModelConfig:

1301 if self.use_async_output_proc:

1302 self.use_async_output_proc = False

1303

1325 - def get_hf_config_sliding_window(

1326 - self) -> Union[Optional[int], list[Optional[int]]]:

1327 - """Get the sliding window size, or None if disabled."""

1328 -

1329 - # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in

1330 - # addition to sliding window size. We check if that field is present

1331 - # and if it's False, return None.

1332 - if (hasattr(self.hf_text_config, "use_sliding_window")

1333 - and not self.hf_text_config.use_sliding_window):

1334 - return None

1304 + def get_sliding_window(self) -> Optional[int]:

1305 + """Get the sliding window size from the HF text config if present."""

1306 return getattr(self.hf_text_config, "sliding_window", None)

1307

1337 - def get_sliding_window(self) -> Optional[Union[int, list[Optional[int]]]]:

1338 - """Get the sliding window size, or None if disabled.

1339 - """

1340 - # If user disables sliding window, return None.

1341 - if self.disable_sliding_window:

1342 - return None

1343 - # Otherwise get the value from the hf config.

1344 - return self.get_hf_config_sliding_window()

1345 -

1308 def get_vocab_size(self) -> int:

1309 return getattr(self.hf_text_config, "vocab_size", 0)

1310

@@ -1762,7 +1724,7 @@ class ModelConfig:

1724 tokenizer_config=tokenizer_config,

1725 max_model_len=max_model_len,

1726 disable_sliding_window=self.disable_sliding_window,

1765 - sliding_window_len=self.get_hf_config_sliding_window(),

1727 + sliding_window=self.get_sliding_window(),

1728 spec_target_max_model_len=self.spec_target_max_model_len,

1729 encoder_config=self.encoder_config)

1730 logger.info("Using max model len %s", max_model_len)

@@ -3305,7 +3267,7 @@ def _get_and_verify_max_len(

3267 tokenizer_config: Optional[dict],

3268 max_model_len: Optional[int],

3269 disable_sliding_window: bool,

3308 - sliding_window_len: Optional[Union[int, list[Optional[int]]]],

3270 + sliding_window: Optional[int],

3271 spec_target_max_model_len: Optional[int] = None,

3272 encoder_config: Optional[Any] = None,

3273 ) -> int:

@@ -3344,13 +3306,10 @@ def _get_and_verify_max_len(

3306

3307 # If sliding window is manually disabled, max_length should be less

3308 # than the sliding window length in the model config.

3347 - if disable_sliding_window and sliding_window_len is not None:

3348 -

3349 - sliding_window_len_min = get_min_sliding_window(sliding_window_len)

3350 - max_len_key = "sliding_window" \

3351 - if sliding_window_len_min < derived_max_model_len else max_len_key

3352 - derived_max_model_len = min(derived_max_model_len,

3353 - sliding_window_len_min)

3309 + if (disable_sliding_window and sliding_window is not None

3310 + and sliding_window < derived_max_model_len):

3311 + max_len_key = "sliding_window"

3312 + derived_max_model_len = sliding_window

3313

3314 # Consider model_max_length in tokenizer_config

3315 if tokenizer_config:

@@ -3451,14 +3410,6 @@ def _get_and_verify_max_len(

3410 return int(max_model_len)

3411

3412

3454 -def get_min_sliding_window(

3455 - sliding_window: Union[int, list[Optional[int]]]) -> int:

3456 - if isinstance(sliding_window, list):

3457 - return min(s for s in sliding_window if s is not None)

3458 -

3459 - return sliding_window

3460 -

3461 -

3413 def get_served_model_name(model: str,

3414 served_model_name: Optional[Union[str, list[str]]]):

3415 """

vllm/engine/arg_utils.py

+9 -1

@@ -39,6 +39,7 @@ from vllm.plugins import load_general_plugins

39 from vllm.ray.lazy_utils import is_ray_initialized

40 from vllm.reasoning import ReasoningParserManager

41 from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3

42 +from vllm.transformers_utils.config import is_interleaved

43 from vllm.transformers_utils.utils import check_gguf_file

44 from vllm.utils import (STR_DUAL_CHUNK_FLASH_ATTN_VAL, FlexibleArgumentParser,

45 GiB_bytes, get_ip, is_in_ray_actor)

@@ -1081,6 +1082,13 @@ class EngineArgs:

1082 "DualChunkFlashAttention is not supported on V1 engine. "

1083 "To run the model in V0 engine, try set 'VLLM_USE_V1=0'")

1084

1085 + sliding_window: Optional[int] = None

1086 + if not is_interleaved(model_config.hf_text_config):

1087 + # Only set CacheConfig.sliding_window if the model is all sliding

1088 + # window. Otherwise CacheConfig.sliding_window will override the

1089 + # global layers in interleaved sliding window models.

1090 + sliding_window = model_config.get_sliding_window()

1091 +

1092 cache_config = CacheConfig(

1093 block_size=self.block_size,

1094 gpu_memory_utilization=self.gpu_memory_utilization,

@@ -1088,7 +1096,7 @@ class EngineArgs:

1096 cache_dtype=self.kv_cache_dtype,

1097 is_attention_free=model_config.is_attention_free,

1098 num_gpu_blocks_override=self.num_gpu_blocks_override,

1091 - sliding_window=model_config.get_sliding_window(),

1099 + sliding_window=sliding_window,

1100 enable_prefix_caching=self.enable_prefix_caching,

1101 prefix_caching_hash_algo=self.prefix_caching_hash_algo,

1102 cpu_offload_gb=self.cpu_offload_gb,

vllm/model_executor/models/commandr.py

+7 -15

@@ -182,21 +182,13 @@ class CohereAttention(nn.Module):

182 )

183

184 # Model v2 has interleaved sliding windows, v1 does not

185 - interleaved_sliding_window = getattr(config,

186 - "interleaved_sliding_window",

187 - None)

188 - self.v1 = interleaved_sliding_window is None

189 -

190 - layer_idx = extract_layer_index(prefix)

191 - layer_has_sliding_window = (

192 - getattr(config, "sliding_window_pattern", False) and

193 - (layer_idx + 1) % self.config.sliding_window_pattern

194 - != 0) or (getattr(config, "layer_types", False)

195 - and config.layer_types[layer_idx] == "sliding_attention")

196 -

197 - self.sliding_window = (interleaved_sliding_window

198 - or config.sliding_window

199 - if layer_has_sliding_window else None)

185 + self.v1 = isinstance(config, CohereConfig)

186 +

187 + self.sliding_window = None

188 + if not self.v1:

189 + layer_idx = extract_layer_index(prefix)

190 + if config.layer_types[layer_idx] == "sliding_attention":

191 + self.sliding_window = config.sliding_window

192

193 self.attn = Attention(self.num_heads,

194 self.head_dim,

vllm/model_executor/models/exaone4.py

+4 -17

@@ -159,25 +159,12 @@ class Exaone4Attention(nn.Module):

159 if quant_config is not None and quant_config.get_name() == "gguf":

160 is_neox_style = False

161

162 - self.apply_all_layers = False # apply rotary embeddings to every layer.

162 layer_idx = extract_layer_index(prefix)

164 - interleaved_sliding_window = getattr(config,

165 - "interleaved_sliding_window",

166 - 4096)

167 - sliding_window_pattern = getattr(config, "sliding_window_pattern",

168 - "LLLG")

169 -

170 - if sliding_window_pattern:

171 - layer_has_sliding_window = (

172 - layer_idx + 1) % sliding_window_pattern.__len__() != 0

173 - else:

174 - layer_has_sliding_window = False

175 - self.apply_all_layers = True

163 + is_sliding = config.layer_types[layer_idx] == "sliding_attention"

164 + self.sliding_window = config.sliding_window if is_sliding else None

165

177 - if layer_has_sliding_window:

178 - self.sliding_window = interleaved_sliding_window

179 - else:

180 - self.sliding_window = None

166 + # apply rotary embeddings to every layer

167 + self.apply_all_layers = not is_sliding

168

169 self.rotary_emb = get_rope(

170 self.head_dim,

vllm/model_executor/models/gemma2.py

+3 -6

@@ -144,13 +144,10 @@ class Gemma2Attention(nn.Module):

144 is_neox_style=True,

145 )

146

147 - # reference:

148 - # https://github.com/huggingface/transformers/blob/54be2d7ae87e873482b984cc956e165ca4dc0ba3/src/transformers/models/gemma2/modeling_gemma2.py#L312 # noqa

147 layer_idx = extract_layer_index(prefix)

150 - use_sliding_window = (layer_idx % 2 == 0 and getattr(

151 - config, "interleaved_sliding_window", None) is not None)

152 - sliding_window = config.interleaved_sliding_window if \

153 - use_sliding_window else None

148 + is_sliding = config.layer_types[layer_idx] == "sliding_attention"

149 + sliding_window = config.sliding_window if is_sliding else None

150 +

151 self.attn = Attention(self.num_heads,

152 self.head_dim,

153 self.scaling,

vllm/model_executor/models/gemma3.py

+4 -10

@@ -146,25 +146,19 @@ class Gemma3Attention(nn.Module):

146 self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)

147 self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)

148

149 - # TODO(woosuk): Add reference to the original HF implementation.

149 layer_idx = extract_layer_index(prefix)

151 - self.is_sliding = (getattr(

152 - config, "interleaved_sliding_window", None) is not None and (bool(

153 - (layer_idx + 1) % config.sliding_window_pattern))) or (

154 - getattr(config, "layer_types", None) is not None

155 - and config.layer_types[layer_idx] == "sliding_attention")

150 + self.is_sliding = config.layer_types[layer_idx] == "sliding_attention"

151 + sliding_window = config.sliding_window if self.is_sliding else None

152 +

153 # Initialize the rotary embedding.

154 if self.is_sliding:

155 # Local attention. Override the values in config.json.

156 self.rope_theta = config.rope_local_base_freq

157 self.rope_scaling = {"rope_type": "default"}

161 - self.sliding_window = (config.interleaved_sliding_window

162 - or config.sliding_window)

158 else:

159 # Global attention. Use the values in config.json.

160 self.rope_theta = config.rope_theta

161 self.rope_scaling = config.rope_scaling

167 - self.sliding_window = None

162 self.rotary_emb = get_rope(

163 self.head_dim,

164 rotary_dim=self.head_dim,

@@ -182,7 +176,7 @@ class Gemma3Attention(nn.Module):

176 cache_config=cache_config,

177 quant_config=quant_config,

178 logits_soft_cap=attn_logits_soft_cap,

185 - per_layer_sliding_window=self.sliding_window,

179 + per_layer_sliding_window=sliding_window,

180 prefix=f"{prefix}.attn")

181

182 def forward(

vllm/model_executor/models/gemma3_mm.py

+2 -4

@@ -502,8 +502,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

502 self.config = config

503 self.quant_config = quant_config

504 self.multimodal_config = multimodal_config

505 - self.sliding_window = getattr(config.text_config,

506 - "interleaved_sliding_window", None)

505

506 self.vision_tower = SiglipVisionModel(config.vision_config,

507 quant_config,

@@ -690,11 +688,11 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

688 global_attn_mask = torch.where(img_mask == 2, 0, global_attn_mask)

689 global_attn_masks.append(global_attn_mask)

690

693 - if self.sliding_window is not None:

691 + if (sliding_window := self.config.sliding_window) is not None:

692 # Create a local causal mask with sliding window (1024).

693 local_attn_mask = torch.ones_like(global_attn_mask)

694 local_attn_mask = torch.tril(local_attn_mask,

697 - diagonal=-self.sliding_window)

695 + diagonal=-sliding_window)

696 local_attn_mask = torch.where(local_attn_mask == 0,

697 global_attn_mask, float("-inf"))

698 local_attn_masks.append(local_attn_mask)

vllm/model_executor/models/gemma3n.py

+6 -7

@@ -313,17 +313,16 @@ class Gemma3nAttention(nn.Module):

313 has_weight=False)

314

315 layer_idx = extract_layer_index(prefix)

316 + is_sliding = config.layer_types[layer_idx] == "sliding_attention"

317 + self.sliding_window = config.sliding_window if is_sliding else None

318

317 - is_sliding_window = (

318 - getattr(config, "interleaved_sliding_window", None) is not None

319 - and config.layer_types[layer_idx] == "sliding_attention")

320 -

321 - if is_sliding_window:

322 - self.sliding_window = config.interleaved_sliding_window

319 + # Initialize the rotary embedding.

320 + if is_sliding:

321 + # Local attention. Override the values in config.json.

322 rope_theta = config.rope_local_base_freq

323 rope_scaling = {"rope_type": "default"}

324 else:

326 - self.sliding_window = None

325 + # Global attention. Use the values in config.json.

326 rope_theta = config.rope_theta

327 rope_scaling = config.rope_scaling

328

vllm/model_executor/models/gritlm.py

+1 -3

@@ -248,9 +248,7 @@ class GritLM(LlamaForCausalLM, SupportsV0Only):

248

249 vllm_config.cache_config.sliding_window = None

250

251 - for attr in ("sliding_window", "interleaved_sliding_window"):

252 - if hasattr(hf_config, attr):

253 - delattr(hf_config, attr)

251 + hf_config.sliding_window = None

252

253 super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)

254

vllm/model_executor/models/llama.py

+5 -12

@@ -167,18 +167,11 @@ class LlamaAttention(nn.Module):

167 rope_scaling=rope_scaling,

168 quant_config=quant_config)

169

170 - if hasattr(config, "interleaved_sliding_window"):

171 - interleaved_sliding_window = config.interleaved_sliding_window

172 - if isinstance(interleaved_sliding_window, int):

173 - sliding_window = interleaved_sliding_window

174 - elif isinstance(interleaved_sliding_window, list):

175 - sw_idx = layer_idx % len(interleaved_sliding_window)

176 - sliding_window = interleaved_sliding_window[sw_idx]

177 - else:

178 - raise ValueError(

179 - f"{type(interleaved_sliding_window)} is not supported.")

180 - else:

181 - sliding_window = None

170 + sliding_window = None

171 + if layer_types := getattr(config, "layer_types", None):

172 + is_sliding = layer_types[layer_idx] == "sliding_attention"

173 + if is_sliding:

174 + sliding_window = config.sliding_window

175

176 self.attn = Attention(

177 self.num_heads,

vllm/model_executor/models/phi4flash.py

+2 -7

@@ -116,13 +116,8 @@ class SambaYAttention(nn.Module):

116 self.Wqkv = nn.Linear(self.hidden_size, op_size, bias=True)

117

118 # disable sliding window for the second half of the model

119 - sliding_window = config.interleaved_sliding_window[layer_idx]

120 - if layer_idx >= config.num_hidden_layers // 2:

121 - assert sliding_window is None, \

122 - "sliding_window must be none for the second decoder"

123 - else:

124 - assert sliding_window is not None, \

125 - "sliding_window must be set for the first decoder"

119 + is_sliding = config.layer_types[layer_idx] == "sliding_attention"

120 + sliding_window = config.sliding_window if is_sliding else None

121

122 assert self.num_heads % 2 == 0, 'num_heads should be even'

123 assert self.num_key_value_heads % 2 == 0, 'num_heads should be even'

vllm/model_executor/models/qwen2.py

+2 -2

@@ -49,6 +49,7 @@ from vllm.model_executor.model_loader.weight_utils import (

49 default_weight_loader, maybe_remap_kv_scale_name)

50 from vllm.model_executor.sampling_metadata import SamplingMetadata

51 from vllm.sequence import IntermediateTensors

52 +from vllm.transformers_utils.config import is_interleaved

54 from .interfaces import SupportsLoRA, SupportsPP

55 from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,

@@ -285,8 +286,7 @@ class Qwen2Model(nn.Module):

286 quant_config = vllm_config.quant_config

287

288 # TODO (@robertgshaw2): see if this can be moved out

288 - if (cache_config.sliding_window is not None

289 - and hasattr(config, "max_window_layers")):

289 + if is_interleaved(vllm_config.model_config.hf_text_config):

290 assert config.max_window_layers == config.num_hidden_layers, (

291 "Sliding window for some but all layers is not supported. "

292 "This model uses sliding window but `max_window_layers` = {} "

vllm/model_executor/models/transformers.py

+7 -45

@@ -16,7 +16,7 @@

16 # limitations under the License.

17 """Wrapper around `transformers` models"""

18 from collections.abc import Iterable, Mapping

19 -from contextlib import contextmanager, nullcontext

19 +from contextlib import contextmanager

20 from typing import Literal, Optional, Union

22 import regex as re

@@ -382,33 +382,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):

382 )

383

384

385 -class ConfigOverride:

386 - """Context manager to temporarily override config attributes."""

387 -

388 - def __init__(self, config: PretrainedConfig, **kwargs):

389 - self.config = config

390 - self.kwargs = kwargs

391 - self.kwargs_original = {}

392 - self.kwargs_delete = set()

393 -

394 - def __enter__(self):

395 - """Override config attributes."""

396 - for key, value in self.kwargs.items():

397 - if not hasattr(self.config, key):

398 - self.kwargs_delete.add(key)

399 - self.kwargs_original[key] = getattr(self.config, key, None)

400 - setattr(self.config, key, value)

401 - return self.config

402 -

403 - def __exit__(self, exc_type, exc_value, traceback):

404 - """Restore original config attributes."""

405 - for key, value in self.kwargs_original.items():

406 - if key in self.kwargs_delete:

407 - delattr(self.config, key)

408 - else:

409 - setattr(self.config, key, value)

410 -

411 -

385 class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):

386 embedding_padding_modules = ["lm_head"]

387 embedding_modules = ["embed_tokens"

@@ -434,21 +407,11 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):

407 # To be updated in child classes for use in `load_weights`

408 self.skip_prefixes: Optional[list[str]] = None

409

437 - # vLLM handles interleaved sliding window attention by creating a new

438 - # interleaved_sliding_window attribute and deleting the sliding_window

439 - # attribute. This breaks the constructors in Transformers so we

440 - # temporarily add the attribute back to construct the model.

441 - config_override = nullcontext()

442 - if hasattr(self.config, "interleaved_sliding_window"):

443 - config_override = ConfigOverride(

444 - self.config,

445 - sliding_window=self.config.interleaved_sliding_window)

446 -

410 # Set correct attn and init on "meta" to delay allocating GPU tensors

411 # TODO: @raushan, use the public `model.set_attn_implementation()`

412 # method once its checks are fixed in Transformers.

413 self.text_config._attn_implementation = "vllm"

451 - with init_on_device_without_buffers("meta"), config_override:

414 + with init_on_device_without_buffers("meta"):

415 self.model: PreTrainedModel = AutoModel.from_config(

416 self.config,

417 torch_dtype=self.model_config.dtype,

@@ -575,11 +538,10 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):

538 attention_instances = {}

539 for i in range(start, end):

540 # Handle interleaved sliding window attention

578 - sliding_window = None

579 - if (hasattr(self.config, "interleaved_sliding_window")

580 - and hasattr(self.config, "sliding_window_pattern")

581 - and ((i + 1) % self.config.sliding_window_pattern > 0)):

582 - sliding_window = self.config.interleaved_sliding_window

541 + per_layer_sliding_window = None

542 + if (hasattr(self.config, "layer_types")

543 + and self.config.layer_types[i] == "sliding_attention"):

544 + per_layer_sliding_window = self.config.sliding_window

545

546 attention_instances[i] = Attention(

547 num_heads=num_heads,

@@ -590,7 +552,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):

552 num_kv_heads=num_kv_heads,

553 cache_config=self.cache_config,

554 quant_config=self.quant_config,

593 - per_layer_sliding_window=sliding_window,

555 + per_layer_sliding_window=per_layer_sliding_window,

556 prefix=f"{i}.attn")

557 return attention_instances

558

vllm/transformers_utils/config.py

+40

@@ -280,6 +280,17 @@ def is_encoder_decoder(config: PretrainedConfig) -> bool:

280 return getattr(config, "is_encoder_decoder", False)

281

282

283 +def is_interleaved(config: PretrainedConfig) -> bool:

284 + """

285 + Detect if the model with this config is used with interleaved attention.

286 + """

287 + text_config = config.get_text_config()

288 + if layer_types := getattr(text_config, "layer_types", None):

289 + interleaved_types = {"full_attention", "sliding_attention"}

290 + return interleaved_types.issubset(layer_types)

291 + return False

292 +

293 +

294 def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:

295 """Remap config attributes to match the expected names."""

296 for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():

@@ -423,6 +434,23 @@ def get_config(

434 raise e

435 config = _maybe_remap_hf_config_attrs(config)

436

437 + # Phi4Flash misuses this config as list[int]. Convert it to int and add

438 + # the layer_types list[str] to make it HF compatible

439 + if (config.model_type == "phi4flash"):

440 + # TODO: Remove after the following PR is merged:

441 + # https://huggingface.co/microsoft/Phi-4-mini-flash-reasoning/discussions/6

442 + if not hasattr(config, "layer_types"):

443 + config.layer_types = [

444 + "sliding_attention" if i < config.num_hidden_layers // 2

445 + and i % 2 == 1 else "full_attention"

446 + for i in range(config.num_hidden_layers)

447 + ]

448 + # TODO: Remove after the following PR is merged:

449 + # https://huggingface.co/microsoft/Phi-4-mini-flash-reasoning/discussions/7

450 + if isinstance(config.sliding_window, list):

451 + config.sliding_window = next(

452 + filter(None, config.sliding_window), None)

453 +

454 elif config_format == ConfigFormat.MISTRAL:

455 # This function loads a params.json config which

456 # should be used when loading models in mistral format

@@ -434,6 +462,18 @@ def get_config(

462 config_dict["max_position_embeddings"] = max_position_embeddings

463

464 config = adapt_config_dict(config_dict)

465 +

466 + # Mistral configs may define sliding_window as list[int]. Convert it

467 + # to int and add the layer_types list[str] to make it HF compatible

468 + if ((sliding_window := getattr(config, "sliding_window", None))

469 + and isinstance(sliding_window, list)):

470 + pattern_repeats = config.num_hidden_layers // len(sliding_window)

471 + layer_types = sliding_window * pattern_repeats

472 + config.layer_types = [

473 + "full_attention" if layer_type is None else "sliding_attention"

474 + for layer_type in layer_types

475 + ]

476 + config.sliding_window = next(filter(None, sliding_window), None)

477 else:

478 supported_formats = [

479 fmt.value for fmt in ConfigFormat if fmt != ConfigFormat.AUTO