| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """This test file includes some cases where it is inappropriate to |
| 4 | only get the `eos_token_id` from the tokenizer as defined by |
| 5 | `BaseRenderer.get_eos_token_id`. |
| 6 | """ |
| 7 | |
| 8 | import json |
| 9 | import math |
| 10 | from types import SimpleNamespace |
| 11 | from typing import cast |
| 12 | from unittest.mock import MagicMock, patch |
| 13 | |
| 14 | import pytest |
| 15 | from transformers import PreTrainedConfig |
| 16 | |
| 17 | from vllm.config.model import ModelConfig |
| 18 | from vllm.tokenizers import get_tokenizer |
| 19 | from vllm.transformers_utils import config as config_module |
| 20 | from vllm.transformers_utils.config import ( |
| 21 | get_safetensors_params_metadata, |
| 22 | mrope_num_dims, |
| 23 | patch_legacy_rope_type, |
| 24 | try_get_generation_config, |
| 25 | uses_mrope, |
| 26 | ) |
| 27 | from vllm.transformers_utils.configs.mistral import adapt_config_dict |
| 28 | |
| 29 | |
| 30 | @pytest.mark.parametrize("layout", ["mixed", "flat"]) |
| 31 | def test_gemma4_dspark_rope_config_preserves_parameters(tmp_path, layout): |
| 32 | """Remove redundant shared entries while preserving per-layer and flat RoPE.""" |
| 33 | from transformers import Gemma4TextConfig |
| 34 | |
| 35 | per_layer = { |
| 36 | "full_attention": { |
| 37 | "rope_type": "proportional", |
| 38 | "partial_rotary_factor": 0.25, |
| 39 | "rope_theta": 1000000.0, |
| 40 | }, |
| 41 | "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0}, |
| 42 | } |
| 43 | rope_parameters: dict[str, object] = dict(per_layer) |
| 44 | if layout == "mixed": |
| 45 | rope_parameters.update(rope_type="default", rope_theta=None) |
| 46 | else: |
| 47 | rope_parameters = {"rope_type": "default", "rope_theta": 12345.0} |
| 48 | (tmp_path / "config.json").write_text( |
| 49 | json.dumps( |
| 50 | { |
| 51 | "model_type": "gemma4_text", |
| 52 | "architectures": ["Gemma4DSparkModel"], |
| 53 | "num_hidden_layers": 1, |
| 54 | "layer_types": ["full_attention"], |
| 55 | "rope_parameters": rope_parameters, |
| 56 | } |
| 57 | ) |
| 58 | ) |
| 59 | _, config = config_module.HFConfigParser().parse( |
| 60 | tmp_path, trust_remote_code=False, max_position_embeddings=8192 |
| 61 | ) |
| 62 | assert isinstance(config, Gemma4TextConfig) |
| 63 | assert config.name_or_path == str(tmp_path) |
| 64 | assert config.max_position_embeddings == 8192 |
| 65 | if layout == "flat": |
| 66 | assert config.rope_parameters["rope_theta"] == 12345.0 |
| 67 | else: |
| 68 | for layer_type, expected in per_layer.items(): |
| 69 | for key, value in expected.items(): |
| 70 | assert config.rope_parameters[layer_type][key] == value |
| 71 | assert set(config.rope_parameters) == set(per_layer) |
| 72 | |
| 73 | |
| 74 | def test_patch_legacy_rope_type_preserves_nope_layers(): |
| 75 | """NoPE layers stay disabled while later RoPE layers are normalized.""" |
| 76 | rope_parameters = { |
| 77 | "full_attention": None, |
| 78 | "sliding_attention": { |
| 79 | "type": "mrope", |
| 80 | "mrope_section": [24, 20, 20], |
| 81 | }, |
| 82 | } |
| 83 | |
| 84 | patch_legacy_rope_type(rope_parameters) |
| 85 | |
| 86 | assert rope_parameters == { |
| 87 | "full_attention": None, |
| 88 | "sliding_attention": { |
| 89 | "type": "mrope", |
| 90 | "rope_type": "default", |
| 91 | "mrope_section": [24, 20, 20], |
| 92 | }, |
| 93 | } |
| 94 | |
| 95 | |
| 96 | def test_patch_legacy_rope_type_normalizes_telechat3_yarn(): |
| 97 | """TeleChat3's RoPE is YaRN with 0.07 in place of the usual 0.1. |
| 98 | |
| 99 | Encoding that as a precomputed attention_factor keeps the config |
| 100 | plain YaRN, which Transformers and every "yarn" guard understand. |
| 101 | `mscale` cannot express it: Transformers only applies mscale when |
| 102 | mscale_all_dim is also truthy. |
| 103 | """ |
| 104 | rope_parameters = { |
| 105 | "type": "telechat3-yarn", |
| 106 | "rope_type": "telechat3-yarn", |
| 107 | "factor": 4.0, |
| 108 | "original_max_position_embeddings": 8192, |
| 109 | } |
| 110 | |
| 111 | patch_legacy_rope_type(rope_parameters) |
| 112 | |
| 113 | assert rope_parameters == { |
| 114 | "rope_type": "yarn", |
| 115 | "factor": 4.0, |
| 116 | "original_max_position_embeddings": 8192, |
| 117 | "attention_factor": pytest.approx(0.07 * math.log(4.0) + 1.0), |
| 118 | } |
| 119 | |
| 120 | |
| 121 | def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling(): |
| 122 | """`yarn.apply_scale: false` must reach the DeepSeek-style attentions. |
| 123 | |
| 124 | Transformers spells it `attention_factor = 1.0`, which DeepseekV2Attention |
| 125 | and its siblings read to select `deepseek_llama_scaling` over |
| 126 | `deepseek_yarn`; without it Mistral-Large-3 runs with a spurious |
| 127 | yarn_get_mscale(factor)^2 attention scaling. |
| 128 | """ |
| 129 | params = { |
| 130 | "dim": 7168, |
| 131 | "n_layers": 61, |
| 132 | "head_dim": 192, |
| 133 | "hidden_dim": 16384, |
| 134 | "n_heads": 128, |
| 135 | "n_kv_heads": 128, |
| 136 | "norm_eps": 1e-5, |
| 137 | "vocab_size": 131072, |
| 138 | "rope_theta": 10000.0, |
| 139 | "max_position_embeddings": 294912, |
| 140 | "q_lora_rank": 1536, |
| 141 | "kv_lora_rank": 512, |
| 142 | "qk_nope_head_dim": 128, |
| 143 | "qk_rope_head_dim": 64, |
| 144 | "v_head_dim": 128, |
| 145 | "moe": { |
| 146 | "num_experts": 128, |
| 147 | "num_experts_per_tok": 4, |
| 148 | "num_shared_experts": 1, |
| 149 | "expert_hidden_dim": 4096, |
| 150 | "first_k_dense_replace": 3, |
| 151 | "route_every_n": 1, |
| 152 | "routed_scale": 1.0, |
| 153 | "num_expert_groups": 1, |
| 154 | "num_expert_groups_per_tok": 1, |
| 155 | }, |
| 156 | "llama_4_scaling": {"beta": 0.1, "original_max_position_embeddings": 8192}, |
| 157 | "yarn": { |
| 158 | "alpha": 1, |
| 159 | "apply_scale": False, |
| 160 | "beta": 32, |
| 161 | "factor": 36, |
| 162 | "original_max_position_embeddings": 8192, |
| 163 | }, |
| 164 | } |
| 165 | |
| 166 | config = adapt_config_dict(params, defaults={}) |
| 167 | |
| 168 | assert config.architectures == ["MistralLarge3ForCausalLM"] |
| 169 | assert config.rope_parameters["attention_factor"] == 1.0 |
| 170 | |
| 171 | |
| 172 | def test_get_llama3_eos_token(): |
| 173 | model_name = "meta-llama/Llama-3.2-1B-Instruct" |
| 174 | |
| 175 | tokenizer = get_tokenizer(model_name) |
| 176 | assert tokenizer.eos_token_id == 128009 |
| 177 | |
| 178 | generation_config = try_get_generation_config(model_name, trust_remote_code=False) |
| 179 | assert generation_config is not None |
| 180 | assert generation_config.eos_token_id == [128001, 128008, 128009] |
| 181 | |
| 182 | |
| 183 | def test_get_blip2_eos_token(): |
| 184 | model_name = "Salesforce/blip2-opt-2.7b" |
| 185 | |
| 186 | tokenizer = get_tokenizer(model_name) |
| 187 | assert tokenizer.eos_token_id == 2 |
| 188 | |
| 189 | generation_config = try_get_generation_config(model_name, trust_remote_code=False) |
| 190 | assert generation_config is not None |
| 191 | assert generation_config.eos_token_id == 50118 |
| 192 | |
| 193 | |
| 194 | def test_model_config_generation_fallback_forwards_code_revision(): |
| 195 | model_config = cast( |
| 196 | ModelConfig, |
| 197 | SimpleNamespace( |
| 198 | generation_config="auto", |
| 199 | hf_config_path=None, |
| 200 | model="org/model", |
| 201 | trust_remote_code=True, |
| 202 | revision="model-pin", |
| 203 | _hf_config_revision=None, |
| 204 | code_revision="code-pin", |
| 205 | config_format="auto", |
| 206 | hf_token=None, |
| 207 | ), |
| 208 | ) |
| 209 | |
| 210 | with ( |
| 211 | patch.object( |
| 212 | config_module.GenerationConfig, |
| 213 | "from_pretrained", |
| 214 | side_effect=OSError, |
| 215 | ), |
| 216 | patch.object( |
| 217 | config_module, |
| 218 | "get_config", |
| 219 | return_value=PreTrainedConfig(), |
| 220 | ) as get_config, |
| 221 | ): |
| 222 | ModelConfig.try_get_generation_config(model_config) |
| 223 | |
| 224 | get_config.assert_called_once_with( |
| 225 | "org/model", |
| 226 | trust_remote_code=True, |
| 227 | revision="model-pin", |
| 228 | code_revision="code-pin", |
| 229 | config_format="auto", |
| 230 | token=None, |
| 231 | ) |
| 232 | |
| 233 | |
| 234 | def test_safetensors_metadata_of_repo_without_safetensors(): |
| 235 | """A repo storing its weights in another format is an answer, not a failure, |
| 236 | so it must not be retried.""" |
| 237 | from huggingface_hub.errors import LocalEntryNotFoundError, NotASafetensorsRepoError |
| 238 | |
| 239 | get_safetensors_metadata = MagicMock( |
| 240 | side_effect=NotASafetensorsRepoError("not a safetensors repo") |
| 241 | ) |
| 242 | api = SimpleNamespace( |
| 243 | get_safetensors_metadata=get_safetensors_metadata, |
| 244 | snapshot_download=MagicMock(side_effect=LocalEntryNotFoundError("no cache")), |
| 245 | ) |
| 246 | |
| 247 | with patch.object(config_module, "hf_api", lambda: api): |
| 248 | assert get_safetensors_params_metadata("some/pytorch-only-model") == {} |
| 249 | |
| 250 | get_safetensors_metadata.assert_called_once() |
| 251 | |
| 252 | |
| 253 | @pytest.mark.parametrize( |
| 254 | ("section_key", "mrope_section", "expected_num_dims"), |
| 255 | [ |
| 256 | ("mrope_section", [16, 24, 24], 3), |
| 257 | ("mrope_section", [16, 16, 16, 16], 4), |
| 258 | # Interleaved M-RoPE takes 2 sections but still consumes 3D positions |
| 259 | ("mrope_section", [32, 32], 3), |
| 260 | # HunYuan-VL checkpoints ship the section under its legacy name |
| 261 | ("xdrope_section", [16, 16, 16, 16], 4), |
| 262 | ], |
| 263 | ) |
| 264 | def test_mrope_num_dims(section_key, mrope_section, expected_num_dims): |
| 265 | config = PreTrainedConfig() |
| 266 | config.rope_parameters = {"rope_type": "default", section_key: mrope_section} |
| 267 | |
| 268 | assert uses_mrope(config) |
| 269 | assert mrope_num_dims(config) == expected_num_dims |
| 270 | |
| 271 | |
| 272 | @pytest.mark.parametrize("section_name", ["mrope_section", "xdrope_section"]) |
| 273 | def test_mrope_num_dims_from_config_attribute(section_name): |
| 274 | """Some configs expose the section as an attribute rather than under |
| 275 | `rope_parameters`.""" |
| 276 | config = PreTrainedConfig() |
| 277 | setattr(config, section_name, [16, 16, 16, 16]) |
| 278 | |
| 279 | assert uses_mrope(config) |
| 280 | assert mrope_num_dims(config) == 4 |
| 281 | |
| 282 | |
| 283 | def test_mrope_num_dims_from_nested_rope_parameters(): |
| 284 | """Sections nested by layer type must be found, not silently defaulted.""" |
| 285 | config = PreTrainedConfig() |
| 286 | config.rope_parameters = { |
| 287 | "full_attention": {"mrope_section": [16, 16, 16, 16]}, |
| 288 | "linear_attention": {"rope_type": "default"}, |
| 289 | } |
| 290 | |
| 291 | assert uses_mrope(config) |
| 292 | assert mrope_num_dims(config) == 4 |
| 293 | |
| 294 | |
| 295 | def test_mrope_num_dims_without_mrope(): |
| 296 | assert mrope_num_dims(PreTrainedConfig()) == 0 |