| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import warnings |
| 5 | from collections.abc import Mapping, Sequence |
| 6 | from dataclasses import dataclass |
| 7 | from typing import Any |
| 8 | |
| 9 | import torch |
| 10 | import torch.nn.functional as F |
| 11 | from transformers import PreTrainedConfig |
| 12 | |
| 13 | from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption |
| 14 | from vllm.config.pooler import SequencePoolingType, TokenPoolingType |
| 15 | from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs |
| 16 | from vllm.multimodal.processing import InputProcessingContext |
| 17 | from vllm.tokenizers import cached_tokenizer_from_config |
| 18 | |
| 19 | from .. import ci_envs |
| 20 | from .registry import HF_EXAMPLE_MODELS |
| 21 | |
| 22 | TokensText = tuple[list[int], str] |
| 23 | |
| 24 | |
| 25 | def check_outputs_equal( |
| 26 | *, |
| 27 | outputs_0_lst: Sequence[TokensText], |
| 28 | outputs_1_lst: Sequence[TokensText], |
| 29 | name_0: str, |
| 30 | name_1: str, |
| 31 | ): |
| 32 | """Compare the two sequences generated by different models, |
| 33 | which should be equal. |
| 34 | """ |
| 35 | assert len(outputs_0_lst) == len(outputs_1_lst) |
| 36 | |
| 37 | for prompt_idx, (outputs_0, outputs_1) in enumerate( |
| 38 | zip(outputs_0_lst, outputs_1_lst) |
| 39 | ): |
| 40 | output_ids_0, output_str_0 = outputs_0 |
| 41 | output_ids_1, output_str_1 = outputs_1 |
| 42 | |
| 43 | # The text and token outputs should exactly match |
| 44 | fail_msg = ( |
| 45 | f"Test{prompt_idx}:" |
| 46 | f"\n{name_0}:\t{output_str_0!r}" |
| 47 | f"\n{name_1}:\t{output_str_1!r}" |
| 48 | ) |
| 49 | |
| 50 | assert output_str_0 == output_str_1, fail_msg |
| 51 | assert output_ids_0 == output_ids_1, fail_msg |
| 52 | |
| 53 | |
| 54 | # Representation of generated sequence as a tuple of |
| 55 | # * Token ID list |
| 56 | # * String |
| 57 | # * List of top sample logprobs for each sampled token |
| 58 | # |
| 59 | # Assumes prompt logprobs were not requested. |
| 60 | TokensTextLogprobs = tuple[ |
| 61 | list[int], str, list[dict[int, float]] | SampleLogprobs | None |
| 62 | ] |
| 63 | |
| 64 | # Allow for tokens to be represented as str's rather than IDs; |
| 65 | # tuple of |
| 66 | # * Token string representations list |
| 67 | # * String |
| 68 | # * Optional list of top sample logprobs for each sampled token |
| 69 | # |
| 70 | # Assumes prompt logprobs were not requested. |
| 71 | TextTextLogprobs = tuple[ |
| 72 | list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None |
| 73 | ] |
| 74 | |
| 75 | # Representation of generated sequence as a tuple of |
| 76 | # * Token ID list |
| 77 | # * String |
| 78 | # * Optional list of top sample logprobs for each sampled token |
| 79 | # * Optional list of top prompt logprobs for each prompt token |
| 80 | # |
| 81 | # Allows prompt logprobs to be requested. |
| 82 | TokensTextLogprobsPromptLogprobs = tuple[ |
| 83 | list[int], |
| 84 | str, |
| 85 | list[dict[int, float]] | SampleLogprobs | None, |
| 86 | list[dict[int, float] | None] | PromptLogprobs | None, |
| 87 | ] |
| 88 | |
| 89 | |
| 90 | def check_logprobs_close( |
| 91 | *, |
| 92 | outputs_0_lst: Sequence[ |
| 93 | TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs |
| 94 | ], |
| 95 | outputs_1_lst: Sequence[ |
| 96 | TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs |
| 97 | ], |
| 98 | name_0: str, |
| 99 | name_1: str, |
| 100 | num_outputs_0_skip_tokens: int = 0, |
| 101 | warn_on_mismatch: bool = True, |
| 102 | always_check_logprobs: bool = False, |
| 103 | ) -> None: |
| 104 | """Compare the logprobs of two sequences generated by different models, |
| 105 | which should be similar but not necessarily equal. |
| 106 | |
| 107 | How sample logprobs are compared: |
| 108 | * `always_check_logprobs == True`: set of highest-logprob token ids |
| 109 | must match between seq0 and seq1 at all sampled token offsets |
| 110 | * `always_check_logprobs == False`: highest-logprob token ids are |
| 111 | only compared at sampled token offsets for which generated token |
| 112 | ids don't match |
| 113 | |
| 114 | Prompt logprobs must be provided either for both input sequences, or |
| 115 | for neither. If prompt logprobs are provided, then highest-logprob |
| 116 | prompt token ids must match between seq0 and seq1 at all prompt token |
| 117 | offsets. |
| 118 | |
| 119 | Args: |
| 120 | outputs_0_lst: First sequence to compare |
| 121 | outputs_0_lst: Second sequence to compare |
| 122 | name_0: sequence #0 name |
| 123 | name_1: sequence #1 name |
| 124 | num_outputs_0_skip_tokens: If > 0, specifies the number of initial |
| 125 | sequence #0 tokens & logprobs to discard |
| 126 | before comparison, i.e. all |
| 127 | of sequence #1 will be compared to |
| 128 | sequence #0 beginning at index |
| 129 | num_outputs_0_skip_tokens |
| 130 | warn_on_mismatch: Issue a warning if there is token-wise or text-wise |
| 131 | mismatch between the two sequences |
| 132 | always_check_logprobs: If true, check logprobs even when tokens match |
| 133 | |
| 134 | """ |
| 135 | assert len(outputs_0_lst) == len(outputs_1_lst) |
| 136 | |
| 137 | # Loop through responses to each prompt. |
| 138 | for prompt_idx, (outputs_0, outputs_1) in enumerate( |
| 139 | zip(outputs_0_lst, outputs_1_lst) |
| 140 | ): |
| 141 | assert len(outputs_0) == len(outputs_1) |
| 142 | if len(outputs_0) == 3: |
| 143 | assert len(outputs_1) == 3 |
| 144 | # Break out tokens, text & sample logprobs |
| 145 | # (prompt logprobs were not provided) |
| 146 | output_ids_0, output_str_0, logprobs_0 = outputs_0 |
| 147 | output_ids_1, output_str_1, logprobs_1 = outputs_1 |
| 148 | elif len(outputs_0) == 4: |
| 149 | assert len(outputs_1) == 4 |
| 150 | # Break out tokens, text, sample logprobs & prompt logprobs |
| 151 | ( |
| 152 | output_ids_0, |
| 153 | output_str_0, |
| 154 | logprobs_0, |
| 155 | prompt_logprobs_0, |
| 156 | ) = outputs_0 |
| 157 | ( |
| 158 | output_ids_1, |
| 159 | output_str_1, |
| 160 | logprobs_1, |
| 161 | prompt_logprobs_1, |
| 162 | ) = outputs_1 |
| 163 | |
| 164 | # Test prompt logprobs closeness |
| 165 | if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None: |
| 166 | # Both sequences' prompt logprobs lists are not `None` |
| 167 | # (although individual list elements may be `None`); |
| 168 | # for each token's logprobs: |
| 169 | for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate( |
| 170 | zip(prompt_logprobs_0, prompt_logprobs_1) |
| 171 | ): |
| 172 | fail_msg = ( |
| 173 | f"Prompt logprobs test:" |
| 174 | f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}" |
| 175 | f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}" |
| 176 | ) |
| 177 | |
| 178 | if logprobs_elem_0 is None: |
| 179 | # If the seq 0 token's logprobs are `None`, |
| 180 | # the seq 1 token's logprobs must be `None` |
| 181 | assert logprobs_elem_1 is None, fail_msg |
| 182 | else: |
| 183 | # If the seq 0 token's logprobs are not `None`, |
| 184 | # the seq 1 token's logprobs must not be `None` |
| 185 | assert logprobs_elem_1 is not None, fail_msg |
| 186 | # Logprobs check: top-k token choices must be the same |
| 187 | assert set(logprobs_elem_0.keys()) == set( |
| 188 | logprobs_elem_1.keys() |
| 189 | ), fail_msg |
| 190 | else: |
| 191 | # Both sequence logprobs lists must be `None` |
| 192 | fail_msg = ( |
| 193 | f"Prompt logprobs test:" |
| 194 | f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}" |
| 195 | f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}" |
| 196 | ) |
| 197 | |
| 198 | assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg |
| 199 | else: |
| 200 | raise ValueError( |
| 201 | f"Outputs tuple must have 3 or 4 elements but " |
| 202 | f"{len(outputs_0)} elements were provided: " |
| 203 | f"{outputs_0}" |
| 204 | ) |
| 205 | |
| 206 | if logprobs_0 is None: |
| 207 | logprobs_0 = [None] * len(output_ids_0) |
| 208 | if logprobs_1 is None: |
| 209 | logprobs_1 = [None] * len(output_ids_1) |
| 210 | |
| 211 | # Skip specified number of initial sequence #0 tokens |
| 212 | # & logprobs, leaving output text as-is for simplicity |
| 213 | # (text mismatches may generate warnings but do not |
| 214 | # cause the test to fail.) |
| 215 | if num_outputs_0_skip_tokens < 0: |
| 216 | raise ValueError("num_outputs_0_skip_tokens must be non-negative") |
| 217 | output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:] |
| 218 | logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:] |
| 219 | |
| 220 | # Loop through generated tokens. |
| 221 | for idx, (output_id_0, output_id_1) in enumerate( |
| 222 | zip(output_ids_0, output_ids_1) |
| 223 | ): |
| 224 | is_tok_mismatch = output_id_0 != output_id_1 |
| 225 | |
| 226 | # If generated tokens don't match |
| 227 | # or it is desired to always check logprobs, |
| 228 | # then |
| 229 | if is_tok_mismatch or always_check_logprobs: |
| 230 | logprobs_elem_0 = logprobs_0[idx] |
| 231 | logprobs_elem_1 = logprobs_1[idx] |
| 232 | |
| 233 | # Each predicted token must be in top N logprobs of the other |
| 234 | fail_msg = ( |
| 235 | f"Test{prompt_idx}:" |
| 236 | f"\nMatched tokens:\t{output_ids_0[:idx]}" |
| 237 | f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}" |
| 238 | f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}" |
| 239 | ) |
| 240 | |
| 241 | assert logprobs_elem_0 is not None, fail_msg |
| 242 | assert logprobs_elem_1 is not None, fail_msg |
| 243 | assert output_id_0 in logprobs_elem_1, fail_msg |
| 244 | assert output_id_1 in logprobs_elem_0, fail_msg |
| 245 | |
| 246 | if warn_on_mismatch and is_tok_mismatch: |
| 247 | with warnings.catch_warnings(): |
| 248 | # This ensures that repeated warnings are shown |
| 249 | # in the output, not just the first occurrence |
| 250 | warnings.simplefilter("always") |
| 251 | |
| 252 | warnings.warn(fail_msg, stacklevel=2) |
| 253 | |
| 254 | # Break out since sequences will now diverge. |
| 255 | break |
| 256 | else: |
| 257 | if output_str_0 != output_str_1 and warn_on_mismatch: |
| 258 | # The token outputs exactly match, |
| 259 | # so the text outputs should exactly match as well |
| 260 | fail_msg = ( |
| 261 | f"Test{prompt_idx}:" |
| 262 | f"\n{name_0}:\t{output_str_0!r}" |
| 263 | f"\n{name_1}:\t{output_str_1!r}" |
| 264 | ) |
| 265 | |
| 266 | with warnings.catch_warnings(): |
| 267 | # This ensures that repeated warnings are shown |
| 268 | # in the output, not just the first occurrence |
| 269 | warnings.simplefilter("always") |
| 270 | |
| 271 | warnings.warn(fail_msg, stacklevel=2) |
| 272 | |
| 273 | |
| 274 | def build_model_context( |
| 275 | model_id: str, |
| 276 | runner: RunnerOption = "auto", |
| 277 | dtype: ModelDType = "auto", |
| 278 | model_config_kwargs: dict[str, Any] | None = None, |
| 279 | mm_processor_kwargs: dict[str, Any] | None = None, |
| 280 | limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None, |
| 281 | mm_processor_cache_gb: int = 0, |
| 282 | ): |
| 283 | """Creates an InputProcessingContext for a given model. |
| 284 | |
| 285 | Args: |
| 286 | model_id: ID of the model being considered. |
| 287 | mm_processor_kwargs: optional processor kwargs for to be leveraged |
| 288 | in the input processor, mapper, dummy data creation, etc. |
| 289 | limit_mm_per_prompt: Multimodal limits. |
| 290 | |
| 291 | Returns: |
| 292 | InputProcessingContext for the model being considered. |
| 293 | |
| 294 | """ |
| 295 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id) |
| 296 | model_info.check_available_online(on_fail="skip") |
| 297 | model_info.check_transformers_version( |
| 298 | on_fail="skip", |
| 299 | check_max_version=False, |
| 300 | check_version_reason="vllm", |
| 301 | ) |
| 302 | |
| 303 | model_config_kwargs = model_config_kwargs or {} |
| 304 | limit_mm_per_prompt = { |
| 305 | modality: dict(limit) if isinstance(limit, Mapping) else limit |
| 306 | for modality, limit in (limit_mm_per_prompt or {}).items() |
| 307 | } |
| 308 | model_config = ModelConfig( |
| 309 | model_id, |
| 310 | runner=runner, |
| 311 | tokenizer=model_info.tokenizer or model_id, |
| 312 | tokenizer_mode=model_info.tokenizer_mode, |
| 313 | revision=model_info.revision, |
| 314 | trust_remote_code=model_info.trust_remote_code, |
| 315 | dtype=dtype, |
| 316 | seed=0, |
| 317 | mm_processor_kwargs=mm_processor_kwargs, |
| 318 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 319 | mm_processor_cache_gb=mm_processor_cache_gb, |
| 320 | hf_overrides=model_info.hf_overrides, |
| 321 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 322 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 323 | enable_mm_embeds=model_info.require_embed_inputs, |
| 324 | enforce_eager=model_info.enforce_eager, |
| 325 | **model_config_kwargs, |
| 326 | ) |
| 327 | |
| 328 | return InputProcessingContext( |
| 329 | model_config, |
| 330 | tokenizer=cached_tokenizer_from_config(model_config), |
| 331 | ) |
| 332 | |
| 333 | |
| 334 | def check_embeddings_close( |
| 335 | *, |
| 336 | embeddings_0_lst: Sequence[list[float]], |
| 337 | embeddings_1_lst: Sequence[list[float]], |
| 338 | name_0: str, |
| 339 | name_1: str, |
| 340 | tol: float = 1e-3, |
| 341 | ) -> None: |
| 342 | assert len(embeddings_0_lst) == len(embeddings_1_lst) |
| 343 | |
| 344 | for prompt_idx, (embeddings_0, embeddings_1) in enumerate( |
| 345 | zip(embeddings_0_lst, embeddings_1_lst) |
| 346 | ): |
| 347 | assert len(embeddings_0) == len(embeddings_1), ( |
| 348 | f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}" |
| 349 | ) |
| 350 | |
| 351 | sim = F.cosine_similarity( |
| 352 | torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0 |
| 353 | ) |
| 354 | |
| 355 | fail_msg = ( |
| 356 | f"Test{prompt_idx}:" |
| 357 | f"\nCosine similarity: \t{sim:.4f}" |
| 358 | f"\n{name_0}:\t{embeddings_0[:16]!r}" |
| 359 | f"\n{name_1}:\t{embeddings_1[:16]!r}" |
| 360 | ) |
| 361 | |
| 362 | assert sim >= 1 - tol, fail_msg |
| 363 | |
| 364 | |
| 365 | def matryoshka_fy(tensor: torch.Tensor, dimensions: int): |
| 366 | tensor = torch.tensor(tensor) |
| 367 | tensor = tensor[..., :dimensions] |
| 368 | tensor = F.normalize(tensor, p=2, dim=1) |
| 369 | return tensor |
| 370 | |
| 371 | |
| 372 | def softmax(data): |
| 373 | if data.shape[-1] == 1: |
| 374 | return F.sigmoid(data) |
| 375 | else: |
| 376 | return F.softmax(data, dim=-1) |
| 377 | |
| 378 | |
| 379 | @dataclass |
| 380 | class ModelInfo: |
| 381 | name: str |
| 382 | revision: str | None = None |
| 383 | architecture: str = "" |
| 384 | dtype: str = "auto" |
| 385 | max_model_len: int | None = None |
| 386 | hf_dtype: str = "float32" |
| 387 | hf_overrides: dict[str, Any] | None = None |
| 388 | seq_pooling_type: SequencePoolingType | None = None |
| 389 | tok_pooling_type: TokenPoolingType | None = None |
| 390 | attn_type: AttnTypeStr | None = None |
| 391 | is_prefix_caching_supported: bool | None = None |
| 392 | is_chunked_prefill_supported: bool | None = None |
| 393 | enable_test: bool = True |
| 394 | |
| 395 | |
| 396 | @dataclass |
| 397 | class EmbedModelInfo(ModelInfo): |
| 398 | mteb_score: float | None = None |
| 399 | # Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL. |
| 400 | mteb_tol: float | None = None |
| 401 | is_matryoshka: bool = False |
| 402 | matryoshka_dimensions: list[int] | None = None |
| 403 | |
| 404 | |
| 405 | @dataclass |
| 406 | class RerankModelInfo(ModelInfo): |
| 407 | mteb_score: float | None = None |
| 408 | # Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL. |
| 409 | mteb_tol: float | None = None |
| 410 | chat_template_name: str | None = None |
| 411 | |
| 412 | |
| 413 | @dataclass |
| 414 | class GenerateModelInfo(ModelInfo): |
| 415 | hf_dtype: str = "auto" |
| 416 | hf_ppl: float | None = None |
| 417 | |
| 418 | |
| 419 | def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs): |
| 420 | # A model family has many models with the same architecture, |
| 421 | # and we don't need to test each one. |
| 422 | if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test: |
| 423 | import pytest |
| 424 | |
| 425 | pytest.skip("Skipping test.") |
| 426 | |
| 427 | # Allow vllm to test using the given dtype, such as float32 |
| 428 | vllm_extra_kwargs = vllm_extra_kwargs or {} |
| 429 | vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype |
| 430 | |
| 431 | # Allow vllm to test using hf_overrides |
| 432 | if model_info.hf_overrides is not None: |
| 433 | vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides |
| 434 | |
| 435 | # Allow changing the head dtype used by vllm in tests |
| 436 | if ci_envs.VLLM_CI_HEAD_DTYPE is not None: |
| 437 | if "hf_overrides" not in vllm_extra_kwargs: |
| 438 | vllm_extra_kwargs["hf_overrides"] = {} |
| 439 | vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE |
| 440 | |
| 441 | # Allow control over whether tests use enforce_eager |
| 442 | if ci_envs.VLLM_CI_ENFORCE_EAGER is not None: |
| 443 | vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER |
| 444 | |
| 445 | return vllm_extra_kwargs |
| 446 | |
| 447 | |
| 448 | def dummy_hf_overrides( |
| 449 | hf_config: PreTrainedConfig, |
| 450 | *, |
| 451 | model_arch: str = "", |
| 452 | exist_overrides: dict[str, Any] | None = None, |
| 453 | use_original_num_layers: bool = False, |
| 454 | ) -> PreTrainedConfig: |
| 455 | """Dummy HF overrides function used to create dummy model |
| 456 | with only minimum nums of layer. |
| 457 | """ |
| 458 | # Copy because this helper is called more than once |
| 459 | # while loading config, and we `.pop()` |
| 460 | exist_overrides = (exist_overrides or {}).copy() |
| 461 | text_config_override = exist_overrides.pop("text_config", None) |
| 462 | hf_config.update(exist_overrides) |
| 463 | |
| 464 | text_config = hf_config.get_text_config() |
| 465 | if text_config_override is not None: |
| 466 | # multimodal test models may override *some* text-model fields |
| 467 | text_config.update(text_config_override) |
| 468 | |
| 469 | # Ensure at least 2 expert per group |
| 470 | # Since `grouped_topk` assumes top-2 |
| 471 | n_group = getattr(text_config, "n_group", None) |
| 472 | # Kimi uses `num_expert_group` instead of `n_group`. |
| 473 | if n_group is None: |
| 474 | n_group = getattr(text_config, "num_expert_group", None) |
| 475 | # InternS1Pro uses `router_n_groups` instead of `n_group`. |
| 476 | if n_group is None: |
| 477 | n_group = getattr(text_config, "router_n_groups", None) |
| 478 | num_experts = n_group * 2 if n_group is not None else 2 |
| 479 | |
| 480 | # we use three layers for Gemma-3n to check |
| 481 | # both normal layer and kv_shared_layer |
| 482 | if use_original_num_layers: |
| 483 | # Use the original number of layers from the config |
| 484 | num_layers = getattr(text_config, "num_layers", 1) |
| 485 | num_hidden_layers = getattr(text_config, "num_hidden_layers", 1) |
| 486 | else: |
| 487 | # Use minimal layers for testing |
| 488 | num_layers = 1 |
| 489 | num_hidden_layers = ( |
| 490 | 3 |
| 491 | if model_arch |
| 492 | in ( |
| 493 | "Gemma3nForConditionalGeneration", |
| 494 | "Gemma4ForCausalLM", |
| 495 | "Gemma4ForConditionalGeneration", |
| 496 | "Gemma4MTPModel", |
| 497 | "DiffusionGemmaForBlockDiffusion", |
| 498 | ) |
| 499 | else 1 |
| 500 | ) |
| 501 | |
| 502 | update_dict = { |
| 503 | "num_layers": num_layers, |
| 504 | # For Gemma-3n |
| 505 | "num_kv_shared_layers": 1, |
| 506 | } |
| 507 | |
| 508 | _hf_config = hf_config |
| 509 | |
| 510 | class DummyConfig: |
| 511 | hf_config = _hf_config |
| 512 | hf_text_config = text_config |
| 513 | |
| 514 | # Keep architecture conversion on the default multimodal path; this |
| 515 | # helper only needs HF-derived fields, not deployment MM limits. |
| 516 | @staticmethod |
| 517 | def _supports_multimodal_for_mm_prefix() -> bool: |
| 518 | return True |
| 519 | |
| 520 | model_arch_config = ModelConfig.get_model_arch_config(DummyConfig) |
| 521 | # Only set MoE related config when the model has MoE layers. |
| 522 | # Otherwise all models detected as MoE by _get_transformers_backend_cls. |
| 523 | if model_arch_config.num_experts > 0: |
| 524 | num_experts_per_tok = 2 |
| 525 | if model_arch in ( |
| 526 | "Llama4ForConditionalGeneration", |
| 527 | "Llama4ForCausalLM", |
| 528 | "EagleLlama4ForCausalLM", |
| 529 | ): |
| 530 | num_experts_per_tok = 1 |
| 531 | elif model_arch == "InternS1ProForConditionalGeneration": |
| 532 | assert n_group is not None |
| 533 | num_experts_per_tok = n_group |
| 534 | update_dict.update( |
| 535 | { |
| 536 | "num_experts": num_experts, |
| 537 | "num_experts_per_tok": num_experts_per_tok, |
| 538 | # Kimi uses `num_experts_per_token`. |
| 539 | "num_experts_per_token": num_experts_per_tok, |
| 540 | "num_local_experts": num_experts, |
| 541 | # Otherwise there will not be any expert layers |
| 542 | "first_k_dense_replace": 0, |
| 543 | # To avoid OOM on DeepSeek-V3 |
| 544 | "n_routed_experts": num_experts, |
| 545 | } |
| 546 | ) |
| 547 | |
| 548 | # Update num_hidden_layers for non-Longcat architectures (Longcat derives it |
| 549 | # from num_layers for its dual-attention layers). |
| 550 | if model_arch not in ( |
| 551 | "LongcatFlashForCausalLM", |
| 552 | "LongCatFlashMTPModel", |
| 553 | "LongcatFlashNgramForCausalLM", |
| 554 | ): |
| 555 | update_dict["num_hidden_layers"] = num_hidden_layers |
| 556 | |
| 557 | text_config.update(update_dict) |
| 558 | |
| 559 | # Keep per-layer metadata consistent with the reduced layer count. |
| 560 | # HY V4 uses this to decide which sparse-attention indexers are shared. |
| 561 | if indexer_types := getattr(text_config, "indexer_types", None): |
| 562 | text_config.update({"indexer_types": indexer_types[:num_hidden_layers]}) |
| 563 | |
| 564 | # Update n_layers and moe configs for Moondream3 model |
| 565 | if model_arch in ("Moondream3ForCausalLM", "HfMoondream"): |
| 566 | text_config.update( |
| 567 | { |
| 568 | "n_layers": num_hidden_layers, |
| 569 | "moe_num_experts": num_experts, |
| 570 | "moe_experts_per_token": 2, |
| 571 | "moe_start_layer": num_hidden_layers, |
| 572 | } |
| 573 | ) |
| 574 | |
| 575 | if hasattr(hf_config, "vision_config"): |
| 576 | vision_config = hf_config.vision_config |
| 577 | vision_config.update( |
| 578 | { |
| 579 | "num_layers": 1, |
| 580 | "num_hidden_layers": 1, |
| 581 | } |
| 582 | ) |
| 583 | |
| 584 | # Keep per-layer metadata consistent with the reduced layer count. |
| 585 | if layer_types := getattr(vision_config, "layer_types", None): |
| 586 | vision_config.update({"layer_types": layer_types[:1]}) |
| 587 | |
| 588 | if model_arch in ("Moondream3ForCausalLM", "HfMoondream"): |
| 589 | vision_config.update({"enc_n_layers": 1}) |
| 590 | |
| 591 | # e.g.: ibm-granite/granite-speech-3.3-2b |
| 592 | if hasattr(hf_config, "encoder_config"): |
| 593 | hf_config.encoder_config.update( |
| 594 | { |
| 595 | "num_layers": 1, |
| 596 | "num_hidden_layers": 1, |
| 597 | } |
| 598 | ) |
| 599 | |
| 600 | # e.g.: Qwen/Qwen2-Audio-7B-Instruct |
| 601 | # audio_config may exist but be None (e.g. audio-less Gemma4 variants). |
| 602 | if getattr(hf_config, "audio_config", None) is not None: |
| 603 | hf_config.audio_config.update( |
| 604 | { |
| 605 | "num_layers": 1, |
| 606 | "num_hidden_layers": 1, |
| 607 | "encoder_layers": 1, |
| 608 | } |
| 609 | ) |
| 610 | |
| 611 | return hf_config |