| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Mapping, Set |
| 5 | from dataclasses import dataclass, field |
| 6 | from typing import Any, Literal |
| 7 | |
| 8 | import pytest |
| 9 | from packaging.version import Version |
| 10 | from transformers import PreTrainedConfig |
| 11 | from transformers import __version__ as TRANSFORMERS_VERSION |
| 12 | |
| 13 | from vllm.config.model import ModelDType, TokenizerMode |
| 14 | |
| 15 | |
| 16 | @dataclass(frozen=True) |
| 17 | class _HfExamplesInfo: |
| 18 | default: str |
| 19 | """The default model to use for testing this architecture.""" |
| 20 | |
| 21 | extras: Mapping[str, str] = field(default_factory=dict) |
| 22 | """Extra models to use for testing this architecture.""" |
| 23 | |
| 24 | tokenizer: str | None = None |
| 25 | """Set the tokenizer to load for this architecture.""" |
| 26 | |
| 27 | tokenizer_mode: TokenizerMode | str = "auto" |
| 28 | """Set the tokenizer type for this architecture.""" |
| 29 | |
| 30 | speculative_model: str | None = None |
| 31 | """ |
| 32 | The default model to use for testing this architecture, which is only used |
| 33 | for speculative decoding. |
| 34 | """ |
| 35 | |
| 36 | speculative_method: str | None = None |
| 37 | """ |
| 38 | The method to use for speculative decoding. |
| 39 | """ |
| 40 | |
| 41 | min_transformers_version: str | None = None |
| 42 | """ |
| 43 | The minimum version of HF Transformers that is required to run this model. |
| 44 | """ |
| 45 | |
| 46 | max_transformers_version: str | None = None |
| 47 | """ |
| 48 | The maximum version of HF Transformers that this model runs on. |
| 49 | """ |
| 50 | |
| 51 | transformers_version_reason: dict[Literal["vllm", "hf"], str] | None = None |
| 52 | """ |
| 53 | The type and reason to skip test for the minimum/maximum version requirement. |
| 54 | vllm: skip all vLLM tests if the version requirement is not met. |
| 55 | hf: only skip tests that uses HF runner if the version requirement is not met. |
| 56 | """ |
| 57 | |
| 58 | require_embed_inputs: bool = False |
| 59 | """ |
| 60 | If `True`, enables prompt and multi-modal embedding inputs while |
| 61 | disabling tokenization. |
| 62 | """ |
| 63 | |
| 64 | dtype: ModelDType = "auto" |
| 65 | """ |
| 66 | The data type for the model weights and activations. |
| 67 | """ |
| 68 | |
| 69 | enforce_eager: bool = False |
| 70 | """ |
| 71 | Whether to enforce eager execution. If True, we will |
| 72 | disable CUDA graph and always execute the model in eager mode. |
| 73 | If False, we will use CUDA graph and eager execution in hybrid. |
| 74 | """ |
| 75 | |
| 76 | enable_prefix_caching: bool = True |
| 77 | """ |
| 78 | Whether to enable prefix caching for the model. If True, we will test the model with |
| 79 | prefix caching enabled. If False, we will test the model without prefix caching. |
| 80 | """ |
| 81 | |
| 82 | is_available_online: bool = True |
| 83 | """ |
| 84 | Set this to `False` if the name of this architecture no longer exists on |
| 85 | the HF repo. To maintain backwards compatibility, we have not removed them |
| 86 | from the main model registry, so without this flag the registry tests will |
| 87 | fail. |
| 88 | """ |
| 89 | |
| 90 | trust_remote_code: bool = False |
| 91 | """The `trust_remote_code` level required to load the model.""" |
| 92 | |
| 93 | hf_overrides: dict[str, Any] = field(default_factory=dict) |
| 94 | """The `hf_overrides` required to load the model.""" |
| 95 | |
| 96 | max_model_len: int | None = None |
| 97 | """ |
| 98 | The maximum model length to use for this model. Some models default to a |
| 99 | length that is too large to fit into memory in CI. |
| 100 | """ |
| 101 | |
| 102 | max_num_batched_tokens: int | None = None |
| 103 | """ |
| 104 | The maximum number of tokens to be processed in a single batch. |
| 105 | """ |
| 106 | |
| 107 | revision: str | None = None |
| 108 | """ |
| 109 | The specific revision (commit hash, tag, or branch) to use for the model. |
| 110 | If not specified, the default revision will be used. |
| 111 | """ |
| 112 | |
| 113 | max_num_seqs: int | None = None |
| 114 | """Maximum number of sequences to be processed in a single iteration.""" |
| 115 | |
| 116 | use_original_num_layers: bool = False |
| 117 | """ |
| 118 | If True, use the original number of layers from the model config |
| 119 | instead of minimal layers for testing. |
| 120 | """ |
| 121 | |
| 122 | def check_transformers_version( |
| 123 | self, |
| 124 | *, |
| 125 | on_fail: Literal["error", "skip", "return"], |
| 126 | check_version_reason: Literal["vllm", "hf"] = "hf", |
| 127 | check_min_version: bool = True, |
| 128 | check_max_version: bool = True, |
| 129 | ) -> str | None: |
| 130 | """If the installed transformers version does not meet the requirements, |
| 131 | perform the given action. |
| 132 | """ |
| 133 | if ( |
| 134 | self.min_transformers_version is None |
| 135 | and self.max_transformers_version is None |
| 136 | ): |
| 137 | return None |
| 138 | |
| 139 | current_version = TRANSFORMERS_VERSION |
| 140 | cur_base_version = Version(current_version).base_version |
| 141 | min_version = self.min_transformers_version |
| 142 | max_version = self.max_transformers_version |
| 143 | msg = f"`transformers=={current_version}` installed, but `transformers" |
| 144 | # Only check the base version for the min/max version, otherwise preview |
| 145 | # models cannot be run because `x.yy.0.dev0`<`x.yy.0` |
| 146 | if min_version and Version(cur_base_version) < Version(min_version): |
| 147 | is_version_valid = False |
| 148 | should_check_version = check_min_version |
| 149 | msg += f">={min_version}` is required to run this model." |
| 150 | elif max_version and Version(cur_base_version) > Version(max_version): |
| 151 | is_version_valid = False |
| 152 | should_check_version = check_max_version |
| 153 | msg += f"<={max_version}` is required to run this model." |
| 154 | else: |
| 155 | is_version_valid = True |
| 156 | should_check_version = False |
| 157 | |
| 158 | # Reasons explain a known incompatibility with a violated version |
| 159 | # bound. They should not skip models when the installed version is |
| 160 | # already within the requested range. |
| 161 | is_reason_applicable = ( |
| 162 | not is_version_valid |
| 163 | and self.transformers_version_reason is not None |
| 164 | and check_version_reason in self.transformers_version_reason |
| 165 | ) |
| 166 | is_transformers_valid = is_version_valid or ( |
| 167 | not should_check_version and not is_reason_applicable |
| 168 | ) |
| 169 | if is_transformers_valid: |
| 170 | return None |
| 171 | elif self.transformers_version_reason: |
| 172 | for reason_type, reason in self.transformers_version_reason.items(): |
| 173 | msg += f" Reason({reason_type}): {reason}" |
| 174 | |
| 175 | if on_fail == "error": |
| 176 | raise RuntimeError(msg) |
| 177 | elif on_fail == "skip": |
| 178 | pytest.skip(msg) |
| 179 | |
| 180 | return msg |
| 181 | |
| 182 | def check_available_online( |
| 183 | self, |
| 184 | *, |
| 185 | on_fail: Literal["error", "skip"], |
| 186 | ) -> None: |
| 187 | """If the model is not available online, perform the given action.""" |
| 188 | if not self.is_available_online: |
| 189 | msg = "Model is not available online" |
| 190 | |
| 191 | if on_fail == "error": |
| 192 | raise RuntimeError(msg) |
| 193 | else: |
| 194 | pytest.skip(msg) |
| 195 | |
| 196 | |
| 197 | _TEXT_GENERATION_EXAMPLE_MODELS = { |
| 198 | # [Decoder-only] |
| 199 | "AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"), |
| 200 | "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"), |
| 201 | "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"), |
| 202 | "AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True), |
| 203 | "BailingMoeForCausalLM": _HfExamplesInfo( |
| 204 | "inclusionAI/Ling-lite-1.5", trust_remote_code=True |
| 205 | ), |
| 206 | "BailingMoeV2ForCausalLM": _HfExamplesInfo( |
| 207 | "inclusionAI/Ling-mini-2.0", trust_remote_code=True |
| 208 | ), |
| 209 | "BailingMoeV2_5ForCausalLM": _HfExamplesInfo( |
| 210 | "inclusionAI/Ring-2.5-1T", trust_remote_code=True |
| 211 | ), |
| 212 | "BailingMoeV3ForCausalLM": _HfExamplesInfo( |
| 213 | "inclusionAI/Ling-3.0-flash", trust_remote_code=True |
| 214 | ), |
| 215 | "BloomForCausalLM": _HfExamplesInfo( |
| 216 | "bigscience/bloom-560m", {"1b": "bigscience/bloomz-1b1"} |
| 217 | ), |
| 218 | "ChatGLMModel": _HfExamplesInfo( |
| 219 | "zai-org/chatglm3-6b", trust_remote_code=True, max_transformers_version="4.48" |
| 220 | ), |
| 221 | "ChatGLMForConditionalGeneration": _HfExamplesInfo( |
| 222 | "thu-coai/ShieldLM-6B-chatglm3", |
| 223 | trust_remote_code=True, |
| 224 | ), |
| 225 | "CohereForCausalLM": _HfExamplesInfo( |
| 226 | "CohereLabs/c4ai-command-r-v01", trust_remote_code=True |
| 227 | ), |
| 228 | "Cohere2ForCausalLM": _HfExamplesInfo( |
| 229 | "CohereLabs/c4ai-command-r7b-12-2024", |
| 230 | trust_remote_code=True, |
| 231 | ), |
| 232 | "Cohere2MoeForCausalLM": _HfExamplesInfo( |
| 233 | "CohereLabs/North-Mini-Code", |
| 234 | trust_remote_code=True, |
| 235 | is_available_online=False, |
| 236 | ), |
| 237 | "CwmForCausalLM": _HfExamplesInfo("facebook/cwm"), |
| 238 | # FIXME: databricks/dbrx-instruct has been deleted |
| 239 | "DbrxForCausalLM": _HfExamplesInfo( |
| 240 | "databricks/dbrx-instruct", is_available_online=False |
| 241 | ), |
| 242 | "DeciLMForCausalLM": _HfExamplesInfo( |
| 243 | "nvidia/Llama-3_3-Nemotron-Super-49B-v1", |
| 244 | trust_remote_code=True, |
| 245 | ), |
| 246 | "DeepseekForCausalLM": _HfExamplesInfo( |
| 247 | "deepseek-ai/deepseek-moe-16b-base", |
| 248 | trust_remote_code=True, |
| 249 | ), |
| 250 | "DeepseekV2ForCausalLM": _HfExamplesInfo( |
| 251 | "deepseek-ai/DeepSeek-V2-Lite-Chat", |
| 252 | trust_remote_code=True, |
| 253 | ), |
| 254 | "DeepseekV3ForCausalLM": _HfExamplesInfo( |
| 255 | "deepseek-ai/DeepSeek-V3", |
| 256 | trust_remote_code=True, |
| 257 | ), |
| 258 | "DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"), |
| 259 | "DeepseekV4ForCausalLM": _HfExamplesInfo( |
| 260 | "deepseek-ai/DeepSeek-V4-Flash", is_available_online=False |
| 261 | ), |
| 262 | "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"), |
| 263 | "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"), |
| 264 | "ExaoneForCausalLM": _HfExamplesInfo( |
| 265 | "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True |
| 266 | ), |
| 267 | "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"), |
| 268 | "ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"), |
| 269 | "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"), |
| 270 | "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"), |
| 271 | "FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"), |
| 272 | "GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"), |
| 273 | "Gemma2ForCausalLM": _HfExamplesInfo( |
| 274 | "google/gemma-2-9b", extras={"tiny": "google/gemma-2-2b-it"} |
| 275 | ), |
| 276 | "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"), |
| 277 | "Gemma4ForCausalLM": _HfExamplesInfo( |
| 278 | "google/gemma-4-E2B-it", |
| 279 | ), |
| 280 | "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"), |
| 281 | "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"), |
| 282 | "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"), |
| 283 | "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"), |
| 284 | "Glm4MoeLiteForCausalLM": _HfExamplesInfo( |
| 285 | "zai-org/GLM-4.7-Flash", |
| 286 | ), |
| 287 | "GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False), |
| 288 | "Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"), |
| 289 | "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"), |
| 290 | "GPTBigCodeForCausalLM": _HfExamplesInfo( |
| 291 | "bigcode/starcoder", |
| 292 | extras={ |
| 293 | "tiny": "bigcode/tiny_starcoder_py", |
| 294 | "santacoder": "bigcode/gpt_bigcode-santacoder", |
| 295 | }, |
| 296 | ), |
| 297 | "GPTJForCausalLM": _HfExamplesInfo( |
| 298 | "Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"} |
| 299 | ), |
| 300 | "GPTNeoXForCausalLM": _HfExamplesInfo( |
| 301 | "EleutherAI/pythia-70m", {"1b": "EleutherAI/pythia-1.4b"} |
| 302 | ), |
| 303 | "GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"), |
| 304 | "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"), |
| 305 | "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"), |
| 306 | "GraniteMoeHybridForCausalLM": _HfExamplesInfo( |
| 307 | "ibm-granite/granite-4.0-tiny-preview" |
| 308 | ), |
| 309 | "GraniteMoeSharedForCausalLM": _HfExamplesInfo( |
| 310 | "ibm-research/moe-7b-1b-active-shared-experts" |
| 311 | ), |
| 312 | "GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"), |
| 313 | "GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"), |
| 314 | "HrmTextForCausalLM": _HfExamplesInfo( |
| 315 | "sapientinc/HRM-Text-1B", |
| 316 | ), |
| 317 | "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"), |
| 318 | "HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"), |
| 319 | "HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True), |
| 320 | # Internal checkpoint, not publicly available yet. |
| 321 | "HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True), |
| 322 | "HyperCLOVAXForCausalLM": _HfExamplesInfo( |
| 323 | "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", |
| 324 | ), |
| 325 | "InklingForCausalLM": _HfExamplesInfo( |
| 326 | "thinkingmachines/Inkling-NVFP4", |
| 327 | tokenizer_mode="inkling", |
| 328 | trust_remote_code=True, |
| 329 | max_model_len=4096, |
| 330 | ), |
| 331 | "InternLM2ForCausalLM": _HfExamplesInfo( |
| 332 | "internlm/internlm2-chat-7b", trust_remote_code=True |
| 333 | ), |
| 334 | "InternLM3ForCausalLM": _HfExamplesInfo( |
| 335 | "internlm/internlm3-8b-instruct", trust_remote_code=True |
| 336 | ), |
| 337 | "IQuestCoderForCausalLM": _HfExamplesInfo( |
| 338 | "IQuestLab/IQuest-Coder-V1-40B-Instruct", trust_remote_code=True |
| 339 | ), |
| 340 | "IQuestLoopCoderForCausalLM": _HfExamplesInfo( |
| 341 | "IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True |
| 342 | ), |
| 343 | "Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"), |
| 344 | "JambaForCausalLM": _HfExamplesInfo( |
| 345 | "ai21labs/AI21-Jamba-1.5-Mini", |
| 346 | extras={ |
| 347 | "tiny": "ai21labs/Jamba-tiny-dev", |
| 348 | "random": "ai21labs/Jamba-tiny-random", |
| 349 | }, |
| 350 | ), |
| 351 | "K2HorizonForCausalLM": _HfExamplesInfo( |
| 352 | "IFM/K2-Horizon-36B", |
| 353 | trust_remote_code=True, |
| 354 | is_available_online=False, |
| 355 | ), |
| 356 | "KimiLinearForCausalLM": _HfExamplesInfo( |
| 357 | "moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True |
| 358 | ), |
| 359 | "LagunaForCausalLM": _HfExamplesInfo("poolside/Laguna-XS.2"), |
| 360 | "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"), |
| 361 | "Lfm2MoeForCausalLM": _HfExamplesInfo( |
| 362 | "LiquidAI/LFM2-8B-A1B", |
| 363 | use_original_num_layers=True, |
| 364 | # Initialize at least one MoE layer |
| 365 | hf_overrides={"num_hidden_layers": 4}, |
| 366 | ), |
| 367 | "LlamaForCausalLM": _HfExamplesInfo( |
| 368 | "meta-llama/Llama-3.2-1B-Instruct", |
| 369 | extras={ |
| 370 | "guard": "meta-llama/Llama-Guard-3-1B", |
| 371 | "hermes": "NousResearch/Hermes-3-Llama-3.1-8B", |
| 372 | "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8", |
| 373 | "fp8_1b": "RedHatAI/Llama-3.2-1B-Instruct-FP8", |
| 374 | "tiny": "hmellor/tiny-random-LlamaForCausalLM", |
| 375 | }, |
| 376 | ), |
| 377 | "LLaMAForCausalLM": _HfExamplesInfo( |
| 378 | "decapoda-research/llama-7b-hf", is_available_online=False |
| 379 | ), |
| 380 | "Llama4ForCausalLM": _HfExamplesInfo( |
| 381 | "meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 382 | ), |
| 383 | "LongcatFlashForCausalLM": _HfExamplesInfo( |
| 384 | "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True |
| 385 | ), |
| 386 | "LongcatFlashNgramForCausalLM": _HfExamplesInfo( |
| 387 | "meituan-longcat/LongCat-Flash-Lite", |
| 388 | trust_remote_code=True, |
| 389 | # Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size) |
| 390 | # so the dummy-weight init test fits in CI memory. |
| 391 | hf_overrides={"ngram_vocab_size_ratio": 1}, |
| 392 | ), |
| 393 | "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"), |
| 394 | "Mamba2ForCausalLM": _HfExamplesInfo( |
| 395 | "mistralai/Mamba-Codestral-7B-v0.1", |
| 396 | extras={ |
| 397 | "random": "yujiepan/mamba2-codestral-v0.1-tiny-random", |
| 398 | }, |
| 399 | ), |
| 400 | "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"), |
| 401 | "MiniCPMForCausalLM": _HfExamplesInfo( |
| 402 | "openbmb/MiniCPM-2B-sft-bf16", trust_remote_code=True |
| 403 | ), |
| 404 | "MiniCPM3ForCausalLM": _HfExamplesInfo( |
| 405 | "openbmb/MiniCPM3-4B", trust_remote_code=True |
| 406 | ), |
| 407 | "MiniCPM4ForCausalLM": _HfExamplesInfo( |
| 408 | "openbmb/MiniCPM4.1-8B", |
| 409 | max_transformers_version="4.57", |
| 410 | transformers_version_reason={ |
| 411 | "hf": "HF remote code imports removed `is_torch_fx_available`; " |
| 412 | "the upstream compatibility shim request was closed as not planned: " |
| 413 | "https://github.com/huggingface/transformers/issues/44561" |
| 414 | }, |
| 415 | trust_remote_code=True, |
| 416 | ), |
| 417 | "MiniMaxM2ForCausalLM": _HfExamplesInfo( |
| 418 | "MiniMaxAI/MiniMax-M2", |
| 419 | trust_remote_code=True, |
| 420 | ), |
| 421 | "MiniMaxM3SparseForCausalLM": _HfExamplesInfo( |
| 422 | "MiniMaxAI/MiniMax-M3", |
| 423 | trust_remote_code=True, |
| 424 | is_available_online=False, |
| 425 | ), |
| 426 | "Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"), |
| 427 | "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"), |
| 428 | "MistralLarge3ForCausalLM": _HfExamplesInfo( |
| 429 | "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4" |
| 430 | ), |
| 431 | "MixtralForCausalLM": _HfExamplesInfo( |
| 432 | "mistralai/Mixtral-8x7B-Instruct-v0.1", |
| 433 | { |
| 434 | "tiny": "axolotl-ai-co/tiny-mixtral-30m", |
| 435 | "tiny-random": "TitanML/tiny-mixtral", |
| 436 | }, |
| 437 | ), |
| 438 | "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"), |
| 439 | "NemotronHForCausalLM": _HfExamplesInfo( |
| 440 | "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True |
| 441 | ), |
| 442 | "NemotronHPuzzleForCausalLM": _HfExamplesInfo( |
| 443 | "", |
| 444 | trust_remote_code=True, |
| 445 | is_available_online=False, |
| 446 | ), |
| 447 | "NanbeigeForCausalLM": _HfExamplesInfo( |
| 448 | "Nanbeige/Nanbeige4.2-3B", |
| 449 | trust_remote_code=True, |
| 450 | ), |
| 451 | "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"), |
| 452 | "Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"), |
| 453 | "Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"), |
| 454 | "OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"), |
| 455 | "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"), |
| 456 | "OPTForCausalLM": _HfExamplesInfo( |
| 457 | "facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"} |
| 458 | ), |
| 459 | "OrionForCausalLM": _HfExamplesInfo( |
| 460 | "OrionStarAI/Orion-14B-Chat", trust_remote_code=True |
| 461 | ), |
| 462 | "PanguEmbeddedForCausalLM": _HfExamplesInfo( |
| 463 | "FreedomIntelligence/openPangu-Embedded-7B-V1.1", trust_remote_code=True |
| 464 | ), |
| 465 | "PanguProMoEV2ForCausalLM": _HfExamplesInfo( |
| 466 | "", |
| 467 | trust_remote_code=True, |
| 468 | is_available_online=False, |
| 469 | ), |
| 470 | "PanguUltraMoEForCausalLM": _HfExamplesInfo( |
| 471 | "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1", |
| 472 | trust_remote_code=True, |
| 473 | is_available_online=False, |
| 474 | ), |
| 475 | "Param2MoEForCausalLM": _HfExamplesInfo( |
| 476 | "bharatgenai/Param2-17B-A2.4B-Thinking", |
| 477 | trust_remote_code=True, |
| 478 | ), |
| 479 | "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"), |
| 480 | "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"), |
| 481 | "PhiMoEForCausalLM": _HfExamplesInfo( |
| 482 | "microsoft/Phi-3.5-MoE-instruct", trust_remote_code=True |
| 483 | ), |
| 484 | "Plamo3ForCausalLM": _HfExamplesInfo( |
| 485 | "pfnet/plamo-3-nict-2b-base", |
| 486 | trust_remote_code=True, |
| 487 | ), |
| 488 | "Qwen2ForCausalLM": _HfExamplesInfo( |
| 489 | "Qwen/Qwen2-0.5B-Instruct", |
| 490 | extras={ |
| 491 | "2.5": "Qwen/Qwen2.5-0.5B-Instruct", |
| 492 | "2.5-1.5B": "Qwen/Qwen2.5-1.5B-Instruct", |
| 493 | }, |
| 494 | ), |
| 495 | "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"), |
| 496 | "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"), |
| 497 | "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"), |
| 498 | "Qwen3_5ForCausalLM": _HfExamplesInfo("codecho/Qwen3.5-0.8B-text-only"), |
| 499 | "Qwen3_5MoeForCausalLM": _HfExamplesInfo( |
| 500 | "codecho/Qwen3.5-35B-A3B-text-only", |
| 501 | extras={"native-prefix": "imdatta0/small_qwen3_5_20b"}, |
| 502 | max_model_len=4096, |
| 503 | ), |
| 504 | "Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False), |
| 505 | "MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"), |
| 506 | "Qwen3NextForCausalLM": _HfExamplesInfo( |
| 507 | "Qwen/Qwen3-Next-80B-A3B-Instruct", |
| 508 | extras={"tiny-random": "tiny-random/qwen3-next-moe"}, |
| 509 | ), |
| 510 | "Rnj1ForCausalLM": _HfExamplesInfo( |
| 511 | "EssentialAI/rnj-1-instruct", |
| 512 | is_available_online=False, |
| 513 | ), |
| 514 | "SarvamMoEForCausalLM": _HfExamplesInfo( |
| 515 | "sarvamai/sarvam-30b", |
| 516 | trust_remote_code=True, |
| 517 | max_model_len=4096, |
| 518 | is_available_online=True, |
| 519 | ), |
| 520 | "SarvamMLAForCausalLM": _HfExamplesInfo( |
| 521 | "sarvamai/sarvam-105b", |
| 522 | trust_remote_code=True, |
| 523 | max_model_len=4096, |
| 524 | is_available_online=True, |
| 525 | max_transformers_version="5.3", |
| 526 | transformers_version_reason={ |
| 527 | "hf": ( |
| 528 | "Transformers v5.4 removed the ignore_keys param from " |
| 529 | "validate_rope(); vLLM patches validate_rope() and is unaffected" |
| 530 | ) |
| 531 | }, |
| 532 | ), |
| 533 | "SeedOssForCausalLM": _HfExamplesInfo( |
| 534 | "ByteDance-Seed/Seed-OSS-36B-Instruct", |
| 535 | trust_remote_code=True, |
| 536 | ), |
| 537 | "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"), |
| 538 | "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"), |
| 539 | "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"), |
| 540 | "Step1ForCausalLM": _HfExamplesInfo( |
| 541 | "stepfun-ai/Step-Audio-EditX", trust_remote_code=True |
| 542 | ), |
| 543 | "Step3p5ForCausalLM": _HfExamplesInfo( |
| 544 | "stepfun-ai/Step-3.5-Flash", |
| 545 | use_original_num_layers=True, |
| 546 | # Initialize at least one MoE layer |
| 547 | hf_overrides={"num_hidden_layers": 4}, |
| 548 | ), |
| 549 | "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3", trust_remote_code=True), |
| 550 | "SolarForCausalLM": _HfExamplesInfo( |
| 551 | "upstage/solar-pro-preview-instruct", trust_remote_code=True |
| 552 | ), |
| 553 | "TeleChat2ForCausalLM": _HfExamplesInfo( |
| 554 | "Tele-AI/TeleChat2-3B", trust_remote_code=True |
| 555 | ), |
| 556 | "TeleChat3ForCausalLM": _HfExamplesInfo( |
| 557 | "Tele-AI/TeleChat3-36B-Thinking", trust_remote_code=True |
| 558 | ), |
| 559 | "TeleFLMForCausalLM": _HfExamplesInfo( |
| 560 | "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True |
| 561 | ), |
| 562 | "VaultGemmaForCausalLM": _HfExamplesInfo("google/vaultgemma-1b"), |
| 563 | "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"), |
| 564 | "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True), |
| 565 | "MiMoV2FlashForCausalLM": _HfExamplesInfo( |
| 566 | "XiaomiMiMo/MiMo-V2-Flash", trust_remote_code=True |
| 567 | ), |
| 568 | "MiMoV2ForCausalLM": _HfExamplesInfo( |
| 569 | "XiaomiMiMo/MiMo-V2.5-Pro", trust_remote_code=True |
| 570 | ), |
| 571 | } |
| 572 | |
| 573 | _EMBEDDING_EXAMPLE_MODELS = { |
| 574 | # [Text-only] |
| 575 | "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"), |
| 576 | "BertForMaskedLM": _HfExamplesInfo("google-bert/bert-base-uncased"), |
| 577 | "BertSpladeSparseEmbeddingModel": _HfExamplesInfo( |
| 578 | "naver/splade-v3", |
| 579 | hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]}, |
| 580 | ), |
| 581 | "BgeM3EmbeddingModel": _HfExamplesInfo("BAAI/bge-m3"), |
| 582 | "DeepseekV3BidirectionalModel": _HfExamplesInfo( |
| 583 | "ai-sage/Giga-Embeddings-instruct-10B-A1.8B-0826", |
| 584 | trust_remote_code=True, |
| 585 | hf_overrides={"model_type": "deepseek_v3", "auto_map": None}, |
| 586 | ), |
| 587 | "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"), |
| 588 | "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"), |
| 589 | "GteModel": _HfExamplesInfo( |
| 590 | "Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True |
| 591 | ), |
| 592 | "GteNewModel": _HfExamplesInfo( |
| 593 | "Alibaba-NLP/gte-base-en-v1.5", |
| 594 | trust_remote_code=True, |
| 595 | hf_overrides={"architectures": ["GteNewModel"]}, |
| 596 | ), |
| 597 | "JinaEmbeddingsV5Model": _HfExamplesInfo( |
| 598 | "jinaai/jina-embeddings-v5-text-small", |
| 599 | trust_remote_code=True, |
| 600 | ), |
| 601 | "LlamaModel": _HfExamplesInfo("llama", is_available_online=False), |
| 602 | "LlamaBidirectionalModel": _HfExamplesInfo( |
| 603 | "nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True |
| 604 | ), |
| 605 | "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"), |
| 606 | "ModernBertModel": _HfExamplesInfo( |
| 607 | "Alibaba-NLP/gte-modernbert-base", trust_remote_code=True |
| 608 | ), |
| 609 | "NomicBertModel": _HfExamplesInfo( |
| 610 | "nomic-ai/nomic-embed-text-v2-moe", trust_remote_code=True |
| 611 | ), |
| 612 | "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"), |
| 613 | "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"), |
| 614 | "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"), |
| 615 | "VoyageQwen3BidirectionalEmbedModel": _HfExamplesInfo( |
| 616 | "voyageai/voyage-4-nano", trust_remote_code=True |
| 617 | ), |
| 618 | "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"), |
| 619 | # [Multimodal] |
| 620 | "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"), |
| 621 | "LlamaNemotronVLModel": _HfExamplesInfo( |
| 622 | "nvidia/llama-nemotron-embed-vl-1b-v2", trust_remote_code=True |
| 623 | ), |
| 624 | "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"), |
| 625 | "Phi3VForCausalLM": _HfExamplesInfo( |
| 626 | "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True |
| 627 | ), |
| 628 | "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), |
| 629 | "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"), |
| 630 | "Terratorch": _HfExamplesInfo( |
| 631 | "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", |
| 632 | dtype="float16", |
| 633 | enforce_eager=True, |
| 634 | require_embed_inputs=True, |
| 635 | # This is to avoid the model going OOM in CI |
| 636 | max_num_seqs=32, |
| 637 | ), |
| 638 | } |
| 639 | |
| 640 | _LATE_INTERACTION_EXAMPLE_MODELS = { |
| 641 | # [Text-only] |
| 642 | "HF_ColBERT": _HfExamplesInfo("answerdotai/answerai-colbert-small-v1"), |
| 643 | "ColBERTModernBertModel": _HfExamplesInfo( |
| 644 | "lightonai/GTE-ModernColBERT-v1", |
| 645 | hf_overrides={"architectures": ["ColBERTModernBertModel"]}, |
| 646 | ), |
| 647 | "ColBERTJinaRobertaModel": _HfExamplesInfo( |
| 648 | "jinaai/jina-colbert-v2", |
| 649 | trust_remote_code=True, |
| 650 | hf_overrides={"architectures": ["ColBERTJinaRobertaModel"]}, |
| 651 | ), |
| 652 | "ColBERTLfm2Model": _HfExamplesInfo( |
| 653 | "LiquidAI/LFM2-ColBERT-350M", |
| 654 | trust_remote_code=True, |
| 655 | hf_overrides={"architectures": ["ColBERTLfm2Model"]}, |
| 656 | ), |
| 657 | "JinaForRanking": _HfExamplesInfo("jinaai/jina-reranker-v3"), |
| 658 | # [Multimodal] |
| 659 | "ColModernVBertForRetrieval": _HfExamplesInfo( |
| 660 | "ModernVBERT/colmodernvbert-merged", |
| 661 | ), |
| 662 | "ColPaliForRetrieval": _HfExamplesInfo("vidore/colpali-v1.3-hf"), |
| 663 | "ColQwen3": _HfExamplesInfo( |
| 664 | "TomoroAI/tomoro-colqwen3-embed-4b", trust_remote_code=True |
| 665 | ), |
| 666 | "OpsColQwen3Model": _HfExamplesInfo( |
| 667 | "OpenSearch-AI/Ops-Colqwen3-4B", trust_remote_code=True |
| 668 | ), |
| 669 | "ColQwen3_5": _HfExamplesInfo( |
| 670 | "athrael-soju/colqwen3.5-4.5B-v3", |
| 671 | trust_remote_code=True, |
| 672 | max_model_len=4096, |
| 673 | ), |
| 674 | "Qwen3VLNemotronEmbedModel": _HfExamplesInfo( |
| 675 | "nvidia/nemotron-colembed-vl-4b-v2", |
| 676 | ), |
| 677 | } |
| 678 | |
| 679 | |
| 680 | _REWARD_EXAMPLE_MODELS = { |
| 681 | "InternLM2ForRewardModel": _HfExamplesInfo( |
| 682 | "internlm/internlm2-1_8b-reward", trust_remote_code=True |
| 683 | ), |
| 684 | "Qwen2ForRewardModel": _HfExamplesInfo( |
| 685 | "Qwen/Qwen2.5-Math-RM-72B", |
| 686 | max_transformers_version="4.53", |
| 687 | transformers_version_reason={ |
| 688 | "hf": "HF model uses remote code that is not compatible with latest Transformers" # noqa: E501 |
| 689 | }, |
| 690 | ), |
| 691 | "Qwen2ForProcessRewardModel": _HfExamplesInfo( |
| 692 | "Qwen/Qwen2.5-Math-PRM-7B", |
| 693 | max_transformers_version="4.53", |
| 694 | transformers_version_reason={ |
| 695 | "hf": "HF model uses remote code that is not compatible with latest Transformers" # noqa: E501 |
| 696 | }, |
| 697 | ), |
| 698 | } |
| 699 | |
| 700 | _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = { |
| 701 | "BertForTokenClassification": _HfExamplesInfo("boltuix/NeuroBERT-NER"), |
| 702 | "ModernBertForTokenClassification": _HfExamplesInfo( |
| 703 | "disham993/electrical-ner-ModernBERT-base" |
| 704 | ), |
| 705 | "OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo( |
| 706 | "openai/privacy-filter", |
| 707 | ), |
| 708 | "RobertaForTokenClassification": _HfExamplesInfo( |
| 709 | "Jean-Baptiste/roberta-large-ner-english" |
| 710 | ), |
| 711 | "XLMRobertaForTokenClassification": _HfExamplesInfo( |
| 712 | "Davlan/xlm-roberta-base-ner-hrl" |
| 713 | ), |
| 714 | } |
| 715 | |
| 716 | _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = { |
| 717 | "BertForSequenceClassification": _HfExamplesInfo( |
| 718 | "cross-encoder/ms-marco-MiniLM-L-6-v2" |
| 719 | ), |
| 720 | "GPT2ForSequenceClassification": _HfExamplesInfo( |
| 721 | "nie3e/sentiment-polish-gpt2-small" |
| 722 | ), |
| 723 | "GteNewForSequenceClassification": _HfExamplesInfo( |
| 724 | "Alibaba-NLP/gte-multilingual-reranker-base", |
| 725 | trust_remote_code=True, |
| 726 | hf_overrides={"architectures": ["GteNewForSequenceClassification"]}, |
| 727 | ), |
| 728 | "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"), |
| 729 | "LlamaBidirectionalForSequenceClassification": _HfExamplesInfo( |
| 730 | "nvidia/llama-nemotron-rerank-1b-v2", trust_remote_code=True |
| 731 | ), |
| 732 | "LlamaNemotronVLForSequenceClassification": _HfExamplesInfo( |
| 733 | "nvidia/llama-nemotron-rerank-vl-1b-v2", trust_remote_code=True |
| 734 | ), |
| 735 | "ModernBertForSequenceClassification": _HfExamplesInfo( |
| 736 | "Alibaba-NLP/gte-reranker-modernbert-base" |
| 737 | ), |
| 738 | "RobertaForSequenceClassification": _HfExamplesInfo( |
| 739 | "cross-encoder/quora-roberta-base" |
| 740 | ), |
| 741 | "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"), |
| 742 | } |
| 743 | |
| 744 | _AUTOMATIC_CONVERTED_MODELS = { |
| 745 | # Use as_seq_cls_model for automatic conversion |
| 746 | "GemmaForSequenceClassification": _HfExamplesInfo( |
| 747 | "BAAI/bge-reranker-v2-gemma", |
| 748 | hf_overrides={ |
| 749 | "architectures": ["GemmaForSequenceClassification"], |
| 750 | "classifier_from_token": ["Yes"], |
| 751 | "method": "no_post_processing", |
| 752 | }, |
| 753 | ), |
| 754 | "LlamaForSequenceClassification": _HfExamplesInfo( |
| 755 | "Skywork/Skywork-Reward-V2-Llama-3.2-1B" |
| 756 | ), |
| 757 | "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), |
| 758 | "Qwen3ForSequenceClassification": _HfExamplesInfo( |
| 759 | "tomaarsen/Qwen3-Reranker-0.6B-seq-cls" |
| 760 | ), |
| 761 | "Qwen3ForTokenClassification": _HfExamplesInfo("bd2lcco/Qwen3-0.6B-finetuned"), |
| 762 | "Qwen3VLForSequenceClassification": _HfExamplesInfo( |
| 763 | "Qwen/Qwen3-VL-Reranker-2B", |
| 764 | is_available_online=False, |
| 765 | hf_overrides={ |
| 766 | "architectures": ["Qwen3VLForSequenceClassification"], |
| 767 | "classifier_from_token": ["no", "yes"], |
| 768 | "is_original_qwen3_reranker": True, |
| 769 | }, |
| 770 | ), |
| 771 | } |
| 772 | |
| 773 | _MULTIMODAL_EXAMPLE_MODELS = { |
| 774 | # [Decoder-only] |
| 775 | "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"), |
| 776 | "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo( |
| 777 | "nvidia/audio-flamingo-3-hf", |
| 778 | transformers_version_reason={ |
| 779 | "vllm": "Needs https://github.com/huggingface/transformers/pull/43538" |
| 780 | }, |
| 781 | ), |
| 782 | "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"), |
| 783 | "BailingMoeV3VLForConditionalGeneration": _HfExamplesInfo( |
| 784 | "inclusionAI/Ling-3.0-flash-VL", |
| 785 | trust_remote_code=True, |
| 786 | is_available_online=True, |
| 787 | use_original_num_layers=True, |
| 788 | hf_overrides={ |
| 789 | "text_config": { |
| 790 | "num_hidden_layers": 6, |
| 791 | "layer_types": ["linear_attention"] * 5 + ["full_attention"], |
| 792 | } |
| 793 | }, |
| 794 | ), |
| 795 | "BeeForConditionalGeneration": _HfExamplesInfo( |
| 796 | "Open-Bee/Bee-8B-RL", |
| 797 | trust_remote_code=True, |
| 798 | ), |
| 799 | "Blip2ForConditionalGeneration": _HfExamplesInfo( |
| 800 | "Salesforce/blip2-opt-2.7b", |
| 801 | extras={"6b": "Salesforce/blip2-opt-6.7b"}, |
| 802 | ), |
| 803 | "Cohere2VisionForConditionalGeneration": _HfExamplesInfo( |
| 804 | "CohereLabs/command-a-vision-07-2025", |
| 805 | extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"}, |
| 806 | ), |
| 807 | "CohereCompassForConditionalGeneration": _HfExamplesInfo( |
| 808 | "CohereLabs/North-Micro-Vision-Instruct", |
| 809 | ), |
| 810 | "Cosmos3ForConditionalGeneration": _HfExamplesInfo( |
| 811 | "nvidia/Cosmos3-Nano", |
| 812 | extras={"super": "nvidia/Cosmos3-Super"}, |
| 813 | max_model_len=4096, |
| 814 | ), |
| 815 | "Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo( |
| 816 | "nvidia/Cosmos3-Edge", |
| 817 | max_model_len=4096, |
| 818 | use_original_num_layers=True, |
| 819 | hf_overrides={ |
| 820 | "text_config": { |
| 821 | "num_hidden_layers": 2, |
| 822 | "hybrid_override_pattern": "*-", |
| 823 | } |
| 824 | }, |
| 825 | ), |
| 826 | "DeepseekVLV2ForCausalLM": _HfExamplesInfo( |
| 827 | "deepseek-ai/deepseek-vl2-tiny", |
| 828 | extras={"fork": "Isotr0py/deepseek-vl2-tiny"}, |
| 829 | max_transformers_version="4.48", |
| 830 | transformers_version_reason={"hf": "HF model is not compatible."}, |
| 831 | ), |
| 832 | "DeepseekOCRForCausalLM": _HfExamplesInfo( |
| 833 | "deepseek-ai/DeepSeek-OCR", |
| 834 | ), |
| 835 | "DeepseekOCR2ForCausalLM": _HfExamplesInfo( |
| 836 | "deepseek-ai/DeepSeek-OCR-2", |
| 837 | ), |
| 838 | "DeepseekV4ForConditionalGeneration": _HfExamplesInfo( |
| 839 | "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", |
| 840 | ), |
| 841 | "DeepseekV41ForCausalLM": _HfExamplesInfo( |
| 842 | "deepseek-ai/DeepSeek-V4.1-Flash", |
| 843 | ), |
| 844 | "Dots3NoteForCausalLM": _HfExamplesInfo( |
| 845 | "dots-studio/dots3-note-prev", |
| 846 | is_available_online=False, |
| 847 | ), |
| 848 | "UnlimitedOCRForCausalLM": _HfExamplesInfo( |
| 849 | "baidu/Unlimited-OCR", |
| 850 | ), |
| 851 | "DotsOCRForCausalLM": _HfExamplesInfo( |
| 852 | "rednote-hilab/dots.ocr", trust_remote_code=True |
| 853 | ), |
| 854 | "Eagle2_5_VLForConditionalGeneration": _HfExamplesInfo( |
| 855 | "nvidia/Eagle2.5-8B", |
| 856 | trust_remote_code=True, |
| 857 | ), |
| 858 | "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"), |
| 859 | "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo( |
| 860 | "baidu/ERNIE-4.5-VL-28B-A3B-PT", |
| 861 | trust_remote_code=True, |
| 862 | revision="refs/pr/17", |
| 863 | ), |
| 864 | "Exaone4_5_ForConditionalGeneration": _HfExamplesInfo( |
| 865 | "LGAI-EXAONE/EXAONE-4.5-33B", |
| 866 | ), |
| 867 | "FireRedASR2ForConditionalGeneration": _HfExamplesInfo( |
| 868 | "allendou/FireRedASR2-LLM-vllm", |
| 869 | trust_remote_code=True, |
| 870 | max_transformers_version="5.1", |
| 871 | transformers_version_reason={ |
| 872 | "vllm": "Incompatible with transformers v5.2+ " |
| 873 | "(dict object has no attribute '__name__').", |
| 874 | }, |
| 875 | ), |
| 876 | "FunASRForConditionalGeneration": _HfExamplesInfo( |
| 877 | "FunAudioLLM/Fun-ASR-Nano-2512-vllm", |
| 878 | trust_remote_code=True, |
| 879 | max_transformers_version="5.15", |
| 880 | transformers_version_reason={ |
| 881 | "vllm": "The official Fun-ASR-Nano vLLM package is validated " |
| 882 | "through transformers v5.15.0; newer versions require separate " |
| 883 | "validation.", |
| 884 | }, |
| 885 | ), |
| 886 | "FunAudioChatForConditionalGeneration": _HfExamplesInfo( |
| 887 | "funaudiochat", is_available_online=False |
| 888 | ), |
| 889 | "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"), |
| 890 | "DiffusionGemmaForBlockDiffusion": _HfExamplesInfo( |
| 891 | "google/diffusiongemma-26B-A4B-it", |
| 892 | trust_remote_code=True, |
| 893 | ), |
| 894 | "Gemma4ForConditionalGeneration": _HfExamplesInfo( |
| 895 | "google/gemma-4-E2B-it", |
| 896 | ), |
| 897 | "Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo( |
| 898 | "google/gemma-4-12B-it", |
| 899 | is_available_online=False, |
| 900 | ), |
| 901 | "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"), |
| 902 | "GlmAsrForConditionalGeneration": _HfExamplesInfo( |
| 903 | "zai-org/GLM-ASR-Nano-2512", |
| 904 | ), |
| 905 | "Granite4VisionForConditionalGeneration": _HfExamplesInfo( |
| 906 | "ibm-granite/granite-vision-4.1-4b", |
| 907 | is_available_online=False, |
| 908 | ), |
| 909 | "GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"), |
| 910 | "GraniteSpeechForConditionalGeneration": _HfExamplesInfo( |
| 911 | "ibm-granite/granite-speech-3.3-2b", |
| 912 | extras={"4.0-1b": "ibm-granite/granite-4.0-1b-speech"}, |
| 913 | ), |
| 914 | "GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo( |
| 915 | "ibm-granite/granite-speech-4.1-2b-plus", |
| 916 | ), |
| 917 | "GLM4VForCausalLM": _HfExamplesInfo( |
| 918 | "zai-org/glm-4v-9b", |
| 919 | trust_remote_code=True, |
| 920 | hf_overrides={"architectures": ["GLM4VForCausalLM"]}, |
| 921 | ), |
| 922 | "Glm4vForConditionalGeneration": _HfExamplesInfo( |
| 923 | "zai-org/GLM-4.1V-9B-Thinking", |
| 924 | extras={"4.6V": "zai-org/GLM-4.6V-Flash"}, |
| 925 | ), |
| 926 | "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"), |
| 927 | "GlmOcrForConditionalGeneration": _HfExamplesInfo( |
| 928 | "zai-org/GLM-OCR", |
| 929 | ), |
| 930 | "Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"), |
| 931 | "H2OVLChatModel": _HfExamplesInfo( |
| 932 | "h2oai/h2ovl-mississippi-800m", |
| 933 | trust_remote_code=True, |
| 934 | extras={"2b": "h2oai/h2ovl-mississippi-2b"}, |
| 935 | max_transformers_version="4.48", |
| 936 | transformers_version_reason={"hf": "HF model is not compatible."}, |
| 937 | ), |
| 938 | "HCXVisionV2ForCausalLM": _HfExamplesInfo( |
| 939 | "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B", |
| 940 | trust_remote_code=True, |
| 941 | revision="a6cdfd3464d1b767259cad23e164eaf39d3e3960", |
| 942 | ), |
| 943 | "HunYuanVLForConditionalGeneration": _HfExamplesInfo( |
| 944 | "tencent/HunyuanOCR", |
| 945 | ), |
| 946 | "Idefics3ForConditionalGeneration": _HfExamplesInfo( |
| 947 | "HuggingFaceM4/Idefics3-8B-Llama3", |
| 948 | extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}, |
| 949 | ), |
| 950 | "InklingForConditionalGeneration": _HfExamplesInfo( |
| 951 | "thinkingmachines/Inkling-NVFP4", |
| 952 | tokenizer_mode="inkling", |
| 953 | trust_remote_code=True, |
| 954 | max_model_len=4096, |
| 955 | ), |
| 956 | "IsaacForConditionalGeneration": _HfExamplesInfo( |
| 957 | "PerceptronAI/Isaac-0.1", |
| 958 | trust_remote_code=True, |
| 959 | extras={"0.2-2B-Preview": "PerceptronAI/Isaac-0.2-2B-Preview"}, |
| 960 | max_transformers_version="4.57", |
| 961 | transformers_version_reason={ |
| 962 | "vllm": ( |
| 963 | "Custom Isaac code is not compatible with Transformers v5. " |
| 964 | "The model should be upstreamed to Transformers for " |
| 965 | "long-term support." |
| 966 | ), |
| 967 | "hf": ( |
| 968 | "Isaac's remote model and processor code import or configure " |
| 969 | "APIs that changed in Transformers v5." |
| 970 | ), |
| 971 | }, |
| 972 | ), |
| 973 | "InternS1ForConditionalGeneration": _HfExamplesInfo( |
| 974 | "internlm/Intern-S1", |
| 975 | trust_remote_code=True, |
| 976 | max_transformers_version="4.57", |
| 977 | transformers_version_reason={ |
| 978 | "vllm": "Custom tokenizer code is not compatible with Transformers v5." |
| 979 | }, |
| 980 | ), |
| 981 | "InternS1ProForConditionalGeneration": _HfExamplesInfo( |
| 982 | "internlm/Intern-S1-Pro", |
| 983 | trust_remote_code=True, |
| 984 | max_transformers_version="5.14.1", |
| 985 | transformers_version_reason={ |
| 986 | "vllm": ( |
| 987 | "Remote video processor code imports " |
| 988 | "`BASE_VIDEO_PROCESSOR_DOCSTRING`, removed in Transformers 5.15." |
| 989 | ) |
| 990 | }, |
| 991 | ), |
| 992 | "InternS2MobiusForConditionalGeneration": _HfExamplesInfo( |
| 993 | "internlm/Intern-S2-Mobius", |
| 994 | trust_remote_code=True, |
| 995 | is_available_online=False, |
| 996 | ), |
| 997 | "InternS2PreviewForConditionalGeneration": _HfExamplesInfo( |
| 998 | "internlm/Intern-S2-Preview", |
| 999 | trust_remote_code=True, |
| 1000 | is_available_online=False, |
| 1001 | ), |
| 1002 | "InternVLChatModel": _HfExamplesInfo( |
| 1003 | "OpenGVLab/InternVL2-1B", |
| 1004 | extras={ |
| 1005 | "2B": "OpenGVLab/InternVL2-2B", |
| 1006 | "3.0": "OpenGVLab/InternVL3-1B", |
| 1007 | "3.5-qwen3": "OpenGVLab/InternVL3_5-1B", |
| 1008 | "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B", |
| 1009 | "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview", |
| 1010 | }, |
| 1011 | trust_remote_code=True, |
| 1012 | ), |
| 1013 | "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"), |
| 1014 | "KananaVForConditionalGeneration": _HfExamplesInfo( |
| 1015 | "kakaocorp/kanana-1.5-v-3b-instruct", |
| 1016 | trust_remote_code=True, |
| 1017 | ), |
| 1018 | "KeyeForConditionalGeneration": _HfExamplesInfo( |
| 1019 | "Kwai-Keye/Keye-VL-8B-Preview", |
| 1020 | trust_remote_code=True, |
| 1021 | ), |
| 1022 | "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo( |
| 1023 | "Kwai-Keye/Keye-VL-1_5-8B", |
| 1024 | trust_remote_code=True, |
| 1025 | ), |
| 1026 | "MoonshotKimiaForCausalLM": _HfExamplesInfo( |
| 1027 | "moonshotai/Kimi-Audio-7B-Instruct", |
| 1028 | tokenizer_mode="kimi_audio", |
| 1029 | trust_remote_code=True, |
| 1030 | ), |
| 1031 | "KimiK25ForConditionalGeneration": _HfExamplesInfo( |
| 1032 | "moonshotai/Kimi-K2.5", |
| 1033 | trust_remote_code=True, |
| 1034 | ), |
| 1035 | "KimiK3ForConditionalGeneration": _HfExamplesInfo( |
| 1036 | "moonshotai/Kimi-K3", |
| 1037 | trust_remote_code=True, |
| 1038 | ), |
| 1039 | "KimiVLForConditionalGeneration": _HfExamplesInfo( |
| 1040 | "moonshotai/Kimi-VL-A3B-Instruct", |
| 1041 | extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"}, |
| 1042 | trust_remote_code=True, |
| 1043 | max_transformers_version="4.53.3", |
| 1044 | transformers_version_reason={ |
| 1045 | "hf": ( |
| 1046 | "HF model uses deprecated transformers API " |
| 1047 | "(PytorchGELUTanh, DynamicCache.seen_tokens, and more). See: " |
| 1048 | "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/discussions/31" |
| 1049 | ) |
| 1050 | }, |
| 1051 | ), |
| 1052 | "LightOnOCRForConditionalGeneration": _HfExamplesInfo( |
| 1053 | "lightonai/LightOnOCR-1B-1025" |
| 1054 | ), |
| 1055 | "Lfm2VlForConditionalGeneration": _HfExamplesInfo( |
| 1056 | "LiquidAI/LFM2-VL-450M", |
| 1057 | ), |
| 1058 | "Llama4ForConditionalGeneration": _HfExamplesInfo( |
| 1059 | "meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 1060 | max_model_len=10240, |
| 1061 | extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"}, |
| 1062 | ), |
| 1063 | "LlavaForConditionalGeneration": _HfExamplesInfo( |
| 1064 | "llava-hf/llava-1.5-7b-hf", |
| 1065 | extras={ |
| 1066 | "mistral": "mistral-community/pixtral-12b", |
| 1067 | "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic", |
| 1068 | }, |
| 1069 | ), |
| 1070 | "LlavaNextForConditionalGeneration": _HfExamplesInfo( |
| 1071 | "llava-hf/llava-v1.6-mistral-7b-hf" |
| 1072 | ), |
| 1073 | "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo( |
| 1074 | "llava-hf/LLaVA-NeXT-Video-7B-hf" |
| 1075 | ), |
| 1076 | "LlavaOnevision2ForConditionalGeneration": _HfExamplesInfo( |
| 1077 | "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct", |
| 1078 | trust_remote_code=True, |
| 1079 | # Keep the init/schema harness eager (these tests never run forward). |
| 1080 | # NOTE: a separate H200-only hang exists in the core FA3 attention |
| 1081 | # construction (after `FlashAttentionImpl.__init__` logs the FA3 |
| 1082 | # version, while building the Qwen3 backbone). enforce_eager does NOT |
| 1083 | # gate that path, so this flag is hygiene, not the fix -- tracked with |
| 1084 | # the maintainer. |
| 1085 | enforce_eager=True, |
| 1086 | ), |
| 1087 | "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo( |
| 1088 | "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" |
| 1089 | ), |
| 1090 | "MiDashengLMModel": _HfExamplesInfo( |
| 1091 | "mispeech/midashenglm-7b", trust_remote_code=True |
| 1092 | ), |
| 1093 | "MiMoV2OmniForCausalLM": _HfExamplesInfo( |
| 1094 | "XiaomiMiMo/MiMo-V2.5-Omni", trust_remote_code=True, is_available_online=False |
| 1095 | ), |
| 1096 | "MiniCPMO": _HfExamplesInfo( |
| 1097 | "openbmb/MiniCPM-o-2_6", |
| 1098 | trust_remote_code=True, |
| 1099 | max_transformers_version="4.57", |
| 1100 | transformers_version_reason={ |
| 1101 | "hf": "Custom processor code is not compatible with Transformers v5." |
| 1102 | }, |
| 1103 | ), |
| 1104 | "MiniCPMV": _HfExamplesInfo( |
| 1105 | "openbmb/MiniCPM-Llama3-V-2_5", |
| 1106 | extras={ |
| 1107 | "2.6": "openbmb/MiniCPM-V-2_6", |
| 1108 | "4.0": "openbmb/MiniCPM-V-4", |
| 1109 | "4.5": "openbmb/MiniCPM-V-4_5", |
| 1110 | }, |
| 1111 | trust_remote_code=True, |
| 1112 | max_transformers_version="4.57", |
| 1113 | transformers_version_reason={ |
| 1114 | "hf": ( |
| 1115 | "MiniCPMV.__init__ does not call self.post_init(), so " |
| 1116 | "`all_tied_weights_keys` is never set; Transformers v5 requires " |
| 1117 | "this attribute in _move_missing_keys_from_meta_to_device." |
| 1118 | ) |
| 1119 | }, |
| 1120 | ), |
| 1121 | "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo( |
| 1122 | "openbmb/MiniCPM-V-4_6", |
| 1123 | ), |
| 1124 | "MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo( |
| 1125 | "MiniMaxAI/MiniMax-M3", |
| 1126 | trust_remote_code=True, |
| 1127 | is_available_online=False, |
| 1128 | ), |
| 1129 | "Mistral3ForConditionalGeneration": _HfExamplesInfo( |
| 1130 | "mistralai/Mistral-Small-3.1-24B-Instruct-2503", |
| 1131 | extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"}, |
| 1132 | ), |
| 1133 | "MolmoForCausalLM": _HfExamplesInfo( |
| 1134 | "allenai/Molmo-7B-D-0924", |
| 1135 | max_transformers_version="4.48", |
| 1136 | transformers_version_reason={ |
| 1137 | "vllm": "Incorrectly-detected `tensorflow` import from processor." |
| 1138 | }, |
| 1139 | extras={"olmo": "allenai/Molmo-7B-O-0924"}, |
| 1140 | trust_remote_code=True, |
| 1141 | ), |
| 1142 | "Moondream3ForCausalLM": _HfExamplesInfo( |
| 1143 | "moondream/moondream3-preview", |
| 1144 | tokenizer="moondream/starmie-v1", |
| 1145 | trust_remote_code=True, |
| 1146 | ), |
| 1147 | "MossAudioModel": _HfExamplesInfo( |
| 1148 | "OpenMOSS-Team/MOSS-Audio-4B-Instruct", |
| 1149 | extras={ |
| 1150 | "4b-thinking": "OpenMOSS-Team/MOSS-Audio-4B-Thinking", |
| 1151 | "8b-instruct": "OpenMOSS-Team/MOSS-Audio-8B-Instruct", |
| 1152 | "8b-thinking": "OpenMOSS-Team/MOSS-Audio-8B-Thinking", |
| 1153 | }, |
| 1154 | trust_remote_code=True, |
| 1155 | ), |
| 1156 | "MossTranscribeDiarizeForConditionalGeneration": _HfExamplesInfo( |
| 1157 | "OpenMOSS-Team/MOSS-Transcribe-Diarize", |
| 1158 | trust_remote_code=True, |
| 1159 | is_available_online=False, |
| 1160 | ), |
| 1161 | "HfMoondream": _HfExamplesInfo( |
| 1162 | "moondream/moondream3-preview", |
| 1163 | tokenizer="moondream/starmie-v1", |
| 1164 | trust_remote_code=True, |
| 1165 | ), |
| 1166 | "Molmo2ForConditionalGeneration": _HfExamplesInfo( |
| 1167 | "allenai/Molmo2-8B", |
| 1168 | extras={"olmo": "allenai/Molmo2-O-7B"}, |
| 1169 | trust_remote_code=True, |
| 1170 | # required by current PrefixLM implementation |
| 1171 | max_num_batched_tokens=31872, |
| 1172 | ), |
| 1173 | "MuseGlimmerForConditionalGeneration": _HfExamplesInfo( |
| 1174 | "meta-models/Muse-Glimmer-30B", |
| 1175 | ), |
| 1176 | "MuseGlimmerForCausalLM": _HfExamplesInfo( |
| 1177 | "meta-models/Muse-Glimmer-30B", |
| 1178 | ), |
| 1179 | "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True), |
| 1180 | "Llama_Nemotron_Nano_VL": _HfExamplesInfo( |
| 1181 | "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1", |
| 1182 | trust_remote_code=True, |
| 1183 | ), |
| 1184 | "NemotronH_Nano_VL_V2": _HfExamplesInfo( |
| 1185 | "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16", |
| 1186 | max_model_len=4096, |
| 1187 | # NemotronH layers are constructed via `hybrid_override_pattern` |
| 1188 | use_original_num_layers=True, |
| 1189 | hf_overrides={ |
| 1190 | "text_config": {"num_hidden_layers": 2, "hybrid_override_pattern": "M*"}, |
| 1191 | }, |
| 1192 | trust_remote_code=True, |
| 1193 | ), |
| 1194 | "NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo( |
| 1195 | "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", |
| 1196 | max_model_len=4096, |
| 1197 | # NemotronH layers are constructed via `hybrid_override_pattern` |
| 1198 | use_original_num_layers=True, |
| 1199 | hf_overrides={ |
| 1200 | "vision_config": PreTrainedConfig( |
| 1201 | args={ |
| 1202 | "min_num_patches": 1, |
| 1203 | "max_num_patches": 12, |
| 1204 | "model": "vit_huge_patch16_224", |
| 1205 | }, |
| 1206 | video_temporal_patch_size=2, |
| 1207 | # TODO(nhaber): This is `true` in the official `config.json`, |
| 1208 | # but this causes a processor exception in the tests due to a known bug |
| 1209 | # with mixed-resolution video when `true`. To be resolved. |
| 1210 | video_maintain_aspect_ratio=False, |
| 1211 | ), |
| 1212 | "text_config": {"num_hidden_layers": 2, "hybrid_override_pattern": "M*"}, |
| 1213 | }, |
| 1214 | trust_remote_code=True, |
| 1215 | ), |
| 1216 | "NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo( |
| 1217 | "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", is_available_online=False |
| 1218 | ), |
| 1219 | # TODO: Change repo id once pertinent archs are public. |
| 1220 | "NemotronH_Omni_Reasoning_V3": _HfExamplesInfo( |
| 1221 | "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", is_available_online=False |
| 1222 | ), |
| 1223 | "OpenCUAForConditionalGeneration": _HfExamplesInfo( |
| 1224 | "xlangai/OpenCUA-7B", |
| 1225 | trust_remote_code=True, |
| 1226 | max_transformers_version="4.57", |
| 1227 | transformers_version_reason={ |
| 1228 | "vllm": "Tokenizer cannot be initialised in Transformers v5." |
| 1229 | }, |
| 1230 | ), |
| 1231 | "OpenPanguVLForConditionalGeneration": _HfExamplesInfo( |
| 1232 | "FreedomIntelligence/openPangu-VL-7B", |
| 1233 | trust_remote_code=True, |
| 1234 | max_model_len=4096, |
| 1235 | enforce_eager=True, |
| 1236 | max_transformers_version="4.57", |
| 1237 | transformers_version_reason={ |
| 1238 | "vllm": ( |
| 1239 | "OpenPanguVLVideoProcessorInitKwargs does not specify total=False, " |
| 1240 | "making all kwargs required. See https://huggingface.co/FreedomIntelligence/openPangu-VL-7B/discussions/2" |
| 1241 | ) |
| 1242 | }, |
| 1243 | ), |
| 1244 | "OpenVLAForActionPrediction": _HfExamplesInfo( |
| 1245 | "openvla/openvla-7b", |
| 1246 | trust_remote_code=True, |
| 1247 | ), |
| 1248 | "Ovis": _HfExamplesInfo( |
| 1249 | "AIDC-AI/Ovis2-1B", |
| 1250 | trust_remote_code=True, |
| 1251 | max_transformers_version="4.53", |
| 1252 | transformers_version_reason={"hf": "HF model is not compatible"}, |
| 1253 | extras={ |
| 1254 | "1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B", |
| 1255 | "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B", |
| 1256 | }, |
| 1257 | ), |
| 1258 | "Ovis2_5": _HfExamplesInfo( |
| 1259 | "AIDC-AI/Ovis2.5-2B", |
| 1260 | trust_remote_code=True, |
| 1261 | max_transformers_version="4.57", |
| 1262 | transformers_version_reason={ |
| 1263 | "hf": "HF remote code reads `is_parallelizable`, which was removed " |
| 1264 | "in Transformers v5." |
| 1265 | }, |
| 1266 | ), |
| 1267 | "Ovis2_6ForCausalLM": _HfExamplesInfo( |
| 1268 | "AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True |
| 1269 | ), |
| 1270 | "Ovis2_6_MoeForCausalLM": _HfExamplesInfo( |
| 1271 | "AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True |
| 1272 | ), |
| 1273 | "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo( |
| 1274 | "PaddlePaddle/PaddleOCR-VL", |
| 1275 | trust_remote_code=True, |
| 1276 | ), |
| 1277 | "PaliGemmaForConditionalGeneration": _HfExamplesInfo( |
| 1278 | "google/paligemma-3b-mix-224", |
| 1279 | extras={"v2": "google/paligemma2-3b-ft-docci-448"}, |
| 1280 | ), |
| 1281 | "Phi3VForCausalLM": _HfExamplesInfo( |
| 1282 | "microsoft/Phi-3-vision-128k-instruct", |
| 1283 | trust_remote_code=True, |
| 1284 | max_transformers_version="4.48", |
| 1285 | transformers_version_reason={ |
| 1286 | "hf": "HF model use deprecated imports which have been removed." |
| 1287 | }, # noqa: E501 |
| 1288 | extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"}, |
| 1289 | ), |
| 1290 | "Phi4ForCausalLMV": _HfExamplesInfo( |
| 1291 | "microsoft/Phi-4-reasoning-vision-15B", |
| 1292 | trust_remote_code=True, |
| 1293 | max_transformers_version="5.3", |
| 1294 | transformers_version_reason={ |
| 1295 | "vllm": ( |
| 1296 | "vllm upgraded transformers above v5.4 where HF model " |
| 1297 | "custom code uses siglip2 internals " |
| 1298 | "(filter_out_non_signature_kwargs) removed " |
| 1299 | "by huggingface/transformers#43514" |
| 1300 | ) |
| 1301 | }, |
| 1302 | ), |
| 1303 | "Phi4MMForCausalLM": _HfExamplesInfo( |
| 1304 | "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True |
| 1305 | ), |
| 1306 | "PixtralForConditionalGeneration": _HfExamplesInfo( |
| 1307 | "mistralai/Pixtral-12B-2409", |
| 1308 | extras={ |
| 1309 | "mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", |
| 1310 | "ministral-3": "mistralai/Ministral-3-3B-Instruct-2512", |
| 1311 | }, |
| 1312 | tokenizer_mode="mistral", |
| 1313 | ), |
| 1314 | "QianfanOCRForConditionalGeneration": _HfExamplesInfo( |
| 1315 | "baidu/Qianfan-OCR", |
| 1316 | ), |
| 1317 | "Qwen2AudioForConditionalGeneration": _HfExamplesInfo( |
| 1318 | "Qwen/Qwen2-Audio-7B-Instruct" |
| 1319 | ), |
| 1320 | "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), |
| 1321 | "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo( |
| 1322 | "Qwen/Qwen2.5-VL-3B-Instruct", |
| 1323 | max_model_len=4096, |
| 1324 | ), |
| 1325 | "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"), |
| 1326 | "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), |
| 1327 | "Qwen3VLForConditionalGeneration": _HfExamplesInfo( |
| 1328 | "Qwen/Qwen3-VL-4B-Instruct", |
| 1329 | max_model_len=4096, |
| 1330 | ), |
| 1331 | "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo( |
| 1332 | "Qwen/Qwen3-VL-30B-A3B-Instruct", |
| 1333 | max_model_len=4096, |
| 1334 | ), |
| 1335 | "Qwen3_5ForConditionalGeneration": _HfExamplesInfo( |
| 1336 | "Qwen/Qwen3.5-0.8B", |
| 1337 | extras={"4b": "Qwen/Qwen3.5-4B"}, |
| 1338 | max_model_len=4096, |
| 1339 | ), |
| 1340 | "Qwen3_5MoeForConditionalGeneration": _HfExamplesInfo( |
| 1341 | "Qwen/Qwen3.5-35B-A3B", |
| 1342 | max_model_len=4096, |
| 1343 | ), |
| 1344 | "Qwen4ExpForConditionalGeneration": _HfExamplesInfo( |
| 1345 | "", |
| 1346 | is_available_online=False, |
| 1347 | ), |
| 1348 | "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo( |
| 1349 | "Qwen/Qwen3-Omni-30B-A3B-Instruct", |
| 1350 | max_model_len=4096, |
| 1351 | ), |
| 1352 | "Qwen3ASRForConditionalGeneration": _HfExamplesInfo( |
| 1353 | "Qwen/Qwen3-ASR-0.6B", |
| 1354 | max_model_len=4096, |
| 1355 | ), |
| 1356 | "Qwen3ASRRealtimeGeneration": _HfExamplesInfo( |
| 1357 | "Qwen/Qwen3-ASR-0.6B", |
| 1358 | max_model_len=4096, |
| 1359 | hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]}, |
| 1360 | ), |
| 1361 | "Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo( |
| 1362 | "Qwen/Qwen3-ForcedAligner-0.6B", |
| 1363 | max_model_len=4096, |
| 1364 | hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]}, |
| 1365 | ), |
| 1366 | "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True), |
| 1367 | "SkyworkR1VChatModel": _HfExamplesInfo( |
| 1368 | "Skywork/Skywork-R1V-38B", |
| 1369 | trust_remote_code=True, |
| 1370 | max_transformers_version="4.57", |
| 1371 | transformers_version_reason={ |
| 1372 | "hf": ( |
| 1373 | "SkyworkChatModel.__init__ does not call self.post_init(), so " |
| 1374 | "`all_tied_weights_keys` is never set; Transformers v5 requires " |
| 1375 | "this attribute in _move_missing_keys_from_meta_to_device." |
| 1376 | ) |
| 1377 | }, |
| 1378 | ), |
| 1379 | "SmolVLMForConditionalGeneration": _HfExamplesInfo( |
| 1380 | "HuggingFaceTB/SmolVLM2-2.2B-Instruct" |
| 1381 | ), |
| 1382 | "Step3VLForConditionalGeneration": _HfExamplesInfo( |
| 1383 | "stepfun-ai/step3", |
| 1384 | trust_remote_code=True, |
| 1385 | max_transformers_version="5.3", |
| 1386 | transformers_version_reason={ |
| 1387 | "hf": ( |
| 1388 | "Transformers v5.4 removed the ignore_keys param from " |
| 1389 | "validate_rope(); vLLM has vendored the config and is unaffected" |
| 1390 | ) |
| 1391 | }, |
| 1392 | ), |
| 1393 | "StepVLForConditionalGeneration": _HfExamplesInfo( |
| 1394 | "stepfun-ai/Step3-VL-10B", trust_remote_code=True |
| 1395 | ), |
| 1396 | "Step3p7ForConditionalGeneration": _HfExamplesInfo( |
| 1397 | "stepfun-ai/Step-3.7-Flash", |
| 1398 | trust_remote_code=True, |
| 1399 | use_original_num_layers=True, |
| 1400 | # The MoE config lives in the nested ``text_config``, so the overrides |
| 1401 | # must be nested too. Use 4 layers to initialize at least one MoE layer |
| 1402 | # and shrink ``moe_num_experts`` (a non-standard key not handled by |
| 1403 | # ``dummy_hf_overrides``) to avoid OOM during init. |
| 1404 | hf_overrides={"text_config": {"num_hidden_layers": 4, "moe_num_experts": 8}}, |
| 1405 | ), |
| 1406 | "UltravoxModel": _HfExamplesInfo( |
| 1407 | "fixie-ai/ultravox-v0_5-llama-3_2-1b", |
| 1408 | trust_remote_code=True, |
| 1409 | ), |
| 1410 | "VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo( |
| 1411 | "microsoft/VibeVoice-ASR-HF", |
| 1412 | ), |
| 1413 | "VoxtralForConditionalGeneration": _HfExamplesInfo( |
| 1414 | "mistralai/Voxtral-Mini-3B-2507", |
| 1415 | tokenizer_mode="mistral", |
| 1416 | ), |
| 1417 | "VoxtralRealtimeGeneration": _HfExamplesInfo( |
| 1418 | "mistralai/Voxtral-Mini-4B-Realtime-2602", |
| 1419 | enforce_eager=True, |
| 1420 | tokenizer_mode="mistral", |
| 1421 | ), |
| 1422 | # [Encoder-decoder] |
| 1423 | "CohereAsrForConditionalGeneration": _HfExamplesInfo( |
| 1424 | "CohereLabs/cohere-transcribe-03-2026", trust_remote_code=True |
| 1425 | ), |
| 1426 | "NemotronParseForConditionalGeneration": _HfExamplesInfo( |
| 1427 | "nvidia/NVIDIA-Nemotron-Parse-v1.2", trust_remote_code=True |
| 1428 | ), |
| 1429 | "WhisperForConditionalGeneration": _HfExamplesInfo( |
| 1430 | "openai/whisper-large-v3-turbo", |
| 1431 | extras={"v3": "openai/whisper-large-v3"}, |
| 1432 | ), |
| 1433 | # [Cross-encoder] |
| 1434 | "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"), |
| 1435 | } |
| 1436 | |
| 1437 | |
| 1438 | _SPECULATIVE_DECODING_EXAMPLE_MODELS = { |
| 1439 | # [Medusa] |
| 1440 | "MedusaModel": _HfExamplesInfo( |
| 1441 | "JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random" |
| 1442 | ), |
| 1443 | # Temporarily disabled. |
| 1444 | # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1. |
| 1445 | # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo( |
| 1446 | # "JackFram/llama-160m", |
| 1447 | # speculative_model="ibm-ai-platform/llama-160m-accelerator" |
| 1448 | # ), |
| 1449 | # [DFlash] |
| 1450 | "DFlashDraftModel": _HfExamplesInfo( |
| 1451 | "Qwen/Qwen3-4B", |
| 1452 | speculative_model="z-lab/Qwen3-4B-DFlash-b16", |
| 1453 | use_original_num_layers=True, # Need all layers since DFlash has >1 layer, |
| 1454 | max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env |
| 1455 | max_num_seqs=32, |
| 1456 | ), |
| 1457 | # LiLiCorr checkpoints are not published yet. |
| 1458 | "LiLiCorrDraftModel": _HfExamplesInfo( |
| 1459 | "Qwen/Qwen3-8B", |
| 1460 | speculative_model="LiLiCorrDraftModel", |
| 1461 | is_available_online=False, |
| 1462 | use_original_num_layers=True, |
| 1463 | ), |
| 1464 | "DFlash2DraftModel": _HfExamplesInfo( |
| 1465 | "Qwen/Qwen3.8-27B", |
| 1466 | speculative_model="z-lab/Qwen3.8-27B-DFlash2", |
| 1467 | is_available_online=False, |
| 1468 | use_original_num_layers=True, |
| 1469 | max_model_len=8192, |
| 1470 | max_num_seqs=32, |
| 1471 | ), |
| 1472 | "DFlashLagunaForCausalLM": _HfExamplesInfo( |
| 1473 | "poolside/Laguna-XS-2.1-NVFP4", |
| 1474 | speculative_model="poolside/Laguna-XS-2.1-DFlash-NVFP4", |
| 1475 | use_original_num_layers=True, |
| 1476 | max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env |
| 1477 | max_num_seqs=32, |
| 1478 | ), |
| 1479 | "DFlashQwen3NextDraftModel": _HfExamplesInfo( |
| 1480 | "Qwen/Qwen3-Coder-Next", |
| 1481 | speculative_model="z-lab/Qwen3-Coder-Next-DFlash", |
| 1482 | use_original_num_layers=True, # DFlash requires all layers |
| 1483 | max_model_len=8192, # Reduce for CI |
| 1484 | max_num_seqs=32, |
| 1485 | ), |
| 1486 | "MuseGlimmerAssistantModel": _HfExamplesInfo( |
| 1487 | "meta-models/Muse-Glimmer-30B", |
| 1488 | speculative_model="meta-models/Muse-Glimmer-30B-assistant", |
| 1489 | max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env |
| 1490 | max_num_seqs=32, |
| 1491 | ), |
| 1492 | "DFlashMuseGlimmerAssistantModel": _HfExamplesInfo( |
| 1493 | "meta-models/Muse-Glimmer-30B", |
| 1494 | speculative_model="meta-models/Muse-Glimmer-30B-assistant", |
| 1495 | max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env |
| 1496 | max_num_seqs=32, |
| 1497 | ), |
| 1498 | # [DSpark] |
| 1499 | "DSparkDraftModel": _HfExamplesInfo( |
| 1500 | "deepseek-ai/DeepSeek-V4-Pro-DSpark", |
| 1501 | speculative_model="deepseek-ai/DeepSeek-V4-Pro-DSpark", # draft in mtp.* |
| 1502 | is_available_online=False, |
| 1503 | use_original_num_layers=True, # DSpark has >1 draft block |
| 1504 | ), |
| 1505 | "DSparkV41DraftModel": _HfExamplesInfo( |
| 1506 | "deepseek-ai/DeepSeek-V4.1-Flash", |
| 1507 | speculative_model="deepseek-ai/DeepSeek-V4.1-Flash", # draft in mtp.* |
| 1508 | is_available_online=False, |
| 1509 | use_original_num_layers=True, # DSpark has >1 draft block |
| 1510 | ), |
| 1511 | "K3DSparkModel": _HfExamplesInfo( |
| 1512 | "moonshotai/Kimi-K3", |
| 1513 | speculative_model="Inferact/Kimi-K3-DSpark", |
| 1514 | use_original_num_layers=True, # DSpark has >1 draft block |
| 1515 | trust_remote_code=True, |
| 1516 | # FIXME: Investigate the NVML failure in CI. |
| 1517 | is_available_online=False, |
| 1518 | ), |
| 1519 | "Qwen3DSparkModel": _HfExamplesInfo( |
| 1520 | "Qwen/Qwen3-8B", |
| 1521 | speculative_model="deepseek-ai/dspark_qwen3_8b_block7", |
| 1522 | is_available_online=False, |
| 1523 | use_original_num_layers=True, # DSpark backbone requires all layers |
| 1524 | ), |
| 1525 | "Qwen3OmniDSparkModel": _HfExamplesInfo( |
| 1526 | "Qwen/Qwen3-Omni-30B-A3B-Instruct", |
| 1527 | speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark", |
| 1528 | is_available_online=False, |
| 1529 | use_original_num_layers=True, |
| 1530 | ), |
| 1531 | "Gemma4DSparkModel": _HfExamplesInfo( |
| 1532 | "google/gemma-4-12B-it", |
| 1533 | speculative_model="deepseek-ai/dspark_gemma4_12b_block7", |
| 1534 | is_available_online=False, |
| 1535 | use_original_num_layers=True, |
| 1536 | ), |
| 1537 | # [Eagle] |
| 1538 | "EagleCohereForCausalLM": _HfExamplesInfo( |
| 1539 | "/host/engines/cohere-moe", |
| 1540 | speculative_model="/host/engines/cohere-moe/eagle", |
| 1541 | tokenizer="/host/engines/cohere-moe", |
| 1542 | trust_remote_code=True, |
| 1543 | is_available_online=False, |
| 1544 | ), |
| 1545 | "EagleDeepSeekMTPModel": _HfExamplesInfo( |
| 1546 | "eagle618/deepseek-v3-random", |
| 1547 | speculative_model="eagle618/eagle-deepseek-v3-random", |
| 1548 | trust_remote_code=True, |
| 1549 | ), |
| 1550 | "EagleLlamaForCausalLM": _HfExamplesInfo( |
| 1551 | "meta-llama/Meta-Llama-3-8B-Instruct", |
| 1552 | trust_remote_code=True, |
| 1553 | speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B", |
| 1554 | tokenizer="meta-llama/Meta-Llama-3-8B-Instruct", |
| 1555 | ), |
| 1556 | "Eagle3DeepseekV2ForCausalLM": _HfExamplesInfo( |
| 1557 | "moonshotai/Kimi-K2.5", |
| 1558 | trust_remote_code=True, |
| 1559 | speculative_model="AQ-MedAI/Kimi-K25-eagle3", |
| 1560 | tokenizer="moonshotai/Kimi-K2.5", |
| 1561 | ), |
| 1562 | "Eagle3DeepseekV3ForCausalLM": _HfExamplesInfo( |
| 1563 | "moonshotai/Kimi-K2.5", |
| 1564 | trust_remote_code=True, |
| 1565 | speculative_model="AQ-MedAI/Kimi-K25-eagle3", |
| 1566 | tokenizer="moonshotai/Kimi-K2.5", |
| 1567 | ), |
| 1568 | "Eagle3LlamaForCausalLM": _HfExamplesInfo( |
| 1569 | "meta-llama/Llama-3.1-8B-Instruct", |
| 1570 | trust_remote_code=True, |
| 1571 | speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B", |
| 1572 | tokenizer="meta-llama/Llama-3.1-8B-Instruct", |
| 1573 | use_original_num_layers=True, |
| 1574 | max_model_len=10240, |
| 1575 | ), |
| 1576 | "Eagle3MiniMaxM2ForCausalLM": _HfExamplesInfo( |
| 1577 | "MiniMaxAI/MiniMax-M2", |
| 1578 | trust_remote_code=True, |
| 1579 | speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B", |
| 1580 | tokenizer="MiniMaxAI/MiniMax-M2", |
| 1581 | ), |
| 1582 | "EagleMistralForCausalLM": _HfExamplesInfo( |
| 1583 | "mistralai/Mistral-Medium-3.5-128B", |
| 1584 | speculative_model="mistralai/Mistral-Medium-3.5-128B-EAGLE", |
| 1585 | is_available_online=False, |
| 1586 | ), |
| 1587 | "EagleMistralLarge3ForCausalLM": _HfExamplesInfo( |
| 1588 | "mistralai/Mistral-Large-3-675B-Instruct-2512", |
| 1589 | speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle", |
| 1590 | ), |
| 1591 | "LlamaForCausalLMEagle3": _HfExamplesInfo( |
| 1592 | "Qwen/Qwen3-8B", |
| 1593 | trust_remote_code=True, |
| 1594 | speculative_model="AngelSlim/Qwen3-8B_eagle3", |
| 1595 | tokenizer="Qwen/Qwen3-8B", |
| 1596 | use_original_num_layers=True, |
| 1597 | ), |
| 1598 | "EagleLlama4ForCausalLM": _HfExamplesInfo( |
| 1599 | "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", |
| 1600 | trust_remote_code=True, |
| 1601 | speculative_model="morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", |
| 1602 | tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 1603 | ), |
| 1604 | "EagleMiniCPMForCausalLM": _HfExamplesInfo( |
| 1605 | "openbmb/MiniCPM-2B-sft-bf16", |
| 1606 | trust_remote_code=True, |
| 1607 | speculative_model="openbmb/MiniCPM-2B-sft-bf16", |
| 1608 | speculative_method="eagle", |
| 1609 | tokenizer="openbmb/MiniCPM-2B-sft-bf16", |
| 1610 | ), |
| 1611 | "Eagle3Qwen2_5vlForCausalLM": _HfExamplesInfo( |
| 1612 | "Qwen/Qwen2.5-VL-7B-Instruct", |
| 1613 | speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl", |
| 1614 | ), |
| 1615 | "Eagle3Qwen3vlForCausalLM": _HfExamplesInfo( |
| 1616 | "Qwen/Qwen3-VL-8B-Instruct", |
| 1617 | speculative_model="taobao-mnn/Qwen3-VL-8B-Instruct-Eagle3", |
| 1618 | ), |
| 1619 | "Eagle3Qwen3ForCausalLM": _HfExamplesInfo( |
| 1620 | "Qwen/Qwen3-8B", |
| 1621 | trust_remote_code=True, |
| 1622 | speculative_model=( |
| 1623 | "inference-optimization/" |
| 1624 | "Qwen3-8B-from-Qwen3-8B_regen-speculators.eagle3-qwen3arch-ckpt1" |
| 1625 | ), |
| 1626 | tokenizer="Qwen/Qwen3-8B", |
| 1627 | use_original_num_layers=True, |
| 1628 | ), |
| 1629 | # [PEagle] |
| 1630 | "PEagleDraftModel": _HfExamplesInfo( |
| 1631 | "Qwen/Qwen3-8B", |
| 1632 | trust_remote_code=True, |
| 1633 | speculative_model="nm-testing/qwen3-8b-peagle-speculators", |
| 1634 | tokenizer="Qwen/Qwen3-8B", |
| 1635 | use_original_num_layers=True, |
| 1636 | ), |
| 1637 | "PeagleLlamaForCausalLM": _HfExamplesInfo( |
| 1638 | "Qwen/Qwen3-8B", |
| 1639 | trust_remote_code=True, |
| 1640 | speculative_model="nm-testing/qwen3-8b-peagle-speculators", |
| 1641 | tokenizer="Qwen/Qwen3-8B", |
| 1642 | use_original_num_layers=True, |
| 1643 | ), |
| 1644 | "PeagleQwen3ForCausalLM": _HfExamplesInfo( |
| 1645 | "Qwen/Qwen3-8B", |
| 1646 | trust_remote_code=True, |
| 1647 | speculative_model=( |
| 1648 | "inference-optimization/Qwen3-8B-speculators.peagle-qwen3arch-ckpt4" |
| 1649 | ), |
| 1650 | tokenizer="Qwen/Qwen3-8B", |
| 1651 | use_original_num_layers=True, |
| 1652 | ), |
| 1653 | # [MTP] |
| 1654 | "BailingMoeV25MTPModel": _HfExamplesInfo( |
| 1655 | "inclusionAI/Ring-2.5-1T", |
| 1656 | speculative_model="inclusionAI/Ring-2.5-1T", |
| 1657 | trust_remote_code=True, |
| 1658 | is_available_online=False, |
| 1659 | ), |
| 1660 | "BailingMoeV3MTPModel": _HfExamplesInfo( |
| 1661 | "inclusionAI/Ling-3.0-flash", |
| 1662 | speculative_model="inclusionAI/Ling-3.0-flash", |
| 1663 | trust_remote_code=True, |
| 1664 | ), |
| 1665 | "DeepSeekMTPModel": _HfExamplesInfo( |
| 1666 | "luccafong/deepseek_mtp_main_random", |
| 1667 | speculative_model="luccafong/deepseek_mtp_draft_random", |
| 1668 | trust_remote_code=True, |
| 1669 | ), |
| 1670 | "DeepseekV32MTPModel": _HfExamplesInfo( |
| 1671 | "nvidia/DeepSeek-V3.2-NVFP4", |
| 1672 | speculative_model="nvidia/DeepSeek-V3.2-NVFP4", |
| 1673 | is_available_online=False, |
| 1674 | ), |
| 1675 | "DeepSeekV4MTPModel": _HfExamplesInfo( |
| 1676 | "deepseek-ai/DeepSeek-V4-Flash", |
| 1677 | speculative_model="deepseek-ai/DeepSeek-V4-Flash", |
| 1678 | trust_remote_code=True, |
| 1679 | is_available_online=False, |
| 1680 | ), |
| 1681 | "Dots3NoteMTPModel": _HfExamplesInfo( |
| 1682 | "dots-studio/dots3-note-prev", |
| 1683 | speculative_model="dots-studio/dots3-note-prev", |
| 1684 | is_available_online=False, |
| 1685 | ), |
| 1686 | "Gemma4MTPModel": _HfExamplesInfo( |
| 1687 | "google/gemma-4-E4B-it", |
| 1688 | speculative_model="google/gemma-4-E4B-it-assistant", |
| 1689 | trust_remote_code=True, |
| 1690 | ), |
| 1691 | "ErnieMTPModel": _HfExamplesInfo( |
| 1692 | "baidu/ERNIE-4.5-21B-A3B-PT", |
| 1693 | trust_remote_code=True, |
| 1694 | speculative_model="baidu/ERNIE-4.5-21B-A3B-PT", |
| 1695 | ), |
| 1696 | "ExaoneMoeMTP": _HfExamplesInfo( |
| 1697 | "LGAI-EXAONE/K-EXAONE-236B-A23B", |
| 1698 | speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B", |
| 1699 | enable_prefix_caching=False, |
| 1700 | ), |
| 1701 | "Exaone4_5_MTP": _HfExamplesInfo( |
| 1702 | "LGAI-EXAONE/EXAONE-4.5-33B", |
| 1703 | speculative_model="LGAI-EXAONE/EXAONE-4.5-33B", |
| 1704 | ), |
| 1705 | "ExtractHiddenStatesModel": _HfExamplesInfo( |
| 1706 | "Qwen/Qwen3-8B", |
| 1707 | speculative_method="extract_hidden_states", |
| 1708 | ), |
| 1709 | "Glm4MoeMTPModel": _HfExamplesInfo( |
| 1710 | "zai-org/GLM-4.5", |
| 1711 | speculative_model="zai-org/GLM-4.5", |
| 1712 | ), |
| 1713 | "Glm4MoeLiteMTPModel": _HfExamplesInfo( |
| 1714 | "zai-org/GLM-4.7-Flash", |
| 1715 | speculative_model="zai-org/GLM-4.7-Flash", |
| 1716 | ), |
| 1717 | "GlmOcrMTPModel": _HfExamplesInfo( |
| 1718 | "zai-org/GLM-OCR", |
| 1719 | speculative_model="zai-org/GLM-OCR", |
| 1720 | ), |
| 1721 | "Glm5NextMTPModel": _HfExamplesInfo( |
| 1722 | "zai-org/GLM-5.3-Flash", |
| 1723 | speculative_model="zai-org/GLM-5.3-Flash", |
| 1724 | ), |
| 1725 | "HYV3MTPModel": _HfExamplesInfo( |
| 1726 | "tencent/Hy3-preview", |
| 1727 | speculative_model="tencent/Hy3-preview", |
| 1728 | ), |
| 1729 | # Internal checkpoint, not publicly available yet. |
| 1730 | "HYV4MTPModel": _HfExamplesInfo( |
| 1731 | "tencent/Hy4-preview", |
| 1732 | speculative_model="tencent/Hy4-preview", |
| 1733 | ), |
| 1734 | "InklingMTPModel": _HfExamplesInfo( |
| 1735 | "thinkingmachines/Inkling-NVFP4", |
| 1736 | speculative_model="thinkingmachines/Inkling-NVFP4", |
| 1737 | tokenizer_mode="inkling", |
| 1738 | trust_remote_code=True, |
| 1739 | max_model_len=4096, |
| 1740 | ), |
| 1741 | "InternS2MobiusMTP": _HfExamplesInfo( |
| 1742 | "internlm/Intern-S2-Mobius", |
| 1743 | trust_remote_code=True, |
| 1744 | speculative_model="internlm/Intern-S2-Mobius", |
| 1745 | is_available_online=False, |
| 1746 | ), |
| 1747 | "KimiK3MTPModel": _HfExamplesInfo( |
| 1748 | "moonshotai/Kimi-K3", |
| 1749 | speculative_model="moonshotai/Kimi-K3", |
| 1750 | trust_remote_code=True, |
| 1751 | is_available_online=False, |
| 1752 | ), |
| 1753 | "LongCatFlashMTPModel": _HfExamplesInfo( |
| 1754 | "meituan-longcat/LongCat-Flash-Chat", |
| 1755 | trust_remote_code=True, |
| 1756 | speculative_model="meituan-longcat/LongCat-Flash-Chat", |
| 1757 | ), |
| 1758 | "MiMoMTPModel": _HfExamplesInfo( |
| 1759 | "XiaomiMiMo/MiMo-7B-RL", |
| 1760 | trust_remote_code=True, |
| 1761 | speculative_model="XiaomiMiMo/MiMo-7B-RL", |
| 1762 | ), |
| 1763 | "MiMoV2MTPModel": _HfExamplesInfo( |
| 1764 | "XiaomiMiMo/MiMo-V2.5-Pro", |
| 1765 | trust_remote_code=True, |
| 1766 | speculative_model="XiaomiMiMo/MiMo-V2.5-Pro", |
| 1767 | is_available_online=False, |
| 1768 | ), |
| 1769 | "MiMoV2OmniMTPModel": _HfExamplesInfo( |
| 1770 | "XiaomiMiMo/MiMo-V2.5-Omni", |
| 1771 | trust_remote_code=True, |
| 1772 | speculative_model="XiaomiMiMo/MiMo-V2.5-Omni", |
| 1773 | is_available_online=False, |
| 1774 | ), |
| 1775 | "MiniMaxM3MTP": _HfExamplesInfo( |
| 1776 | "MiniMaxAI/MiniMax-M3", |
| 1777 | trust_remote_code=True, |
| 1778 | is_available_online=False, |
| 1779 | ), |
| 1780 | "NemotronHMTPModel": _HfExamplesInfo( |
| 1781 | "nvidia/Nemotron-Super-Placeholder", |
| 1782 | speculative_model="nvidia/Nemotron-Super-Placeholder", |
| 1783 | is_available_online=False, |
| 1784 | ), |
| 1785 | "OpenPanguMTPModel": _HfExamplesInfo( |
| 1786 | "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1", |
| 1787 | trust_remote_code=True, |
| 1788 | is_available_online=False, |
| 1789 | ), |
| 1790 | "Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"), |
| 1791 | "Qwen3_5MTP": _HfExamplesInfo( |
| 1792 | "Qwen/Qwen3.5-0.8B", |
| 1793 | speculative_model="Qwen/Qwen3.5-0.8B", |
| 1794 | ), |
| 1795 | "Qwen3_5MoeMTP": _HfExamplesInfo( |
| 1796 | "Qwen/Qwen3.5-35B-A3B", |
| 1797 | speculative_model="Qwen/Qwen3.5-35B-A3B", |
| 1798 | ), |
| 1799 | "Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False), |
| 1800 | "Step3p5MTP": _HfExamplesInfo( |
| 1801 | "stepfun-ai/Step-3.5-Flash", |
| 1802 | speculative_model="stepfun-ai/Step-3.5-Flash", |
| 1803 | use_original_num_layers=True, |
| 1804 | # Initialize at least one MoE layer |
| 1805 | hf_overrides={"num_hidden_layers": 4}, |
| 1806 | is_available_online=False, |
| 1807 | ), |
| 1808 | } |
| 1809 | |
| 1810 | _TRANSFORMERS_BACKEND_MODELS = { |
| 1811 | "TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"), |
| 1812 | "TransformersForSequenceClassification": _HfExamplesInfo( |
| 1813 | "papluca/xlm-roberta-base-language-detection", |
| 1814 | ), |
| 1815 | "TransformersForCausalLM": _HfExamplesInfo( |
| 1816 | "hmellor/Ilama-3.2-1B", trust_remote_code=True |
| 1817 | ), |
| 1818 | "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"), |
| 1819 | "TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"), |
| 1820 | "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo( |
| 1821 | "Qwen/Qwen3-VL-30B-A3B-Instruct" |
| 1822 | ), |
| 1823 | "TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"), |
| 1824 | "TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"), |
| 1825 | "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"), |
| 1826 | "TransformersMultiModalForSequenceClassification": _HfExamplesInfo( |
| 1827 | "google/gemma-3-4b-it" |
| 1828 | ), |
| 1829 | } |
| 1830 | |
| 1831 | _EXAMPLE_MODELS = { |
| 1832 | **_TEXT_GENERATION_EXAMPLE_MODELS, |
| 1833 | **_EMBEDDING_EXAMPLE_MODELS, |
| 1834 | **_LATE_INTERACTION_EXAMPLE_MODELS, |
| 1835 | **_REWARD_EXAMPLE_MODELS, |
| 1836 | **_TOKEN_CLASSIFICATION_EXAMPLE_MODELS, |
| 1837 | **_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS, |
| 1838 | **_MULTIMODAL_EXAMPLE_MODELS, |
| 1839 | **_SPECULATIVE_DECODING_EXAMPLE_MODELS, |
| 1840 | **_TRANSFORMERS_BACKEND_MODELS, |
| 1841 | } |
| 1842 | |
| 1843 | |
| 1844 | class HfExampleModels: |
| 1845 | def __init__(self, hf_models: Mapping[str, _HfExamplesInfo]) -> None: |
| 1846 | super().__init__() |
| 1847 | |
| 1848 | self.hf_models = hf_models |
| 1849 | |
| 1850 | def get_supported_archs(self) -> Set[str]: |
| 1851 | return self.hf_models.keys() |
| 1852 | |
| 1853 | def get_hf_info(self, model_arch: str) -> _HfExamplesInfo: |
| 1854 | try: |
| 1855 | return self.hf_models[model_arch] |
| 1856 | except KeyError: |
| 1857 | raise ValueError( |
| 1858 | f"No example model defined for {model_arch}; please update this file." |
| 1859 | ) from None |
| 1860 | |
| 1861 | def find_hf_info(self, model_id: str) -> _HfExamplesInfo: |
| 1862 | for info in self.hf_models.values(): |
| 1863 | if info.default == model_id: |
| 1864 | return info |
| 1865 | |
| 1866 | # Fallback to extras |
| 1867 | for info in self.hf_models.values(): |
| 1868 | if any(extra == model_id for extra in info.extras.values()): |
| 1869 | return info |
| 1870 | |
| 1871 | raise ValueError( |
| 1872 | f"No example model defined for {model_id}; please update this file." |
| 1873 | ) |
| 1874 | |
| 1875 | |
| 1876 | HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS) |
| 1877 | AUTO_EXAMPLE_MODELS = HfExampleModels(_AUTOMATIC_CONVERTED_MODELS) |