| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | import logging |
| 6 | import os |
| 7 | from dataclasses import MISSING, Field, asdict, dataclass, field |
| 8 | from datetime import timedelta |
| 9 | from pathlib import Path |
| 10 | from types import SimpleNamespace |
| 11 | from typing import cast |
| 12 | from unittest.mock import patch |
| 13 | |
| 14 | import pydantic |
| 15 | import pytest |
| 16 | import torch |
| 17 | from huggingface_hub import ResolvedRevision |
| 18 | from pydantic import ValidationError |
| 19 | |
| 20 | import vllm.config.vllm as vllm_config_module |
| 21 | import vllm.envs as envs |
| 22 | from vllm.compilation.backends import VllmBackend |
| 23 | from vllm.config import ( |
| 24 | AttentionConfig, |
| 25 | CacheConfig, |
| 26 | CompilationConfig, |
| 27 | DeviceConfig, |
| 28 | EngramConfig, |
| 29 | HiSparseConfig, |
| 30 | KernelConfig, |
| 31 | KVTransferConfig, |
| 32 | ModelConfig, |
| 33 | ObservabilityConfig, |
| 34 | ParallelConfig, |
| 35 | PoolerConfig, |
| 36 | SchedulerConfig, |
| 37 | SpeculativeConfig, |
| 38 | VllmConfig, |
| 39 | WatermarkConfig, |
| 40 | update_config, |
| 41 | ) |
| 42 | from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig |
| 43 | from vllm.config.kernel import IrOpPriorityConfig |
| 44 | from vllm.config.load import LoadConfig |
| 45 | from vllm.config.mamba import MambaBackendEnum |
| 46 | from vllm.config.speculative import _validate_qwen3_omni_dspark |
| 47 | from vllm.config.utils import get_field |
| 48 | from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel |
| 49 | from vllm.platforms import current_platform |
| 50 | from vllm.transformers_utils.config import ( |
| 51 | _patch_hf_transformers_nested_rope_validation, |
| 52 | get_pooling_config, |
| 53 | try_get_dense_modules, |
| 54 | ) |
| 55 | from vllm.v1.attention.backend import AttentionCGSupport |
| 56 | |
| 57 | DEVICE_TYPE = current_platform.device_type |
| 58 | |
| 59 | |
| 60 | def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch): |
| 61 | calls = [] |
| 62 | |
| 63 | def original_validate_rope(config, *args, **kwargs): |
| 64 | calls.append(config) |
| 65 | |
| 66 | from transformers import PreTrainedConfig |
| 67 | |
| 68 | monkeypatch.setattr(PreTrainedConfig, "validate_rope", original_validate_rope) |
| 69 | _patch_hf_transformers_nested_rope_validation() |
| 70 | |
| 71 | nested_rope_parameters = { |
| 72 | "full_attention": {"rope_type": "default"}, |
| 73 | "original_max_position_embeddings": 32768, |
| 74 | } |
| 75 | PreTrainedConfig.validate_rope( |
| 76 | SimpleNamespace(rope_parameters=nested_rope_parameters) |
| 77 | ) |
| 78 | assert nested_rope_parameters == {"full_attention": {"rope_type": "default"}} |
| 79 | |
| 80 | flat_rope_parameters = { |
| 81 | "rope_type": "linear", |
| 82 | "factor": 8.0, |
| 83 | "rope_theta": 500000.0, |
| 84 | } |
| 85 | PreTrainedConfig.validate_rope( |
| 86 | SimpleNamespace(rope_parameters=flat_rope_parameters) |
| 87 | ) |
| 88 | assert flat_rope_parameters == { |
| 89 | "rope_type": "linear", |
| 90 | "factor": 8.0, |
| 91 | "rope_theta": 500000.0, |
| 92 | } |
| 93 | assert len(calls) == 2 |
| 94 | |
| 95 | |
| 96 | def test_dspark_adaptive_verification_separates_graph_cache(): |
| 97 | config = object.__new__(SpeculativeConfig) |
| 98 | config.method = "dspark" |
| 99 | config.draft_model_config = None |
| 100 | config.enable_adaptive_verification = False |
| 101 | fixed_hash = config.compute_hash() |
| 102 | config.enable_adaptive_verification = True |
| 103 | assert config.compute_hash() != fixed_hash |
| 104 | |
| 105 | |
| 106 | def _write_json(path: Path, value: object) -> None: |
| 107 | path.parent.mkdir(parents=True, exist_ok=True) |
| 108 | path.write_text(json.dumps(value), encoding="utf-8") |
| 109 | |
| 110 | |
| 111 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test") |
| 112 | @pytest.mark.parametrize( |
| 113 | ("is_mm_prefix_lm", "is_multimodal_model", "expected"), |
| 114 | [ |
| 115 | pytest.param(True, True, True, id="multimodal-prefix-lm"), |
| 116 | pytest.param(False, True, False, id="multimodal-causal"), |
| 117 | pytest.param(True, False, False, id="text-prefix-lm"), |
| 118 | pytest.param(None, True, False, id="missing-model-config"), |
| 119 | ], |
| 120 | ) |
| 121 | def test_rocm_mm_prefix_lm_disables_chunked_mm_input( |
| 122 | is_mm_prefix_lm: bool | None, |
| 123 | is_multimodal_model: bool, |
| 124 | expected: bool, |
| 125 | ) -> None: |
| 126 | from vllm.platforms.rocm import RocmPlatform |
| 127 | |
| 128 | config = SimpleNamespace( |
| 129 | compilation_config=SimpleNamespace(cudagraph_mode=CUDAGraphMode.NONE), |
| 130 | parallel_config=SimpleNamespace( |
| 131 | prefill_context_parallel_size=1, |
| 132 | worker_cls="test-worker", |
| 133 | ), |
| 134 | model_config=( |
| 135 | None |
| 136 | if is_mm_prefix_lm is None |
| 137 | else SimpleNamespace(is_mm_prefix_lm=is_mm_prefix_lm) |
| 138 | ), |
| 139 | scheduler_config=SimpleNamespace( |
| 140 | is_multimodal_model=is_multimodal_model, |
| 141 | disable_chunked_mm_input=False, |
| 142 | ), |
| 143 | ) |
| 144 | |
| 145 | RocmPlatform.check_and_update_config(config) |
| 146 | |
| 147 | assert config.scheduler_config.disable_chunked_mm_input is expected |
| 148 | |
| 149 | |
| 150 | def _sampling_replay_config( |
| 151 | *, |
| 152 | return_sampling_mask: bool = True, |
| 153 | use_v2_model_runner: bool = True, |
| 154 | speculative_method: str | None = None, |
| 155 | rejection_sample_method: str = "standard", |
| 156 | adaptive: bool = False, |
| 157 | is_diffusion: bool = False, |
| 158 | logits_processors: list[str] | None = None, |
| 159 | logprobs_mode: str = "processed_logprobs", |
| 160 | ): |
| 161 | speculative_config = None |
| 162 | if speculative_method is not None: |
| 163 | speculative_config = SimpleNamespace( |
| 164 | method=speculative_method, |
| 165 | enable_adaptive_verification=adaptive, |
| 166 | rejection_sample_method=rejection_sample_method, |
| 167 | ) |
| 168 | return SimpleNamespace( |
| 169 | model_config=SimpleNamespace( |
| 170 | return_sampling_mask=return_sampling_mask, |
| 171 | is_diffusion=is_diffusion, |
| 172 | logits_processors=logits_processors or [], |
| 173 | logprobs_mode=logprobs_mode, |
| 174 | ), |
| 175 | use_v2_model_runner=use_v2_model_runner, |
| 176 | speculative_config=speculative_config, |
| 177 | ) |
| 178 | |
| 179 | |
| 180 | @pytest.mark.parametrize( |
| 181 | ("config", "message"), |
| 182 | [ |
| 183 | (_sampling_replay_config(), None), |
| 184 | *( |
| 185 | (_sampling_replay_config(speculative_method=method), None) |
| 186 | for method in ( |
| 187 | "mtp", |
| 188 | "eagle", |
| 189 | "eagle3", |
| 190 | "dflash", |
| 191 | "dspark", |
| 192 | "draft_model", |
| 193 | ) |
| 194 | ), |
| 195 | *( |
| 196 | ( |
| 197 | _sampling_replay_config( |
| 198 | speculative_method="mtp", |
| 199 | rejection_sample_method=rejection_sample_method, |
| 200 | ), |
| 201 | None, |
| 202 | ) |
| 203 | for rejection_sample_method in ("standard", "block", "synthetic") |
| 204 | ), |
| 205 | ( |
| 206 | _sampling_replay_config( |
| 207 | return_sampling_mask=False, speculative_method="dspark", adaptive=True |
| 208 | ), |
| 209 | None, |
| 210 | ), |
| 211 | ( |
| 212 | _sampling_replay_config(speculative_method="dspark", adaptive=True), |
| 213 | "requires fixed verification boundaries", |
| 214 | ), |
| 215 | ( |
| 216 | _sampling_replay_config(is_diffusion=True), |
| 217 | "does not support diffusion models", |
| 218 | ), |
| 219 | ( |
| 220 | _sampling_replay_config(logits_processors=["custom"]), |
| 221 | "does not support custom logits processors", |
| 222 | ), |
| 223 | ( |
| 224 | _sampling_replay_config(logprobs_mode="raw_logprobs"), |
| 225 | "requires logprobs_mode='processed_logprobs'", |
| 226 | ), |
| 227 | ( |
| 228 | _sampling_replay_config(use_v2_model_runner=False), |
| 229 | "requires Model Runner V2", |
| 230 | ), |
| 231 | ], |
| 232 | ) |
| 233 | def test_sampling_replay_config(config, message): |
| 234 | if message is None: |
| 235 | VllmConfig._verify_sampling_replay_config(config) |
| 236 | else: |
| 237 | with pytest.raises(ValueError, match=message): |
| 238 | VllmConfig._verify_sampling_replay_config(config) |
| 239 | |
| 240 | |
| 241 | def test_kda_recoverssm_derivation_is_revalidated(): |
| 242 | config = SimpleNamespace( |
| 243 | cache_config=SimpleNamespace( |
| 244 | use_replayssm=True, |
| 245 | use_kda_recoverssm=False, |
| 246 | mamba_cache_mode="none", |
| 247 | replayssm_buffer_len=16, |
| 248 | ), |
| 249 | num_speculative_tokens=3, |
| 250 | model_config=SimpleNamespace( |
| 251 | supports_replayssm=True, |
| 252 | architecture="KimiLinearForCausalLM", |
| 253 | ), |
| 254 | mamba_config=SimpleNamespace( |
| 255 | backend=MambaBackendEnum.TRITON, |
| 256 | enable_stochastic_rounding=False, |
| 257 | ), |
| 258 | parallel_config=SimpleNamespace(pipeline_parallel_size=1), |
| 259 | kv_transfer_config=None, |
| 260 | use_v2_model_runner=True, |
| 261 | ) |
| 262 | |
| 263 | VllmConfig.validate_mamba_cached_kernel(config) |
| 264 | assert config.cache_config.use_replayssm |
| 265 | assert config.cache_config.use_kda_recoverssm |
| 266 | |
| 267 | config.cache_config.mamba_cache_mode = "align" |
| 268 | VllmConfig.validate_mamba_cached_kernel(config) |
| 269 | config.use_v2_model_runner = False |
| 270 | with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"): |
| 271 | VllmConfig.validate_mamba_cached_kernel(config) |
| 272 | config.use_v2_model_runner = True |
| 273 | config.cache_config.mamba_cache_mode = "none" |
| 274 | |
| 275 | config.model_config.architecture = "NemotronHForCausalLM" |
| 276 | config.mamba_config.backend = MambaBackendEnum.FLASHINFER |
| 277 | VllmConfig.validate_mamba_cached_kernel(config) |
| 278 | assert not config.cache_config.use_kda_recoverssm |
| 279 | |
| 280 | config.model_config.architecture = "KimiLinearForCausalLM" |
| 281 | config.parallel_config.pipeline_parallel_size = 2 |
| 282 | with pytest.raises(ValueError, match="pipeline_parallel_size=1"): |
| 283 | VllmConfig.validate_mamba_cached_kernel(config) |
| 284 | |
| 285 | |
| 286 | def test_mamba_cache_mode_all_is_rejected(): |
| 287 | """The removed 'all' mode must fail validation instead of being ignored.""" |
| 288 | with pytest.raises(ValidationError, match="mamba_cache_mode"): |
| 289 | CacheConfig(mamba_cache_mode="all") |
| 290 | |
| 291 | |
| 292 | def test_per_request_spec_decode_metrics_requires_spec_decode(): |
| 293 | # The flag only makes sense with speculative decoding configured; enabling |
| 294 | # it without --speculative-config should fail fast rather than silently |
| 295 | # produce no metrics. |
| 296 | for level in ("summary", "detailed"): |
| 297 | with pytest.raises(ValueError, match="speculative"): |
| 298 | VllmConfig( |
| 299 | observability_config=ObservabilityConfig( |
| 300 | per_request_spec_decode_metrics=level |
| 301 | ) |
| 302 | ) |
| 303 | |
| 304 | |
| 305 | @pytest.mark.parametrize( |
| 306 | "kv_transfer_config", |
| 307 | [ |
| 308 | KVTransferConfig( |
| 309 | kv_connector="NixlConnector", |
| 310 | kv_role="kv_both", |
| 311 | ), |
| 312 | KVTransferConfig( |
| 313 | kv_connector="MultiConnector", |
| 314 | kv_role="kv_both", |
| 315 | kv_connector_extra_config={ |
| 316 | "connectors": [ |
| 317 | { |
| 318 | "kv_connector": "NixlConnector", |
| 319 | "kv_role": "kv_both", |
| 320 | }, |
| 321 | { |
| 322 | "kv_connector": "OffloadingConnector", |
| 323 | "kv_role": "kv_both", |
| 324 | }, |
| 325 | ] |
| 326 | }, |
| 327 | ), |
| 328 | ], |
| 329 | ) |
| 330 | def test_pd_dcp_interleave_size_is_adjusted_to_block_size( |
| 331 | caplog, disable_log_dedup, kv_transfer_config |
| 332 | ): |
| 333 | config = VllmConfig( |
| 334 | cache_config=CacheConfig(block_size=16), |
| 335 | device_config=DeviceConfig(device="cpu"), |
| 336 | parallel_config=ParallelConfig( |
| 337 | tensor_parallel_size=2, |
| 338 | decode_context_parallel_size=2, |
| 339 | cp_kv_cache_interleave_size=3, |
| 340 | distributed_executor_backend="mp", |
| 341 | ), |
| 342 | kv_transfer_config=kv_transfer_config, |
| 343 | ) |
| 344 | |
| 345 | kv_cache_config = SimpleNamespace( |
| 346 | kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))] |
| 347 | ) |
| 348 | with caplog.at_level(logging.INFO): |
| 349 | config.adjust_dcp_kv_cache_interleave_size(kv_cache_config) |
| 350 | |
| 351 | assert config.parallel_config.cp_kv_cache_interleave_size == 16 |
| 352 | assert "automatically adjusted from 3 to block_size 16" in caplog.text |
| 353 | |
| 354 | |
| 355 | def test_kv_offloading_does_not_adjust_dcp_interleave_size(): |
| 356 | config = VllmConfig( |
| 357 | cache_config=CacheConfig(block_size=16), |
| 358 | device_config=DeviceConfig(device="cpu"), |
| 359 | parallel_config=ParallelConfig( |
| 360 | tensor_parallel_size=2, |
| 361 | decode_context_parallel_size=2, |
| 362 | cp_kv_cache_interleave_size=1, |
| 363 | distributed_executor_backend="mp", |
| 364 | ), |
| 365 | kv_transfer_config=KVTransferConfig( |
| 366 | kv_connector="OffloadingConnector", |
| 367 | kv_role="kv_both", |
| 368 | ), |
| 369 | ) |
| 370 | |
| 371 | kv_cache_config = SimpleNamespace( |
| 372 | kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))] |
| 373 | ) |
| 374 | config.adjust_dcp_kv_cache_interleave_size(kv_cache_config) |
| 375 | |
| 376 | assert config.parallel_config.cp_kv_cache_interleave_size == 1 |
| 377 | |
| 378 | |
| 379 | def test_kv_offloading_does_not_skip_dcp_interleave_validation(): |
| 380 | config = SimpleNamespace( |
| 381 | cache_config=SimpleNamespace( |
| 382 | block_size=16, |
| 383 | mamba_cache_mode="none", |
| 384 | ), |
| 385 | parallel_config=SimpleNamespace( |
| 386 | decode_context_parallel_size=2, |
| 387 | cp_kv_cache_interleave_size=3, |
| 388 | ), |
| 389 | scheduler_config=SimpleNamespace(disable_chunked_mm_input=False), |
| 390 | kv_transfer_config=KVTransferConfig( |
| 391 | kv_connector="OffloadingConnector", |
| 392 | kv_role="kv_both", |
| 393 | ), |
| 394 | ) |
| 395 | |
| 396 | with pytest.raises(AssertionError, match="divisible by"): |
| 397 | VllmConfig.validate_block_size(config) |
| 398 | |
| 399 | |
| 400 | def test_nixl_dcp_check_skipped_for_submodel_config(): |
| 401 | # with_hf_config() builds a submodule view of the config (e.g. a |
| 402 | # multimodal model's text stack) whose architecture list is empty. |
| 403 | # Re-running VllmConfig validation on it is unsafe: the NIXL DCP check |
| 404 | # reads use_mla, which resolves the architecture registry. |
| 405 | model_config = ModelConfig("Qwen/Qwen2-VL-2B-Instruct", max_model_len=2048) |
| 406 | vllm_config = VllmConfig( |
| 407 | model_config=model_config, |
| 408 | device_config=DeviceConfig(device="cpu"), |
| 409 | kv_transfer_config=KVTransferConfig( |
| 410 | kv_connector="NixlConnector", |
| 411 | kv_role="kv_both", |
| 412 | ), |
| 413 | ) |
| 414 | submodel_config = vllm_config.with_hf_config(model_config.hf_text_config) |
| 415 | assert submodel_config.model_config.is_submodel_config |
| 416 | assert submodel_config.model_config.architectures == [] |
| 417 | assert submodel_config.model_config.use_mla is False |
| 418 | |
| 419 | |
| 420 | def test_nixl_dcp_check_rejects_non_mla_model_with_dcp(monkeypatch): |
| 421 | # Pretend the model has a single KV head so the DCP feasibility checks |
| 422 | # pass and the MLA-only assert is what actually fires. |
| 423 | monkeypatch.setattr(ModelConfig, "get_total_num_kv_heads", lambda self: 1) |
| 424 | with pytest.raises(ValidationError, match="only supported for MLA models"): |
| 425 | VllmConfig( |
| 426 | model_config=ModelConfig("Qwen/Qwen3-0.6B", max_model_len=2048), |
| 427 | device_config=DeviceConfig(device="cpu"), |
| 428 | parallel_config=ParallelConfig( |
| 429 | tensor_parallel_size=2, |
| 430 | decode_context_parallel_size=2, |
| 431 | distributed_executor_backend="mp", |
| 432 | ), |
| 433 | kv_transfer_config=KVTransferConfig( |
| 434 | kv_connector="NixlConnector", |
| 435 | kv_role="kv_both", |
| 436 | ), |
| 437 | ) |
| 438 | |
| 439 | |
| 440 | def test_compile_config_repr_succeeds(): |
| 441 | # setup: VllmBackend mutates the config object |
| 442 | config = VllmConfig() |
| 443 | backend = VllmBackend(config) |
| 444 | backend.configure_post_pass() |
| 445 | |
| 446 | # test that repr(config) succeeds |
| 447 | val = repr(config) |
| 448 | assert "VllmConfig" in val |
| 449 | assert "inductor_passes" in val |
| 450 | |
| 451 | |
| 452 | @pytest.mark.parametrize( |
| 453 | ("env_value", "expected"), |
| 454 | [ |
| 455 | (None, None), |
| 456 | ("0", False), |
| 457 | ("1", True), |
| 458 | ], |
| 459 | ) |
| 460 | def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected): |
| 461 | if env_value is None: |
| 462 | monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) |
| 463 | else: |
| 464 | monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", env_value) |
| 465 | |
| 466 | assert envs.VLLM_USE_V2_MODEL_RUNNER is expected |
| 467 | |
| 468 | |
| 469 | def test_hisparse_requires_v2_model_runner(): |
| 470 | config = object.__new__(VllmConfig) |
| 471 | config.attention_config = AttentionConfig(hisparse_config=HiSparseConfig()) |
| 472 | |
| 473 | with patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", None): |
| 474 | assert config.use_v2_model_runner |
| 475 | with ( |
| 476 | patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", False), |
| 477 | pytest.raises(ValueError, match="requires Model Runner V2"), |
| 478 | ): |
| 479 | _ = config.use_v2_model_runner |
| 480 | |
| 481 | |
| 482 | def test_hisparse_rejects_decode_context_parallelism(monkeypatch): |
| 483 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 484 | monkeypatch.setattr(current_platform, "device_count", lambda: 2) |
| 485 | with pytest.raises(ValueError, match="decode context parallelism"): |
| 486 | VllmConfig( |
| 487 | attention_config=AttentionConfig(hisparse_config=HiSparseConfig()), |
| 488 | parallel_config=ParallelConfig( |
| 489 | tensor_parallel_size=2, |
| 490 | decode_context_parallel_size=2, |
| 491 | ), |
| 492 | ) |
| 493 | |
| 494 | |
| 495 | def test_hisparse_rejects_pipeline_parallelism(monkeypatch): |
| 496 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 497 | monkeypatch.setattr(current_platform, "device_count", lambda: 2) |
| 498 | with pytest.raises(ValueError, match="pipeline parallelism"): |
| 499 | VllmConfig( |
| 500 | attention_config=AttentionConfig(hisparse_config=HiSparseConfig()), |
| 501 | parallel_config=ParallelConfig(pipeline_parallel_size=2), |
| 502 | ) |
| 503 | |
| 504 | |
| 505 | def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch): |
| 506 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 507 | monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True) |
| 508 | with pytest.raises(ValueError, match="requires the hybrid KV cache manager"): |
| 509 | VllmConfig( |
| 510 | attention_config=AttentionConfig(hisparse_config=HiSparseConfig()), |
| 511 | scheduler_config=SchedulerConfig( |
| 512 | max_model_len=2048, |
| 513 | is_encoder_decoder=False, |
| 514 | disable_hybrid_kv_cache_manager=True, |
| 515 | ), |
| 516 | ) |
| 517 | |
| 518 | |
| 519 | def test_hisparse_rejects_disabled_full_isl_reservation(monkeypatch): |
| 520 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 521 | with pytest.raises(ValueError, match="requires --scheduler-reserve-full-isl"): |
| 522 | VllmConfig( |
| 523 | attention_config=AttentionConfig(hisparse_config=HiSparseConfig()), |
| 524 | scheduler_config=SchedulerConfig( |
| 525 | max_model_len=2048, |
| 526 | is_encoder_decoder=False, |
| 527 | scheduler_reserve_full_isl=False, |
| 528 | ), |
| 529 | ) |
| 530 | |
| 531 | |
| 532 | def test_hisparse_rejects_non_cuda(monkeypatch): |
| 533 | monkeypatch.setattr(current_platform, "is_cuda", lambda: False) |
| 534 | with pytest.raises(ValueError, match="requires NVIDIA CUDA"): |
| 535 | VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig())) |
| 536 | |
| 537 | |
| 538 | @pytest.mark.parametrize( |
| 539 | "kv_transfer_config", |
| 540 | [ |
| 541 | KVTransferConfig( |
| 542 | kv_connector="HiSparseConnector", |
| 543 | kv_role="kv_both", |
| 544 | kv_connector_extra_config={"host_pool_gib": 128}, |
| 545 | ), |
| 546 | KVTransferConfig( |
| 547 | kv_connector="MultiConnector", |
| 548 | kv_role="kv_both", |
| 549 | kv_connector_extra_config={ |
| 550 | "connectors": [ |
| 551 | { |
| 552 | "kv_connector": "OffloadingConnector", |
| 553 | "kv_role": "kv_both", |
| 554 | "kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30}, |
| 555 | }, |
| 556 | { |
| 557 | "kv_connector": "HiSparseConnector", |
| 558 | "kv_role": "kv_both", |
| 559 | "kv_connector_extra_config": {"host_pool_gib": 128}, |
| 560 | }, |
| 561 | ] |
| 562 | }, |
| 563 | ), |
| 564 | KVTransferConfig( |
| 565 | kv_connector="MultiConnector", |
| 566 | kv_role="kv_consumer", |
| 567 | kv_connector_extra_config={ |
| 568 | "connectors": [ |
| 569 | {"kv_connector": "NixlConnector", "kv_role": "kv_consumer"}, |
| 570 | { |
| 571 | "kv_connector": "HiSparseConnector", |
| 572 | "kv_role": "kv_both", |
| 573 | "kv_connector_extra_config": {"host_pool_gib": 128}, |
| 574 | }, |
| 575 | ] |
| 576 | }, |
| 577 | ), |
| 578 | ], |
| 579 | ids=["standalone", "multi-connector", "pd-decode"], |
| 580 | ) |
| 581 | def test_hisparse_connector_implies_attention_config(monkeypatch, kv_transfer_config): |
| 582 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 583 | monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True) |
| 584 | config = VllmConfig( |
| 585 | kv_transfer_config=kv_transfer_config, |
| 586 | # Skip the HMA auto-detect block: it imports the connector class, |
| 587 | # which pulls the platform's compiled attention extensions. |
| 588 | scheduler_config=SchedulerConfig( |
| 589 | max_model_len=2048, |
| 590 | is_encoder_decoder=False, |
| 591 | disable_hybrid_kv_cache_manager=False, |
| 592 | ), |
| 593 | ) |
| 594 | assert isinstance(config.attention_config.hisparse_config, HiSparseConfig) |
| 595 | |
| 596 | |
| 597 | def test_hisparse_connector_preserves_explicit_attention_config(monkeypatch): |
| 598 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 599 | monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True) |
| 600 | config = VllmConfig( |
| 601 | attention_config=AttentionConfig( |
| 602 | hisparse_config=HiSparseConfig(device_buffer_size=512) |
| 603 | ), |
| 604 | kv_transfer_config=KVTransferConfig( |
| 605 | kv_connector="HiSparseConnector", |
| 606 | kv_role="kv_both", |
| 607 | kv_connector_extra_config={"host_pool_gib": 128}, |
| 608 | ), |
| 609 | scheduler_config=SchedulerConfig( |
| 610 | max_model_len=2048, |
| 611 | is_encoder_decoder=False, |
| 612 | disable_hybrid_kv_cache_manager=False, |
| 613 | ), |
| 614 | ) |
| 615 | assert config.attention_config.hisparse_config.device_buffer_size == 512 |
| 616 | |
| 617 | |
| 618 | def test_hisparse_connector_without_cuda_still_rejected(monkeypatch): |
| 619 | monkeypatch.setattr(current_platform, "is_cuda", lambda: False) |
| 620 | with pytest.raises(ValueError, match="requires NVIDIA CUDA"): |
| 621 | VllmConfig( |
| 622 | kv_transfer_config=KVTransferConfig( |
| 623 | kv_connector="HiSparseConnector", |
| 624 | kv_role="kv_both", |
| 625 | kv_connector_extra_config={"host_pool_gib": 128}, |
| 626 | ) |
| 627 | ) |
| 628 | |
| 629 | |
| 630 | def test_no_hisparse_connector_keeps_attention_config_unset(monkeypatch): |
| 631 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 632 | config = VllmConfig( |
| 633 | kv_transfer_config=KVTransferConfig( |
| 634 | kv_connector="OffloadingConnector", kv_role="kv_both" |
| 635 | ) |
| 636 | ) |
| 637 | assert config.attention_config.hisparse_config is None |
| 638 | |
| 639 | |
| 640 | def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch): |
| 641 | """ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled |
| 642 | MRV1 paths and off breakable cudagraphs by default.""" |
| 643 | from vllm.config.vllm import ( |
| 644 | ROCM_DEFAULT_MRV1_ARCHITECTURES, |
| 645 | default_breakable_cudagraph_architectures, |
| 646 | ) |
| 647 | from vllm.platforms import current_platform |
| 648 | |
| 649 | monkeypatch.setattr(current_platform, "is_rocm", lambda: True) |
| 650 | # The lookup is lru_cached against a fixed platform. |
| 651 | default_breakable_cudagraph_architectures.cache_clear() |
| 652 | try: |
| 653 | assert "DeepseekV32ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES |
| 654 | assert "DeepseekV4ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES |
| 655 | assert "GlmMoeDsaForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES |
| 656 | |
| 657 | breakable_architectures = default_breakable_cudagraph_architectures() |
| 658 | assert "DeepseekV32ForCausalLM" not in breakable_architectures |
| 659 | assert "DeepseekV32MTPModel" not in breakable_architectures |
| 660 | assert "GlmMoeDsaForCausalLM" not in breakable_architectures |
| 661 | # V4.1 cannot torch.compile and the ROCm sparse SWA backend only |
| 662 | # supports uniform-batch CUDA graphs, so it must opt in or the |
| 663 | # default FULL_AND_PIECEWISE serve path cannot start. |
| 664 | assert "DeepseekV41ForCausalLM" in breakable_architectures |
| 665 | |
| 666 | # The carve-out takes effect via the runner-selection property |
| 667 | # (warning_once args must be hashable for its lru_cache). |
| 668 | monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) |
| 669 | config = SimpleNamespace( |
| 670 | model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]), |
| 671 | attention_config=AttentionConfig(), |
| 672 | ) |
| 673 | config._get_v1_model_runner_unsupported_features = lambda: [] |
| 674 | assert VllmConfig.use_v2_model_runner.fget(config) is False |
| 675 | finally: |
| 676 | default_breakable_cudagraph_architectures.cache_clear() |
| 677 | |
| 678 | |
| 679 | def test_rocm_mrv1_default_yields_to_v1_unsupported_config(monkeypatch): |
| 680 | """The ROCm V1 default is a speed preference, not a capability claim. |
| 681 | |
| 682 | DSpark runs only on V2, so pinning DeepSeek V4 to V1 would fail config |
| 683 | validation instead of serving it. With nothing V1 refuses, it still holds. |
| 684 | """ |
| 685 | from vllm.platforms import current_platform |
| 686 | |
| 687 | monkeypatch.setattr(current_platform, "is_rocm", lambda: True) |
| 688 | monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) |
| 689 | monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) |
| 690 | |
| 691 | config = SimpleNamespace( |
| 692 | model_config=SimpleNamespace( |
| 693 | architectures=["DeepseekV4ForCausalLM"], is_diffusion=False |
| 694 | ), |
| 695 | attention_config=AttentionConfig(), |
| 696 | parallel_config=SimpleNamespace( |
| 697 | prefill_context_parallel_size=1, |
| 698 | pipeline_parallel_size=1, |
| 699 | enable_batch_sharded_sampling=False, |
| 700 | ), |
| 701 | scheduler_config=SimpleNamespace(async_scheduling=False), |
| 702 | speculative_config=None, |
| 703 | ) |
| 704 | config._dflash_needs_multi_kv_group = lambda: False |
| 705 | config._is_dflash_candidate_draft = lambda: False |
| 706 | config._get_v2_model_runner_unsupported_features = lambda: [] |
| 707 | # The real predicate, so the test also pins where dspark lands in it. |
| 708 | config._get_v1_model_runner_unsupported_features = lambda: ( |
| 709 | VllmConfig._get_v1_model_runner_unsupported_features(config) |
| 710 | ) |
| 711 | |
| 712 | assert VllmConfig.use_v2_model_runner.fget(config) is False |
| 713 | |
| 714 | config.speculative_config = SimpleNamespace( |
| 715 | method="dspark", enable_adaptive_verification=False |
| 716 | ) |
| 717 | assert VllmConfig.use_v2_model_runner.fget(config) is True |
| 718 | |
| 719 | # Yielding is not the same as selecting V2: the later checks still run, so |
| 720 | # a config neither runner can serve lands on V1 and fails validation there. |
| 721 | config._get_v2_model_runner_unsupported_features = lambda: ["sequence parallelism"] |
| 722 | assert VllmConfig.use_v2_model_runner.fget(config) is False |
| 723 | |
| 724 | |
| 725 | @pytest.mark.parametrize( |
| 726 | ("model", "architecture"), |
| 727 | [ |
| 728 | ("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM"), |
| 729 | ("zai-org/GLM-5.2-FP8", "GlmMoeDsaForCausalLM"), |
| 730 | ("nvidia/DeepSeek-V3.2-NVFP4", "DeepseekV32ForCausalLM"), |
| 731 | ], |
| 732 | ) |
| 733 | @pytest.mark.parametrize("with_mtp", [False, True], ids=["no-mtp", "mtp"]) |
| 734 | def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( |
| 735 | monkeypatch, model, architecture, with_mtp |
| 736 | ): |
| 737 | from vllm.compilation.breakable_cudagraph import ( |
| 738 | is_breakable_cudagraph_enabled, |
| 739 | ) |
| 740 | from vllm.config.vllm import default_breakable_cudagraph_architectures |
| 741 | from vllm.platforms import current_platform |
| 742 | |
| 743 | monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) |
| 744 | monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) |
| 745 | monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) |
| 746 | monkeypatch.setattr(current_platform, "is_rocm", lambda: False) |
| 747 | default_breakable_cudagraph_architectures.cache_clear() |
| 748 | |
| 749 | model_config = SimpleNamespace( |
| 750 | model=model, |
| 751 | architectures=[architecture], |
| 752 | runner_type="generate", |
| 753 | is_moe=True, |
| 754 | is_hybrid=False, |
| 755 | is_attention_free=False, |
| 756 | is_diffusion=False, |
| 757 | ) |
| 758 | config = SimpleNamespace( |
| 759 | model_config=model_config, |
| 760 | attention_config=AttentionConfig(), |
| 761 | speculative_config=SimpleNamespace(method="mtp") if with_mtp else None, |
| 762 | parallel_config=SimpleNamespace(prefill_context_parallel_size=1), |
| 763 | compilation_config=CompilationConfig( |
| 764 | cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE |
| 765 | ), |
| 766 | ) |
| 767 | config._dflash_needs_multi_kv_group = lambda: False |
| 768 | config._get_v2_model_runner_unsupported_features = lambda: [] |
| 769 | config._uses_breakable_cudagraph_by_default = lambda: ( |
| 770 | VllmConfig._uses_breakable_cudagraph_by_default(config) |
| 771 | ) |
| 772 | |
| 773 | try: |
| 774 | assert VllmConfig.use_v2_model_runner.fget(config) |
| 775 | assert VllmConfig._maybe_enable_breakable_cudagraph(config) |
| 776 | assert is_breakable_cudagraph_enabled() |
| 777 | assert config.compilation_config.mode == CompilationMode.NONE |
| 778 | assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs() |
| 779 | finally: |
| 780 | os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) |
| 781 | default_breakable_cudagraph_architectures.cache_clear() |
| 782 | |
| 783 | |
| 784 | @pytest.mark.parametrize( |
| 785 | ("architecture", "is_rocm", "expected"), |
| 786 | [ |
| 787 | ("DeepseekV32ForCausalLM", False, True), |
| 788 | ("DeepseekV32ForCausalLM", True, False), |
| 789 | ("DeepseekV32MTPModel", False, True), |
| 790 | ("DeepseekV32MTPModel", True, False), |
| 791 | ("GlmMoeDsaForCausalLM", False, True), |
| 792 | ("GlmMoeDsaForCausalLM", True, False), |
| 793 | ("Qwen4ExpForCausalLM", False, True), |
| 794 | ("Qwen4ExpForCausalLM", True, False), |
| 795 | ("Qwen4ExpForConditionalGeneration", False, True), |
| 796 | ("Qwen4ExpForConditionalGeneration", True, False), |
| 797 | ("Qwen4ExpMTP", False, True), |
| 798 | ("Qwen4ExpMTP", True, False), |
| 799 | ], |
| 800 | ) |
| 801 | def test_breakable_cudagraph_platform_default( |
| 802 | monkeypatch, architecture, is_rocm, expected |
| 803 | ): |
| 804 | from vllm.config.vllm import default_breakable_cudagraph_architectures |
| 805 | from vllm.platforms import current_platform |
| 806 | |
| 807 | monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) |
| 808 | monkeypatch.setattr(current_platform, "is_rocm", lambda: is_rocm) |
| 809 | default_breakable_cudagraph_architectures.cache_clear() |
| 810 | config = SimpleNamespace( |
| 811 | model_config=SimpleNamespace(architectures=[architecture]), |
| 812 | compilation_config=CompilationConfig(), |
| 813 | ) |
| 814 | config._uses_breakable_cudagraph_by_default = lambda: ( |
| 815 | VllmConfig._uses_breakable_cudagraph_by_default(config) |
| 816 | ) |
| 817 | |
| 818 | try: |
| 819 | assert VllmConfig._maybe_enable_breakable_cudagraph(config) is expected |
| 820 | if expected: |
| 821 | assert config.compilation_config.mode == CompilationMode.NONE |
| 822 | finally: |
| 823 | os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) |
| 824 | default_breakable_cudagraph_architectures.cache_clear() |
| 825 | |
| 826 | |
| 827 | @pytest.mark.parametrize( |
| 828 | "case,hidden,heads,intermediate,tp,expected", |
| 829 | [ |
| 830 | ("bf16", 2048, (16, 8), 6144, 1, True), |
| 831 | ("bi-off", 2048, (16, 8), 6144, 1, False), |
| 832 | ("opt-out", 2048, (16, 8), 6144, 1, False), |
| 833 | ("eager", 2048, (16, 8), 6144, 1, False), |
| 834 | ("sp", 2048, (16, 8), 6144, 1, False), |
| 835 | ("fp16", 2048, (16, 8), 6144, 1, False), |
| 836 | ("quantized", 2048, (16, 8), 6144, 1, False), |
| 837 | ("untuned", 4096, (32, 32), 11008, 1, False), |
| 838 | ("tp4-tuned", 4096, (8, 2), 12288, 4, True), |
| 839 | ("list-intermediate", 2048, (16, 8), [2048, 4096], 1, False), |
| 840 | ("no-table", 2048, None, 6144, 1, False), |
| 841 | ], |
| 842 | ) |
| 843 | def test_batch_invariant_breakable_cudagraph( |
| 844 | monkeypatch, case, hidden, heads, intermediate, tp, expected |
| 845 | ): |
| 846 | from vllm.config.vllm import default_breakable_cudagraph_architectures |
| 847 | from vllm.model_executor.determinism import batch_invariant_configs as bi_configs |
| 848 | |
| 849 | monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0" if case == "bi-off" else "1") |
| 850 | monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) |
| 851 | if case == "opt-out": |
| 852 | monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") |
| 853 | monkeypatch.setattr(current_platform, "is_cuda", lambda: True) |
| 854 | monkeypatch.setattr(current_platform, "is_rocm", lambda: False) |
| 855 | table = None if case == "no-table" else {(12288, 2048): None, (6144, 4096): None} |
| 856 | monkeypatch.setattr(current_platform, "get_device_capability", lambda: None) |
| 857 | monkeypatch.setattr(bi_configs, "_get_tuned_matmul_arch_family", lambda cap: "test") |
| 858 | monkeypatch.setattr( |
| 859 | bi_configs, |
| 860 | "_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS", |
| 861 | {"test": table} if table else {}, |
| 862 | ) |
| 863 | default_breakable_cudagraph_architectures.cache_clear() |
| 864 | config = object.__new__(VllmConfig) |
| 865 | config.model_config = SimpleNamespace( |
| 866 | architectures=["Qwen3ForCausalLM"], |
| 867 | enforce_eager=case == "eager", |
| 868 | dtype=torch.float16 if case == "fp16" else torch.bfloat16, |
| 869 | quantization="fp8" if case == "quantized" else None, |
| 870 | hf_text_config=SimpleNamespace(intermediate_size=intermediate), |
| 871 | get_hidden_size=lambda: hidden, |
| 872 | get_head_size=lambda: 128, |
| 873 | get_num_attention_heads=lambda pc: heads[0], |
| 874 | get_num_kv_heads=lambda pc: heads[1], |
| 875 | ) |
| 876 | config.parallel_config = SimpleNamespace(tensor_parallel_size=tp) |
| 877 | config.compilation_config = ( |
| 878 | CompilationConfig(pass_config=PassConfig(enable_sp=True)) |
| 879 | if case == "sp" |
| 880 | else CompilationConfig() |
| 881 | ) |
| 882 | try: |
| 883 | assert config._maybe_enable_breakable_cudagraph() is expected |
| 884 | if expected: |
| 885 | assert config.compilation_config.mode == CompilationMode.NONE |
| 886 | finally: |
| 887 | os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) |
| 888 | default_breakable_cudagraph_architectures.cache_clear() |
| 889 | |
| 890 | |
| 891 | @pytest.mark.parametrize( |
| 892 | ("model_type", "expected_architecture"), |
| 893 | [ |
| 894 | ("deepseek_v32", "DeepseekV32MTPModel"), |
| 895 | ("glm_moe_dsa", "DeepseekV32MTPModel"), |
| 896 | ("deepseek_v3", "DeepSeekMTPModel"), |
| 897 | ], |
| 898 | ) |
| 899 | def test_dsa_models_select_matching_mtp(model_type, expected_architecture): |
| 900 | from transformers import PreTrainedConfig |
| 901 | |
| 902 | hf_config = PreTrainedConfig( |
| 903 | architectures=["DeepseekV32ForCausalLM"], |
| 904 | num_nextn_predict_layers=1, |
| 905 | ) |
| 906 | hf_config.model_type = model_type |
| 907 | |
| 908 | SpeculativeConfig.hf_config_override(hf_config) |
| 909 | |
| 910 | assert hf_config.architectures == [expected_architecture] |
| 911 | |
| 912 | |
| 913 | def test_v2_model_runner_supports_extract_hidden_states(): |
| 914 | config = VllmConfig() |
| 915 | config.speculative_config = cast( |
| 916 | SpeculativeConfig, |
| 917 | SimpleNamespace( |
| 918 | method="extract_hidden_states", |
| 919 | parallel_drafting=False, |
| 920 | enable_adaptive_verification=False, |
| 921 | ), |
| 922 | ) |
| 923 | |
| 924 | assert config._get_v2_model_runner_unsupported_features() == [] |
| 925 | |
| 926 | |
| 927 | def test_v2_model_runner_supports_custom_logits_processors(): |
| 928 | config = VllmConfig() |
| 929 | config.model_config = cast( |
| 930 | ModelConfig, SimpleNamespace(logits_processors=["a.b:C"]) |
| 931 | ) |
| 932 | |
| 933 | assert config._get_v2_model_runner_unsupported_features() == [] |
| 934 | |
| 935 | |
| 936 | @pytest.mark.parametrize("architecture", ["DFlash2DraftModel", "LiLiCorrDraftModel"]) |
| 937 | def test_dflash_candidate_draft_forces_v2_model_runner(architecture): |
| 938 | """A DFlash2 draft must reach the V2 speculator, the only one that runs its |
| 939 | candidate selector; on V1 it would draft as DFlash1 without raising.""" |
| 940 | |
| 941 | def config(method, architectures): |
| 942 | return SimpleNamespace( |
| 943 | speculative_config=SimpleNamespace( |
| 944 | method=method, |
| 945 | draft_model_config=SimpleNamespace(architectures=architectures), |
| 946 | ) |
| 947 | ) |
| 948 | |
| 949 | assert VllmConfig._is_dflash_candidate_draft(config("dflash", [architecture])) |
| 950 | assert not VllmConfig._is_dflash_candidate_draft( |
| 951 | config("dflash", ["DFlashDraftModel"]) |
| 952 | ) |
| 953 | assert not VllmConfig._is_dflash_candidate_draft(config("eagle", [architecture])) |
| 954 | assert not VllmConfig._is_dflash_candidate_draft( |
| 955 | SimpleNamespace(speculative_config=None) |
| 956 | ) |
| 957 | assert not VllmConfig._is_dflash_candidate_draft( |
| 958 | SimpleNamespace( |
| 959 | speculative_config=SimpleNamespace(method="dflash", draft_model_config=None) |
| 960 | ) |
| 961 | ) |
| 962 | |
| 963 | |
| 964 | @pytest.mark.parametrize( |
| 965 | ("use_v2_model_runner", "expected_capture_sizes"), |
| 966 | [ |
| 967 | (False, [4, 8, 12, 16]), |
| 968 | (True, list(range(1, 17))), |
| 969 | ], |
| 970 | ) |
| 971 | def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( |
| 972 | use_v2_model_runner, |
| 973 | expected_capture_sizes, |
| 974 | ): |
| 975 | compilation_config = CompilationConfig( |
| 976 | cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, |
| 977 | cudagraph_capture_sizes=list(range(1, 17)), |
| 978 | ) |
| 979 | compilation_config.max_cudagraph_capture_size = 16 |
| 980 | compilation_config.post_init_cudagraph_sizes() |
| 981 | |
| 982 | cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes( |
| 983 | AttentionCGSupport.ALWAYS, |
| 984 | "FakeAttentionBackend", |
| 985 | uniform_decode_query_len=4, |
| 986 | use_v2_model_runner=use_v2_model_runner, |
| 987 | tensor_parallel_size=1, |
| 988 | ) |
| 989 | |
| 990 | assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE |
| 991 | assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes |
| 992 | |
| 993 | |
| 994 | @pytest.mark.parametrize( |
| 995 | ("mode", "piecewise_capture_available", "attention_support", "expected"), |
| 996 | [ |
| 997 | ("PIECEWISE", False, "ALWAYS", "NONE"), |
| 998 | ("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"), |
| 999 | ("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"), |
| 1000 | ("FULL_DECODE_ONLY", False, "NEVER", "NONE"), |
| 1001 | ("FULL_AND_PIECEWISE", True, "UNIFORM_BATCH", "FULL_AND_PIECEWISE"), |
| 1002 | ("FULL", True, "UNIFORM_BATCH", "FULL_DECODE_ONLY"), |
| 1003 | ], |
| 1004 | ) |
| 1005 | def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider( |
| 1006 | mode, piecewise_capture_available, attention_support, expected |
| 1007 | ): |
| 1008 | compilation_config = CompilationConfig( |
| 1009 | mode=CompilationMode.VLLM_COMPILE, |
| 1010 | cudagraph_mode=CUDAGraphMode[mode], |
| 1011 | use_inductor_graph_partition=True, |
| 1012 | ) |
| 1013 | |
| 1014 | resolved = compilation_config.resolve_cudagraph_mode_and_sizes( |
| 1015 | AttentionCGSupport[attention_support], |
| 1016 | "FakeAttentionBackend", |
| 1017 | piecewise_capture_available=piecewise_capture_available, |
| 1018 | ) |
| 1019 | |
| 1020 | assert resolved.name == expected |
| 1021 | assert compilation_config.cudagraph_mode == resolved |
| 1022 | |
| 1023 | |
| 1024 | @pytest.mark.skipif( |
| 1025 | not current_platform.is_cuda_alike(), reason="Requires CUDA graph support" |
| 1026 | ) |
| 1027 | @pytest.mark.parametrize( |
| 1028 | "engine_kwargs", |
| 1029 | [ |
| 1030 | {"runner": "pooling", "convert": "embed"}, |
| 1031 | pytest.param( |
| 1032 | {"prefill_context_parallel_size": 2, "tensor_parallel_size": 2}, |
| 1033 | marks=pytest.mark.skipif( |
| 1034 | not current_platform.is_rocm(), reason="ROCm PCP graph restriction" |
| 1035 | ), |
| 1036 | ), |
| 1037 | ], |
| 1038 | ) |
| 1039 | def test_late_piecewise_restrictions_without_compilation(monkeypatch, engine_kwargs): |
| 1040 | """Late compatibility overrides must not restore unavailable piecewise graphs.""" |
| 1041 | from vllm.engine.arg_utils import EngineArgs |
| 1042 | |
| 1043 | monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") |
| 1044 | monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") |
| 1045 | config = EngineArgs( |
| 1046 | model="facebook/opt-125m", |
| 1047 | compilation_config=CompilationConfig(mode=CompilationMode.NONE), |
| 1048 | **engine_kwargs, |
| 1049 | ).create_engine_config() |
| 1050 | |
| 1051 | assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE |
| 1052 | assert config.compilation_config.cudagraph_capture_sizes == [] |
| 1053 | assert config.compilation_config.max_cudagraph_capture_size == 0 |
| 1054 | |
| 1055 | |
| 1056 | def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): |
| 1057 | """Cudagraph memory profiling uses a minimal KV cache, so the Mamba |
| 1058 | block-count guard must only fire for the real cache sizing.""" |
| 1059 | kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4) |
| 1060 | |
| 1061 | compilation_config = CompilationConfig( |
| 1062 | cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, |
| 1063 | ) |
| 1064 | with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"): |
| 1065 | compilation_config.resolve_cudagraph_mode_and_sizes( |
| 1066 | AttentionCGSupport.ALWAYS, |
| 1067 | "FakeAttentionBackend", |
| 1068 | uniform_decode_query_len=1, |
| 1069 | use_v2_model_runner=True, |
| 1070 | tensor_parallel_size=1, |
| 1071 | kv_cache_config=kv_cache_config, |
| 1072 | max_num_reqs=256, |
| 1073 | ) |
| 1074 | |
| 1075 | compilation_config = CompilationConfig( |
| 1076 | cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, |
| 1077 | ) |
| 1078 | cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes( |
| 1079 | AttentionCGSupport.ALWAYS, |
| 1080 | "FakeAttentionBackend", |
| 1081 | uniform_decode_query_len=1, |
| 1082 | use_v2_model_runner=True, |
| 1083 | tensor_parallel_size=1, |
| 1084 | kv_cache_config=kv_cache_config, |
| 1085 | max_num_reqs=256, |
| 1086 | is_profiling=True, |
| 1087 | ) |
| 1088 | assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE |
| 1089 | |
| 1090 | |
| 1091 | @pytest.mark.parametrize( |
| 1092 | ("graph_mode", "should_raise"), |
| 1093 | [ |
| 1094 | (CUDAGraphMode.FULL_DECODE_ONLY, False), |
| 1095 | (CUDAGraphMode.FULL_AND_PIECEWISE, False), |
| 1096 | (CUDAGraphMode.NONE, True), |
| 1097 | (CUDAGraphMode.PIECEWISE, True), |
| 1098 | ], |
| 1099 | ) |
| 1100 | def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise): |
| 1101 | config = SimpleNamespace( |
| 1102 | speculative_config=SimpleNamespace(enable_adaptive_verification=True), |
| 1103 | lora_config=None, |
| 1104 | compilation_config=CompilationConfig(cudagraph_mode=graph_mode), |
| 1105 | parallel_config=SimpleNamespace(pipeline_parallel_size=1), |
| 1106 | ) |
| 1107 | |
| 1108 | if should_raise: |
| 1109 | with pytest.raises(ValueError, match="requires full CUDA graphs"): |
| 1110 | VllmConfig._validate_adaptive_verification(config) |
| 1111 | else: |
| 1112 | VllmConfig._validate_adaptive_verification(config) |
| 1113 | |
| 1114 | |
| 1115 | @pytest.mark.parametrize( |
| 1116 | ("model_config", "expected"), |
| 1117 | [ |
| 1118 | ( |
| 1119 | SimpleNamespace( |
| 1120 | model="Qwen/Qwen3-32B", |
| 1121 | architectures=["Qwen3ForCausalLM"], |
| 1122 | runner_type="generate", |
| 1123 | is_moe=False, |
| 1124 | is_quantized=False, |
| 1125 | ), |
| 1126 | True, |
| 1127 | ), |
| 1128 | ( |
| 1129 | SimpleNamespace( |
| 1130 | model="Qwen/Qwen2-7B-Instruct", |
| 1131 | architectures=["Qwen2ForCausalLM"], |
| 1132 | runner_type="generate", |
| 1133 | is_moe=False, |
| 1134 | is_quantized=False, |
| 1135 | ), |
| 1136 | True, |
| 1137 | ), |
| 1138 | ( |
| 1139 | SimpleNamespace( |
| 1140 | model="meta-llama/Llama-3.2-1B", |
| 1141 | architectures=["LlamaForCausalLM"], |
| 1142 | runner_type="generate", |
| 1143 | is_moe=False, |
| 1144 | is_quantized=False, |
| 1145 | ), |
| 1146 | True, |
| 1147 | ), |
| 1148 | ( |
| 1149 | SimpleNamespace( |
| 1150 | model="mistralai/Mistral-7B-v0.1", |
| 1151 | architectures=["MistralForCausalLM"], |
| 1152 | runner_type="generate", |
| 1153 | is_moe=False, |
| 1154 | is_quantized=False, |
| 1155 | ), |
| 1156 | True, |
| 1157 | ), |
| 1158 | ( |
| 1159 | SimpleNamespace( |
| 1160 | model="facebook/opt-125m", |
| 1161 | architectures=["OPTForCausalLM"], |
| 1162 | runner_type="generate", |
| 1163 | is_moe=False, |
| 1164 | is_quantized=False, |
| 1165 | ), |
| 1166 | True, |
| 1167 | ), |
| 1168 | ( |
| 1169 | SimpleNamespace( |
| 1170 | model="google/gemma-2-2b", |
| 1171 | architectures=["Gemma2ForCausalLM"], |
| 1172 | runner_type="generate", |
| 1173 | is_moe=False, |
| 1174 | is_quantized=False, |
| 1175 | ), |
| 1176 | True, |
| 1177 | ), |
| 1178 | ( |
| 1179 | SimpleNamespace( |
| 1180 | model="deepseek-ai/DeepSeek-V2-Lite-Chat", |
| 1181 | architectures=["DeepseekV2ForCausalLM"], |
| 1182 | runner_type="generate", |
| 1183 | is_moe=True, |
| 1184 | is_quantized=False, |
| 1185 | ), |
| 1186 | True, |
| 1187 | ), |
| 1188 | ( |
| 1189 | SimpleNamespace( |
| 1190 | model="deepseek-ai/DeepSeek-V2-Chat", |
| 1191 | architectures=["DeepseekV2ForCausalLM"], |
| 1192 | runner_type="generate", |
| 1193 | is_moe=True, |
| 1194 | is_quantized=False, |
| 1195 | ), |
| 1196 | True, |
| 1197 | ), |
| 1198 | ( |
| 1199 | SimpleNamespace( |
| 1200 | model="deepseek-ai/DeepSeek-V3", |
| 1201 | architectures=["DeepseekV3ForCausalLM"], |
| 1202 | runner_type="generate", |
| 1203 | is_moe=True, |
| 1204 | is_quantized=False, |
| 1205 | ), |
| 1206 | True, |
| 1207 | ), |
| 1208 | ( |
| 1209 | SimpleNamespace( |
| 1210 | model="deepseek-ai/DeepSeek-V4-Flash", |
| 1211 | architectures=["DeepseekV4ForCausalLM"], |
| 1212 | runner_type="generate", |
| 1213 | is_moe=True, |
| 1214 | is_quantized=True, |
| 1215 | ), |
| 1216 | True, |
| 1217 | ), |
| 1218 | ( |
| 1219 | SimpleNamespace( |
| 1220 | model="Qwen/Qwen1.5-MoE-A2.7B", |
| 1221 | architectures=["Qwen2MoeForCausalLM"], |
| 1222 | runner_type="generate", |
| 1223 | is_moe=True, |
| 1224 | is_quantized=False, |
| 1225 | ), |
| 1226 | True, |
| 1227 | ), |
| 1228 | ( |
| 1229 | SimpleNamespace( |
| 1230 | model="Qwen/Qwen1.5-MoE-A2.7B-Chat", |
| 1231 | architectures=["Qwen2MoeForCausalLM"], |
| 1232 | runner_type="generate", |
| 1233 | is_moe=True, |
| 1234 | is_quantized=False, |
| 1235 | ), |
| 1236 | True, |
| 1237 | ), |
| 1238 | ( |
| 1239 | SimpleNamespace( |
| 1240 | model="ibm-research/PowerMoE-3b", |
| 1241 | architectures=["GraniteMoeForCausalLM"], |
| 1242 | runner_type="generate", |
| 1243 | is_moe=True, |
| 1244 | is_quantized=False, |
| 1245 | ), |
| 1246 | True, |
| 1247 | ), |
| 1248 | ( |
| 1249 | SimpleNamespace( |
| 1250 | model="thinkingmachines/Inkling", |
| 1251 | architectures=["InklingForCausalLM"], |
| 1252 | runner_type="generate", |
| 1253 | is_moe=True, |
| 1254 | is_quantized=False, |
| 1255 | ), |
| 1256 | True, |
| 1257 | ), |
| 1258 | ( |
| 1259 | SimpleNamespace( |
| 1260 | model="thinkingmachines/Inkling", |
| 1261 | architectures=["InklingForConditionalGeneration"], |
| 1262 | runner_type="generate", |
| 1263 | is_moe=True, |
| 1264 | is_quantized=False, |
| 1265 | ), |
| 1266 | True, |
| 1267 | ), |
| 1268 | ( |
| 1269 | SimpleNamespace( |
| 1270 | model="mistralai/Mixtral-8x7B-Instruct-v0.1", |
| 1271 | architectures=["MixtralForCausalLM"], |
| 1272 | runner_type="generate", |
| 1273 | is_moe=True, |
| 1274 | is_quantized=False, |
| 1275 | ), |
| 1276 | True, |
| 1277 | ), |
| 1278 | ( |
| 1279 | SimpleNamespace( |
| 1280 | model="Qwen/Qwen3-1.7B-FP8", |
| 1281 | architectures=["Qwen3ForCausalLM"], |
| 1282 | runner_type="generate", |
| 1283 | is_moe=False, |
| 1284 | is_quantized=True, |
| 1285 | ), |
| 1286 | True, |
| 1287 | ), |
| 1288 | ( |
| 1289 | SimpleNamespace( |
| 1290 | model="Qwen/Qwen3.5-4B", |
| 1291 | architectures=["Qwen3_5ForConditionalGeneration"], |
| 1292 | runner_type="generate", |
| 1293 | is_moe=False, |
| 1294 | is_quantized=False, |
| 1295 | is_hybrid=True, |
| 1296 | ), |
| 1297 | True, |
| 1298 | ), |
| 1299 | ( |
| 1300 | SimpleNamespace( |
| 1301 | model="state-spaces/mamba-130m-hf", |
| 1302 | architectures=["MambaForCausalLM"], |
| 1303 | runner_type="generate", |
| 1304 | is_moe=False, |
| 1305 | is_quantized=False, |
| 1306 | is_attention_free=True, |
| 1307 | ), |
| 1308 | True, |
| 1309 | ), |
| 1310 | ( |
| 1311 | SimpleNamespace( |
| 1312 | model="sentence-transformers/all-MiniLM-L6-v2", |
| 1313 | architectures=["BertModel"], |
| 1314 | runner_type="pooling", |
| 1315 | is_multimodal_model=False, |
| 1316 | is_moe=False, |
| 1317 | is_quantized=False, |
| 1318 | ), |
| 1319 | True, |
| 1320 | ), |
| 1321 | ( |
| 1322 | SimpleNamespace( |
| 1323 | model="Qwen/Qwen3-Embedding-0.6B", |
| 1324 | architectures=["Qwen3ForCausalLM"], |
| 1325 | runner_type="pooling", |
| 1326 | is_multimodal_model=False, |
| 1327 | is_moe=False, |
| 1328 | is_quantized=False, |
| 1329 | ), |
| 1330 | True, |
| 1331 | ), |
| 1332 | ( |
| 1333 | SimpleNamespace( |
| 1334 | model="TomoroAI/tomoro-colqwen3-embed-4b", |
| 1335 | architectures=["ColQwen3"], |
| 1336 | runner_type="pooling", |
| 1337 | is_multimodal_model=True, |
| 1338 | is_moe=False, |
| 1339 | is_quantized=False, |
| 1340 | ), |
| 1341 | True, |
| 1342 | ), |
| 1343 | ], |
| 1344 | ) |
| 1345 | def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch): |
| 1346 | from vllm.platforms import current_platform |
| 1347 | |
| 1348 | # The expectations below are the platform-independent defaults; ROCm's |
| 1349 | # DeepSeek carve-out is covered by test_rocm_keeps_compiled_deepseek_defaults. |
| 1350 | monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) |
| 1351 | monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) |
| 1352 | monkeypatch.setattr(current_platform, "is_rocm", lambda: False) |
| 1353 | config = SimpleNamespace( |
| 1354 | model_config=model_config, |
| 1355 | attention_config=AttentionConfig(), |
| 1356 | ) |
| 1357 | config._get_v2_model_runner_unsupported_features = lambda: [] |
| 1358 | |
| 1359 | assert VllmConfig.use_v2_model_runner.fget(config) is expected |
| 1360 | |
| 1361 | |
| 1362 | def test_v1_model_runner_rejects_v2_only_features(): |
| 1363 | config = SimpleNamespace( |
| 1364 | parallel_config=ParallelConfig( |
| 1365 | prefill_context_parallel_size=2, |
| 1366 | distributed_executor_backend="mp", |
| 1367 | ), |
| 1368 | scheduler_config=SchedulerConfig.default_factory(async_scheduling=False), |
| 1369 | speculative_config=None, |
| 1370 | model_config=None, |
| 1371 | ) |
| 1372 | config._dflash_needs_multi_kv_group = lambda: False |
| 1373 | config._is_dflash_candidate_draft = lambda: False |
| 1374 | config._get_v1_model_runner_unsupported_features = lambda: ( |
| 1375 | VllmConfig._get_v1_model_runner_unsupported_features(config) |
| 1376 | ) |
| 1377 | |
| 1378 | with pytest.raises(ValueError, match="prefill context parallel"): |
| 1379 | VllmConfig._validate_v1_model_runner(config) |
| 1380 | |
| 1381 | |
| 1382 | def test_batch_sharded_sampling_rejects_return_sampling_mask(): |
| 1383 | """The batch-sharded gather drops sampling masks, so the combination must |
| 1384 | fail loudly instead of returning ``sampling_mask=None``.""" |
| 1385 | config = SimpleNamespace( |
| 1386 | parallel_config=SimpleNamespace( |
| 1387 | enable_batch_sharded_sampling=True, tensor_parallel_size=2 |
| 1388 | ), |
| 1389 | scheduler_config=SimpleNamespace(max_num_seqs=8), |
| 1390 | model_config=SimpleNamespace(max_logprobs=20, return_sampling_mask=True), |
| 1391 | speculative_config=None, |
| 1392 | ) |
| 1393 | |
| 1394 | with pytest.raises(ValueError, match="sampling masks"): |
| 1395 | VllmConfig._validate_batch_sharded_sampling(config) |
| 1396 | |
| 1397 | |
| 1398 | @pytest.mark.skip_global_cleanup |
| 1399 | def test_with_hf_config_populates_missing_architectures_from_causal_lm_mapping( |
| 1400 | monkeypatch, |
| 1401 | ): |
| 1402 | monkeypatch.setattr( |
| 1403 | vllm_config_module, |
| 1404 | "replace", |
| 1405 | lambda self, **kwargs: SimpleNamespace(**kwargs), |
| 1406 | ) |
| 1407 | cfg = SimpleNamespace( |
| 1408 | model_config=SimpleNamespace( |
| 1409 | is_multimodal_model=False, |
| 1410 | hf_config=SimpleNamespace(), |
| 1411 | get_model_arch_config=lambda: "arch-config", |
| 1412 | ) |
| 1413 | ) |
| 1414 | hf_config = SimpleNamespace(model_type="mistral", architectures=None) |
| 1415 | |
| 1416 | updated = VllmConfig.with_hf_config(cfg, hf_config) |
| 1417 | |
| 1418 | assert updated.model_config.hf_config.architectures == ["MistralForCausalLM"] |
| 1419 | assert hf_config.architectures is None |
| 1420 | |
| 1421 | |
| 1422 | @pytest.mark.skip_global_cleanup |
| 1423 | def test_with_hf_config_preserves_explicit_architectures_override(monkeypatch): |
| 1424 | monkeypatch.setattr( |
| 1425 | vllm_config_module, |
| 1426 | "replace", |
| 1427 | lambda self, **kwargs: SimpleNamespace(**kwargs), |
| 1428 | ) |
| 1429 | cfg = SimpleNamespace( |
| 1430 | model_config=SimpleNamespace( |
| 1431 | is_multimodal_model=False, |
| 1432 | hf_config=SimpleNamespace(), |
| 1433 | get_model_arch_config=lambda: "arch-config", |
| 1434 | ) |
| 1435 | ) |
| 1436 | hf_config = SimpleNamespace(model_type="mistral", architectures=None) |
| 1437 | |
| 1438 | updated = VllmConfig.with_hf_config( |
| 1439 | cfg, |
| 1440 | hf_config, |
| 1441 | architectures=["Ministral3ForCausalLM"], |
| 1442 | ) |
| 1443 | |
| 1444 | assert updated.model_config.hf_config.architectures == ["Ministral3ForCausalLM"] |
| 1445 | |
| 1446 | |
| 1447 | @pytest.mark.skip_global_cleanup |
| 1448 | def test_with_hf_config_leaves_unknown_model_type_without_architectures( |
| 1449 | monkeypatch, |
| 1450 | ): |
| 1451 | monkeypatch.setattr( |
| 1452 | vllm_config_module, |
| 1453 | "replace", |
| 1454 | lambda self, **kwargs: SimpleNamespace(**kwargs), |
| 1455 | ) |
| 1456 | cfg = SimpleNamespace( |
| 1457 | model_config=SimpleNamespace( |
| 1458 | is_multimodal_model=False, |
| 1459 | hf_config=SimpleNamespace(), |
| 1460 | get_model_arch_config=lambda: "arch-config", |
| 1461 | ) |
| 1462 | ) |
| 1463 | hf_config = SimpleNamespace( |
| 1464 | model_type="not_a_real_model", |
| 1465 | architectures=None, |
| 1466 | ) |
| 1467 | |
| 1468 | updated = VllmConfig.with_hf_config(cfg, hf_config) |
| 1469 | |
| 1470 | assert updated.model_config.hf_config.architectures is None |
| 1471 | |
| 1472 | |
| 1473 | @pytest.mark.parametrize( |
| 1474 | "checkpoint_tensors,tied", |
| 1475 | [ |
| 1476 | # The checkpoint has an lm_head of its own, so it must win over the config |
| 1477 | (["model.embed_tokens.weight", "lm_head.weight"], False), |
| 1478 | (["model.embed_tokens.weight"], True), |
| 1479 | # Contents unknown (not safetensors), so the config must be left alone |
| 1480 | ([], True), |
| 1481 | ], |
| 1482 | ) |
| 1483 | def test_maybe_untie_word_embeddings(tmp_path, checkpoint_tensors, tied): |
| 1484 | import torch |
| 1485 | from safetensors.torch import save_file |
| 1486 | |
| 1487 | if checkpoint_tensors: |
| 1488 | save_file( |
| 1489 | {name: torch.zeros(2, 2) for name in checkpoint_tensors}, |
| 1490 | tmp_path / "model.safetensors", |
| 1491 | ) |
| 1492 | |
| 1493 | text_config = SimpleNamespace(tie_word_embeddings=True) |
| 1494 | model_config = SimpleNamespace( |
| 1495 | model=str(tmp_path), |
| 1496 | revision=None, |
| 1497 | hf_config=SimpleNamespace( |
| 1498 | tie_word_embeddings=True, |
| 1499 | get_text_config=lambda: text_config, |
| 1500 | ), |
| 1501 | word_embeddings_untied_by_checkpoint=False, |
| 1502 | ) |
| 1503 | |
| 1504 | ModelConfig.maybe_untie_word_embeddings(model_config) |
| 1505 | |
| 1506 | # Both levels must agree, since different callers read different ones |
| 1507 | assert model_config.hf_config.tie_word_embeddings is tied |
| 1508 | assert text_config.tie_word_embeddings is tied |
| 1509 | assert model_config.word_embeddings_untied_by_checkpoint is not tied |
| 1510 | |
| 1511 | |
| 1512 | def test_async_scheduling_with_pipeline_parallelism_is_allowed(): |
| 1513 | cfg = VllmConfig( |
| 1514 | scheduler_config=SchedulerConfig( |
| 1515 | max_model_len=8192, |
| 1516 | is_encoder_decoder=False, |
| 1517 | async_scheduling=True, |
| 1518 | ), |
| 1519 | parallel_config=ParallelConfig( |
| 1520 | pipeline_parallel_size=2, |
| 1521 | distributed_executor_backend="mp", |
| 1522 | nnodes=2, |
| 1523 | ), |
| 1524 | ) |
| 1525 | assert cfg.scheduler_config.async_scheduling is True |
| 1526 | |
| 1527 | |
| 1528 | def test_v1_model_runner_drops_async_scheduling_with_pipeline_parallelism(monkeypatch): |
| 1529 | """PP>1 must stay buildable whenever the V1 model runner is selected, such |
| 1530 | as the external_launcher fallback; async scheduling is dropped instead.""" |
| 1531 | monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") |
| 1532 | |
| 1533 | cfg = VllmConfig( |
| 1534 | scheduler_config=SchedulerConfig( |
| 1535 | max_model_len=8192, |
| 1536 | is_encoder_decoder=False, |
| 1537 | ), |
| 1538 | parallel_config=ParallelConfig( |
| 1539 | pipeline_parallel_size=2, |
| 1540 | distributed_executor_backend="mp", |
| 1541 | nnodes=2, |
| 1542 | ), |
| 1543 | ) |
| 1544 | assert cfg.scheduler_config.async_scheduling is False |
| 1545 | |
| 1546 | |
| 1547 | def test_v1_model_runner_rejects_pipeline_parallelism_with_async_scheduling(): |
| 1548 | """Only the async combination desyncs the grammar FSM (#45014), so plain |
| 1549 | PP>1 must stay usable on the V1 model runner.""" |
| 1550 | config = SimpleNamespace( |
| 1551 | parallel_config=ParallelConfig( |
| 1552 | pipeline_parallel_size=2, |
| 1553 | distributed_executor_backend="mp", |
| 1554 | ), |
| 1555 | scheduler_config=SchedulerConfig.default_factory(async_scheduling=False), |
| 1556 | speculative_config=None, |
| 1557 | model_config=None, |
| 1558 | ) |
| 1559 | config._dflash_needs_multi_kv_group = lambda: False |
| 1560 | config._is_dflash_candidate_draft = lambda: False |
| 1561 | |
| 1562 | assert VllmConfig._get_v1_model_runner_unsupported_features(config) == [] |
| 1563 | |
| 1564 | config.scheduler_config.async_scheduling = True |
| 1565 | unsupported = VllmConfig._get_v1_model_runner_unsupported_features(config) |
| 1566 | assert "pipeline parallelism with async scheduling" in unsupported |
| 1567 | |
| 1568 | |
| 1569 | def test_data_parallel_rpc_port_has_fixed_default(): |
| 1570 | assert ParallelConfig().data_parallel_rpc_port == 29550 |
| 1571 | |
| 1572 | |
| 1573 | def test_all2all_backend_has_portable_default(): |
| 1574 | assert ParallelConfig().all2all_backend == "allgather_reducescatter" |
| 1575 | |
| 1576 | |
| 1577 | def test_dp_group_uses_configured_timeout_without_current_config(monkeypatch): |
| 1578 | monkeypatch.setattr(vllm_config_module, "_current_vllm_config", None) |
| 1579 | config = ParallelConfig(cpu_distributed_timeout_seconds=30) |
| 1580 | with ( |
| 1581 | patch( |
| 1582 | "vllm.distributed.utils.rendezvous", |
| 1583 | return_value=iter([(torch.distributed.HashStore(), 0, 1)]), |
| 1584 | ), |
| 1585 | patch("vllm.distributed.utils.init_gloo_process_group") as init_group, |
| 1586 | ): |
| 1587 | config.stateless_init_dp_group() |
| 1588 | assert init_group.call_args.kwargs["timeout"] == timedelta(seconds=30) |
| 1589 | |
| 1590 | |
| 1591 | @pytest.mark.parametrize( |
| 1592 | "dp_size, across_dp, expected", |
| 1593 | [(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)], |
| 1594 | ) |
| 1595 | def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: int): |
| 1596 | parallel = ParallelConfig( |
| 1597 | tensor_parallel_size=4, |
| 1598 | data_parallel_size=dp_size, |
| 1599 | distributed_executor_backend="mp", |
| 1600 | ) |
| 1601 | config = EngramConfig(embedding_across_dp=across_dp) |
| 1602 | assert config.get_parallel_size(parallel) == expected |
| 1603 | |
| 1604 | |
| 1605 | @pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"]) |
| 1606 | def test_engram_rejects_elastic_cross_dp(option): |
| 1607 | parallel = ParallelConfig( |
| 1608 | tensor_parallel_size=4, |
| 1609 | data_parallel_size=2, |
| 1610 | distributed_executor_backend="mp", |
| 1611 | ) |
| 1612 | parallel.enable_elastic_ep = True |
| 1613 | with pytest.raises(ValueError, match=f"{option}.*elastic EP"): |
| 1614 | EngramConfig(**{option: True}).verify_parallel_config(parallel) |
| 1615 | |
| 1616 | |
| 1617 | def test_engram_dp_shared_memory_requires_cpu_offload(): |
| 1618 | with pytest.raises(ValueError, match="requires cpu_offload"): |
| 1619 | EngramConfig(cpu_offload=False, dp_shared_memory=True) |
| 1620 | |
| 1621 | |
| 1622 | @pytest.mark.skip_global_cleanup |
| 1623 | @pytest.mark.parametrize( |
| 1624 | "cpu_offload,use_thp,dp_shared_memory,dp_size,elastic_ep,expected", |
| 1625 | [ |
| 1626 | (True, False, None, 2, False, True), |
| 1627 | (False, False, None, 2, False, False), |
| 1628 | (True, False, None, 1, False, False), |
| 1629 | (True, False, None, 2, True, False), |
| 1630 | # use_thp backs private tables; sharing would silently ignore it. |
| 1631 | (True, True, None, 2, False, False), |
| 1632 | (True, True, False, 2, False, False), |
| 1633 | ], |
| 1634 | ) |
| 1635 | def test_engram_dp_shared_memory_defaults_when_supported( |
| 1636 | cpu_offload, use_thp, dp_shared_memory, dp_size, elastic_ep, expected |
| 1637 | ): |
| 1638 | """Resolve sharing only where supported, preserving an explicit false.""" |
| 1639 | parallel = ParallelConfig(data_parallel_size=dp_size) |
| 1640 | parallel.enable_elastic_ep = elastic_ep |
| 1641 | config = EngramConfig( |
| 1642 | cpu_offload=cpu_offload, |
| 1643 | use_thp=use_thp, |
| 1644 | dp_shared_memory=dp_shared_memory, |
| 1645 | ) |
| 1646 | config.resolve_dp_shared_memory(parallel) |
| 1647 | assert config.dp_shared_memory is expected |
| 1648 | |
| 1649 | |
| 1650 | @pytest.mark.skip_global_cleanup |
| 1651 | def test_engram_thp_rejects_explicit_shared_memory(): |
| 1652 | with pytest.raises( |
| 1653 | ValueError, match="use_thp requires cpu_offload=True and dp_shared_memory=False" |
| 1654 | ): |
| 1655 | EngramConfig(use_thp=True, dp_shared_memory=True) |
| 1656 | |
| 1657 | |
| 1658 | @pytest.mark.parametrize( |
| 1659 | "dp_size,load_format,multithread,error", |
| 1660 | [ |
| 1661 | (1, "auto", False, "requires data_parallel_size > 1"), |
| 1662 | (2, "auto", False, None), |
| 1663 | (2, "safetensors", True, None), |
| 1664 | ], |
| 1665 | ) |
| 1666 | def test_engram_dp_shared_memory_config_validation( |
| 1667 | monkeypatch, dp_size, load_format, multithread, error |
| 1668 | ): |
| 1669 | """Reject invalid shared configs before distributed init; allow threaded loads.""" |
| 1670 | monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True) |
| 1671 | config = cast( |
| 1672 | VllmConfig, |
| 1673 | SimpleNamespace( |
| 1674 | model_config=SimpleNamespace( |
| 1675 | architecture="DeepseekV41ForCausalLM", |
| 1676 | hf_text_config=SimpleNamespace(engram_layer_ids=[1]), |
| 1677 | ), |
| 1678 | speculative_config=None, |
| 1679 | engram_config=EngramConfig(cpu_offload=True, dp_shared_memory=True), |
| 1680 | parallel_config=ParallelConfig(data_parallel_size=dp_size), |
| 1681 | load_config=LoadConfig( |
| 1682 | load_format=load_format, |
| 1683 | model_loader_extra_config={"enable_multithread_load": multithread}, |
| 1684 | ), |
| 1685 | ), |
| 1686 | ) |
| 1687 | if error: |
| 1688 | with pytest.raises(ValueError, match=error): |
| 1689 | VllmConfig._resolve_and_verify_engram_config(config) |
| 1690 | else: |
| 1691 | VllmConfig._resolve_and_verify_engram_config(config) |
| 1692 | |
| 1693 | |
| 1694 | @pytest.mark.parametrize( |
| 1695 | "architecture, ple_layers, platform, supported", |
| 1696 | [ |
| 1697 | ("DeepseekV41ForCausalLM", [1], "cuda", True), |
| 1698 | ("DeepseekV41ForCausalLM", [1], "rocm", True), |
| 1699 | ("DeepseekV41ForCausalLM", [], "cuda", False), |
| 1700 | ("DeepseekV41ForCausalLM", [1], "cpu", True), |
| 1701 | ("Qwen4ExpForCausalLM", [1], "cuda", True), |
| 1702 | ("Qwen4ExpForCausalLM", [1], "rocm", True), |
| 1703 | ("Qwen4ExpForConditionalGeneration", [1], "cuda", True), |
| 1704 | ("Qwen4ExpForConditionalGeneration", [1], "rocm", True), |
| 1705 | ("Qwen4ExpForCausalLM", [], "cuda", False), |
| 1706 | ("Qwen4ExpForCausalLM", None, "cuda", False), |
| 1707 | ("Qwen4ExpForCausalLM", [1], "cpu", True), |
| 1708 | ("LlamaForCausalLM", [1], "cuda", False), |
| 1709 | ("Qwen4ExpMTP", [], "cuda", False), |
| 1710 | (None, None, "cuda", False), |
| 1711 | ], |
| 1712 | ) |
| 1713 | def test_engram_model_support( |
| 1714 | monkeypatch, architecture, ple_layers, platform, supported |
| 1715 | ): |
| 1716 | """A similarly named HF field must not enable unsupported implementations.""" |
| 1717 | monkeypatch.setattr(current_platform, "is_cuda", lambda: platform == "cuda") |
| 1718 | monkeypatch.setattr( |
| 1719 | current_platform, "is_cuda_alike", lambda: platform in ("cuda", "rocm") |
| 1720 | ) |
| 1721 | model = ( |
| 1722 | cast( |
| 1723 | ModelConfig, |
| 1724 | SimpleNamespace( |
| 1725 | architecture=architecture, |
| 1726 | hf_text_config=SimpleNamespace( |
| 1727 | ple_layer_ids=ple_layers, engram_layer_ids=ple_layers |
| 1728 | ), |
| 1729 | ), |
| 1730 | ) |
| 1731 | if architecture is not None |
| 1732 | else None |
| 1733 | ) |
| 1734 | config = EngramConfig(cpu_offload=False, embedding_across_dp=False) |
| 1735 | if supported: |
| 1736 | config.verify_model_config(model) |
| 1737 | else: |
| 1738 | with pytest.raises(ValueError, match="requires a model with supported Engram"): |
| 1739 | config.verify_model_config(model) |
| 1740 | |
| 1741 | resolved = cast( |
| 1742 | VllmConfig, |
| 1743 | SimpleNamespace( |
| 1744 | model_config=model, |
| 1745 | speculative_config=None, |
| 1746 | parallel_config=ParallelConfig(), |
| 1747 | load_config=LoadConfig(load_format="dummy"), |
| 1748 | engram_config=None, |
| 1749 | ), |
| 1750 | ) |
| 1751 | VllmConfig._resolve_and_verify_engram_config(resolved) |
| 1752 | assert (resolved.engram_config is not None) == supported |
| 1753 | if supported: |
| 1754 | assert resolved.engram_config.cpu_offload is True |
| 1755 | |
| 1756 | |
| 1757 | def test_engram_cpu_offload_default(): |
| 1758 | assert EngramConfig().cpu_offload is True |
| 1759 | |
| 1760 | |
| 1761 | def test_engram_config_defaults_to_none(): |
| 1762 | config = VllmConfig() |
| 1763 | assert config.engram_config is None |
| 1764 | assert config.compute_hash() |
| 1765 | |
| 1766 | |
| 1767 | @pytest.mark.parametrize("explicit", [False, True]) |
| 1768 | @pytest.mark.parametrize("enable_dbo,ubatch_size", [(True, 0), (False, 2)]) |
| 1769 | def test_deepseek_engram_rejects_microbatching(explicit, enable_dbo, ubatch_size): |
| 1770 | """Reject overlapping Engram staging even without an explicit EngramConfig.""" |
| 1771 | config = cast( |
| 1772 | VllmConfig, |
| 1773 | SimpleNamespace( |
| 1774 | model_config=SimpleNamespace( |
| 1775 | architecture="DeepseekV41ForCausalLM", |
| 1776 | hf_text_config=SimpleNamespace(engram_layer_ids=[1]), |
| 1777 | ), |
| 1778 | speculative_config=None, |
| 1779 | engram_config=EngramConfig(cpu_offload=False) if explicit else None, |
| 1780 | parallel_config=ParallelConfig( |
| 1781 | enable_dbo=enable_dbo, ubatch_size=ubatch_size |
| 1782 | ), |
| 1783 | ), |
| 1784 | ) |
| 1785 | with pytest.raises( |
| 1786 | ValueError, match="Engram does not support DBO or microbatching" |
| 1787 | ): |
| 1788 | VllmConfig._resolve_and_verify_engram_config(config) |
| 1789 | |
| 1790 | |
| 1791 | def test_engram_explicit_config_requires_supported_model(): |
| 1792 | """Explicit all-false settings still opt into model validation.""" |
| 1793 | with pytest.raises(ValueError, match="requires a model with supported Engram"): |
| 1794 | VllmConfig(engram_config=EngramConfig(cpu_offload=False)) |
| 1795 | |
| 1796 | |
| 1797 | @pytest.mark.parametrize("target_has_ple", [False, True]) |
| 1798 | @pytest.mark.parametrize("explicit", [False, True]) |
| 1799 | def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, explicit): |
| 1800 | """MTP may inherit cross-DP sharding without having its own PLE layers.""" |
| 1801 | monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True) |
| 1802 | target = SimpleNamespace( |
| 1803 | architecture="Qwen4ExpForCausalLM", |
| 1804 | hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []), |
| 1805 | ) |
| 1806 | draft = SimpleNamespace(architecture="Qwen4ExpMTP") |
| 1807 | config = cast( |
| 1808 | VllmConfig, |
| 1809 | SimpleNamespace( |
| 1810 | model_config=draft, |
| 1811 | speculative_config=SimpleNamespace( |
| 1812 | draft_model_config=draft, target_model_config=target |
| 1813 | ), |
| 1814 | engram_config=EngramConfig(embedding_across_dp=True) if explicit else None, |
| 1815 | parallel_config=ParallelConfig(), |
| 1816 | load_config=LoadConfig(load_format="dummy"), |
| 1817 | ), |
| 1818 | ) |
| 1819 | if explicit and not target_has_ple: |
| 1820 | with pytest.raises(ValueError, match="requires a model with supported Engram"): |
| 1821 | VllmConfig._resolve_and_verify_engram_config(config) |
| 1822 | else: |
| 1823 | VllmConfig._resolve_and_verify_engram_config(config) |
| 1824 | assert (config.engram_config is not None) == target_has_ple |
| 1825 | if target_has_ple: |
| 1826 | assert config.engram_config.cpu_offload is True |
| 1827 | |
| 1828 | |
| 1829 | def test_engram_hash_tracks_execution_options(): |
| 1830 | configs = [ |
| 1831 | EngramConfig(cpu_offload=True), |
| 1832 | EngramConfig(cpu_offload=False), |
| 1833 | EngramConfig(cpu_offload=True, embedding_across_dp=True), |
| 1834 | EngramConfig(cpu_offload=True, dp_shared_memory=True), |
| 1835 | ] |
| 1836 | assert len({config.compute_hash() for config in configs}) == len(configs) |
| 1837 | |
| 1838 | |
| 1839 | @pytest.mark.parametrize("port", [1, 29550, 65535]) |
| 1840 | def test_data_parallel_rpc_port_accepts_valid_ports(port: int): |
| 1841 | assert ParallelConfig(data_parallel_rpc_port=port).data_parallel_rpc_port == port |
| 1842 | |
| 1843 | |
| 1844 | @pytest.mark.parametrize("port", [-1, 0, 65536]) |
| 1845 | def test_data_parallel_rpc_port_rejects_invalid_ports(port: int): |
| 1846 | with pytest.raises(ValidationError): |
| 1847 | ParallelConfig(data_parallel_rpc_port=port) |
| 1848 | |
| 1849 | |
| 1850 | def test_reconfigure_for_independent_dp_rank_on_multinode_dense_model(): |
| 1851 | parallel_config = ParallelConfig( |
| 1852 | tensor_parallel_size=8, |
| 1853 | data_parallel_size=2, |
| 1854 | data_parallel_size_local=1, |
| 1855 | data_parallel_rank=1, |
| 1856 | distributed_executor_backend="mp", |
| 1857 | nnodes=2, |
| 1858 | node_rank=1, |
| 1859 | ) |
| 1860 | |
| 1861 | assert parallel_config.nnodes_within_dp == 1 |
| 1862 | assert parallel_config.node_rank_within_dp == 0 |
| 1863 | |
| 1864 | parallel_config.reconfigure_for_independent_dp_rank() |
| 1865 | |
| 1866 | assert parallel_config.data_parallel_size == 1 |
| 1867 | assert parallel_config.data_parallel_size_local == 1 |
| 1868 | assert parallel_config.data_parallel_rank == 0 |
| 1869 | assert parallel_config.data_parallel_index == 1 |
| 1870 | assert parallel_config.nnodes == 1 |
| 1871 | assert parallel_config.node_rank == 0 |
| 1872 | assert parallel_config.world_size == 8 |
| 1873 | |
| 1874 | |
| 1875 | @pytest.mark.parametrize("data_parallel_size", [4, 8]) |
| 1876 | def test_nnodes_within_dp_when_replicas_outnumber_nodes(data_parallel_size): |
| 1877 | """A replica that fits on one node spans one node, never zero. |
| 1878 | |
| 1879 | External LB pins ``data_parallel_size_local`` to 1, so the ratio rounds |
| 1880 | down to 0 as soon as there are more DP replicas than nodes. Anything |
| 1881 | dividing by ``nnodes_within_dp`` then raises ZeroDivisionError. |
| 1882 | """ |
| 1883 | parallel_config = ParallelConfig( |
| 1884 | data_parallel_size=data_parallel_size, |
| 1885 | data_parallel_size_local=1, |
| 1886 | data_parallel_external_lb=True, |
| 1887 | distributed_executor_backend="mp", |
| 1888 | nnodes=2, |
| 1889 | node_rank=1, |
| 1890 | ) |
| 1891 | |
| 1892 | assert parallel_config.nnodes_within_dp == 1 |
| 1893 | assert parallel_config.node_rank_within_dp == 0 |
| 1894 | assert parallel_config.local_world_size == parallel_config.world_size |
| 1895 | |
| 1896 | |
| 1897 | @pytest.mark.parametrize("nnodes", [4, 9]) |
| 1898 | def test_nnodes_within_dp_rejects_uneven_internal_lb(nnodes): |
| 1899 | parallel_config = ParallelConfig( |
| 1900 | data_parallel_size=8, |
| 1901 | data_parallel_size_local=1, |
| 1902 | distributed_executor_backend="mp", |
| 1903 | nnodes=nnodes, |
| 1904 | ) |
| 1905 | |
| 1906 | with pytest.raises(ValueError, match="Invalid data parallel configuration"): |
| 1907 | _ = parallel_config.nnodes_within_dp |
| 1908 | |
| 1909 | |
| 1910 | def test_draft_model_enables_async_scheduling_by_default(): |
| 1911 | parallel_config = ParallelConfig(distributed_executor_backend="uni") |
| 1912 | model_config = ModelConfig("Qwen/Qwen3-0.6B", max_model_len=2048) |
| 1913 | speculative_config = SpeculativeConfig( |
| 1914 | method="draft_model", |
| 1915 | model="Qwen/Qwen3-0.6B", |
| 1916 | num_speculative_tokens=3, |
| 1917 | target_model_config=model_config, |
| 1918 | target_parallel_config=parallel_config, |
| 1919 | ) |
| 1920 | cfg = VllmConfig( |
| 1921 | model_config=model_config, |
| 1922 | scheduler_config=SchedulerConfig( |
| 1923 | max_model_len=2048, |
| 1924 | is_encoder_decoder=False, |
| 1925 | ), |
| 1926 | parallel_config=parallel_config, |
| 1927 | speculative_config=speculative_config, |
| 1928 | ) |
| 1929 | |
| 1930 | assert cfg.scheduler_config.async_scheduling is True |
| 1931 | |
| 1932 | |
| 1933 | def test_dflash_allows_async_scheduling(tmp_path: Path): |
| 1934 | """DFlash must stay async-schedulable: it was the only parallel-drafting |
| 1935 | method excluded from the allowlist.""" |
| 1936 | from transformers import LlamaConfig |
| 1937 | |
| 1938 | common = dict( |
| 1939 | hidden_size=128, |
| 1940 | intermediate_size=256, |
| 1941 | num_hidden_layers=2, |
| 1942 | num_attention_heads=4, |
| 1943 | num_key_value_heads=2, |
| 1944 | vocab_size=256, |
| 1945 | max_position_embeddings=2048, |
| 1946 | ) |
| 1947 | target_path = tmp_path / "target" |
| 1948 | draft_path = tmp_path / "draft" |
| 1949 | _write_json( |
| 1950 | target_path / "config.json", |
| 1951 | LlamaConfig(architectures=["LlamaForCausalLM"], **common).to_dict(), |
| 1952 | ) |
| 1953 | _write_json( |
| 1954 | draft_path / "config.json", |
| 1955 | dict( |
| 1956 | common, |
| 1957 | architectures=["DFlash2DraftModel"], |
| 1958 | model_type="qwen3", |
| 1959 | num_hidden_layers=1, |
| 1960 | layer_types=["sliding_attention"], |
| 1961 | sliding_window=128, |
| 1962 | dflash_config=dict( |
| 1963 | block_size=4, |
| 1964 | mask_token_id=255, |
| 1965 | target_layer_ids=[0], |
| 1966 | conv_group_size=2, |
| 1967 | conv_kernel_size=2, |
| 1968 | selector_rank=8, |
| 1969 | selector_top_k=4, |
| 1970 | ), |
| 1971 | ), |
| 1972 | ) |
| 1973 | parallel_config = ParallelConfig(distributed_executor_backend="uni") |
| 1974 | model_config = ModelConfig( |
| 1975 | model=str(target_path), tokenizer_mode="skip", max_model_len=2048 |
| 1976 | ) |
| 1977 | speculative_config = SpeculativeConfig( |
| 1978 | method="dflash", |
| 1979 | model=str(draft_path), |
| 1980 | num_speculative_tokens=3, |
| 1981 | target_model_config=model_config, |
| 1982 | target_parallel_config=parallel_config, |
| 1983 | ) |
| 1984 | cfg = VllmConfig( |
| 1985 | model_config=model_config, |
| 1986 | scheduler_config=SchedulerConfig( |
| 1987 | max_model_len=2048, |
| 1988 | is_encoder_decoder=False, |
| 1989 | ), |
| 1990 | parallel_config=parallel_config, |
| 1991 | speculative_config=speculative_config, |
| 1992 | ) |
| 1993 | |
| 1994 | assert speculative_config.method == "dflash" |
| 1995 | assert cfg.scheduler_config.async_scheduling is True |
| 1996 | |
| 1997 | |
| 1998 | @pytest.mark.skip_global_cleanup |
| 1999 | @pytest.mark.parametrize("tp_size", [1, 2]) |
| 2000 | @pytest.mark.parametrize("target_ep", [False, True], ids=["ep-off", "ep-on"]) |
| 2001 | @pytest.mark.parametrize( |
| 2002 | ("method", "draft_is_moe"), |
| 2003 | [ |
| 2004 | pytest.param("draft_model", False, id="dense-draft"), |
| 2005 | pytest.param("eagle", False, id="eagle"), |
| 2006 | pytest.param("eagle3", False, id="eagle3"), |
| 2007 | pytest.param("draft_model", True, id="moe-draft"), |
| 2008 | pytest.param("mtp", True, id="mtp"), |
| 2009 | pytest.param("dspark", True, id="dspark"), |
| 2010 | ], |
| 2011 | ) |
| 2012 | def test_draft_inherits_ep_only_for_moe( |
| 2013 | tmp_path: Path, |
| 2014 | tp_size: int, |
| 2015 | target_ep: bool, |
| 2016 | method: str, |
| 2017 | draft_is_moe: bool, |
| 2018 | ): |
| 2019 | """Validate final draft configs without loading weights or mocking validation.""" |
| 2020 | from transformers import LlamaConfig, MixtralConfig |
| 2021 | |
| 2022 | from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config |
| 2023 | |
| 2024 | common = dict( |
| 2025 | hidden_size=128, |
| 2026 | intermediate_size=256, |
| 2027 | num_hidden_layers=2, |
| 2028 | num_attention_heads=4, |
| 2029 | num_key_value_heads=2, |
| 2030 | vocab_size=128, |
| 2031 | max_position_embeddings=2048, |
| 2032 | ) |
| 2033 | if method in ("mtp", "dspark"): |
| 2034 | target_hf_config = DeepseekV4Config( |
| 2035 | architectures=["DeepseekV4ForCausalLM"], |
| 2036 | n_routed_experts=4, |
| 2037 | num_experts_per_tok=2, |
| 2038 | num_nextn_predict_layers=1, |
| 2039 | compress_ratios=[1, 1], |
| 2040 | head_dim=32, |
| 2041 | **common, |
| 2042 | ) |
| 2043 | else: |
| 2044 | target_hf_config = MixtralConfig( |
| 2045 | architectures=["MixtralForCausalLM"], num_local_experts=4, **common |
| 2046 | ) |
| 2047 | target_path = tmp_path / "target" |
| 2048 | draft_path = tmp_path / "draft" |
| 2049 | _write_json(target_path / "config.json", target_hf_config.to_dict()) |
| 2050 | _write_json( |
| 2051 | draft_path / "config.json", |
| 2052 | LlamaConfig(architectures=["LlamaForCausalLM"], **common).to_dict(), |
| 2053 | ) |
| 2054 | target_model_config = ModelConfig( |
| 2055 | model=str(target_path), tokenizer_mode="skip", max_model_len=2048 |
| 2056 | ) |
| 2057 | target_parallel_config = ParallelConfig( |
| 2058 | tensor_parallel_size=tp_size, |
| 2059 | enable_expert_parallel=target_ep, |
| 2060 | distributed_executor_backend="mp", |
| 2061 | ) |
| 2062 | target_model_config.verify_with_parallel_config(target_parallel_config) |
| 2063 | speculative_config = SpeculativeConfig( |
| 2064 | method=method, |
| 2065 | model=str(target_path if draft_is_moe else draft_path), |
| 2066 | num_speculative_tokens=1, |
| 2067 | target_model_config=target_model_config, |
| 2068 | target_parallel_config=target_parallel_config, |
| 2069 | ) |
| 2070 | |
| 2071 | assert speculative_config.method == method |
| 2072 | assert speculative_config.draft_model_config.is_moe is draft_is_moe |
| 2073 | assert speculative_config.draft_parallel_config.enable_expert_parallel is ( |
| 2074 | target_ep and draft_is_moe |
| 2075 | ) |
| 2076 | assert speculative_config.draft_parallel_config.tensor_parallel_size == tp_size |
| 2077 | assert target_parallel_config.enable_expert_parallel is target_ep |
| 2078 | assert target_parallel_config.tensor_parallel_size == tp_size |
| 2079 | |
| 2080 | |
| 2081 | @pytest.mark.skip_global_cleanup |
| 2082 | @pytest.mark.parametrize("target_ep", [False, True], ids=["ep-off", "ep-on"]) |
| 2083 | @pytest.mark.parametrize("pass_none", [False, True], ids=["omitted", "explicit-none"]) |
| 2084 | def test_draft_parallel_config_preserves_ep_without_model( |
| 2085 | target_ep: bool, pass_none: bool |
| 2086 | ): |
| 2087 | """Legacy callers without draft model information keep EP inheritance.""" |
| 2088 | target_parallel_config = ParallelConfig( |
| 2089 | tensor_parallel_size=2, |
| 2090 | enable_expert_parallel=target_ep, |
| 2091 | distributed_executor_backend="mp", |
| 2092 | ) |
| 2093 | if pass_none: |
| 2094 | draft_parallel_config = SpeculativeConfig.create_draft_parallel_config( |
| 2095 | target_parallel_config, 2, draft_model_config=None |
| 2096 | ) |
| 2097 | else: |
| 2098 | draft_parallel_config = SpeculativeConfig.create_draft_parallel_config( |
| 2099 | target_parallel_config, 2 |
| 2100 | ) |
| 2101 | |
| 2102 | assert draft_parallel_config.enable_expert_parallel is target_ep |
| 2103 | assert draft_parallel_config.tensor_parallel_size == 2 |
| 2104 | assert target_parallel_config.enable_expert_parallel is target_ep |
| 2105 | |
| 2106 | |
| 2107 | @pytest.mark.parametrize( |
| 2108 | ("method", "parallel_drafting", "expected_slots"), |
| 2109 | [ |
| 2110 | pytest.param("eagle3", False, 0, id="eagle3"), |
| 2111 | pytest.param("eagle3", True, 7, id="p-eagle"), |
| 2112 | pytest.param("dflash", True, 8, id="dflash"), |
| 2113 | pytest.param("dspark", True, 7, id="dspark"), |
| 2114 | pytest.param("mtp", False, 0, id="mtp"), |
| 2115 | pytest.param("ngram", False, 0, id="ngram"), |
| 2116 | pytest.param("draft_model", False, 1, id="draft-model"), |
| 2117 | pytest.param("draft_model", True, 8, id="pard"), |
| 2118 | ], |
| 2119 | ) |
| 2120 | def test_max_num_new_slots_for_drafting(method, parallel_drafting, expected_slots): |
| 2121 | speculative_config = SpeculativeConfig( |
| 2122 | model="ngram", |
| 2123 | num_speculative_tokens=8, |
| 2124 | ) |
| 2125 | speculative_config.method = method |
| 2126 | speculative_config.parallel_drafting = parallel_drafting |
| 2127 | |
| 2128 | assert speculative_config.max_num_new_slots_for_drafting == expected_slots |
| 2129 | |
| 2130 | |
| 2131 | @dataclass |
| 2132 | class _TestConfigFields: |
| 2133 | a: int |
| 2134 | b: dict = field(default_factory=dict) |
| 2135 | c: str = "default" |
| 2136 | |
| 2137 | |
| 2138 | def test_get_field(): |
| 2139 | b = get_field(_TestConfigFields, "b") |
| 2140 | assert isinstance(b, Field) |
| 2141 | assert b.default is MISSING |
| 2142 | assert b.default_factory is dict |
| 2143 | |
| 2144 | c = get_field(_TestConfigFields, "c") |
| 2145 | assert isinstance(c, Field) |
| 2146 | assert c.default == "default" |
| 2147 | assert c.default_factory is MISSING |
| 2148 | |
| 2149 | |
| 2150 | @dataclass |
| 2151 | class _TestNestedConfig: |
| 2152 | a: _TestConfigFields = field(default_factory=lambda: _TestConfigFields(a=0)) |
| 2153 | |
| 2154 | |
| 2155 | @dataclass |
| 2156 | class _TestDerivedConfigFields(_TestConfigFields): |
| 2157 | pass |
| 2158 | |
| 2159 | |
| 2160 | def test_update_config(): |
| 2161 | # Simple update |
| 2162 | config1 = _TestConfigFields(a=0) |
| 2163 | new_config1 = update_config(config1, {"a": 42}) |
| 2164 | assert new_config1.a == 42 |
| 2165 | # Nonexistent field |
| 2166 | with pytest.raises(ValueError, match=r"_TestConfigFields\.nonexistent"): |
| 2167 | new_config1 = update_config(config1, {"nonexistent": 1}) |
| 2168 | # Nested update with dataclass |
| 2169 | config2 = _TestNestedConfig() |
| 2170 | new_inner_config = _TestConfigFields(a=1, c="new_value") |
| 2171 | new_config2 = update_config(config2, {"a": new_inner_config}) |
| 2172 | assert new_config2.a == new_inner_config |
| 2173 | # Declared field type, not the live value's subtype, defines valid overrides |
| 2174 | config_with_derived = _TestNestedConfig(a=_TestDerivedConfigFields(a=0)) |
| 2175 | new_config2 = update_config(config_with_derived, {"a": new_inner_config}) |
| 2176 | assert new_config2.a is new_inner_config |
| 2177 | # Nested update with unrelated dataclass |
| 2178 | with pytest.raises(ValueError, match=r"_TestNestedConfig\.a"): |
| 2179 | update_config(config2, {"a": _TestNestedConfig()}) |
| 2180 | # Nested update with dict |
| 2181 | config3 = _TestNestedConfig() |
| 2182 | new_config3 = update_config(config3, {"a": {"c": "new_value"}}) |
| 2183 | assert new_config3.a.c == "new_value" |
| 2184 | # Nested update with invalid type |
| 2185 | with pytest.raises(ValueError, match=r"_TestNestedConfig\.a"): |
| 2186 | update_config(config3, {"a": "new_value"}) |
| 2187 | # Invalid nested field preserves its full path |
| 2188 | with pytest.raises(ValueError, match=r"_TestNestedConfig\.a\.nonexistent"): |
| 2189 | update_config(config3, {"a": {"nonexistent": 1}}) |
| 2190 | |
| 2191 | |
| 2192 | @pytest.mark.parametrize( |
| 2193 | ("model_id", "expected_runner_type", "expected_convert_type"), |
| 2194 | [ |
| 2195 | ("distilbert/distilgpt2", "generate", "none"), |
| 2196 | ("intfloat/multilingual-e5-small", "pooling", "none"), |
| 2197 | ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"), |
| 2198 | ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"), |
| 2199 | ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"), |
| 2200 | ("openai/whisper-small", "generate", "none"), |
| 2201 | ], |
| 2202 | ) |
| 2203 | def test_auto_runner(model_id, expected_runner_type, expected_convert_type): |
| 2204 | config = ModelConfig(model_id, runner="auto") |
| 2205 | |
| 2206 | assert config.runner_type == expected_runner_type |
| 2207 | assert config.convert_type == expected_convert_type |
| 2208 | |
| 2209 | |
| 2210 | @pytest.mark.parametrize( |
| 2211 | ("model_id", "expected_runner_type", "expected_convert_type"), |
| 2212 | [ |
| 2213 | ("distilbert/distilgpt2", "pooling", "embed"), |
| 2214 | ("intfloat/multilingual-e5-small", "pooling", "none"), |
| 2215 | ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"), |
| 2216 | ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "none"), |
| 2217 | ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "none"), |
| 2218 | ("openai/whisper-small", "pooling", "embed"), |
| 2219 | ], |
| 2220 | ) |
| 2221 | def test_pooling_runner(model_id, expected_runner_type, expected_convert_type): |
| 2222 | config = ModelConfig(model_id, runner="pooling") |
| 2223 | |
| 2224 | assert config.runner_type == expected_runner_type |
| 2225 | assert config.convert_type == expected_convert_type |
| 2226 | |
| 2227 | |
| 2228 | @pytest.mark.parametrize( |
| 2229 | ("model_id", "expected_runner_type", "expected_convert_type"), |
| 2230 | [ |
| 2231 | ("Qwen/Qwen2.5-1.5B-Instruct", "draft", "none"), |
| 2232 | ], |
| 2233 | ) |
| 2234 | def test_draft_runner(model_id, expected_runner_type, expected_convert_type): |
| 2235 | config = ModelConfig(model_id, runner="draft") |
| 2236 | |
| 2237 | assert config.runner_type == expected_runner_type |
| 2238 | assert config.convert_type == expected_convert_type |
| 2239 | |
| 2240 | |
| 2241 | MODEL_IDS_EXPECTED = [ |
| 2242 | ("Qwen/Qwen1.5-7B", 32768), |
| 2243 | ("mistralai/Mistral-7B-v0.1", 4096), |
| 2244 | ("mistralai/Mistral-7B-Instruct-v0.2", 32768), |
| 2245 | ] |
| 2246 | |
| 2247 | |
| 2248 | @pytest.mark.parametrize("model_id_expected", MODEL_IDS_EXPECTED) |
| 2249 | def test_disable_sliding_window(model_id_expected): |
| 2250 | model_id, expected = model_id_expected |
| 2251 | model_config = ModelConfig(model_id, disable_sliding_window=True) |
| 2252 | assert model_config.max_model_len == expected |
| 2253 | |
| 2254 | |
| 2255 | @pytest.mark.skipif( |
| 2256 | current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." |
| 2257 | ) |
| 2258 | def test_get_pooling_config(): |
| 2259 | model_id = "sentence-transformers/all-MiniLM-L12-v2" |
| 2260 | model_config = ModelConfig(model_id) |
| 2261 | |
| 2262 | assert model_config.pooler_config is not None |
| 2263 | assert model_config.pooler_config.use_activation |
| 2264 | assert model_config.pooler_config.seq_pooling_type == "MEAN" |
| 2265 | assert model_config.pooler_config.tok_pooling_type == "ALL" |
| 2266 | |
| 2267 | |
| 2268 | @pytest.mark.parametrize( |
| 2269 | ("pooling_module_type", "normalize_module_type", "pooling_config", "expected"), |
| 2270 | [ |
| 2271 | ( |
| 2272 | "sentence_transformers.models.Pooling", |
| 2273 | "sentence_transformers.models.Normalize", |
| 2274 | {"pooling_mode_mean_tokens": True}, |
| 2275 | {"use_activation": True, "seq_pooling_type": "MEAN"}, |
| 2276 | ), |
| 2277 | ( |
| 2278 | "sentence_transformers.sentence_transformer.modules.pooling.Pooling", |
| 2279 | "sentence_transformers.sentence_transformer.modules.normalize.Normalize", |
| 2280 | {"pooling_mode": "lasttoken"}, |
| 2281 | {"use_activation": True, "seq_pooling_type": "LAST"}, |
| 2282 | ), |
| 2283 | ( |
| 2284 | "sentence_transformers.sentence_transformer.modules.pooling.Pooling", |
| 2285 | "sentence_transformers.base.modules.normalize.Normalize", |
| 2286 | {"pooling_mode": "mean"}, |
| 2287 | {"use_activation": True, "seq_pooling_type": "MEAN"}, |
| 2288 | ), |
| 2289 | ], |
| 2290 | ) |
| 2291 | def test_get_pooling_config_supports_sentence_transformers_schemas( |
| 2292 | tmp_path, |
| 2293 | caplog, |
| 2294 | pooling_module_type, |
| 2295 | normalize_module_type, |
| 2296 | pooling_config, |
| 2297 | expected, |
| 2298 | ): |
| 2299 | modules = [ |
| 2300 | {"idx": 1, "name": "1", "path": "1_Pooling", "type": pooling_module_type}, |
| 2301 | { |
| 2302 | "idx": 2, |
| 2303 | "name": "2", |
| 2304 | "path": "2_Normalize", |
| 2305 | "type": normalize_module_type, |
| 2306 | }, |
| 2307 | ] |
| 2308 | _write_json(tmp_path / "modules.json", modules) |
| 2309 | _write_json(tmp_path / "1_Pooling" / "config.json", pooling_config) |
| 2310 | |
| 2311 | with caplog.at_level(logging.WARNING): |
| 2312 | config = get_pooling_config(str(tmp_path)) |
| 2313 | |
| 2314 | assert config == expected |
| 2315 | assert "Unable to determine Sentence Transformers pooling type" not in caplog.text |
| 2316 | |
| 2317 | |
| 2318 | def test_get_pooling_config_warns_when_pooling_mode_is_unknown(tmp_path, caplog): |
| 2319 | modules = [ |
| 2320 | { |
| 2321 | "idx": 1, |
| 2322 | "name": "1", |
| 2323 | "path": "1_Pooling", |
| 2324 | "type": "sentence_transformers.models.Pooling", |
| 2325 | } |
| 2326 | ] |
| 2327 | _write_json(tmp_path / "modules.json", modules) |
| 2328 | _write_json( |
| 2329 | tmp_path / "1_Pooling" / "config.json", |
| 2330 | {"pooling_mode": "unsupported"}, |
| 2331 | ) |
| 2332 | |
| 2333 | with caplog.at_level(logging.WARNING): |
| 2334 | config = get_pooling_config(str(tmp_path)) |
| 2335 | |
| 2336 | assert config == {"use_activation": False} |
| 2337 | assert "Unable to determine Sentence Transformers pooling type" in caplog.text |
| 2338 | |
| 2339 | |
| 2340 | @pytest.mark.parametrize( |
| 2341 | "dense_module_type", |
| 2342 | [ |
| 2343 | "sentence_transformers.models.Dense", |
| 2344 | "sentence_transformers.base.modules.dense.Dense", |
| 2345 | ], |
| 2346 | ) |
| 2347 | def test_get_dense_modules_supports_sentence_transformers_schemas( |
| 2348 | tmp_path, dense_module_type |
| 2349 | ): |
| 2350 | modules = [{"idx": 2, "name": "2", "path": "2_Dense", "type": dense_module_type}] |
| 2351 | dense_config = { |
| 2352 | "in_features": 768, |
| 2353 | "out_features": 3072, |
| 2354 | "bias": False, |
| 2355 | "activation_function": "torch.nn.modules.linear.Identity", |
| 2356 | } |
| 2357 | _write_json(tmp_path / "modules.json", modules) |
| 2358 | _write_json(tmp_path / "2_Dense" / "config.json", dense_config) |
| 2359 | |
| 2360 | assert try_get_dense_modules(str(tmp_path)) == [ |
| 2361 | {**dense_config, "folder": "2_Dense"} |
| 2362 | ] |
| 2363 | |
| 2364 | |
| 2365 | @pytest.mark.skipif( |
| 2366 | current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." |
| 2367 | ) |
| 2368 | def test_get_pooling_config_from_args(): |
| 2369 | model_id = "sentence-transformers/all-MiniLM-L12-v2" |
| 2370 | pooler_config = PoolerConfig(seq_pooling_type="CLS", use_activation=False) |
| 2371 | model_config = ModelConfig(model_id, pooler_config=pooler_config) |
| 2372 | |
| 2373 | assert asdict(model_config.pooler_config) == asdict(pooler_config) |
| 2374 | |
| 2375 | |
| 2376 | @pytest.mark.parametrize( |
| 2377 | ("model_id", "default_pooling_type", "pooling_type"), |
| 2378 | [ |
| 2379 | ("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", "LAST", "LAST"), # LLM |
| 2380 | ("intfloat/e5-small", "CLS", "MEAN"), # BertModel |
| 2381 | ], |
| 2382 | ) |
| 2383 | def test_default_seq_pooling_type(model_id, default_pooling_type, pooling_type): |
| 2384 | model_config = ModelConfig(model_id) |
| 2385 | assert model_config._model_info.default_seq_pooling_type == default_pooling_type |
| 2386 | assert model_config.pooler_config.seq_pooling_type == pooling_type |
| 2387 | |
| 2388 | |
| 2389 | @pytest.mark.parametrize( |
| 2390 | ("model_id", "default_pooling_type", "pooling_type"), |
| 2391 | [ |
| 2392 | ("Qwen/Qwen2.5-Math-RM-72B", "ALL", "ALL"), # reward |
| 2393 | ("Qwen/Qwen2.5-Math-PRM-7B", "STEP", "STEP"), # step reward |
| 2394 | ], |
| 2395 | ) |
| 2396 | def test_default_tok_pooling_type(model_id, default_pooling_type, pooling_type): |
| 2397 | model_config = ModelConfig(model_id) |
| 2398 | assert model_config._model_info.default_tok_pooling_type == default_pooling_type |
| 2399 | assert model_config.pooler_config.tok_pooling_type == pooling_type |
| 2400 | |
| 2401 | |
| 2402 | @pytest.mark.parametrize( |
| 2403 | ("model_id", "expected_is_moe_model"), |
| 2404 | [ |
| 2405 | ("RedHatAI/Qwen3-8B-speculator.eagle3", False), |
| 2406 | ("RedHatAI/Llama-3.1-8B-Instruct-NVFP4", False), |
| 2407 | ("RedHatAI/Llama-3.2-1B-FP8", False), |
| 2408 | ("RedHatAI/Mistral-Small-24B-Instruct-2501-quantized.w8a8", False), |
| 2409 | ("RedHatAI/gpt-oss-20b", True), |
| 2410 | ("RedHatAI/DeepSeek-V2.5-1210-FP8", True), |
| 2411 | ("RedHatAI/Llama-4-Scout-17B-16E-Instruct", True), |
| 2412 | ("RedHatAI/Mixtral-8x7B-Instruct-v0.1", True), |
| 2413 | ], |
| 2414 | ) |
| 2415 | def test_moe_model_detection(model_id, expected_is_moe_model): |
| 2416 | model_config = ModelConfig(model_id) |
| 2417 | # Just check that is_moe field exists and is a boolean |
| 2418 | assert model_config.is_moe == expected_is_moe_model |
| 2419 | |
| 2420 | |
| 2421 | @pytest.mark.parametrize( |
| 2422 | ("model_id", "quantized"), |
| 2423 | [ |
| 2424 | ("RedHatAI/Qwen3-8B-speculator.eagle3", False), |
| 2425 | ("RedHatAI/Llama-3.1-8B-Instruct-NVFP4", True), |
| 2426 | ("RedHatAI/Llama-3.2-1B-FP8", True), |
| 2427 | ("RedHatAI/Mistral-Small-24B-Instruct-2501-quantized.w8a8", True), |
| 2428 | ("RedHatAI/gpt-oss-20b", True), |
| 2429 | ("RedHatAI/DeepSeek-V2.5-1210-FP8", True), |
| 2430 | ("RedHatAI/Mixtral-8x7B-Instruct-v0.1", False), |
| 2431 | ], |
| 2432 | ) |
| 2433 | def test_is_quantized(model_id, quantized): |
| 2434 | model_config = ModelConfig(model_id) |
| 2435 | # Just check that quantized field exists and is a boolean |
| 2436 | assert model_config.is_quantized == quantized |
| 2437 | |
| 2438 | |
| 2439 | @pytest.mark.skipif( |
| 2440 | current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." |
| 2441 | ) |
| 2442 | def test_get_bert_tokenization_sentence_transformer_config(): |
| 2443 | model_id = "BAAI/bge-base-en-v1.5" |
| 2444 | bge_model_config = ModelConfig(model_id) |
| 2445 | |
| 2446 | bert_bge_model_config = bge_model_config._get_encoder_config() |
| 2447 | |
| 2448 | assert bert_bge_model_config["max_seq_length"] == 512 |
| 2449 | assert bert_bge_model_config["do_lower_case"] |
| 2450 | |
| 2451 | |
| 2452 | def test_rope_customization(): |
| 2453 | TEST_ROPE_PARAMETERS = { |
| 2454 | "rope_theta": 16_000_000.0, |
| 2455 | "rope_type": "dynamic", |
| 2456 | "factor": 2.0, |
| 2457 | } |
| 2458 | LLAMA_ROPE_PARAMETERS = {"rope_theta": 500000.0, "rope_type": "default"} |
| 2459 | LONGCHAT_ROPE_PARAMETERS = {"rope_type": "linear", "factor": 8.0} |
| 2460 | |
| 2461 | llama_model_config = ModelConfig("meta-llama/Meta-Llama-3-8B-Instruct") |
| 2462 | assert ( |
| 2463 | getattr(llama_model_config.hf_config, "rope_parameters", None) |
| 2464 | == LLAMA_ROPE_PARAMETERS |
| 2465 | ) |
| 2466 | assert llama_model_config.max_model_len == 8192 |
| 2467 | |
| 2468 | llama_model_config = ModelConfig( |
| 2469 | "meta-llama/Meta-Llama-3-8B-Instruct", |
| 2470 | hf_overrides={"rope_parameters": TEST_ROPE_PARAMETERS}, |
| 2471 | ) |
| 2472 | assert ( |
| 2473 | getattr(llama_model_config.hf_config, "rope_parameters", None) |
| 2474 | == TEST_ROPE_PARAMETERS |
| 2475 | ) |
| 2476 | assert llama_model_config.max_model_len == 16384 |
| 2477 | |
| 2478 | longchat_model_config = ModelConfig("lmsys/longchat-13b-16k") |
| 2479 | # Check if LONGCHAT_ROPE_PARAMETERS entries are in longchat_model_config |
| 2480 | assert all( |
| 2481 | longchat_model_config.hf_config.rope_parameters.get(key) == value |
| 2482 | for key, value in LONGCHAT_ROPE_PARAMETERS.items() |
| 2483 | ) |
| 2484 | assert longchat_model_config.max_model_len == 16384 |
| 2485 | |
| 2486 | longchat_model_config = ModelConfig( |
| 2487 | "lmsys/longchat-13b-16k", |
| 2488 | hf_overrides={ |
| 2489 | "rope_parameters": TEST_ROPE_PARAMETERS, |
| 2490 | }, |
| 2491 | ) |
| 2492 | assert ( |
| 2493 | getattr(longchat_model_config.hf_config, "rope_parameters", None) |
| 2494 | == TEST_ROPE_PARAMETERS |
| 2495 | ) |
| 2496 | assert longchat_model_config.max_model_len == 4096 |
| 2497 | |
| 2498 | |
| 2499 | def test_nested_hf_overrides(): |
| 2500 | """Test that nested hf_overrides work correctly.""" |
| 2501 | # Test with a model that has text_config |
| 2502 | model_config = ModelConfig( |
| 2503 | "Qwen/Qwen2-VL-2B-Instruct", |
| 2504 | hf_overrides={ |
| 2505 | "text_config": { |
| 2506 | "hidden_size": 1024, |
| 2507 | }, |
| 2508 | }, |
| 2509 | ) |
| 2510 | assert model_config.hf_config.text_config.hidden_size == 1024 |
| 2511 | |
| 2512 | # Test with deeply nested overrides |
| 2513 | model_config = ModelConfig( |
| 2514 | "Qwen/Qwen2-VL-2B-Instruct", |
| 2515 | hf_overrides={ |
| 2516 | "text_config": { |
| 2517 | "hidden_size": 2048, |
| 2518 | "num_attention_heads": 16, |
| 2519 | }, |
| 2520 | "vision_config": { |
| 2521 | "hidden_size": 512, |
| 2522 | }, |
| 2523 | }, |
| 2524 | ) |
| 2525 | assert model_config.hf_config.text_config.hidden_size == 2048 |
| 2526 | assert model_config.hf_config.text_config.num_attention_heads == 16 |
| 2527 | assert model_config.hf_config.vision_config.hidden_size == 512 |
| 2528 | |
| 2529 | |
| 2530 | def test_model_class_overrides_registers_target(): |
| 2531 | """`model_class_overrides` redirects an architecture to a custom class.""" |
| 2532 | from vllm.model_executor.models import ModelRegistry |
| 2533 | |
| 2534 | arch = "_TestModelClassOverrideArch" |
| 2535 | target = "vllm.model_executor.models.llama:LlamaForCausalLM" |
| 2536 | assert arch not in ModelRegistry.models |
| 2537 | |
| 2538 | model_config = ModelConfig( |
| 2539 | "facebook/opt-125m", |
| 2540 | model_class_overrides={arch: target}, |
| 2541 | ) |
| 2542 | try: |
| 2543 | # Accessing `.registry` is the chokepoint that applies the overrides; |
| 2544 | # it has already run during construction. |
| 2545 | registered = model_config.registry.models[arch] |
| 2546 | assert registered.module_name == "vllm.model_executor.models.llama" |
| 2547 | assert registered.class_name == "LlamaForCausalLM" |
| 2548 | # Idempotent: a second access does not re-register or error out. |
| 2549 | assert model_config.registry.models[arch] is registered |
| 2550 | finally: |
| 2551 | ModelRegistry.models.pop(arch, None) |
| 2552 | |
| 2553 | |
| 2554 | @pytest.mark.skipif( |
| 2555 | current_platform.is_rocm(), reason="Encoder Decoder models not supported on ROCm." |
| 2556 | ) |
| 2557 | @pytest.mark.parametrize( |
| 2558 | ("model_id", "is_encoder_decoder"), |
| 2559 | [ |
| 2560 | ("facebook/opt-125m", False), |
| 2561 | ("openai/whisper-tiny", True), |
| 2562 | ("meta-llama/Llama-3.2-1B-Instruct", False), |
| 2563 | ], |
| 2564 | ) |
| 2565 | def test_is_encoder_decoder(model_id, is_encoder_decoder): |
| 2566 | config = ModelConfig(model_id) |
| 2567 | |
| 2568 | assert config.is_encoder_decoder == is_encoder_decoder |
| 2569 | |
| 2570 | |
| 2571 | @pytest.mark.parametrize( |
| 2572 | ("model_id", "uses_mrope"), |
| 2573 | [ |
| 2574 | ("facebook/opt-125m", False), |
| 2575 | ("Qwen/Qwen2-VL-2B-Instruct", True), |
| 2576 | ], |
| 2577 | ) |
| 2578 | def test_uses_mrope(model_id, uses_mrope): |
| 2579 | config = ModelConfig(model_id) |
| 2580 | |
| 2581 | assert config.uses_mrope == uses_mrope |
| 2582 | |
| 2583 | |
| 2584 | def test_generation_config_loading(): |
| 2585 | model_id = "Qwen/Qwen2.5-1.5B-Instruct" |
| 2586 | |
| 2587 | # When set generation_config to "vllm", the default generation config |
| 2588 | # will not be loaded. |
| 2589 | model_config = ModelConfig(model_id, generation_config="vllm") |
| 2590 | assert model_config.get_diff_sampling_param() == {} |
| 2591 | |
| 2592 | # When set generation_config to "auto", the default generation config |
| 2593 | # should be loaded. |
| 2594 | model_config = ModelConfig(model_id, generation_config="auto") |
| 2595 | |
| 2596 | correct_generation_config = { |
| 2597 | "repetition_penalty": 1.1, |
| 2598 | "temperature": 0.7, |
| 2599 | "top_p": 0.8, |
| 2600 | "top_k": 20, |
| 2601 | } |
| 2602 | |
| 2603 | assert model_config.get_diff_sampling_param() == correct_generation_config |
| 2604 | |
| 2605 | # The generation config could be overridden by the user. |
| 2606 | override_generation_config = {"temperature": 0.5, "top_k": 5} |
| 2607 | |
| 2608 | model_config = ModelConfig( |
| 2609 | model_id, |
| 2610 | generation_config="auto", |
| 2611 | override_generation_config=override_generation_config, |
| 2612 | ) |
| 2613 | |
| 2614 | override_result = correct_generation_config.copy() |
| 2615 | override_result.update(override_generation_config) |
| 2616 | |
| 2617 | assert model_config.get_diff_sampling_param() == override_result |
| 2618 | |
| 2619 | # When generation_config is set to "vllm" and override_generation_config |
| 2620 | # is set, the override_generation_config should be used directly. |
| 2621 | model_config = ModelConfig( |
| 2622 | model_id, |
| 2623 | generation_config="vllm", |
| 2624 | override_generation_config=override_generation_config, |
| 2625 | ) |
| 2626 | |
| 2627 | assert model_config.get_diff_sampling_param() == override_generation_config |
| 2628 | |
| 2629 | |
| 2630 | @pytest.mark.parametrize( |
| 2631 | "pt_load_map_location", |
| 2632 | [ |
| 2633 | DEVICE_TYPE, |
| 2634 | {"": DEVICE_TYPE}, |
| 2635 | ], |
| 2636 | ) |
| 2637 | def test_load_config_pt_load_map_location(pt_load_map_location): |
| 2638 | load_config = LoadConfig(pt_load_map_location=pt_load_map_location) |
| 2639 | config = VllmConfig(load_config=load_config) |
| 2640 | |
| 2641 | assert config.load_config.pt_load_map_location == pt_load_map_location |
| 2642 | |
| 2643 | |
| 2644 | @pytest.mark.parametrize( |
| 2645 | ("model_id", "max_model_len", "expected_max_len", "should_raise"), |
| 2646 | [ |
| 2647 | ("BAAI/bge-reranker-base", None, 512, False), |
| 2648 | ("BAAI/bge-reranker-base", 256, 256, False), |
| 2649 | ("BAAI/bge-reranker-base", 513, 512, True), |
| 2650 | ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", None, 131072, False), |
| 2651 | ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", 131073, 131072, True), |
| 2652 | ], |
| 2653 | ) |
| 2654 | def test_get_and_verify_max_len( |
| 2655 | model_id, max_model_len, expected_max_len, should_raise |
| 2656 | ): |
| 2657 | """Test get_and_verify_max_len with different configurations.""" |
| 2658 | model_config = ModelConfig(model_id) |
| 2659 | |
| 2660 | if should_raise: |
| 2661 | with pytest.raises(ValueError): |
| 2662 | model_config.get_and_verify_max_len(max_model_len) |
| 2663 | else: |
| 2664 | actual_max_len = model_config.get_and_verify_max_len(max_model_len) |
| 2665 | assert actual_max_len == expected_max_len |
| 2666 | |
| 2667 | |
| 2668 | @pytest.mark.parametrize("max_model_len", [None, 1024]) |
| 2669 | @pytest.mark.parametrize( |
| 2670 | ("rope_parameters", "expected_max_len"), |
| 2671 | [ |
| 2672 | ({"rope_type": "default"}, 4096), |
| 2673 | ({"rope_type": "linear", "factor": 2.0}, 8192), |
| 2674 | ({"rope_type": "longrope"}, 2048), |
| 2675 | ], |
| 2676 | ) |
| 2677 | def test_get_and_verify_max_len_with_nope_layers( |
| 2678 | max_model_len, rope_parameters, expected_max_len |
| 2679 | ): |
| 2680 | """NoPE layers do not prevent deriving or scaling the context length.""" |
| 2681 | from transformers import PreTrainedConfig |
| 2682 | |
| 2683 | from vllm.config.model import _get_and_verify_max_len |
| 2684 | from vllm.transformers_utils.model_arch_config_convertor import ( |
| 2685 | ModelArchConfigConvertorBase, |
| 2686 | ) |
| 2687 | |
| 2688 | hf_config = PreTrainedConfig( |
| 2689 | max_position_embeddings=4096, |
| 2690 | original_max_position_embeddings=2048, |
| 2691 | ) |
| 2692 | hf_config.rope_parameters = { |
| 2693 | "full_attention": None, |
| 2694 | "sliding_attention": rope_parameters, |
| 2695 | } |
| 2696 | model_arch_config = ModelArchConfigConvertorBase(hf_config, hf_config).convert() |
| 2697 | |
| 2698 | actual_max_len = _get_and_verify_max_len( |
| 2699 | hf_config=hf_config, |
| 2700 | model_arch_config=model_arch_config, |
| 2701 | tokenizer_config=None, |
| 2702 | max_model_len=max_model_len, |
| 2703 | disable_sliding_window=False, |
| 2704 | sliding_window=None, |
| 2705 | ) |
| 2706 | |
| 2707 | assert actual_max_len == (max_model_len or expected_max_len) |
| 2708 | assert hf_config.rope_parameters["full_attention"] is None |
| 2709 | |
| 2710 | |
| 2711 | @pytest.mark.parametrize( |
| 2712 | ("rope_type", "factor", "expected_max_len"), |
| 2713 | [ |
| 2714 | # TeleChat3-36B-Thinking: 32768 already scaled from 8192 by 4 |
| 2715 | ("yarn", 4.0, 32768), |
| 2716 | # sarvam-105b: declares factor 40 but only serves 131072 of it |
| 2717 | ("deepseek_yarn", 40.0, 32768), |
| 2718 | ("deepseek_llama_scaling", 40.0, 32768), |
| 2719 | # Non-YaRN scaling still multiplies |
| 2720 | ("linear", 4.0, 131072), |
| 2721 | ], |
| 2722 | ) |
| 2723 | def test_get_and_verify_max_len_yarn_is_already_scaled( |
| 2724 | rope_type, factor, expected_max_len |
| 2725 | ): |
| 2726 | """YaRN variants must not re-apply `factor` to max_position_embeddings. |
| 2727 | |
| 2728 | Transformers treats max_position_embeddings as the final context length |
| 2729 | for every YaRN variant, so scaling it again overstates the limit and lets |
| 2730 | requests past the end of the cos/sin cache. |
| 2731 | """ |
| 2732 | from transformers import PreTrainedConfig |
| 2733 | |
| 2734 | from vllm.config.model import _get_and_verify_max_len |
| 2735 | from vllm.transformers_utils.model_arch_config_convertor import ( |
| 2736 | ModelArchConfigConvertorBase, |
| 2737 | ) |
| 2738 | |
| 2739 | hf_config = PreTrainedConfig(max_position_embeddings=32768) |
| 2740 | hf_config.rope_parameters = { |
| 2741 | "rope_type": rope_type, |
| 2742 | "factor": factor, |
| 2743 | "original_max_position_embeddings": 8192, |
| 2744 | } |
| 2745 | model_arch_config = ModelArchConfigConvertorBase(hf_config, hf_config).convert() |
| 2746 | |
| 2747 | actual_max_len = _get_and_verify_max_len( |
| 2748 | hf_config=hf_config, |
| 2749 | model_arch_config=model_arch_config, |
| 2750 | tokenizer_config=None, |
| 2751 | max_model_len=None, |
| 2752 | disable_sliding_window=False, |
| 2753 | sliding_window=None, |
| 2754 | ) |
| 2755 | |
| 2756 | assert actual_max_len == expected_max_len |
| 2757 | |
| 2758 | |
| 2759 | class MockConfig: |
| 2760 | """Simple mock object for testing maybe_pull_model_tokenizer_for_runai.""" |
| 2761 | |
| 2762 | def __init__(self, model: str, tokenizer: str): |
| 2763 | self.model = model |
| 2764 | self.tokenizer = tokenizer |
| 2765 | self.model_weights = None |
| 2766 | |
| 2767 | |
| 2768 | @pytest.mark.parametrize( |
| 2769 | "s3_url", |
| 2770 | [ |
| 2771 | "s3://example-bucket-1/model/", |
| 2772 | "s3://example-bucket-2/model/", |
| 2773 | ], |
| 2774 | ) |
| 2775 | @patch("vllm.transformers_utils.runai_utils.ObjectStorageModel.pull_files") |
| 2776 | def test_s3_url_model_tokenizer_paths(mock_pull_files, s3_url): |
| 2777 | """Test that S3 URLs create deterministic local directories for model and |
| 2778 | tokenizer.""" |
| 2779 | # Mock pull_files to avoid actually downloading files during tests |
| 2780 | mock_pull_files.return_value = None |
| 2781 | |
| 2782 | # Create first mock and run the method |
| 2783 | config1 = MockConfig(model=s3_url, tokenizer=s3_url) |
| 2784 | ModelConfig.maybe_pull_model_tokenizer_for_runai(config1, s3_url, s3_url) |
| 2785 | |
| 2786 | # Check that model and tokenizer point to existing directories |
| 2787 | assert os.path.exists(config1.model), ( |
| 2788 | f"Model directory does not exist: {config1.model}" |
| 2789 | ) |
| 2790 | assert os.path.isdir(config1.model), ( |
| 2791 | f"Model path is not a directory: {config1.model}" |
| 2792 | ) |
| 2793 | assert os.path.exists(config1.tokenizer), ( |
| 2794 | f"Tokenizer directory does not exist: {config1.tokenizer}" |
| 2795 | ) |
| 2796 | assert os.path.isdir(config1.tokenizer), ( |
| 2797 | f"Tokenizer path is not a directory: {config1.tokenizer}" |
| 2798 | ) |
| 2799 | |
| 2800 | # Verify that the paths are different from the original S3 URL |
| 2801 | assert config1.model != s3_url, "Model path should be converted to local directory" |
| 2802 | assert config1.tokenizer != s3_url, ( |
| 2803 | "Tokenizer path should be converted to local directory" |
| 2804 | ) |
| 2805 | |
| 2806 | # Store the original paths |
| 2807 | created_model_dir = config1.model |
| 2808 | create_tokenizer_dir = config1.tokenizer |
| 2809 | |
| 2810 | # Create a new mock and run the method with the same S3 URL |
| 2811 | config2 = MockConfig(model=s3_url, tokenizer=s3_url) |
| 2812 | ModelConfig.maybe_pull_model_tokenizer_for_runai(config2, s3_url, s3_url) |
| 2813 | |
| 2814 | # Check that the new directories exist |
| 2815 | assert os.path.exists(config2.model), ( |
| 2816 | f"Model directory does not exist: {config2.model}" |
| 2817 | ) |
| 2818 | assert os.path.isdir(config2.model), ( |
| 2819 | f"Model path is not a directory: {config2.model}" |
| 2820 | ) |
| 2821 | assert os.path.exists(config2.tokenizer), ( |
| 2822 | f"Tokenizer directory does not exist: {config2.tokenizer}" |
| 2823 | ) |
| 2824 | assert os.path.isdir(config2.tokenizer), ( |
| 2825 | f"Tokenizer path is not a directory: {config2.tokenizer}" |
| 2826 | ) |
| 2827 | |
| 2828 | # Verify that the paths are deterministic (same as before) |
| 2829 | assert config2.model == created_model_dir, ( |
| 2830 | f"Model paths are not deterministic. " |
| 2831 | f"Original: {created_model_dir}, New: {config2.model}" |
| 2832 | ) |
| 2833 | assert config2.tokenizer == create_tokenizer_dir, ( |
| 2834 | f"Tokenizer paths are not deterministic. " |
| 2835 | f"Original: {create_tokenizer_dir}, New: {config2.tokenizer}" |
| 2836 | ) |
| 2837 | |
| 2838 | |
| 2839 | @patch("vllm.transformers_utils.runai_utils.ObjectStorageModel.pull_files") |
| 2840 | def test_s3_url_different_models_create_different_directories(mock_pull_files): |
| 2841 | """Test that different S3 URLs create different local directories.""" |
| 2842 | # Mock pull_files to avoid actually downloading files during tests |
| 2843 | mock_pull_files.return_value = None |
| 2844 | |
| 2845 | s3_url1 = "s3://example-bucket-1/model/" |
| 2846 | s3_url2 = "s3://example-bucket-2/model/" |
| 2847 | |
| 2848 | # Create mocks with different S3 URLs and run the method |
| 2849 | config1 = MockConfig(model=s3_url1, tokenizer=s3_url1) |
| 2850 | ModelConfig.maybe_pull_model_tokenizer_for_runai(config1, s3_url1, s3_url1) |
| 2851 | |
| 2852 | config2 = MockConfig(model=s3_url2, tokenizer=s3_url2) |
| 2853 | ModelConfig.maybe_pull_model_tokenizer_for_runai(config2, s3_url2, s3_url2) |
| 2854 | |
| 2855 | # Verify that different URLs produce different directories |
| 2856 | assert config1.model != config2.model, ( |
| 2857 | f"Different S3 URLs should create different model directories. " |
| 2858 | f"URL1 model: {config1.model}, URL2 model: {config2.model}" |
| 2859 | ) |
| 2860 | assert config1.tokenizer != config2.tokenizer, ( |
| 2861 | f"Different S3 URLs should create different tokenizer directories. " |
| 2862 | f"URL1 tokenizer: {config1.tokenizer}, " |
| 2863 | f"URL2 tokenizer: {config2.tokenizer}" |
| 2864 | ) |
| 2865 | |
| 2866 | # Verify that both sets of directories exist |
| 2867 | assert os.path.exists(config1.model) and os.path.isdir(config1.model) |
| 2868 | assert os.path.exists(config1.tokenizer) and os.path.isdir(config1.tokenizer) |
| 2869 | assert os.path.exists(config2.model) and os.path.isdir(config2.model) |
| 2870 | assert os.path.exists(config2.tokenizer) and os.path.isdir(config2.tokenizer) |
| 2871 | |
| 2872 | |
| 2873 | @patch("vllm.transformers_utils.runai_utils.ObjectStorageModel.pull_files") |
| 2874 | def test_s3_url_different_model_and_tokenizer(mock_pull_files): |
| 2875 | """Test that when model and tokenizer are different cloud URIs, |
| 2876 | pull_files receives the correct URI for each.""" |
| 2877 | mock_pull_files.return_value = None |
| 2878 | |
| 2879 | model_url = "s3://bucket/model/" |
| 2880 | tokenizer_url = "s3://bucket/tokenizer/" |
| 2881 | |
| 2882 | config = MockConfig(model=model_url, tokenizer=tokenizer_url) |
| 2883 | ModelConfig.maybe_pull_model_tokenizer_for_runai(config, model_url, tokenizer_url) |
| 2884 | |
| 2885 | # pull_files should be called twice: once for model, once for tokenizer |
| 2886 | assert mock_pull_files.call_count == 2 |
| 2887 | # First call: model URI with allow_pattern |
| 2888 | assert mock_pull_files.call_args_list[0][0][0] == model_url |
| 2889 | # Second call: tokenizer URI with ignore_pattern |
| 2890 | assert mock_pull_files.call_args_list[1][0][0] == tokenizer_url |
| 2891 | |
| 2892 | |
| 2893 | @pytest.mark.parametrize( |
| 2894 | ("model_id", "expected_attn_type", "expected_result", "reason"), |
| 2895 | [ |
| 2896 | # pooling models |
| 2897 | ( |
| 2898 | "jason9693/Qwen2.5-1.5B-apeach", |
| 2899 | "decoder", |
| 2900 | True, |
| 2901 | "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501 |
| 2902 | ), |
| 2903 | ( |
| 2904 | "Qwen/Qwen3-Embedding-0.6B", |
| 2905 | "decoder", |
| 2906 | True, |
| 2907 | "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501 |
| 2908 | ), |
| 2909 | ( |
| 2910 | "Qwen/Qwen2.5-Math-PRM-7B", |
| 2911 | "decoder", |
| 2912 | False, |
| 2913 | "Pooling models with causal attn and LAST/STEP pooling do not support chunked prefill.", # noqa: E501 |
| 2914 | ), |
| 2915 | ( |
| 2916 | "internlm/internlm2-1_8b-reward", |
| 2917 | "decoder", |
| 2918 | True, |
| 2919 | "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501 |
| 2920 | ), |
| 2921 | ( |
| 2922 | "BAAI/bge-base-en", |
| 2923 | "encoder_only", |
| 2924 | False, |
| 2925 | "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501 |
| 2926 | ), |
| 2927 | ( |
| 2928 | "boltuix/NeuroBERT-NER", |
| 2929 | "encoder_only", |
| 2930 | False, |
| 2931 | "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501 |
| 2932 | ), |
| 2933 | ( |
| 2934 | "papluca/xlm-roberta-base-language-detection", |
| 2935 | "encoder_only", |
| 2936 | False, |
| 2937 | "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501 |
| 2938 | ), |
| 2939 | ( |
| 2940 | "Alibaba-NLP/gte-Qwen2-1.5B-instruct", |
| 2941 | "encoder_only", |
| 2942 | False, |
| 2943 | "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501 |
| 2944 | ), |
| 2945 | ( |
| 2946 | "intfloat/e5-small", |
| 2947 | "encoder_only", |
| 2948 | False, |
| 2949 | "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501 |
| 2950 | ), |
| 2951 | # multimodal models |
| 2952 | ( |
| 2953 | "openai/clip-vit-base-patch32", |
| 2954 | "decoder", |
| 2955 | True, |
| 2956 | "Pooling models with causal attn and LAST/ALL pooling support chunked prefill.", # noqa: E501 |
| 2957 | ), |
| 2958 | ( |
| 2959 | "google/siglip-base-patch16-224", |
| 2960 | "encoder_only", |
| 2961 | False, |
| 2962 | "Pooling models with bidirectional attn do not support chunked prefill.", # noqa: E501 |
| 2963 | ), |
| 2964 | # generate models |
| 2965 | ( |
| 2966 | "Qwen/Qwen3-0.6B", |
| 2967 | "decoder", |
| 2968 | True, |
| 2969 | "Generative models support chunked prefill.", # noqa: E501 |
| 2970 | ), |
| 2971 | ( |
| 2972 | "Qwen/Qwen3-Next-80B-A3B-Instruct", |
| 2973 | "hybrid", |
| 2974 | True, |
| 2975 | "Generative models support chunked prefill.", # noqa: E501 |
| 2976 | ), |
| 2977 | ( |
| 2978 | "ibm-granite/granite-4.0-h-small", |
| 2979 | "hybrid", |
| 2980 | True, |
| 2981 | "Generative models support chunked prefill.", # noqa: E501 |
| 2982 | ), |
| 2983 | ( |
| 2984 | "state-spaces/mamba-130m-hf", |
| 2985 | "attention_free", |
| 2986 | True, |
| 2987 | "Generative models support chunked prefill.", # noqa: E501 |
| 2988 | ), |
| 2989 | # encoder_decoder models |
| 2990 | ( |
| 2991 | "openai/whisper-small", |
| 2992 | "encoder_decoder", |
| 2993 | False, |
| 2994 | "Encoder decoder models do not support chunked prefill.", # noqa: E501 |
| 2995 | ), |
| 2996 | ], |
| 2997 | ) |
| 2998 | def test_is_chunked_prefill_supported( |
| 2999 | model_id: str, |
| 3000 | expected_attn_type: str, |
| 3001 | expected_result: bool, |
| 3002 | reason: str, |
| 3003 | caplog_vllm, |
| 3004 | ): |
| 3005 | model_config = ModelConfig(model_id, trust_remote_code=True) |
| 3006 | assert model_config.attn_type == expected_attn_type |
| 3007 | with caplog_vllm.at_level(level=logging.DEBUG, logger="vllm"): |
| 3008 | assert model_config.is_chunked_prefill_supported == expected_result |
| 3009 | assert reason in caplog_vllm.text |
| 3010 | |
| 3011 | |
| 3012 | @pytest.mark.parametrize( |
| 3013 | ("model_id", "expected_attn_type", "expected_result", "reason"), |
| 3014 | [ |
| 3015 | # pooling models |
| 3016 | ( |
| 3017 | "jason9693/Qwen2.5-1.5B-apeach", |
| 3018 | "decoder", |
| 3019 | True, |
| 3020 | "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501 |
| 3021 | ), |
| 3022 | ( |
| 3023 | "Qwen/Qwen3-Embedding-0.6B", |
| 3024 | "decoder", |
| 3025 | True, |
| 3026 | "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501 |
| 3027 | ), |
| 3028 | ( |
| 3029 | "Qwen/Qwen2.5-Math-PRM-7B", |
| 3030 | "decoder", |
| 3031 | False, |
| 3032 | "Pooling models with causal attn and LAST/STEP pooling do not support prefix caching.", # noqa: E501 |
| 3033 | ), |
| 3034 | ( |
| 3035 | "internlm/internlm2-1_8b-reward", |
| 3036 | "decoder", |
| 3037 | True, |
| 3038 | "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501 |
| 3039 | ), |
| 3040 | ( |
| 3041 | "BAAI/bge-base-en", |
| 3042 | "encoder_only", |
| 3043 | False, |
| 3044 | "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501 |
| 3045 | ), |
| 3046 | ( |
| 3047 | "boltuix/NeuroBERT-NER", |
| 3048 | "encoder_only", |
| 3049 | False, |
| 3050 | "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501 |
| 3051 | ), |
| 3052 | ( |
| 3053 | "papluca/xlm-roberta-base-language-detection", |
| 3054 | "encoder_only", |
| 3055 | False, |
| 3056 | "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501 |
| 3057 | ), |
| 3058 | ( |
| 3059 | "Alibaba-NLP/gte-Qwen2-1.5B-instruct", |
| 3060 | "encoder_only", |
| 3061 | False, |
| 3062 | "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501 |
| 3063 | ), |
| 3064 | ( |
| 3065 | "intfloat/e5-small", |
| 3066 | "encoder_only", |
| 3067 | False, |
| 3068 | "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501 |
| 3069 | ), |
| 3070 | # multimodal models |
| 3071 | ( |
| 3072 | "openai/clip-vit-base-patch32", |
| 3073 | "decoder", |
| 3074 | True, |
| 3075 | "Pooling models with causal attn and LAST/ALL pooling support prefix caching.", # noqa: E501 |
| 3076 | ), |
| 3077 | ( |
| 3078 | "google/siglip-base-patch16-224", |
| 3079 | "encoder_only", |
| 3080 | False, |
| 3081 | "Pooling models with bidirectional attn do not support prefix caching.", # noqa: E501 |
| 3082 | ), |
| 3083 | # generate models |
| 3084 | ( |
| 3085 | "Qwen/Qwen3-0.6B", |
| 3086 | "decoder", |
| 3087 | True, |
| 3088 | "Generative models support prefix caching.", # noqa: E501 |
| 3089 | ), |
| 3090 | ( |
| 3091 | "Qwen/Qwen3-Next-80B-A3B-Instruct", |
| 3092 | "hybrid", |
| 3093 | True, |
| 3094 | "Generative hybrid models support prefix caching.", # noqa: E501 |
| 3095 | ), |
| 3096 | ( |
| 3097 | "ibm-granite/granite-4.0-h-small", |
| 3098 | "hybrid", |
| 3099 | True, |
| 3100 | "Generative hybrid models support prefix caching.", # noqa: E501 |
| 3101 | ), |
| 3102 | ( |
| 3103 | "state-spaces/mamba-130m-hf", |
| 3104 | "attention_free", |
| 3105 | False, |
| 3106 | "Attention free models do not support prefix caching since the feature is still experimental.", # noqa: E501 |
| 3107 | ), |
| 3108 | # encoder_decoder models |
| 3109 | ( |
| 3110 | "openai/whisper-small", |
| 3111 | "encoder_decoder", |
| 3112 | False, |
| 3113 | "Encoder decoder models do not support prefix caching.", # noqa: E501 |
| 3114 | ), |
| 3115 | ], |
| 3116 | ) |
| 3117 | def test_is_prefix_caching_supported( |
| 3118 | model_id: str, |
| 3119 | expected_attn_type: str, |
| 3120 | expected_result: bool, |
| 3121 | reason: str, |
| 3122 | caplog_vllm, |
| 3123 | ): |
| 3124 | model_config = ModelConfig(model_id, trust_remote_code=True) |
| 3125 | assert model_config.attn_type == expected_attn_type |
| 3126 | with caplog_vllm.at_level(level=logging.DEBUG, logger="vllm"): |
| 3127 | assert model_config.is_prefix_caching_supported == expected_result |
| 3128 | assert reason in caplog_vllm.text |
| 3129 | |
| 3130 | |
| 3131 | @pytest.mark.parametrize( |
| 3132 | ("backend", "custom_ops", "expected"), |
| 3133 | [ |
| 3134 | ("eager", [], True), |
| 3135 | ("eager", ["+fused_layernorm"], True), |
| 3136 | ("eager", ["all", "-fused_layernorm"], False), |
| 3137 | ("inductor", [], False), |
| 3138 | ("inductor", ["none", "+fused_layernorm"], True), |
| 3139 | ("inductor", ["none", "-fused_layernorm"], False), |
| 3140 | ], |
| 3141 | ) |
| 3142 | def test_is_custom_op_enabled(backend: str, custom_ops: list[str], expected: bool): |
| 3143 | """Test that is_custom_op_enabled works correctly.""" |
| 3144 | config = VllmConfig( |
| 3145 | compilation_config=CompilationConfig(backend=backend, custom_ops=custom_ops) |
| 3146 | ) |
| 3147 | assert config.compilation_config.is_custom_op_enabled("fused_layernorm") is expected |
| 3148 | |
| 3149 | |
| 3150 | def test_vllm_config_defaults_are_none(): |
| 3151 | """Verify that optimization-level defaults are None when not set by user.""" |
| 3152 | # Test all optimization levels to ensure defaults work correctly |
| 3153 | for opt_level in OptimizationLevel: |
| 3154 | config = object.__new__(VllmConfig) |
| 3155 | config.compilation_config = CompilationConfig() |
| 3156 | config.optimization_level = opt_level |
| 3157 | config.model_config = None |
| 3158 | |
| 3159 | # Use the global optimization level defaults |
| 3160 | default_config = OPTIMIZATION_LEVEL_TO_CONFIG[opt_level] |
| 3161 | |
| 3162 | # Verify that all pass_config values are None before defaults are applied |
| 3163 | for pass_k in default_config["compilation_config"]["pass_config"]: |
| 3164 | assert getattr(config.compilation_config.pass_config, pass_k) is None |
| 3165 | |
| 3166 | # Verify that other config values are None before defaults are applied |
| 3167 | for k in default_config["compilation_config"]: |
| 3168 | if k != "pass_config": |
| 3169 | assert getattr(config.compilation_config, k) is None |
| 3170 | |
| 3171 | |
| 3172 | def test_validate_mamba_align_subblock_prefill(): |
| 3173 | """Align mode permits configured prefill chunks smaller than a block.""" |
| 3174 | config = SimpleNamespace( |
| 3175 | cache_config=SimpleNamespace( |
| 3176 | block_size=11392, |
| 3177 | mamba_cache_mode="align", |
| 3178 | ), |
| 3179 | parallel_config=SimpleNamespace( |
| 3180 | decode_context_parallel_size=1, |
| 3181 | ), |
| 3182 | scheduler_config=SimpleNamespace( |
| 3183 | max_num_batched_tokens=8192, |
| 3184 | long_prefill_token_threshold=4096, |
| 3185 | disable_chunked_mm_input=False, |
| 3186 | ), |
| 3187 | kv_transfer_config=None, |
| 3188 | ) |
| 3189 | |
| 3190 | VllmConfig.validate_block_size(config) |
| 3191 | |
| 3192 | |
| 3193 | @pytest.mark.parametrize( |
| 3194 | ("model_id", "compilation_config", "optimization_level"), |
| 3195 | [ |
| 3196 | ( |
| 3197 | None, |
| 3198 | CompilationConfig(backend="eager", custom_ops=["+quant_fp8"]), |
| 3199 | OptimizationLevel.O0, |
| 3200 | ), |
| 3201 | (None, CompilationConfig(), OptimizationLevel.O0), |
| 3202 | (None, CompilationConfig(), OptimizationLevel.O1), |
| 3203 | (None, CompilationConfig(), OptimizationLevel.O2), |
| 3204 | (None, CompilationConfig(), OptimizationLevel.O3), |
| 3205 | ( |
| 3206 | "RedHatAI/Qwen3-8B-speculator.eagle3", |
| 3207 | CompilationConfig(backend="inductor", custom_ops=["+quant_fp8"]), |
| 3208 | OptimizationLevel.O2, |
| 3209 | ), |
| 3210 | ( |
| 3211 | "RedHatAI/Qwen3-8B-speculator.eagle3", |
| 3212 | CompilationConfig(), |
| 3213 | OptimizationLevel.O0, |
| 3214 | ), |
| 3215 | ( |
| 3216 | "RedHatAI/Qwen3-8B-speculator.eagle3", |
| 3217 | CompilationConfig(), |
| 3218 | OptimizationLevel.O1, |
| 3219 | ), |
| 3220 | ( |
| 3221 | "RedHatAI/Qwen3-8B-speculator.eagle3", |
| 3222 | CompilationConfig(), |
| 3223 | OptimizationLevel.O2, |
| 3224 | ), |
| 3225 | ( |
| 3226 | "RedHatAI/Qwen3-8B-speculator.eagle3", |
| 3227 | CompilationConfig(), |
| 3228 | OptimizationLevel.O3, |
| 3229 | ), |
| 3230 | ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O0), |
| 3231 | ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O1), |
| 3232 | ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O2), |
| 3233 | ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O3), |
| 3234 | ], |
| 3235 | ) |
| 3236 | def test_vllm_config_defaults(model_id, compilation_config, optimization_level): |
| 3237 | """Test that optimization-level defaults are correctly applied.""" |
| 3238 | model_config = None |
| 3239 | if model_id is not None: |
| 3240 | model_config = ModelConfig(model_id) |
| 3241 | vllm_config = VllmConfig( |
| 3242 | model_config=model_config, |
| 3243 | compilation_config=compilation_config, |
| 3244 | optimization_level=optimization_level, |
| 3245 | ) |
| 3246 | else: |
| 3247 | vllm_config = VllmConfig( |
| 3248 | compilation_config=compilation_config, |
| 3249 | optimization_level=optimization_level, |
| 3250 | ) |
| 3251 | # Use the global optimization level defaults |
| 3252 | default_config = OPTIMIZATION_LEVEL_TO_CONFIG[optimization_level] |
| 3253 | |
| 3254 | # Verify pass_config defaults (nested under compilation_config) |
| 3255 | pass_config_dict = default_config["compilation_config"]["pass_config"] |
| 3256 | for pass_k, pass_v in pass_config_dict.items(): |
| 3257 | actual = getattr(vllm_config.compilation_config.pass_config, pass_k) |
| 3258 | expected = pass_v(vllm_config) if callable(pass_v) else pass_v |
| 3259 | assert actual == expected, ( |
| 3260 | f"pass_config.{pass_k}: expected {expected}, got {actual}" |
| 3261 | ) |
| 3262 | |
| 3263 | # Verify other compilation_config defaults |
| 3264 | compilation_config_dict = default_config["compilation_config"] |
| 3265 | for k, v in compilation_config_dict.items(): |
| 3266 | if k == "pass_config": |
| 3267 | continue |
| 3268 | actual = getattr(vllm_config.compilation_config, k) |
| 3269 | expected = v(vllm_config) if callable(v) else v |
| 3270 | # On platforms without static graph support, __post_init__ forces |
| 3271 | # cudagraph_mode to NONE; expect that instead of the level default. |
| 3272 | if k == "cudagraph_mode" and not current_platform.support_static_graph_mode(): |
| 3273 | expected = CUDAGraphMode.NONE |
| 3274 | assert actual == expected, ( |
| 3275 | f"compilation_config.{k}: expected {expected}, got {actual}" |
| 3276 | ) |
| 3277 | |
| 3278 | |
| 3279 | def test_vllm_config_callable_defaults(): |
| 3280 | """Test that callable defaults work in the config system. |
| 3281 | |
| 3282 | Verifies that lambdas in default configs can inspect VllmConfig properties |
| 3283 | (e.g., is_quantized, is_model_moe) to conditionally set optimization flags. |
| 3284 | """ |
| 3285 | config_no_model = VllmConfig(optimization_level=OptimizationLevel.O2) |
| 3286 | |
| 3287 | # Callable that checks if model exists |
| 3288 | has_model = lambda cfg: cfg.model_config is not None |
| 3289 | assert has_model(config_no_model) is False |
| 3290 | |
| 3291 | # Test with quantized model |
| 3292 | quantized_model = ModelConfig("RedHatAI/Llama-3.2-1B-FP8") |
| 3293 | config_quantized = VllmConfig( |
| 3294 | model_config=quantized_model, optimization_level=OptimizationLevel.O2 |
| 3295 | ) |
| 3296 | enable_if_quantized = lambda cfg: ( |
| 3297 | cfg.model_config is not None and cfg.model_config.is_quantized |
| 3298 | ) |
| 3299 | assert enable_if_quantized(config_quantized) is True |
| 3300 | assert enable_if_quantized(config_no_model) is False |
| 3301 | |
| 3302 | # Test with MoE model |
| 3303 | moe_model = ModelConfig("deepseek-ai/DeepSeek-V2-Lite") |
| 3304 | config_moe = VllmConfig( |
| 3305 | model_config=moe_model, optimization_level=OptimizationLevel.O2 |
| 3306 | ) |
| 3307 | enable_if_sequential = lambda cfg: ( |
| 3308 | cfg.model_config is not None and not cfg.model_config.is_moe |
| 3309 | ) |
| 3310 | assert enable_if_sequential(config_moe) is False |
| 3311 | assert enable_if_sequential(config_quantized) is True |
| 3312 | |
| 3313 | |
| 3314 | @pytest.mark.skipif( |
| 3315 | not current_platform.support_static_graph_mode(), |
| 3316 | reason="Explicit overrides may be force-overwritten without static graph support.", |
| 3317 | ) |
| 3318 | def test_vllm_config_explicit_overrides(): |
| 3319 | """Test that explicit property overrides work correctly with callable defaults. |
| 3320 | |
| 3321 | When users explicitly set configuration properties, those values |
| 3322 | take precedence over callable defaults, across different models and |
| 3323 | optimization levels. |
| 3324 | """ |
| 3325 | from vllm.config.compilation import PassConfig |
| 3326 | |
| 3327 | quantized_model = ModelConfig("RedHatAI/Llama-3.2-1B-FP8") |
| 3328 | moe_model = ModelConfig("deepseek-ai/DeepSeek-V2-Lite") |
| 3329 | regular_model = ModelConfig("Qwen/Qwen1.5-7B") |
| 3330 | |
| 3331 | # Explicit compilation mode override on O0 (where default is NONE) |
| 3332 | compilation_config = CompilationConfig(mode=CompilationMode.VLLM_COMPILE) |
| 3333 | config = VllmConfig( |
| 3334 | optimization_level=OptimizationLevel.O0, |
| 3335 | compilation_config=compilation_config, |
| 3336 | ) |
| 3337 | assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3338 | assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE |
| 3339 | |
| 3340 | # Explicit pass config flags to override defaults |
| 3341 | pass_config = PassConfig(eliminate_noops=True, fuse_attn_quant=True) |
| 3342 | compilation_config = CompilationConfig(pass_config=pass_config) |
| 3343 | config = VllmConfig( |
| 3344 | optimization_level=OptimizationLevel.O0, |
| 3345 | compilation_config=compilation_config, |
| 3346 | ) |
| 3347 | assert config.compilation_config.pass_config.eliminate_noops is True |
| 3348 | assert config.compilation_config.pass_config.fuse_attn_quant is True |
| 3349 | |
| 3350 | # Explicit cudagraph mode override on quantized model at O2 |
| 3351 | pass_config = PassConfig(enable_qk_norm_rope_fusion=True) |
| 3352 | compilation_config = CompilationConfig( |
| 3353 | cudagraph_mode=CUDAGraphMode.NONE, pass_config=pass_config |
| 3354 | ) |
| 3355 | config = VllmConfig( |
| 3356 | model_config=quantized_model, |
| 3357 | optimization_level=OptimizationLevel.O2, |
| 3358 | compilation_config=compilation_config, |
| 3359 | ) |
| 3360 | assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE |
| 3361 | assert config.compilation_config.pass_config.enable_qk_norm_rope_fusion is ( |
| 3362 | current_platform.is_cuda_alike() or current_platform.is_xpu() |
| 3363 | ) |
| 3364 | # Mode should still use default for O2 |
| 3365 | assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3366 | |
| 3367 | # Different optimization levels with same model |
| 3368 | config_o0 = VllmConfig( |
| 3369 | model_config=regular_model, optimization_level=OptimizationLevel.O0 |
| 3370 | ) |
| 3371 | config_o2 = VllmConfig( |
| 3372 | model_config=regular_model, optimization_level=OptimizationLevel.O2 |
| 3373 | ) |
| 3374 | assert config_o0.compilation_config.mode == CompilationMode.NONE |
| 3375 | assert config_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3376 | assert config_o0.compilation_config.cudagraph_mode == CUDAGraphMode.NONE |
| 3377 | assert ( |
| 3378 | config_o2.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE |
| 3379 | ) |
| 3380 | |
| 3381 | # Same optimization level across different model types |
| 3382 | config_moe_o2 = VllmConfig( |
| 3383 | model_config=moe_model, optimization_level=OptimizationLevel.O2 |
| 3384 | ) |
| 3385 | config_regular_o2 = VllmConfig( |
| 3386 | model_config=regular_model, optimization_level=OptimizationLevel.O2 |
| 3387 | ) |
| 3388 | config_quantized_o2 = VllmConfig( |
| 3389 | model_config=quantized_model, optimization_level=OptimizationLevel.O2 |
| 3390 | ) |
| 3391 | # All should have same base compilation settings at O2 |
| 3392 | assert config_moe_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3393 | assert config_regular_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3394 | assert config_quantized_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3395 | assert ( |
| 3396 | config_moe_o2.compilation_config.cudagraph_mode |
| 3397 | == CUDAGraphMode.FULL_AND_PIECEWISE |
| 3398 | ) |
| 3399 | assert ( |
| 3400 | config_regular_o2.compilation_config.cudagraph_mode |
| 3401 | == CUDAGraphMode.FULL_AND_PIECEWISE |
| 3402 | ) |
| 3403 | |
| 3404 | # Override one field but not others |
| 3405 | pass_config = PassConfig(eliminate_noops=False) |
| 3406 | compilation_config = CompilationConfig(pass_config=pass_config) |
| 3407 | config = VllmConfig( |
| 3408 | model_config=regular_model, |
| 3409 | optimization_level=OptimizationLevel.O2, |
| 3410 | compilation_config=compilation_config, |
| 3411 | ) |
| 3412 | # Explicit override should be respected |
| 3413 | assert config.compilation_config.pass_config.eliminate_noops is False |
| 3414 | # Other fields should still use defaults |
| 3415 | assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE |
| 3416 | assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE |
| 3417 | |
| 3418 | |
| 3419 | def test_fusion_pass_op_priority(): |
| 3420 | """This test checks that custom op enablement & IR op priority |
| 3421 | correctly control default fusions""" |
| 3422 | # Default config, O2, rms_norm+quant fusion disabled |
| 3423 | cfg1 = VllmConfig() |
| 3424 | assert not cfg1.compilation_config.pass_config.fuse_norm_quant |
| 3425 | |
| 3426 | # rms_norm manually enabled, O1, rms_norm+quant fusion enabled |
| 3427 | cfg2 = VllmConfig( |
| 3428 | optimization_level=OptimizationLevel.O1, |
| 3429 | compilation_config=CompilationConfig( |
| 3430 | custom_ops=["+rms_norm"], |
| 3431 | ), |
| 3432 | ) |
| 3433 | assert cfg2.compilation_config.pass_config.fuse_norm_quant |
| 3434 | |
| 3435 | # using custom kernel for RMSNorm via IR: |
| 3436 | # Note that vLLM IR only supports the non-residual rms_norm for now; |
| 3437 | # soon this will be resolved. |
| 3438 | cfg3 = VllmConfig( |
| 3439 | kernel_config=KernelConfig( |
| 3440 | ir_op_priority=IrOpPriorityConfig(rms_norm=["vllm_c"]) |
| 3441 | ) |
| 3442 | ) |
| 3443 | assert cfg3.compilation_config.pass_config.fuse_norm_quant |
| 3444 | |
| 3445 | # block-fp8 model should enable quant_fp8 automatically |
| 3446 | cfg4 = VllmConfig(model_config=ModelConfig("Qwen/Qwen3-4B-FP8")) |
| 3447 | assert "+quant_fp8" in cfg4.compilation_config.custom_ops |
| 3448 | assert cfg4.compilation_config.pass_config.fuse_norm_quant |
| 3449 | |
| 3450 | |
| 3451 | def test_scheduler_config_init(): |
| 3452 | with pytest.raises(ValidationError): |
| 3453 | # Positional InitVars missing |
| 3454 | # (InitVars cannot have defaults otherwise they will become attributes) |
| 3455 | SchedulerConfig() |
| 3456 | |
| 3457 | with pytest.raises(AttributeError): |
| 3458 | # InitVar does not become an attribute |
| 3459 | print(SchedulerConfig.default_factory().max_model_len) |
| 3460 | |
| 3461 | |
| 3462 | @pytest.mark.parametrize( |
| 3463 | ( |
| 3464 | "model_id", |
| 3465 | "data_parallel_size", |
| 3466 | "external_lb", |
| 3467 | "expected_needs_coordinator", |
| 3468 | ), |
| 3469 | [ |
| 3470 | # Non-MoE model with DP=1 should not need coordinator |
| 3471 | ("facebook/opt-125m", 1, False, False), |
| 3472 | # Non-MoE model with DP>1 internal LB should need coordinator |
| 3473 | ("facebook/opt-125m", 2, False, True), |
| 3474 | # MoE model with DP=1 should not need coordinator |
| 3475 | ("mistralai/Mixtral-8x7B-Instruct-v0.1", 1, False, False), |
| 3476 | # MoE model with DP>1 internal LB should need both coordinator |
| 3477 | # and wave coordination |
| 3478 | ("mistralai/Mixtral-8x7B-Instruct-v0.1", 2, False, True), |
| 3479 | # MoE model with DP>1 external LB needs coordinator for wave coordination |
| 3480 | # (wave coordination runs in coordinator process) |
| 3481 | ("mistralai/Mixtral-8x7B-Instruct-v0.1", 2, True, True), |
| 3482 | ], |
| 3483 | ) |
| 3484 | def test_needs_dp_coordination( |
| 3485 | model_id, |
| 3486 | data_parallel_size, |
| 3487 | external_lb, |
| 3488 | expected_needs_coordinator, |
| 3489 | ): |
| 3490 | """Test that DP coordinator and wave coordination are configured correctly.""" |
| 3491 | from vllm.config import ParallelConfig |
| 3492 | |
| 3493 | model_config = ModelConfig(model_id) |
| 3494 | parallel_config = ParallelConfig( |
| 3495 | data_parallel_size=data_parallel_size, |
| 3496 | data_parallel_external_lb=external_lb, |
| 3497 | ) |
| 3498 | vllm_config = VllmConfig(model_config=model_config, parallel_config=parallel_config) |
| 3499 | |
| 3500 | assert vllm_config.needs_dp_coordinator == expected_needs_coordinator |
| 3501 | |
| 3502 | |
| 3503 | def test_fault_tolerance_requires_single_api_server(): |
| 3504 | """Fault tolerance assumes one AsyncMPClient manages all engines, so it |
| 3505 | is incompatible with API server scale-out (_api_process_count > 1).""" |
| 3506 | with pytest.raises(ValueError, match="single API server"): |
| 3507 | ParallelConfig(enable_fault_tolerance=True, _api_process_count=2) |
| 3508 | |
| 3509 | # Single API server (the FT-supported topology) is accepted. |
| 3510 | ParallelConfig(enable_fault_tolerance=True, _api_process_count=1) |
| 3511 | |
| 3512 | |
| 3513 | def test_renderer_num_workers_with_mm_cache(): |
| 3514 | """Disallow renderer_num_workers > 1 with the mm processor cache only for |
| 3515 | pooling models, whose preprocessing runs on the renderer workers.""" |
| 3516 | mm_model = "Qwen/Qwen2-VL-2B-Instruct" |
| 3517 | |
| 3518 | # Should raise: pooling + multi-worker + cache enabled (default cache_gb=4) |
| 3519 | with pytest.raises(ValueError, match="renderer-num-workers"): |
| 3520 | ModelConfig(mm_model, runner="pooling", renderer_num_workers=4) |
| 3521 | |
| 3522 | # Should raise: pooling + multi-worker + explicit cache size |
| 3523 | with pytest.raises(ValueError, match="renderer-num-workers"): |
| 3524 | ModelConfig( |
| 3525 | mm_model, |
| 3526 | runner="pooling", |
| 3527 | renderer_num_workers=2, |
| 3528 | mm_processor_cache_gb=1.0, |
| 3529 | ) |
| 3530 | |
| 3531 | # Should pass: pooling + multi-worker + cache disabled |
| 3532 | config = ModelConfig( |
| 3533 | mm_model, runner="pooling", renderer_num_workers=4, mm_processor_cache_gb=0 |
| 3534 | ) |
| 3535 | assert config.renderer_num_workers == 4 |
| 3536 | |
| 3537 | # Should pass: generate models preprocess on the dedicated mm executor |
| 3538 | config = ModelConfig(mm_model, renderer_num_workers=4) |
| 3539 | assert config.renderer_num_workers == 4 |
| 3540 | |
| 3541 | # Should pass: single worker + cache enabled (default) |
| 3542 | config = ModelConfig(mm_model, renderer_num_workers=1) |
| 3543 | assert config.renderer_num_workers == 1 |
| 3544 | |
| 3545 | |
| 3546 | def test_eagle_draft_model_config(): |
| 3547 | """Test that EagleDraft model config is correctly set.""" |
| 3548 | target_model_config = ModelConfig( |
| 3549 | "meta-llama/Meta-Llama-3-8B-Instruct", trust_remote_code=True |
| 3550 | ) |
| 3551 | speculative_config = SpeculativeConfig( |
| 3552 | model="yuhuili/EAGLE-LLaMA3-Instruct-8B", |
| 3553 | num_speculative_tokens=1, |
| 3554 | target_model_config=target_model_config, |
| 3555 | target_parallel_config=ParallelConfig(), |
| 3556 | ) |
| 3557 | draft_model_config = speculative_config.draft_model_config |
| 3558 | assert draft_model_config.hf_config.architectures == ["EagleLlamaForCausalLM"] |
| 3559 | assert draft_model_config.hf_text_config.architectures == ["EagleLlamaForCausalLM"] |
| 3560 | assert draft_model_config.hf_config.model_type == "eagle" |
| 3561 | assert draft_model_config.hf_text_config.model_type == "eagle" |
| 3562 | assert draft_model_config.architectures == ["EagleLlamaForCausalLM"] |
| 3563 | assert draft_model_config.architecture == "EagleLlamaForCausalLM" |
| 3564 | |
| 3565 | |
| 3566 | def test_draft_sample_method_probabilistic_is_accepted(): |
| 3567 | speculative_config = SpeculativeConfig( |
| 3568 | method="ngram", |
| 3569 | num_speculative_tokens=1, |
| 3570 | draft_sample_method="probabilistic", |
| 3571 | ) |
| 3572 | assert speculative_config.draft_sample_method == "probabilistic" |
| 3573 | |
| 3574 | |
| 3575 | @pytest.mark.parametrize("disable_eagle_block_drop", [False, True]) |
| 3576 | def test_eagle_block_drop_can_be_disabled_without_disabling_eagle( |
| 3577 | disable_eagle_block_drop: bool, |
| 3578 | ): |
| 3579 | # Start from an ngram config to avoid loading model metadata: these predicates |
| 3580 | # depend only on the speculative method and the new switch. |
| 3581 | speculative_config = SpeculativeConfig( |
| 3582 | method="ngram", |
| 3583 | num_speculative_tokens=3, |
| 3584 | disable_eagle_block_drop=disable_eagle_block_drop, |
| 3585 | ) |
| 3586 | speculative_config.method = "eagle3" |
| 3587 | |
| 3588 | assert speculative_config.use_eagle() |
| 3589 | assert speculative_config.use_eagle_block_drop() is not disable_eagle_block_drop |
| 3590 | |
| 3591 | |
| 3592 | def test_draft_sample_method_gumbel_is_rejected(): |
| 3593 | with pytest.raises(ValidationError): |
| 3594 | SpeculativeConfig( |
| 3595 | method="ngram", |
| 3596 | num_speculative_tokens=1, |
| 3597 | draft_sample_method="gumbel", |
| 3598 | ) |
| 3599 | |
| 3600 | |
| 3601 | def _watermarked_vllm_config() -> VllmConfig: |
| 3602 | config = object.__new__(VllmConfig) |
| 3603 | config.watermark_config = WatermarkConfig(key=42) |
| 3604 | config.attention_config = AttentionConfig() |
| 3605 | config.speculative_config = None |
| 3606 | return config |
| 3607 | |
| 3608 | |
| 3609 | def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log_dedup): |
| 3610 | config = _watermarked_vllm_config() |
| 3611 | config.watermark_config = WatermarkConfig( |
| 3612 | key=42, allow_target_only_watermarking=True |
| 3613 | ) |
| 3614 | config.speculative_config = SimpleNamespace( |
| 3615 | method="mtp", |
| 3616 | draft_sample_method="probabilistic", |
| 3617 | rejection_sample_method="standard", |
| 3618 | parallel_drafting=False, |
| 3619 | ) |
| 3620 | |
| 3621 | with caplog_vllm.at_level(logging.WARNING): |
| 3622 | config._check_watermarking_unsupported() |
| 3623 | |
| 3624 | assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text |
| 3625 | |
| 3626 | |
| 3627 | def test_speculative_watermarking_without_context_dedup_does_not_warn( |
| 3628 | caplog_vllm, disable_log_dedup |
| 3629 | ): |
| 3630 | config = _watermarked_vllm_config() |
| 3631 | config.watermark_config = WatermarkConfig( |
| 3632 | algorithm="dual_key_gumbel", key=42, deduplicate_contexts="none" |
| 3633 | ) |
| 3634 | config.speculative_config = SimpleNamespace( |
| 3635 | method="mtp", |
| 3636 | draft_sample_method="probabilistic", |
| 3637 | rejection_sample_method="standard", |
| 3638 | parallel_drafting=False, |
| 3639 | ) |
| 3640 | |
| 3641 | caplog_vllm.clear() |
| 3642 | with caplog_vllm.at_level(logging.WARNING): |
| 3643 | config._check_watermarking_unsupported() |
| 3644 | |
| 3645 | assert "dedup" not in caplog_vllm.text.lower() |
| 3646 | |
| 3647 | |
| 3648 | def test_speculative_context_dedup_logs_no_unsupported_warning( |
| 3649 | caplog_vllm, disable_log_dedup |
| 3650 | ): |
| 3651 | config = _watermarked_vllm_config() |
| 3652 | config.watermark_config = WatermarkConfig( |
| 3653 | algorithm="dual_key_gumbel", key=42, deduplicate_contexts="single_turn" |
| 3654 | ) |
| 3655 | config.speculative_config = SimpleNamespace( |
| 3656 | method="mtp", |
| 3657 | draft_sample_method="probabilistic", |
| 3658 | rejection_sample_method="standard", |
| 3659 | parallel_drafting=False, |
| 3660 | ) |
| 3661 | |
| 3662 | caplog_vllm.clear() |
| 3663 | with caplog_vllm.at_level(logging.WARNING): |
| 3664 | config._check_watermarking_unsupported() |
| 3665 | |
| 3666 | assert "dedup" not in caplog_vllm.text.lower() |
| 3667 | |
| 3668 | |
| 3669 | def test_gumbel_rejects_speculative_decoding_without_target_only(): |
| 3670 | config = _watermarked_vllm_config() |
| 3671 | config.watermark_config = WatermarkConfig( |
| 3672 | algorithm="gumbel", key=42, allow_target_only_watermarking=False |
| 3673 | ) |
| 3674 | config.speculative_config = SimpleNamespace( |
| 3675 | method="mtp", |
| 3676 | draft_sample_method="probabilistic", |
| 3677 | rejection_sample_method="standard", |
| 3678 | parallel_drafting=False, |
| 3679 | ) |
| 3680 | |
| 3681 | with pytest.raises(ValueError, match="'gumbel'.*allow_target_only_watermarking"): |
| 3682 | config._check_watermarking_unsupported() |
| 3683 | |
| 3684 | |
| 3685 | def test_dual_key_gumbel_warns_that_configured_alpha_is_unused( |
| 3686 | caplog_vllm, disable_log_dedup |
| 3687 | ): |
| 3688 | config = _watermarked_vllm_config() |
| 3689 | config.watermark_config = WatermarkConfig( |
| 3690 | algorithm="dual_key_gumbel", key=42, alpha=0.25 |
| 3691 | ) |
| 3692 | config.speculative_config = SimpleNamespace( |
| 3693 | method="mtp", |
| 3694 | draft_sample_method="probabilistic", |
| 3695 | rejection_sample_method="standard", |
| 3696 | parallel_drafting=False, |
| 3697 | ) |
| 3698 | |
| 3699 | with caplog_vllm.at_level(logging.WARNING): |
| 3700 | config._check_watermarking_unsupported() |
| 3701 | |
| 3702 | assert "The configured alpha=0.25 is not used" in caplog_vllm.text |
| 3703 | |
| 3704 | |
| 3705 | @pytest.mark.parametrize( |
| 3706 | ("alpha", "speculative"), |
| 3707 | [(0.1, True), (0.25, False)], |
| 3708 | ids=["default-alpha", "no-specdec"], |
| 3709 | ) |
| 3710 | def test_dual_key_gumbel_alpha_warning_is_not_emitted( |
| 3711 | caplog_vllm, disable_log_dedup, alpha, speculative |
| 3712 | ): |
| 3713 | config = _watermarked_vllm_config() |
| 3714 | config.watermark_config = WatermarkConfig( |
| 3715 | algorithm="dual_key_gumbel", key=42, alpha=alpha |
| 3716 | ) |
| 3717 | if speculative: |
| 3718 | config.speculative_config = SimpleNamespace( |
| 3719 | method="mtp", |
| 3720 | draft_sample_method="probabilistic", |
| 3721 | rejection_sample_method="standard", |
| 3722 | parallel_drafting=False, |
| 3723 | ) |
| 3724 | |
| 3725 | with caplog_vllm.at_level(logging.WARNING): |
| 3726 | config._check_watermarking_unsupported() |
| 3727 | |
| 3728 | assert "is not used" not in caplog_vllm.text |
| 3729 | |
| 3730 | |
| 3731 | def test_dual_key_gumbel_requires_probabilistic_drafting(): |
| 3732 | config = _watermarked_vllm_config() |
| 3733 | config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42) |
| 3734 | config.speculative_config = SimpleNamespace( |
| 3735 | method="mtp", |
| 3736 | draft_sample_method="greedy", |
| 3737 | rejection_sample_method="standard", |
| 3738 | parallel_drafting=False, |
| 3739 | ) |
| 3740 | |
| 3741 | with pytest.raises(ValueError, match="draft_sample_method='probabilistic'"): |
| 3742 | config._check_watermarking_unsupported() |
| 3743 | |
| 3744 | |
| 3745 | @pytest.mark.parametrize("method", ["eagle", "eagle3", "mtp"]) |
| 3746 | def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(method): |
| 3747 | config = _watermarked_vllm_config() |
| 3748 | config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42) |
| 3749 | config.speculative_config = SimpleNamespace( |
| 3750 | method=method, |
| 3751 | draft_sample_method="probabilistic", |
| 3752 | rejection_sample_method="standard", |
| 3753 | parallel_drafting=False, |
| 3754 | ) |
| 3755 | |
| 3756 | config._check_watermarking_unsupported() |
| 3757 | |
| 3758 | |
| 3759 | def test_dual_key_gumbel_supports_dspark(): |
| 3760 | config = _watermarked_vllm_config() |
| 3761 | config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42) |
| 3762 | config.speculative_config = SimpleNamespace( |
| 3763 | method="dspark", |
| 3764 | draft_sample_method="probabilistic", |
| 3765 | rejection_sample_method="standard", |
| 3766 | parallel_drafting=True, |
| 3767 | ) |
| 3768 | |
| 3769 | config._check_watermarking_unsupported() |
| 3770 | |
| 3771 | |
| 3772 | def test_dual_key_gumbel_rejects_non_autoregressive_speculation(): |
| 3773 | config = _watermarked_vllm_config() |
| 3774 | config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42) |
| 3775 | config.speculative_config = SimpleNamespace( |
| 3776 | method="ngram", |
| 3777 | draft_sample_method="probabilistic", |
| 3778 | rejection_sample_method="standard", |
| 3779 | parallel_drafting=False, |
| 3780 | ) |
| 3781 | |
| 3782 | with pytest.raises(ValueError, match="autoregressive model-based"): |
| 3783 | config._check_watermarking_unsupported() |
| 3784 | |
| 3785 | |
| 3786 | @pytest.mark.parametrize( |
| 3787 | ("overrides", "match"), |
| 3788 | [ |
| 3789 | ({"rejection_sample_method": "synthetic"}, "rejection_sample_method"), |
| 3790 | ({"rejection_sample_method": "block"}, "rejection_sample_method"), |
| 3791 | ({"parallel_drafting": True}, "Parallel speculative drafting"), |
| 3792 | ], |
| 3793 | ) |
| 3794 | def test_dual_key_gumbel_rejects_incompatible_speculative_modes(overrides, match): |
| 3795 | config = _watermarked_vllm_config() |
| 3796 | config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42) |
| 3797 | values = { |
| 3798 | "method": "mtp", |
| 3799 | "draft_sample_method": "probabilistic", |
| 3800 | "rejection_sample_method": "standard", |
| 3801 | "parallel_drafting": False, |
| 3802 | **overrides, |
| 3803 | } |
| 3804 | config.speculative_config = SimpleNamespace(**values) |
| 3805 | |
| 3806 | with pytest.raises(ValueError, match=match): |
| 3807 | config._check_watermarking_unsupported() |
| 3808 | |
| 3809 | |
| 3810 | def test_gumbel_watermark_rejects_beam_search(): |
| 3811 | with pytest.raises(ValueError, match="Beam search is not supported"): |
| 3812 | _watermarked_vllm_config()._check_watermarking_unsupported(beam_search=True) |
| 3813 | |
| 3814 | |
| 3815 | def test_gumbel_watermark_rejects_custom_sampler(): |
| 3816 | with pytest.raises(ValueError, match="custom samplers are not supported"): |
| 3817 | _watermarked_vllm_config()._check_watermarking_unsupported(custom_sampler=True) |
| 3818 | |
| 3819 | |
| 3820 | def test_watermark_key_must_fit_in_64_bits(): |
| 3821 | with pytest.raises(ValueError, match="64 bits"): |
| 3822 | WatermarkConfig(key=2**64) |
| 3823 | |
| 3824 | |
| 3825 | @pytest.mark.parametrize("alpha", [-0.1, 1.1]) |
| 3826 | def test_watermark_alpha_must_be_a_probability(alpha): |
| 3827 | with pytest.raises(ValidationError): |
| 3828 | WatermarkConfig(key=42, algorithm="dual_key_gumbel", alpha=alpha) |
| 3829 | |
| 3830 | |
| 3831 | def test_unknown_watermark_prf_is_rejected(): |
| 3832 | with pytest.raises(ValidationError): |
| 3833 | pydantic.TypeAdapter(WatermarkConfig).validate_python( |
| 3834 | {"key": 42, "prf": "unsupported"} |
| 3835 | ) |
| 3836 | |
| 3837 | |
| 3838 | def test_watermark_key_is_excluded_from_serialization(): |
| 3839 | serialized = pydantic.TypeAdapter(WatermarkConfig).dump_python( |
| 3840 | WatermarkConfig(key=42), mode="json" |
| 3841 | ) |
| 3842 | |
| 3843 | assert "key" not in serialized |
| 3844 | |
| 3845 | |
| 3846 | def test_watermarking_forces_model_runner_v2(monkeypatch): |
| 3847 | monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") |
| 3848 | |
| 3849 | with patch("vllm.config.vllm.logger.info_once") as info_once: |
| 3850 | assert _watermarked_vllm_config().use_v2_model_runner |
| 3851 | |
| 3852 | info_once.assert_called_once_with( |
| 3853 | "Watermarking requires Model Runner V2 and overrides " |
| 3854 | "VLLM_USE_V2_MODEL_RUNNER=0." |
| 3855 | ) |
| 3856 | |
| 3857 | |
| 3858 | @patch("vllm.config.speculative.ModelConfig") |
| 3859 | def test_mtp_draft_uses_model_weights_not_local_cache(mock_model_config_cls): |
| 3860 | """Regression test: MTP + runai_streamer should use model_weights (original |
| 3861 | S3 URL) for the draft model, not model (local cache dir set by |
| 3862 | pull_runai_model_from_obj_storage).""" |
| 3863 | from unittest.mock import MagicMock |
| 3864 | |
| 3865 | s3_url = "s3://my-bucket/Qwen3-35B-A3B-FP8" |
| 3866 | local_cache = "/root/.cache/vllm/assets/model_streamer/abcd1234" |
| 3867 | |
| 3868 | mock_draft = MagicMock() |
| 3869 | mock_draft.model = local_cache |
| 3870 | mock_draft.hf_config.model_type = "deepseek_mtp" |
| 3871 | mock_draft.hf_config.n_predict = None |
| 3872 | mock_draft.max_model_len = 4096 |
| 3873 | mock_model_config_cls.return_value = mock_draft |
| 3874 | |
| 3875 | target_config = MagicMock() |
| 3876 | target_config.model = local_cache |
| 3877 | target_config.model_weights = s3_url |
| 3878 | target_config.hf_text_config.model_type = "deepseek_v3" |
| 3879 | target_config.quantization = None |
| 3880 | target_config.max_model_len = 4096 |
| 3881 | |
| 3882 | SpeculativeConfig( |
| 3883 | method="mtp", |
| 3884 | num_speculative_tokens=1, |
| 3885 | target_model_config=target_config, |
| 3886 | target_parallel_config=ParallelConfig(), |
| 3887 | ) |
| 3888 | |
| 3889 | actual_model = mock_model_config_cls.call_args.kwargs["model"] |
| 3890 | assert actual_model == s3_url |
| 3891 | |
| 3892 | |
| 3893 | def _make_qwen3_omni_dspark_configs(): |
| 3894 | text_config = SimpleNamespace( |
| 3895 | hidden_size=2048, |
| 3896 | num_hidden_layers=48, |
| 3897 | num_attention_heads=32, |
| 3898 | num_key_value_heads=4, |
| 3899 | head_dim=128, |
| 3900 | vocab_size=152064, |
| 3901 | ) |
| 3902 | target_model_config = SimpleNamespace( |
| 3903 | hf_config=SimpleNamespace( |
| 3904 | model_type="qwen3_omni_moe", |
| 3905 | architectures=["Qwen3OmniMoeForConditionalGeneration"], |
| 3906 | thinker_config=SimpleNamespace(text_config=text_config), |
| 3907 | ), |
| 3908 | hf_text_config=text_config, |
| 3909 | architectures=["Qwen3OmniMoeForConditionalGeneration"], |
| 3910 | get_hidden_size=lambda: 2048, |
| 3911 | get_total_num_hidden_layers=lambda: 48, |
| 3912 | get_vocab_size=lambda: 152064, |
| 3913 | ) |
| 3914 | draft_hf_config = SimpleNamespace( |
| 3915 | model_type="qwen3", |
| 3916 | architectures=["Qwen3OmniDSparkModel"], |
| 3917 | hidden_size=2048, |
| 3918 | num_attention_heads=32, |
| 3919 | num_key_value_heads=4, |
| 3920 | head_dim=128, |
| 3921 | block_size=7, |
| 3922 | target_hidden_size=2048, |
| 3923 | target_layer_ids=[1, 9, 17, 25, 33], |
| 3924 | use_aux_hidden_state=True, |
| 3925 | markov_rank=64, |
| 3926 | markov_head_type="vanilla", |
| 3927 | sample_from_anchor=True, |
| 3928 | dspark_bonus_anchor=False, |
| 3929 | vocab_size=152064, |
| 3930 | draft_vocab_size=32000, |
| 3931 | mask_token_id=151669, |
| 3932 | rope_parameters={"rope_type": "default", "rope_theta": 1000000.0}, |
| 3933 | ) |
| 3934 | draft_model_config = SimpleNamespace( |
| 3935 | hf_config=draft_hf_config, |
| 3936 | architectures=["Qwen3OmniDSparkModel"], |
| 3937 | ) |
| 3938 | return target_model_config, draft_model_config |
| 3939 | |
| 3940 | |
| 3941 | @pytest.mark.skip_global_cleanup |
| 3942 | def test_qwen3_omni_dspark_checkpoint_contract_is_accepted(): |
| 3943 | target_config, draft_config = _make_qwen3_omni_dspark_configs() |
| 3944 | _validate_qwen3_omni_dspark(target_config, draft_config, 7) |
| 3945 | |
| 3946 | |
| 3947 | @pytest.mark.skip_global_cleanup |
| 3948 | def test_qwen3_omni_dspark_rejects_generic_qwen3_architecture(): |
| 3949 | target_config, draft_config = _make_qwen3_omni_dspark_configs() |
| 3950 | draft_config.hf_config.architectures = ["Qwen3DSparkModel"] |
| 3951 | draft_config.architectures = ["Qwen3DSparkModel"] |
| 3952 | |
| 3953 | with pytest.raises(ValueError, match="must be converted first"): |
| 3954 | _validate_qwen3_omni_dspark(target_config, draft_config, 7) |
| 3955 | |
| 3956 | |
| 3957 | @pytest.mark.parametrize( |
| 3958 | ("field", "value", "error"), |
| 3959 | [ |
| 3960 | ("block_size", 5, "trained block_size"), |
| 3961 | ("target_hidden_size", 4096, "target_hidden_size"), |
| 3962 | ("hidden_size", 4096, "draft hidden_size"), |
| 3963 | ("num_attention_heads", 16, "num_attention_heads"), |
| 3964 | ("num_key_value_heads", 8, "num_key_value_heads"), |
| 3965 | ("head_dim", 64, "head_dim"), |
| 3966 | ("target_layer_ids", [7, 48], "zero-based text-layer"), |
| 3967 | ("target_layer_ids", [23, 7], "strictly increasing"), |
| 3968 | ("use_aux_hidden_state", False, "use_aux_hidden_state=true"), |
| 3969 | ("markov_rank", 0, "markov_rank"), |
| 3970 | ("markov_head_type", "gated", "markov_head_type='vanilla'"), |
| 3971 | ("sample_from_anchor", False, "sample_from_anchor=true"), |
| 3972 | ("dspark_bonus_anchor", True, "dspark_bonus_anchor=false"), |
| 3973 | ("vocab_size", 0, "input vocab_size must be a positive integer"), |
| 3974 | ], |
| 3975 | ) |
| 3976 | @pytest.mark.skip_global_cleanup |
| 3977 | def test_qwen3_omni_dspark_rejects_incompatible_checkpoint_fields(field, value, error): |
| 3978 | target_config, draft_config = _make_qwen3_omni_dspark_configs() |
| 3979 | setattr(draft_config.hf_config, field, value) |
| 3980 | |
| 3981 | with pytest.raises(ValueError, match=error): |
| 3982 | _validate_qwen3_omni_dspark(target_config, draft_config, 7) |
| 3983 | |
| 3984 | |
| 3985 | @pytest.mark.skip_global_cleanup |
| 3986 | def test_qwen3_omni_dspark_rejects_mrope_draft_positions(): |
| 3987 | target_config, draft_config = _make_qwen3_omni_dspark_configs() |
| 3988 | draft_config.hf_config.rope_parameters["mrope_section"] = [24, 20, 20] |
| 3989 | |
| 3990 | with pytest.raises(ValueError, match="logical 1-D RoPE"): |
| 3991 | _validate_qwen3_omni_dspark(target_config, draft_config, 7) |
| 3992 | |
| 3993 | |
| 3994 | @pytest.mark.skip_global_cleanup |
| 3995 | def test_qwen3_omni_dspark_allows_draft_only_noise_token_row(): |
| 3996 | target_config, draft_config = _make_qwen3_omni_dspark_configs() |
| 3997 | draft_config.hf_config.vocab_size = 152065 |
| 3998 | draft_config.hf_config.draft_vocab_size = 152064 |
| 3999 | draft_config.hf_config.mask_token_id = 152064 |
| 4000 | |
| 4001 | _validate_qwen3_omni_dspark(target_config, draft_config, 7) |
| 4002 | |
| 4003 | |
| 4004 | @pytest.mark.skip_global_cleanup |
| 4005 | def test_qwen3_omni_dspark_allows_smaller_input_vocabulary(): |
| 4006 | target_config, draft_config = _make_qwen3_omni_dspark_configs() |
| 4007 | draft_config.hf_config.vocab_size = 151936 |
| 4008 | draft_config.hf_config.mask_token_id = 151669 |
| 4009 | |
| 4010 | _validate_qwen3_omni_dspark(target_config, draft_config, 7) |
| 4011 | |
| 4012 | |
| 4013 | def test_ir_op_priority_default(): |
| 4014 | """Test that IR op priority defaults are set correctly.""" |
| 4015 | from vllm.config.kernel import IrOpPriorityConfig |
| 4016 | |
| 4017 | # Assert default is applied to ops |
| 4018 | priority_config = IrOpPriorityConfig.with_default(["vllm_c", "native"]) |
| 4019 | assert priority_config.rms_norm == ["vllm_c", "native"] |
| 4020 | assert priority_config.fused_add_rms_norm == ["vllm_c", "native"] |
| 4021 | |
| 4022 | # Assert single ops override the default |
| 4023 | priority_config = IrOpPriorityConfig.with_default( |
| 4024 | ["native"], rms_norm=["oink", "native"] |
| 4025 | ) |
| 4026 | assert priority_config.rms_norm == ["oink", "native"] |
| 4027 | assert priority_config.fused_add_rms_norm == ["native"] |
| 4028 | |
| 4029 | |
| 4030 | @pytest.mark.parametrize("mode", [CompilationMode.NONE, CompilationMode.VLLM_COMPILE]) |
| 4031 | @pytest.mark.parametrize("backend", ["inductor", "eager"]) |
| 4032 | def test_ir_op_platform_defaults_support_sparse_gelu(mode, backend): |
| 4033 | """Worker initialization must not select an unregistered sparse GELU provider.""" |
| 4034 | from vllm import ir |
| 4035 | |
| 4036 | config = SimpleNamespace( |
| 4037 | compilation_config=CompilationConfig(mode=mode, backend=backend) |
| 4038 | ) |
| 4039 | priority = current_platform.get_default_ir_op_priority(config) |
| 4040 | expected = ["triton", "native"] if current_platform.is_cuda() else ["native"] |
| 4041 | |
| 4042 | with priority.set_priority(): |
| 4043 | assert ir.ops.gelu_and_mul_sparse.get_priority() == expected |
| 4044 | |
| 4045 | |
| 4046 | def test_ir_op_priority_str(): |
| 4047 | """Test that passing a comma-delimited string works.""" |
| 4048 | from vllm.config.kernel import IrOpPriorityConfig |
| 4049 | |
| 4050 | priority_config = IrOpPriorityConfig(rms_norm="vllm_c") |
| 4051 | assert priority_config.rms_norm == ["vllm_c"] |
| 4052 | |
| 4053 | priority_config = IrOpPriorityConfig(rms_norm="vllm_c,native") |
| 4054 | assert priority_config.rms_norm == ["vllm_c", "native"] |
| 4055 | |
| 4056 | priority_config = IrOpPriorityConfig(rms_norm=" native, vllm_c ") |
| 4057 | assert priority_config.rms_norm == ["native", "vllm_c"] |
| 4058 | |
| 4059 | with pytest.raises(pydantic.ValidationError): |
| 4060 | # must be list of only strings |
| 4061 | priority_config = IrOpPriorityConfig(rms_norm=["vllm_c", 4, "native"]) |
| 4062 | |
| 4063 | |
| 4064 | def test_ir_op_priority_ctx(): |
| 4065 | """Test that the priority-setting context sets priority correctly.""" |
| 4066 | from vllm import ir |
| 4067 | from vllm.config.kernel import IrOpPriorityConfig |
| 4068 | |
| 4069 | priority = IrOpPriorityConfig.with_default(["native"], rms_norm=["vllm_c"]) |
| 4070 | priority2 = IrOpPriorityConfig.with_default( |
| 4071 | ["native"], fused_add_rms_norm=["vllm_c"] |
| 4072 | ) |
| 4073 | with priority.set_priority(): |
| 4074 | assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"] |
| 4075 | assert ir.ops.fused_add_rms_norm.get_priority() == ["native"] |
| 4076 | with priority2.set_priority(): |
| 4077 | assert ir.ops.rms_norm.get_priority() == ["native"] |
| 4078 | assert ir.ops.fused_add_rms_norm.get_priority() == ["vllm_c", "native"] |
| 4079 | |
| 4080 | # context restored |
| 4081 | assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"] |
| 4082 | assert ir.ops.fused_add_rms_norm.get_priority() == ["native"] |
| 4083 | |
| 4084 | with pytest.raises(ValueError), priority2.set_priority(): |
| 4085 | assert ir.ops.rms_norm.get_priority() == ["native"] |
| 4086 | assert ir.ops.fused_add_rms_norm.get_priority() == ["vllm_c", "native"] |
| 4087 | |
| 4088 | raise ValueError |
| 4089 | |
| 4090 | # context restored even after exception |
| 4091 | assert ir.ops.rms_norm.get_priority() == ["vllm_c", "native"] |
| 4092 | assert ir.ops.fused_add_rms_norm.get_priority() == ["native"] |
| 4093 | |
| 4094 | |
| 4095 | def test_load_config_rejects_invalid_safetensors_load_strategy(): |
| 4096 | with pytest.raises(pydantic.ValidationError): |
| 4097 | LoadConfig(safetensors_load_strategy="not_a_real_strategy") |
| 4098 | |
| 4099 | |
| 4100 | @pytest.mark.parametrize("bad_load_format", [None, 123]) |
| 4101 | def test_load_config_rejects_non_string_load_format(bad_load_format): |
| 4102 | with pytest.raises(pydantic.ValidationError): |
| 4103 | LoadConfig(load_format=bad_load_format) |
| 4104 | |
| 4105 | |
| 4106 | # A real Qwen3-0.6B model revision that is used in the test below. |
| 4107 | REVISION = "c1899de289a04d12100db370d81485cdf75e47ca" |
| 4108 | |
| 4109 | |
| 4110 | @patch("vllm.config.model.resolve_revision", return_value=ResolvedRevision(REVISION)) |
| 4111 | def test_revision_resolved_for_model(mock_resolve): |
| 4112 | model = "Qwen/Qwen3-0.6B" |
| 4113 | config = ModelConfig(model) |
| 4114 | assert isinstance(config.revision, ResolvedRevision) |
| 4115 | assert config.revision.resolved == REVISION |
| 4116 | mock_resolve.assert_any_call(model, None, config.hf_token) |
| 4117 | |
| 4118 | |
| 4119 | @pytest.mark.parametrize( |
| 4120 | ("layer_types", "expected_attention"), |
| 4121 | [ |
| 4122 | # Qwen3-Next / Qwen3.5 spell their attention layers "full_attention". |
| 4123 | (["linear_attention", "full_attention"], 1), |
| 4124 | # GLM-5.3-Flash and Qwen4-Exp use sparse attention, which still caches |
| 4125 | # every token and so must count as attention. |
| 4126 | (["linear_attention", "deepseek_sparse_attention"], 1), |
| 4127 | (["linear_attention", "qwen_sparse_attention"], 1), |
| 4128 | (["linear_attention", "linear_attention"], 0), |
| 4129 | ], |
| 4130 | ) |
| 4131 | def test_hybrid_layer_counts_sparse_attention_as_attention( |
| 4132 | layer_types, expected_attention |
| 4133 | ): |
| 4134 | """Sparse-attention layer types consume a full attention KV cache.""" |
| 4135 | model_config = object.__new__(ModelConfig) |
| 4136 | model_config.hf_config = SimpleNamespace() |
| 4137 | model_config.hf_text_config = SimpleNamespace(layer_types=layer_types) |
| 4138 | model_config.model_arch_config = SimpleNamespace(text_model_type="glm5_next") |
| 4139 | |
| 4140 | parallel_config = SimpleNamespace() |
| 4141 | with ( |
| 4142 | patch.object(ModelConfig, "is_hybrid", True), |
| 4143 | patch.object(ModelConfig, "has_noops", False), |
| 4144 | patch.object(ModelConfig, "is_attention_free", False), |
| 4145 | patch.object( |
| 4146 | ModelConfig, |
| 4147 | "get_layers_start_end_indices", |
| 4148 | return_value=(0, len(layer_types)), |
| 4149 | ), |
| 4150 | ): |
| 4151 | assert ( |
| 4152 | model_config.get_num_layers_by_block_type(parallel_config, "attention") |
| 4153 | == expected_attention |
| 4154 | ) |
| 4155 | assert ( |
| 4156 | model_config.get_num_layers_by_block_type( |
| 4157 | parallel_config, "linear_attention" |
| 4158 | ) |
| 4159 | == len(layer_types) - expected_attention |
| 4160 | ) |