@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2836dd73f1

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 22:35 UTC

[Model][CI] Let more pooling models support v1 (#21747)

Signed-off-by: wang.yuqi <noooop@126.com>

wang.yuqi committed Jul 31, 2025 at 16:51 UTC 2836dd73f13015ee386c544760ca0d16888203f3

8 files changed +14 -48

tests/models/language/pooling/test_classification.py

-8

@@ -6,14 +6,6 @@ from transformers import AutoModelForSequenceClassification

7 from vllm.platforms import current_platform

9 -# TODO: enable when float32 is supported by V1

10 -# @pytest.fixture(autouse=True)

11 -# def v1(run_with_both_engines):

12 -# # Simple autouse wrapper to run both engines for each test

13 -# # This can be promoted up to conftest.py to run for every

14 -# # test in a package

15 -# pass

16 -

10 @pytest.mark.parametrize(

11 "model",

tests/models/language/pooling/test_gte.py

+4 -14

@@ -56,17 +56,10 @@ MODELS = [

56 enable_test=False),

57 ]

59 -V1FlashAttentionImpNotSupported = [

60 - "Alibaba-NLP/gte-Qwen2-1.5B-instruct", "Alibaba-NLP/gte-modernbert-base"

61 -]

62 -

60 @pytest.mark.parametrize("model_info", MODELS)

65 -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo,

66 - monkeypatch) -> None:

67 - if model_info.name in V1FlashAttentionImpNotSupported:

68 - monkeypatch.setenv("VLLM_USE_V1", "0")

69 -

61 +def test_embed_models_mteb(hf_runner, vllm_runner,

62 + model_info: EmbedModelInfo) -> None:

63 vllm_extra_kwargs: dict[str, Any] = {}

64 if model_info.architecture == "GteNewModel":

65 vllm_extra_kwargs["hf_overrides"] = {"architectures": ["GteNewModel"]}

@@ -77,11 +70,8 @@ def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo,

71 @pytest.mark.parametrize("model_info", MODELS)

72 def test_embed_models_correctness(hf_runner, vllm_runner,

80 - model_info: EmbedModelInfo, example_prompts,

81 - monkeypatch) -> None:

82 - if model_info.name in V1FlashAttentionImpNotSupported:

83 - monkeypatch.setenv("VLLM_USE_V1", "0")

84 -

73 + model_info: EmbedModelInfo,

74 + example_prompts) -> None:

75 vllm_extra_kwargs: dict[str, Any] = {}

76 if model_info.architecture == "GteNewModel":

77 vllm_extra_kwargs["hf_overrides"] = {"architectures": ["GteNewModel"]}

tests/models/language/pooling/test_jina.py

-13

@@ -4,7 +4,6 @@ from functools import partial

5 import pytest

7 -import vllm.envs as envs

7 from vllm import PoolingParams

9 from ...utils import EmbedModelInfo, RerankModelInfo

@@ -24,14 +23,6 @@ RERANK_MODELS = [

23 ]

27 -@pytest.fixture(autouse=True)

28 -def v1(run_with_both_engines):

29 - # Simple autouse wrapper to run both engines for each test

30 - # This can be promoted up to conftest.py to run for every

31 - # test in a package

32 - pass

33 -

34 -

26 @pytest.mark.parametrize("model_info", EMBEDDING_MODELS)

27 def test_embed_models_mteb(hf_runner, vllm_runner,

28 model_info: EmbedModelInfo) -> None:

@@ -63,10 +54,6 @@ def test_embed_models_correctness(hf_runner, vllm_runner,

54 @pytest.mark.parametrize("model_info", RERANK_MODELS)

55 def test_rerank_models_mteb(hf_runner, vllm_runner,

56 model_info: RerankModelInfo) -> None:

66 - if (model_info.architecture == "XLMRobertaForSequenceClassification"

67 - and envs.VLLM_USE_V1):

68 - pytest.skip("Not supported yet")

69 -

57 mteb_test_rerank_models(hf_runner, vllm_runner, model_info)

tests/models/language/pooling/test_qwen3_reranker.py

-6

@@ -83,9 +83,6 @@ def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:

83 }

84 }

86 - if model_info.name == "Qwen/Qwen3-Reranker-4B":

87 - vllm_extra_kwargs["max_num_seqs"] = 1

88 -

86 mteb_test_rerank_models(Qwen3RerankerHfRunner, vllm_runner, model_info,

87 vllm_extra_kwargs)

@@ -106,9 +103,6 @@ def test_rerank_models_mteb_tp(vllm_runner,

103 "tensor_parallel_size": 2,

104 }

105

109 - if model_info.name == "Qwen/Qwen3-Reranker-4B":

110 - vllm_extra_kwargs["max_num_seqs"] = 1

111 -

106 mteb_test_rerank_models(Qwen3RerankerHfRunner,

107 vllm_runner,

108 model_info,

vllm/config.py

@@ -776,6 +776,9 @@ class ModelConfig:

776 raise ValueError(

777 "`override_neuron_config` is only supported on Neuron.")

778

779 + # Avoid running try_verify_and_update_config multiple times

780 + self.config_updated = False

781 +

782 self._verify_quantization()

783 self._verify_cuda_graph()

784 self._verify_bnb_config()

@@ -4914,6 +4917,11 @@ class VllmConfig:

4917 if self.model_config is None:

4918 return

4919

4920 + # Avoid running try_verify_and_update_config multiple times

4921 + if getattr(self.model_config, "config_updated", False):

4922 + return

4923 + self.model_config.config_updated = True

4924 +

4925 architecture = self.model_config.architecture

4926 if architecture is None:

4927 return

vllm/model_executor/models/bert_with_rope.py

+1 -4

@@ -8,7 +8,6 @@ from torch import nn

8 from transformers import PretrainedConfig

10 from vllm.attention import Attention, AttentionType

11 -from vllm.compilation.decorators import support_torch_compile

11 from vllm.config import CacheConfig, VllmConfig

12 from vllm.distributed import (divide, get_tensor_model_parallel_rank,

13 get_tensor_model_parallel_world_size,

@@ -26,7 +25,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope

25 from vllm.model_executor.layers.vocab_parallel_embedding import (

26 VocabParallelEmbedding)

27 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

29 -from vllm.model_executor.models import SupportsV0Only

28 from vllm.model_executor.models.interfaces import SupportsQuant

29 from vllm.model_executor.models.utils import WeightsMapper

30 from vllm.model_executor.utils import set_weight_attrs

@@ -360,7 +358,6 @@ class BertWithRopeBlock(nn.Module):

358 return hidden_states

359

360

363 -@support_torch_compile

361 class BertWithRopeEncoder(nn.Module):

362

363 def __init__(self,

@@ -394,7 +391,7 @@ class BertWithRopeEncoder(nn.Module):

391 return hidden_states

392

393

397 -class BertWithRope(nn.Module, SupportsV0Only, SupportsQuant):

394 +class BertWithRope(nn.Module, SupportsQuant):

395 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})

396

397 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

vllm/model_executor/models/config.py

+1 -1

@@ -93,7 +93,7 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):

93 config.num_hidden_layers = config.n_layer

95 head_dim = config.hidden_size // config.num_attention_heads

96 - rotary_emb_dim = head_dim * config.rotary_emb_fraction

96 + rotary_emb_dim = int(head_dim * config.rotary_emb_fraction)

97 max_trained_positions = getattr(config, "max_trained_positions", 2048)

98 config.rotary_kwargs = {

99 "head_size": head_dim,

vllm/model_executor/models/modernbert.py

-2

@@ -8,7 +8,6 @@ from torch import nn

8 from transformers import ModernBertConfig

10 from vllm.attention import Attention, AttentionType

11 -from vllm.compilation.decorators import support_torch_compile

11 from vllm.config import VllmConfig

12 from vllm.distributed import get_tensor_model_parallel_world_size

13 from vllm.model_executor.layers.linear import (QKVParallelLinear,

@@ -200,7 +199,6 @@ class ModernBertEncoderLayer(nn.Module):

199 return hidden_states

200

201

203 -@support_torch_compile

202 class ModernBertModel(nn.Module):

203 hf_to_vllm_mapper = WeightsMapper(

204 orig_to_new_prefix={"layers.": "encoder_layer.layers."})