@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 83609791d2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model] Add Qwen2 PRM model support (#12202)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 20, 2025 at 14:59 UTC 83609791d2ceeb628e0d1f5ea60a64c132eb083c
5 files changed +45 -13
docs/source/models/supported_models.md
+5
@@ -470,6 +470,11 @@ of the whole prompt are extracted from the normalized hidden state corresponding
470 - `Qwen/Qwen2.5-Math-RM-72B`, etc.
471 - ✅︎
472 - ✅︎
473 +* - `Qwen2ForProcessRewardModel`
474 + - Qwen2-based
475 + - `Qwen/Qwen2.5-Math-PRM-7B`, `Qwen/Qwen2.5-Math-PRM-72B`, etc.
476 + - ✅︎
477 + - ✅︎
478 ```
479
480 If your model is not in the above list, we will try to automatically convert the model using
tests/models/embedding/language/test_embedding.py
+5 -4
@@ -17,14 +17,15 @@ from ..utils import check_embeddings_close
17 marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
18 pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
19 pytest.param("intfloat/multilingual-e5-large"),
20 - # [Encoder-decoder]
21 - pytest.param("intfloat/e5-mistral-7b-instruct",
22 - marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
20 + # [Decoder-only]
21 pytest.param("BAAI/bge-multilingual-gemma2",
22 marks=[pytest.mark.core_model]),
25 - pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
23 + pytest.param("intfloat/e5-mistral-7b-instruct",
24 + marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
25 pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
26 pytest.param("Alibaba-NLP/gte-Qwen2-7B-instruct"),
27 + pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
28 + # [Encoder-decoder]
29 pytest.param("sentence-transformers/stsb-roberta-base-v2"),
30 ],
31 )
tests/models/registry.py
+1
@@ -155,6 +155,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
155 "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
156 "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
157 "Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
158 + "Qwen2ForProcessRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-PRM-7B"),
159 "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), # noqa: E501
160 "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"), # noqa: E501
161 "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"), # noqa: E501
vllm/model_executor/models/qwen2_rm.py
+33 -9
@@ -12,7 +12,7 @@ from vllm.attention import AttentionMetadata
12 from vllm.config import VllmConfig
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 RowParallelLinear)
15 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
15 +from vllm.model_executor.layers.pooler import Pooler, PoolingType, SimplePooler
16 from vllm.model_executor.pooling_metadata import PoolingMetadata
17 from vllm.sequence import IntermediateTensors, PoolerOutput
18
@@ -32,7 +32,7 @@ class ReLU(nn.Module):
32 return self.activation(input)
33
34
35 -class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
35 +class Qwen2RewardBaseModel(nn.Module, SupportsLoRA, SupportsPP):
36 packed_modules_mapping = {
37 "qkv_proj": [
38 "q_proj",
@@ -60,7 +60,6 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
60 config = vllm_config.model_config.hf_config
61 quant_config = vllm_config.quant_config
62 lora_config = vllm_config.lora_config
63 - pooler_config = vllm_config.model_config.pooler_config
63
64 self.config = config
65 self.lora_config = lora_config
@@ -74,14 +73,11 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
73 config.hidden_size,
74 quant_config=quant_config),
75 ReLU(),
77 - RowParallelLinear(config.hidden_size, 1,
76 + RowParallelLinear(config.hidden_size,
77 + config.num_labels,
78 quant_config=quant_config),
79 )
80 - self._pooler = Pooler.from_config_with_defaults(
81 - pooler_config,
82 - pooling_type=PoolingType.ALL,
83 - normalize=False,
84 - softmax=False)
80 + self._pooler: SimplePooler
81 self.make_empty_intermediate_tensors = (
82 self.model.make_empty_intermediate_tensors)
83
@@ -115,3 +111,31 @@ class Qwen2ForRewardModel(nn.Module, SupportsLoRA, SupportsPP):
111 loader = AutoWeightsLoader(self,
112 ignore_unexpected_prefixes=["lm_head."])
113 return loader.load_weights(weights)
114 +
115 +
116 +class Qwen2ForRewardModel(Qwen2RewardBaseModel):
117 +
118 + def __init__(self, *, vllm_config, prefix=""):
119 + vllm_config.model_config.hf_config.num_labels = 1
120 + super().__init__(vllm_config=vllm_config, prefix=prefix)
121 + pooler_config = vllm_config.model_config.pooler_config
122 + self._pooler = Pooler.from_config_with_defaults(
123 + pooler_config,
124 + pooling_type=PoolingType.ALL,
125 + normalize=False,
126 + softmax=False)
127 +
128 +
129 +class Qwen2ForProcessRewardModel(Qwen2RewardBaseModel):
130 +
131 + def __init__(self, *, vllm_config, prefix=""):
132 + vllm_config.model_config.hf_config.num_labels = 2
133 + super().__init__(vllm_config=vllm_config, prefix=prefix)
134 + pooler_config = vllm_config.model_config.pooler_config
135 + self._pooler = Pooler.from_config_with_defaults(
136 + pooler_config,
137 + pooling_type=PoolingType.STEP,
138 + normalize=False,
139 + softmax=True,
140 + step_tag_id=151651,
141 + )
vllm/model_executor/models/registry.py
+1
@@ -127,6 +127,7 @@ _EMBEDDING_MODELS = {
127 "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
128 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
129 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
130 + "Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
131 "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
132 # [Multimodal]
133 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501