A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model] Support InternLM2 Reward models (#11571)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Dec 28, 2024 at 14:14 UTC
d34be24bb196cb0cce167257c97449f0cd6858f7
4 files changed
+67
-1
docs/source/models/supported_models.md
+5
@@ -450,6 +450,11 @@ of the whole prompt are extracted from the normalized hidden state corresponding
450
- Example HF Models
451
- :ref:`LoRA <lora-adapter>`
452
- :ref:`PP <distributed-serving>`
453
+ * - :code:`InternLM2ForRewardModel`
454
+ - InternLM2-based
455
+ - :code:`internlm/internlm2-1_8b-reward`, :code:`internlm/internlm2-7b-reward`, etc.
456
+ - ✅︎
457
+ - ✅︎
458
* - :code:`LlamaForCausalLM`
459
- Llama-based
460
- :code:`peiyi9979/math-shepherd-mistral-7b-prm`, etc.
tests/models/registry.py
+2
@@ -140,6 +140,8 @@ _EMBEDDING_EXAMPLE_MODELS = {
140
"BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
141
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
142
"GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
143
+ "InternLM2ForRewardModel": _HfExamplesInfo("internlm/internlm2-1_8b-reward",
144
+ trust_remote_code=True),
145
"JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"), # noqa: E501
146
"LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
147
"MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
vllm/model_executor/models/internlm2.py
+59
-1
@@ -18,14 +18,16 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
18
QKVParallelLinear,
19
RowParallelLinear)
20
from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.rotary_embedding import get_rope
24
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
26
ParallelLMHead, VocabParallelEmbedding)
27
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28
+from vllm.model_executor.pooling_metadata import PoolingMetadata
29
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
-from vllm.sequence import IntermediateTensors
30
+from vllm.sequence import IntermediateTensors, PoolerOutput
31
32
from .interfaces import SupportsLoRA, SupportsPP
33
from .utils import (is_pp_missing_parameter,
@@ -433,3 +435,59 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
435
weight_loader(param, loaded_weight)
436
loaded_params.add(name)
437
return loaded_params
438
+
439
+
440
+class InternLM2ForRewardModel(InternLM2ForCausalLM):
441
+
442
+ def __init__(
443
+ self,
444
+ *,
445
+ vllm_config: VllmConfig,
446
+ prefix: str = "",
447
+ model_type: Type[InternLM2Model] = InternLM2Model,
448
+ ):
449
+ super().__init__(vllm_config=vllm_config,
450
+ prefix=prefix,
451
+ model_type=model_type)
452
+
453
+ for attr in ("output", "logits_processor", "sampler"):
454
+ delattr(self, attr)
455
+
456
+ config = vllm_config.model_config.hf_config
457
+ self.v_head = RowParallelLinear(
458
+ config.hidden_size,
459
+ 1,
460
+ bias=False,
461
+ input_is_parallel=False,
462
+ prefix=maybe_prefix(prefix, "v_head"),
463
+ )
464
+
465
+ pooler_config = vllm_config.model_config.pooler_config
466
+ self._pooler = Pooler.from_config_with_defaults(
467
+ pooler_config,
468
+ pooling_type=PoolingType.ALL,
469
+ normalize=False,
470
+ softmax=False,
471
+ )
472
+
473
+ def forward(
474
+ self,
475
+ input_ids: torch.Tensor,
476
+ positions: torch.Tensor,
477
+ kv_caches: List[torch.Tensor],
478
+ attn_metadata: AttentionMetadata,
479
+ intermediate_tensors: Optional[IntermediateTensors] = None,
480
+ inputs_embeds: Optional[torch.Tensor] = None,
481
+ ) -> Union[torch.Tensor, IntermediateTensors]:
482
+ hidden_states = self.model(input_ids, positions, kv_caches,
483
+ attn_metadata, intermediate_tensors,
484
+ inputs_embeds)
485
+ logits, _ = self.v_head(hidden_states)
486
+ return logits
487
+
488
+ def pooler(
489
+ self,
490
+ hidden_states: torch.Tensor,
491
+ pooling_metadata: PoolingMetadata,
492
+ ) -> Optional[PoolerOutput]:
493
+ return self._pooler(hidden_states, pooling_metadata)
vllm/model_executor/models/registry.py
+1
@@ -113,6 +113,7 @@ _EMBEDDING_MODELS = {
113
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
114
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
115
"GritLM": ("gritlm", "GritLM"),
116
+ "InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
117
"JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"), # noqa: E501
118
"LlamaModel": ("llama", "LlamaForCausalLM"),
119
**{