@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc2ef1f77c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Support Qwen2.5-Math-RM-72B (#8896)

Zilin Zhu committed Sep 29, 2024 at 12:19 UTC bc2ef1f77c1578612198f60ec392731efb3847c5
3 files changed +170
vllm/model_executor/layers/pooler.py
+7
@@ -11,6 +11,7 @@ from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
11 class PoolingType(IntEnum):
12 """Enumeration for different types of pooling methods."""
13 LAST = 0
14 + ALL = 1
15
16
17 class Pooler(nn.Module):
@@ -43,6 +44,12 @@ class Pooler(nn.Module):
44 if self.pooling_type == PoolingType.LAST:
45 last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
46 pooled_data = hidden_states[last_token_flat_indices]
47 + elif self.pooling_type == PoolingType.ALL:
48 + offset = 0
49 + pooled_data = []
50 + for prompt_len in prompt_lens:
51 + pooled_data.append(hidden_states[offset:offset + prompt_len])
52 + offset += prompt_len
53 else:
54 raise ValueError(f"Invalid pooling type: {self.pooling_type}")
55
vllm/model_executor/models/__init__.py
+1
@@ -74,6 +74,7 @@ _GENERATION_MODELS = {
74
75 _EMBEDDING_MODELS = {
76 "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
77 + "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
78 }
79
80 _MULTIMODAL_MODELS = {
vllm/model_executor/models/qwen2_rm.py new
+162
@@ -0,0 +1,162 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
4 +# Copyright 2024 The Qwen team.
5 +# Copyright 2023 The vLLM team.
6 +"""Inference-only Qwen2-RM model compatible with HuggingFace weights."""
7 +from typing import Iterable, List, Optional, Tuple
8 +
9 +import torch
10 +from torch import nn
11 +from transformers import Qwen2Config
12 +
13 +from vllm.attention import AttentionMetadata
14 +from vllm.config import CacheConfig, LoRAConfig
15 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 + RowParallelLinear)
17 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
18 +from vllm.model_executor.layers.quantization.base_config import (
19 + QuantizationConfig)
20 +from vllm.model_executor.model_loader.weight_utils import (
21 + default_weight_loader, maybe_remap_kv_scale_name)
22 +from vllm.model_executor.models.qwen2 import Qwen2Model
23 +from vllm.model_executor.pooling_metadata import PoolingMetadata
24 +from vllm.sequence import IntermediateTensors, PoolerOutput
25 +
26 +from .utils import is_pp_missing_parameter
27 +
28 +
29 +class ReLU(nn.Module):
30 +
31 + def __init__(self):
32 + super().__init__()
33 + self.activation = nn.ReLU()
34 +
35 + def forward(self, input):
36 + input, _ = input
37 + return self.activation(input)
38 +
39 +
40 +class Qwen2ForRewardModel(nn.Module):
41 + packed_modules_mapping = {
42 + "qkv_proj": [
43 + "q_proj",
44 + "k_proj",
45 + "v_proj",
46 + ],
47 + "gate_up_proj": [
48 + "gate_proj",
49 + "up_proj",
50 + ],
51 + }
52 +
53 + # LoRA specific attributes
54 + supported_lora_modules = [
55 + "qkv_proj",
56 + "o_proj",
57 + "gate_up_proj",
58 + "down_proj",
59 + ]
60 + embedding_modules = {}
61 + embedding_padding_modules = []
62 +
63 + def __init__(
64 + self,
65 + config: Qwen2Config,
66 + cache_config: Optional[CacheConfig] = None,
67 + quant_config: Optional[QuantizationConfig] = None,
68 + lora_config: Optional[LoRAConfig] = None,
69 + ) -> None:
70 + # TODO (@robertgshaw2): see if this can be moved out
71 + if (cache_config.sliding_window is not None
72 + and hasattr(config, "max_window_layers")):
73 + raise ValueError("Sliding window for some but all layers is not "
74 + "supported. This model uses sliding window "
75 + "but `max_window_layers` = %s is less than "
76 + "`num_hidden_layers` = %s. Please open an issue "
77 + "to discuss this feature." % (
78 + config.max_window_layers,
79 + config.num_hidden_layers,
80 + ))
81 +
82 + super().__init__()
83 +
84 + self.config = config
85 + self.lora_config = lora_config
86 +
87 + self.quant_config = quant_config
88 + self.model = Qwen2Model(config, cache_config, quant_config)
89 +
90 + self.score = nn.Sequential(
91 + ColumnParallelLinear(config.hidden_size,
92 + config.hidden_size,
93 + quant_config=quant_config),
94 + ReLU(),
95 + RowParallelLinear(config.hidden_size, 1,
96 + quant_config=quant_config),
97 + )
98 + self._pooler = Pooler(pooling_type=PoolingType.ALL, normalize=False)
99 +
100 + def forward(
101 + self,
102 + input_ids: torch.Tensor,
103 + positions: torch.Tensor,
104 + kv_caches: List[torch.Tensor],
105 + attn_metadata: AttentionMetadata,
106 + intermediate_tensors: Optional[IntermediateTensors] = None,
107 + ) -> torch.Tensor:
108 + hidden_states = self.model(input_ids, positions, kv_caches,
109 + attn_metadata, intermediate_tensors)
110 + logits, _ = self.score(hidden_states)
111 + return logits
112 +
113 + def pooler(
114 + self,
115 + hidden_states: torch.Tensor,
116 + pooling_metadata: PoolingMetadata,
117 + ) -> Optional[PoolerOutput]:
118 + return self._pooler(hidden_states, pooling_metadata)
119 +
120 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
121 + stacked_params_mapping = [
122 + # (param_name, shard_name, shard_id)
123 + ("qkv_proj", "q_proj", "q"),
124 + ("qkv_proj", "k_proj", "k"),
125 + ("qkv_proj", "v_proj", "v"),
126 + ("gate_up_proj", "gate_proj", 0),
127 + ("gate_up_proj", "up_proj", 1),
128 + ]
129 + params_dict = dict(self.named_parameters(remove_duplicate=False))
130 + for name, loaded_weight in weights:
131 + # Skip loading lm_head for embedding model
132 + if name == "lm_head.weight":
133 + continue
134 + if "rotary_emb.inv_freq" in name:
135 + continue
136 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
137 + if weight_name not in name:
138 + continue
139 + name = name.replace(weight_name, param_name)
140 + # Skip loading extra bias for GPTQ models.
141 + if name.endswith(".bias") and name not in params_dict:
142 + continue
143 + if is_pp_missing_parameter(name, self):
144 + continue
145 + param = params_dict[name]
146 + weight_loader = param.weight_loader
147 + weight_loader(param, loaded_weight, shard_id)
148 + break
149 + else:
150 + # Skip loading extra bias for GPTQ models.
151 + if name.endswith(".bias") and name not in params_dict:
152 + continue
153 + # Remapping the name of FP8 kv-scale.
154 + name = maybe_remap_kv_scale_name(name, params_dict)
155 + if name is None:
156 + continue
157 + if is_pp_missing_parameter(name, self):
158 + continue
159 + param = params_dict[name]
160 + weight_loader = getattr(param, "weight_loader",
161 + default_weight_loader)
162 + weight_loader(param, loaded_weight)