@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a57d13cc96
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

add QWen-7b (#685)

Co-authored-by: wq.chu <wq.chu@tianrang-inc.com>

Qing committed Aug 9, 2023 at 04:50 UTC a57d13cc967d04478cebcfa2c3440fb167b21c6d
6 files changed +396 -11
vllm/model_executor/model_loader.py
+1
@@ -23,6 +23,7 @@ _MODEL_REGISTRY = {
23 "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
24 "MPTForCausalLM": MPTForCausalLM,
25 "OPTForCausalLM": OPTForCausalLM,
26 + "QWenLMHeadModel": QWenLMHeadModel,
27 "RWForCausalLM": FalconForCausalLM,
28 }
29
vllm/model_executor/models/__init__.py
+5 -11
@@ -9,17 +9,11 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
9 from vllm.model_executor.models.llama import LlamaForCausalLM
10 from vllm.model_executor.models.mpt import MPTForCausalLM
11 from vllm.model_executor.models.opt import OPTForCausalLM
12 +from vllm.model_executor.models.qwen import QWenLMHeadModel
13
14 __all__ = [
14 - "BaiChuanForCausalLM",
15 - "BaichuanForCausalLM",
16 - "BloomForCausalLM",
17 - "FalconForCausalLM",
18 - "GPT2LMHeadModel",
19 - "GPTBigCodeForCausalLM",
20 - "GPTJForCausalLM",
21 - "GPTNeoXForCausalLM",
22 - "LlamaForCausalLM",
23 - "MPTForCausalLM",
24 - "OPTForCausalLM",
15 + "BaiChuanForCausalLM", "BaichuanForCausalLM", "BloomForCausalLM",
16 + "FalconForCausalLM", "GPT2LMHeadModel", "GPTBigCodeForCausalLM",
17 + "GPTJForCausalLM", "GPTNeoXForCausalLM", "LlamaForCausalLM",
18 + "MPTForCausalLM", "OPTForCausalLM", "QWenLMHeadModel"
19 ]
vllm/model_executor/models/qwen.py new
+316
@@ -0,0 +1,316 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
4 +# Copyright (c) Alibaba Cloud.
5 +# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6 +"""Inference-only QWen model compatible with HuggingFace weights.
7 +
8 +The input of the model is flattened to a 1D tensor of tokens. The model uses
9 +InputMetadata to extract the original 2D shape of the input.
10 +"""
11 +from typing import Dict, List, Optional, Tuple
12 +
13 +import torch
14 +from torch import nn
15 +
16 +from vllm.model_executor.input_metadata import InputMetadata
17 +from vllm.model_executor.layers.activation import SiluAndMul
18 +from vllm.model_executor.layers.layernorm import RMSNorm
19 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
20 +from vllm.model_executor.layers.sampler import Sampler
21 +from vllm.model_executor.weight_utils import (
22 + hf_model_weights_iterator,
23 + load_tensor_parallel_weights,
24 +)
25 +from vllm.model_executor.parallel_utils.parallel_state import (
26 + get_tensor_model_parallel_rank,
27 + get_tensor_model_parallel_world_size,
28 +)
29 +from vllm.model_executor.parallel_utils.tensor_parallel import (
30 + VocabParallelEmbedding,
31 + ColumnParallelLinear,
32 + RowParallelLinear,
33 +)
34 +from vllm.sequence import SequenceOutputs
35 +from vllm.transformers_utils.configs.qwen import QWenConfig
36 +
37 +KVCache = Tuple[torch.Tensor, torch.Tensor]
38 +
39 +
40 +class QWenMLP(nn.Module):
41 +
42 + def __init__(
43 + self,
44 + hidden_size: int,
45 + intermediate_size: int,
46 + hidden_act: str = "silu",
47 + ):
48 + super().__init__()
49 + self.gate_up_proj = ColumnParallelLinear(
50 + hidden_size,
51 + 2 * intermediate_size,
52 + bias=False,
53 + gather_output=False,
54 + perform_initialization=False,
55 + )
56 + self.c_proj = RowParallelLinear(
57 + intermediate_size,
58 + hidden_size,
59 + bias=False,
60 + input_is_parallel=True,
61 + perform_initialization=False,
62 + )
63 + if hidden_act != "silu":
64 + raise ValueError(f"Unsupported activation: {hidden_act}. "
65 + "Only silu is supported for now.")
66 + self.act_fn = SiluAndMul()
67 +
68 + def forward(self, x):
69 + gate_up, _ = self.gate_up_proj(x)
70 + x = self.act_fn(gate_up)
71 + x, _ = self.c_proj(x)
72 + return x
73 +
74 +
75 +class QWenAttention(nn.Module):
76 +
77 + def __init__(self, hidden_size: int, num_heads: int,
78 + max_position_embeddings: int):
79 + super().__init__()
80 + self.hidden_size = hidden_size
81 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
82 + )
83 + self.total_num_heads = num_heads
84 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
85 + self.num_heads = (self.total_num_heads //
86 + tensor_model_parallel_world_size)
87 + self.head_dim = hidden_size // self.total_num_heads
88 +
89 + # pylint: disable=invalid-name
90 + self.c_attn = ColumnParallelLinear(
91 + hidden_size,
92 + 3 * hidden_size,
93 + bias=True,
94 + gather_output=False,
95 + perform_initialization=False,
96 + )
97 + self.c_proj = RowParallelLinear(
98 + self.total_num_heads * self.head_dim,
99 + hidden_size,
100 + bias=False,
101 + input_is_parallel=True,
102 + perform_initialization=False,
103 + )
104 + self.scaling = self.head_dim**-0.5
105 + self.attn = PagedAttentionWithRoPE(
106 + self.num_heads,
107 + self.head_dim,
108 + self.scaling,
109 + rotary_dim=self.head_dim,
110 + max_position=max_position_embeddings,
111 + )
112 +
113 + def forward(
114 + self,
115 + positions: torch.Tensor,
116 + hidden_states: torch.Tensor,
117 + kv_cache: KVCache,
118 + input_metadata: InputMetadata,
119 + cache_event: Optional[torch.cuda.Event],
120 + ) -> torch.Tensor:
121 + qkv, _ = self.c_attn(hidden_states)
122 + q, k, v = qkv.chunk(chunks=3, dim=-1)
123 +
124 + k_cache, v_cache = kv_cache
125 + attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
126 + input_metadata, cache_event)
127 +
128 + output, _ = self.c_proj(attn_output)
129 + return output
130 +
131 +
132 +class QWenBlock(nn.Module):
133 +
134 + def __init__(self, config: QWenConfig):
135 + super().__init__()
136 + self.ln_1 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
137 +
138 + self.attn = QWenAttention(config.n_embd, config.num_attention_heads,
139 + config.max_position_embeddings)
140 +
141 + self.ln_2 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
142 +
143 + self.mlp = QWenMLP(config.n_embd, config.ffn_hidden_size // 2)
144 +
145 + def forward(
146 + self,
147 + positions: torch.Tensor,
148 + hidden_states: torch.Tensor,
149 + kv_cache: KVCache,
150 + input_metadata: InputMetadata,
151 + cache_event: Optional[torch.cuda.Event],
152 + ) -> torch.Tensor:
153 + # Self Attention
154 + residual = hidden_states
155 + hidden_states = self.ln_1(hidden_states)
156 + hidden_states = self.attn(
157 + positions=positions,
158 + hidden_states=hidden_states,
159 + kv_cache=kv_cache,
160 + input_metadata=input_metadata,
161 + cache_event=cache_event,
162 + )
163 + hidden_states = residual + hidden_states
164 +
165 + # Fully Connected
166 + residual = hidden_states
167 + hidden_states = self.ln_2(hidden_states)
168 + hidden_states = self.mlp(hidden_states)
169 + hidden_states = residual + hidden_states
170 + return hidden_states
171 +
172 +
173 +class QWenModel(nn.Module):
174 +
175 + def __init__(self, config: QWenConfig):
176 + super().__init__()
177 + self.config = config
178 + self.vocab_size = config.vocab_size
179 +
180 + vocab_size = ((config.vocab_size + 63) // 64) * 64
181 + self.wte = VocabParallelEmbedding(vocab_size,
182 + config.n_embd,
183 + perform_initialization=False)
184 + self.h = nn.ModuleList(
185 + [QWenBlock(config) for _ in range(config.num_hidden_layers)])
186 + self.ln_f = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
187 +
188 + def forward(
189 + self,
190 + input_ids: torch.Tensor,
191 + positions: torch.Tensor,
192 + kv_caches: List[KVCache],
193 + input_metadata: InputMetadata,
194 + cache_events: Optional[List[torch.cuda.Event]],
195 + ) -> torch.Tensor:
196 + hidden_states = self.wte(input_ids)
197 + for i in range(len(self.h)):
198 + if cache_events is None:
199 + cache_event = None
200 + else:
201 + cache_event = cache_events[i]
202 + layer = self.h[i]
203 + hidden_states = layer(
204 + positions,
205 + hidden_states,
206 + kv_caches[i],
207 + input_metadata,
208 + cache_event,
209 + )
210 + hidden_states = self.ln_f(hidden_states)
211 + return hidden_states
212 +
213 +
214 +class QWenLMHeadModel(nn.Module):
215 +
216 + def __init__(self, config: QWenConfig):
217 + super().__init__()
218 + self.config = config
219 + self.transformer = QWenModel(config)
220 + vocab_size = ((config.vocab_size + 63) // 64) * 64
221 + self.lm_head = ColumnParallelLinear(
222 + config.n_embd,
223 + vocab_size,
224 + bias=False,
225 + gather_output=False,
226 + perform_initialization=False,
227 + )
228 + self.sampler = Sampler(config.vocab_size)
229 +
230 + def forward(
231 + self,
232 + input_ids: torch.Tensor,
233 + positions: torch.Tensor,
234 + kv_caches: List[KVCache],
235 + input_metadata: InputMetadata,
236 + cache_events: Optional[List[torch.cuda.Event]],
237 + ) -> Dict[int, SequenceOutputs]:
238 + hidden_states = self.transformer(input_ids, positions, kv_caches,
239 + input_metadata, cache_events)
240 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
241 + input_metadata)
242 + return next_tokens
243 +
244 + _column_parallel_weights = ["wte.weight", "lm_head.weight"]
245 + _row_parallel_weights = ["c_proj.weight"]
246 +
247 + def load_weights(
248 + self,
249 + model_name_or_path: str,
250 + cache_dir: Optional[str] = None,
251 + use_np_cache: bool = False,
252 + ):
253 + tp_world_size = get_tensor_model_parallel_world_size()
254 + tp_rank = get_tensor_model_parallel_rank()
255 + state_dict = self.state_dict()
256 +
257 + for name, loaded_weight in hf_model_weights_iterator(
258 + model_name_or_path, cache_dir, use_np_cache):
259 + if "rotary_emb.inv_freq" in name:
260 + continue
261 +
262 + if "wte" in name or "lm_head" in name:
263 + # Consider padding in the vocab size.
264 + param = state_dict[name]
265 + padded_vocab_size = param.shape[0] * tp_world_size
266 + num_extra_rows = padded_vocab_size - self.config.vocab_size
267 + extra_rows = torch.empty(num_extra_rows,
268 + loaded_weight.shape[1])
269 + extra_rows = extra_rows.to(loaded_weight)
270 + loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
271 +
272 + if "c_attn" in name:
273 + total_num_heads = self.config.num_attention_heads
274 + hidden_size = self.config.hidden_size
275 + head_size = hidden_size // total_num_heads
276 + num_heads = total_num_heads // tp_world_size
277 + head_start = tp_rank * num_heads
278 + head_end = (tp_rank + 1) * num_heads
279 +
280 + if "weight" in name:
281 + loaded_weight = loaded_weight.view(3, total_num_heads,
282 + head_size, hidden_size)
283 + loaded_weight = loaded_weight[:, head_start:head_end, :, :]
284 + loaded_weight = loaded_weight.reshape(-1, hidden_size)
285 + elif "bias" in name:
286 + loaded_weight = loaded_weight.view(3, total_num_heads,
287 + head_size)
288 + loaded_weight = loaded_weight[:, head_start:head_end, :]
289 + loaded_weight = loaded_weight.reshape(-1)
290 +
291 + is_gate_up_weight = False
292 + for stride_id, weight_name in enumerate(["w2", "w1"]):
293 + if weight_name not in name:
294 + continue
295 + param = state_dict[name.replace(weight_name, "gate_up_proj")]
296 + shard_size = param.shape[0] // 2
297 + loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
298 + (tp_rank + 1)]
299 + param_slice = param.data[shard_size * stride_id:shard_size *
300 + (stride_id + 1)]
301 + assert param_slice.shape == loaded_weight.shape
302 + param_slice.copy_(loaded_weight)
303 + is_gate_up_weight = True
304 + break
305 + if is_gate_up_weight:
306 + continue
307 +
308 + param = state_dict[name]
309 + load_tensor_parallel_weights(
310 + param,
311 + loaded_weight,
312 + name,
313 + self._column_parallel_weights,
314 + self._row_parallel_weights,
315 + tp_rank,
316 + )
vllm/transformers_utils/config.py
+1
@@ -5,6 +5,7 @@ from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
5 _CONFIG_REGISTRY = {
6 "mpt": MPTConfig,
7 "baichuan": BaiChuanConfig,
8 + "qwen": QWenConfig,
9 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
10 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
11 }
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,5 +1,6 @@
1 from vllm.transformers_utils.configs.mpt import MPTConfig
2 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3 +from vllm.transformers_utils.configs.qwen import QWenConfig
4 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
5 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
6 # `FalconConfig` class from the official HuggingFace transformers library.
@@ -8,5 +9,6 @@ from vllm.transformers_utils.configs.falcon import RWConfig
9 __all__ = [
10 "MPTConfig",
11 "BaiChuanConfig",
12 + "QWenConfig",
13 "RWConfig",
14 ]
vllm/transformers_utils/configs/qwen.py new
+71
@@ -0,0 +1,71 @@
1 +# Copyright (c) Alibaba Cloud.
2 +# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
3 +
4 +from transformers import PretrainedConfig
5 +
6 +
7 +class QWenConfig(PretrainedConfig):
8 + model_type = "qwen"
9 + keys_to_ignore_at_inference = ["past_key_values"]
10 + attribute_map = {
11 + "hidden_size": "n_embd",
12 + "num_attention_heads": "n_head",
13 + "max_position_embeddings": "n_positions",
14 + "num_hidden_layers": "n_layer",
15 + }
16 +
17 + def __init__(
18 + self,
19 + vocab_size=151851,
20 + n_embd=4096,
21 + n_layer=32,
22 + n_head=32,
23 + n_inner=None,
24 + embd_pdrop=0.0,
25 + attn_pdrop=0.0,
26 + layer_norm_epsilon=1e-5,
27 + initializer_range=0.02,
28 + scale_attn_weights=True,
29 + use_cache=True,
30 + eos_token_id=151643,
31 + apply_residual_connection_post_layernorm=False,
32 + bf16=True,
33 + kv_channels=128,
34 + rotary_pct=1.0,
35 + rotary_emb_base=10000,
36 + use_dynamic_ntk=False,
37 + use_logn_attn=False,
38 + use_flash_attn=True,
39 + ffn_hidden_size=22016,
40 + no_bias=True,
41 + tie_word_embeddings=False,
42 + **kwargs,
43 + ):
44 + self.eos_token_id = eos_token_id
45 + super().__init__(eos_token_id=eos_token_id,
46 + tie_word_embeddings=tie_word_embeddings,
47 + **kwargs)
48 +
49 + self.vocab_size = vocab_size
50 + self.n_embd = n_embd
51 + self.n_layer = n_layer
52 + self.n_head = n_head
53 + self.n_inner = n_inner
54 + self.embd_pdrop = embd_pdrop
55 + self.attn_pdrop = attn_pdrop
56 + self.layer_norm_epsilon = layer_norm_epsilon
57 + self.initializer_range = initializer_range
58 + self.scale_attn_weights = scale_attn_weights
59 + self.use_cache = use_cache
60 + self.apply_residual_connection_post_layernorm = (
61 + apply_residual_connection_post_layernorm)
62 + self.bf16 = bf16
63 + self.kv_channels = kv_channels
64 + self.rotary_pct = rotary_pct
65 + self.rotary_emb_base = rotary_emb_base
66 + self.use_dynamic_ntk = use_dynamic_ntk
67 + self.use_logn_attn = use_logn_attn
68 + self.use_flash_attn = use_flash_attn
69 + self.ffn_hidden_size = ffn_hidden_size
70 + self.no_bias = no_bias
71 + self.tie_word_embeddings = tie_word_embeddings