@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7f579eb97
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Support Yi model (#1567)

Roy committed Nov 7, 2023 at 07:26 UTC e7f579eb97f8129acec28804fb74d61f4f9b1ddf
6 files changed +496
vllm/model_executor/model_loader.py
+1
@@ -33,6 +33,7 @@ _MODEL_REGISTRY = {
33 "OPTForCausalLM": OPTForCausalLM,
34 "QWenLMHeadModel": QWenLMHeadModel,
35 "RWForCausalLM": FalconForCausalLM,
36 + "YiForCausalLM": YiForCausalLM,
37 }
38
39 # FIXME(woosuk): Remove this once all models support quantization.
vllm/model_executor/models/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.model_executor.models.mistral import MistralForCausalLM
13 from vllm.model_executor.models.mpt import MptForCausalLM
14 from vllm.model_executor.models.opt import OPTForCausalLM
15 from vllm.model_executor.models.qwen import QWenLMHeadModel
16 +from vllm.model_executor.models.yi import YiForCausalLM
17
18 __all__ = [
19 "AquilaForCausalLM",
@@ -30,4 +31,5 @@ __all__ = [
31 "OPTForCausalLM",
32 "QWenLMHeadModel",
33 "MistralForCausalLM",
34 + "YiForCausalLM",
35 ]
vllm/model_executor/models/yi.py new
+426
@@ -0,0 +1,426 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights.
24 +
25 +The input of the model is flattened to a 1D tensor of tokens. The model uses
26 +InputMetadata to extract the original 2D shape of the input.
27 +"""
28 +from typing import Any, Dict, List, Optional, Tuple
29 +
30 +import torch
31 +from torch import nn
32 +from vllm.transformers_utils.configs.yi import YiConfig
33 +
34 +from vllm.model_executor.input_metadata import InputMetadata
35 +from vllm.model_executor.layers.activation import SiluAndMul
36 +from vllm.model_executor.layers.layernorm import RMSNorm
37 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38 +from vllm.model_executor.layers.sampler import Sampler
39 +from vllm.model_executor.layers.quantized_linear import ParallelLinear
40 +from vllm.model_executor.parallel_utils.parallel_state import (
41 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 +from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43 +from vllm.model_executor.quantization_utils import QuantizationConfig
44 +from vllm.model_executor.weight_utils import (
45 + convert_pyslice_to_tensor, hf_model_weights_iterator,
46 + load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
47 +from vllm.sequence import SamplerOutput
48 +
49 +KVCache = Tuple[torch.Tensor, torch.Tensor]
50 +
51 +
52 +class YiMLP(nn.Module):
53 +
54 + def __init__(
55 + self,
56 + hidden_size: int,
57 + intermediate_size: int,
58 + hidden_act: str,
59 + quant_config: Optional[QuantizationConfig] = None,
60 + ) -> None:
61 + super().__init__()
62 + self.gate_up_proj = ParallelLinear.column(hidden_size,
63 + 2 * intermediate_size,
64 + bias=False,
65 + gather_output=False,
66 + quant_config=quant_config)
67 + self.down_proj = ParallelLinear.row(intermediate_size,
68 + hidden_size,
69 + bias=False,
70 + input_is_parallel=True,
71 + quant_config=quant_config)
72 + if hidden_act != "silu":
73 + raise ValueError(f"Unsupported activation: {hidden_act}. "
74 + "Only silu is supported for now.")
75 + self.act_fn = SiluAndMul()
76 +
77 + def forward(self, x):
78 + gate_up, _ = self.gate_up_proj(x)
79 + x = self.act_fn(gate_up)
80 + x, _ = self.down_proj(x)
81 + return x
82 +
83 +
84 +class YiAttention(nn.Module):
85 +
86 + def __init__(
87 + self,
88 + hidden_size: int,
89 + num_heads: int,
90 + num_kv_heads: int,
91 + rope_theta: float = 10000,
92 + rope_scaling: Optional[Dict[str, Any]] = None,
93 + max_position_embeddings: int = 8192,
94 + quant_config: Optional[QuantizationConfig] = None,
95 + ) -> None:
96 + super().__init__()
97 + self.hidden_size = hidden_size
98 + tp_size = get_tensor_model_parallel_world_size()
99 + self.total_num_heads = num_heads
100 + assert self.total_num_heads % tp_size == 0
101 + self.num_heads = self.total_num_heads // tp_size
102 + self.total_num_kv_heads = num_kv_heads
103 + if self.total_num_kv_heads >= tp_size:
104 + # Number of KV heads is greater than TP size, so we partition
105 + # the KV heads across multiple tensor parallel GPUs.
106 + assert self.total_num_kv_heads % tp_size == 0
107 + else:
108 + # Number of KV heads is less than TP size, so we replicate
109 + # the KV heads across multiple tensor parallel GPUs.
110 + assert tp_size % self.total_num_kv_heads == 0
111 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112 + num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
113 + self.head_dim = hidden_size // self.total_num_heads
114 + self.q_size = self.num_heads * self.head_dim
115 + self.kv_size = self.num_kv_heads * self.head_dim
116 + self.scaling = self.head_dim**-0.5
117 + self.rope_theta = rope_theta
118 + self.max_position_embeddings = max_position_embeddings
119 +
120 + self.qkv_proj = ParallelLinear.column(
121 + hidden_size,
122 + (self.total_num_heads +
123 + 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
124 + self.head_dim,
125 + bias=False,
126 + gather_output=False,
127 + quant_config=quant_config,
128 + )
129 + self.o_proj = ParallelLinear.row(
130 + self.total_num_heads * self.head_dim,
131 + hidden_size,
132 + bias=False,
133 + input_is_parallel=True,
134 + quant_config=quant_config,
135 + )
136 + self.attn = PagedAttentionWithRoPE(
137 + self.num_heads,
138 + self.head_dim,
139 + self.scaling,
140 + base=self.rope_theta,
141 + max_position=self.max_position_embeddings,
142 + rotary_dim=self.head_dim,
143 + num_kv_heads=self.num_kv_heads,
144 + rope_scaling=rope_scaling)
145 +
146 + def forward(
147 + self,
148 + positions: torch.Tensor,
149 + hidden_states: torch.Tensor,
150 + kv_cache: KVCache,
151 + input_metadata: InputMetadata,
152 + cache_event: Optional[torch.cuda.Event],
153 + ) -> torch.Tensor:
154 + qkv, _ = self.qkv_proj(hidden_states)
155 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
156 + k_cache, v_cache = kv_cache
157 + attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
158 + input_metadata, cache_event)
159 + output, _ = self.o_proj(attn_output)
160 + return output
161 +
162 +
163 +class YiDecoderLayer(nn.Module):
164 +
165 + def __init__(
166 + self,
167 + config: YiConfig,
168 + quant_config: Optional[QuantizationConfig] = None,
169 + ) -> None:
170 + super().__init__()
171 + self.hidden_size = config.hidden_size
172 + # Requires transformers > 4.32.0
173 + rope_theta = getattr(config, "rope_theta", 10000)
174 + rope_scaling = getattr(config, "rope_scaling", None)
175 + max_position_embeddings = getattr(config, "max_position_embeddings",
176 + 8192)
177 + self.self_attn = YiAttention(
178 + hidden_size=self.hidden_size,
179 + num_heads=config.num_attention_heads,
180 + num_kv_heads=config.num_key_value_heads,
181 + rope_theta=rope_theta,
182 + rope_scaling=rope_scaling,
183 + max_position_embeddings=max_position_embeddings,
184 + quant_config=quant_config,
185 + )
186 + self.mlp = YiMLP(
187 + hidden_size=self.hidden_size,
188 + intermediate_size=config.intermediate_size,
189 + hidden_act=config.hidden_act,
190 + quant_config=quant_config,
191 + )
192 + self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
193 + self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
194 +
195 + def forward(
196 + self,
197 + positions: torch.Tensor,
198 + hidden_states: torch.Tensor,
199 + kv_cache: KVCache,
200 + input_metadata: InputMetadata,
201 + cache_event: Optional[torch.cuda.Event],
202 + ) -> torch.Tensor:
203 + # Self Attention
204 + residual = hidden_states
205 + hidden_states = self.ln1(hidden_states)
206 + hidden_states = self.self_attn(
207 + positions=positions,
208 + hidden_states=hidden_states,
209 + kv_cache=kv_cache,
210 + input_metadata=input_metadata,
211 + cache_event=cache_event,
212 + )
213 + hidden_states = residual + hidden_states
214 +
215 + # Fully Connected
216 + residual = hidden_states
217 + hidden_states = self.ln2(hidden_states)
218 + hidden_states = self.mlp(hidden_states)
219 + hidden_states = residual + hidden_states
220 + return hidden_states
221 +
222 +
223 +class YiModel(nn.Module):
224 +
225 + def __init__(
226 + self,
227 + config: YiConfig,
228 + quant_config: Optional[QuantizationConfig] = None,
229 + ) -> None:
230 + super().__init__()
231 + self.config = config
232 + self.padding_idx = config.pad_token_id
233 + self.vocab_size = config.vocab_size
234 +
235 + vocab_size = ((config.vocab_size + 63) // 64) * 64
236 + self.embed_tokens = VocabParallelEmbedding(
237 + vocab_size,
238 + config.hidden_size,
239 + )
240 + self.layers = nn.ModuleList([
241 + YiDecoderLayer(config, quant_config)
242 + for _ in range(config.num_hidden_layers)
243 + ])
244 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
245 +
246 + def forward(
247 + self,
248 + input_ids: torch.Tensor,
249 + positions: torch.Tensor,
250 + kv_caches: List[KVCache],
251 + input_metadata: InputMetadata,
252 + cache_events: Optional[List[torch.cuda.Event]],
253 + ) -> torch.Tensor:
254 + hidden_states = self.embed_tokens(input_ids)
255 + for i in range(len(self.layers)):
256 + if cache_events is None:
257 + cache_event = None
258 + else:
259 + cache_event = cache_events[i]
260 + layer = self.layers[i]
261 + hidden_states = layer(
262 + positions,
263 + hidden_states,
264 + kv_caches[i],
265 + input_metadata,
266 + cache_event,
267 + )
268 + hidden_states = self.norm(hidden_states)
269 + return hidden_states
270 +
271 +
272 +class YiForCausalLM(nn.Module):
273 +
274 + def __init__(
275 + self,
276 + config: YiConfig,
277 + quant_config: Optional[QuantizationConfig] = None,
278 + ) -> None:
279 + super().__init__()
280 + self.config = config
281 + self.quant_config = quant_config
282 + self.model = YiModel(config, quant_config)
283 + vocab_size = ((config.vocab_size + 63) // 64) * 64
284 + # NOTE: The LM head is not quantized.
285 + self.lm_head = ParallelLinear.column(config.hidden_size,
286 + vocab_size,
287 + bias=False,
288 + gather_output=False,
289 + quant_config=None)
290 + self.sampler = Sampler(config.vocab_size)
291 +
292 + def forward(
293 + self,
294 + input_ids: torch.Tensor,
295 + positions: torch.Tensor,
296 + kv_caches: List[KVCache],
297 + input_metadata: InputMetadata,
298 + cache_events: Optional[List[torch.cuda.Event]],
299 + ) -> SamplerOutput:
300 + hidden_states = self.model(input_ids, positions, kv_caches,
301 + input_metadata, cache_events)
302 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303 + input_metadata)
304 + return next_tokens
305 +
306 + _column_parallel_layers = []
307 + _row_parallel_layers = ["o_proj", "down_proj"]
308 +
309 + def load_weights(self,
310 + model_name_or_path: str,
311 + cache_dir: Optional[str] = None,
312 + load_format: str = "auto",
313 + revision: Optional[str] = None):
314 + if self.quant_config is None:
315 + col_weight_suffixes = ["weight"]
316 + row_weight_suffixes = ["weight"]
317 + else:
318 + col_weight_suffixes = (
319 + self.quant_config.get_col_parallel_tensor_names())
320 + row_weight_suffixes = (
321 + self.quant_config.get_row_parallel_tensor_names())
322 +
323 + column_parallel_weights: List[str] = []
324 + for layer in self._column_parallel_layers:
325 + for suffix in col_weight_suffixes:
326 + column_parallel_weights.append(f"{layer}.{suffix}")
327 + row_parallel_weights: List[str] = []
328 + for layer in self._row_parallel_layers:
329 + for suffix in row_weight_suffixes:
330 + row_parallel_weights.append(f"{layer}.{suffix}")
331 +
332 + tp_size = get_tensor_model_parallel_world_size()
333 + tp_rank = get_tensor_model_parallel_rank()
334 + q_proj_shard_size = (self.config.hidden_size // tp_size)
335 + num_kv_heads_replicas = max(1,
336 + tp_size // self.config.num_key_value_heads)
337 + num_kv_heads_per_gpu = max(1,
338 + self.config.num_key_value_heads // tp_size)
339 + kv_proj_shard_size = (self.config.hidden_size //
340 + self.config.num_attention_heads *
341 + num_kv_heads_per_gpu)
342 + attention_weight_specs = [
343 + # (weight_name, shard_size, offset)
344 + ("q_proj", q_proj_shard_size, 0),
345 + ("k_proj", kv_proj_shard_size, q_proj_shard_size),
346 + ("v_proj", kv_proj_shard_size,
347 + q_proj_shard_size + kv_proj_shard_size),
348 + ]
349 + state_dict = self.state_dict()
350 +
351 + for name, loaded_weight in hf_model_weights_iterator(
352 + model_name_or_path, cache_dir, load_format, revision):
353 + if "rotary_emb.inv_freq" in name:
354 + continue
355 +
356 + packed_dim = None
357 + is_transposed = False
358 + if self.quant_config is not None:
359 + packed_dim = self.quant_config.get_packed_dim(name)
360 + is_transposed = self.quant_config.is_transposed(name)
361 + if is_transposed:
362 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
363 + loaded_weight = loaded_weight.T
364 +
365 + is_attention_weight = False
366 + for weight_name, shard_size, offset in attention_weight_specs:
367 + if weight_name not in name:
368 + continue
369 + param = state_dict[name.replace(weight_name, "qkv_proj")]
370 + if is_transposed:
371 + param = param.T
372 +
373 + if packed_dim is not None:
374 + shard_dim = 0 if not is_transposed else 1
375 + if packed_dim == shard_dim:
376 + shard_size //= self.quant_config.pack_factor
377 + offset //= self.quant_config.pack_factor
378 +
379 + if weight_name in ["k_proj", "v_proj"]:
380 + shard_id = tp_rank // num_kv_heads_replicas
381 + else:
382 + shard_id = tp_rank
383 + loaded_weight = loaded_weight[shard_size *
384 + shard_id:shard_size *
385 + (shard_id + 1)]
386 + param_slice = param.data[offset:offset + shard_size]
387 + assert param_slice.shape == loaded_weight.shape
388 +
389 + param_slice.copy_(loaded_weight)
390 + is_attention_weight = True
391 + break
392 + if is_attention_weight:
393 + continue
394 +
395 + is_gate_up_weight = False
396 + for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
397 + if weight_name not in name:
398 + continue
399 + param = state_dict[name.replace(weight_name, "gate_up_proj")]
400 + if is_transposed:
401 + param = param.T
402 +
403 + shard_size = param.shape[0] // 2
404 + loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
405 + (tp_rank + 1)]
406 + param_slice = param.data[shard_size * stride_id:shard_size *
407 + (stride_id + 1)]
408 + assert param_slice.shape == loaded_weight.shape
409 + param_slice.copy_(loaded_weight)
410 + is_gate_up_weight = True
411 + break
412 + if is_gate_up_weight:
413 + continue
414 +
415 + param = state_dict[name]
416 + if is_transposed:
417 + param = param.T
418 +
419 + if "embed_tokens" in name or "lm_head" in name:
420 + load_padded_tensor_parallel_vocab(param, loaded_weight,
421 + tp_rank)
422 + continue
423 +
424 + load_tensor_parallel_weights(param, loaded_weight, name,
425 + column_parallel_weights,
426 + row_parallel_weights, tp_rank)
vllm/transformers_utils/config.py
+1
@@ -11,6 +11,7 @@ _CONFIG_REGISTRY = {
11 "qwen": QWenConfig,
12 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
13 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
14 + "yi": YiConfig,
15 }
16
17
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,10 +5,12 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
5 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
6 # `FalconConfig` class from the official HuggingFace transformers library.
7 from vllm.transformers_utils.configs.falcon import RWConfig
8 +from vllm.transformers_utils.configs.yi import YiConfig
9
10 __all__ = [
11 "BaiChuanConfig",
12 "AquilaConfig",
13 "QWenConfig",
14 "RWConfig",
15 + "YiConfig",
16 ]
vllm/transformers_utils/configs/yi.py new
+64
@@ -0,0 +1,64 @@
1 +""" Yi model configuration"""
2 +from transformers.configuration_utils import PretrainedConfig
3 +from transformers.utils import logging
4 +
5 +logger = logging.get_logger(__name__)
6 +
7 +Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8 +
9 +
10 +class YiConfig(PretrainedConfig):
11 + r"""
12 + Reference:
13 + https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14 + """
15 + model_type = "Yi"
16 + keys_to_ignore_at_inference = ["past_key_values"]
17 +
18 + def __init__(
19 + self,
20 + vocab_size=64000,
21 + hidden_size=4096,
22 + intermediate_size=11008,
23 + num_hidden_layers=32,
24 + num_attention_heads=32,
25 + num_key_value_heads=4,
26 + hidden_act="silu",
27 + max_position_embeddings=4096,
28 + initializer_range=0.02,
29 + rms_norm_eps=1e-5,
30 + use_cache=True,
31 + pad_token_id=0,
32 + bos_token_id=1,
33 + eos_token_id=2,
34 + tie_word_embeddings=False,
35 + output_attentions=False,
36 + rope_theta=5000000.0,
37 + **kwargs,
38 + ):
39 + self.vocab_size = vocab_size
40 + self.max_position_embeddings = max_position_embeddings
41 + self.hidden_size = hidden_size
42 + self.intermediate_size = intermediate_size
43 + self.num_hidden_layers = num_hidden_layers
44 + self.num_attention_heads = num_attention_heads
45 +
46 + # for backward compatibility
47 + if num_key_value_heads is None:
48 + num_key_value_heads = num_attention_heads
49 +
50 + self.num_key_value_heads = num_key_value_heads
51 + self.hidden_act = hidden_act
52 + self.initializer_range = initializer_range
53 + self.rms_norm_eps = rms_norm_eps
54 + self.use_cache = use_cache
55 + self.output_attentions = output_attentions
56 + self.rope_theta = rope_theta
57 +
58 + super().__init__(
59 + pad_token_id=pad_token_id,
60 + bos_token_id=bos_token_id,
61 + eos_token_id=eos_token_id,
62 + tie_word_embeddings=tie_word_embeddings,
63 + **kwargs,
64 + )