@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b4543c8f6b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Model] add minicpm (#3893)

ywfang committed Apr 8, 2024 at 18:28 UTC b4543c8f6bf67a7f1a0d6d0fd6cf5697c7eeaabb
4 files changed +543
README.md
+1
@@ -70,6 +70,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
70 - InternLM2 (`internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.)
71 - Jais (`core42/jais-13b`, `core42/jais-13b-chat`, `core42/jais-30b-v3`, `core42/jais-30b-chat-v3`, etc.)
72 - LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
73 +- MiniCPM (`openbmb/MiniCPM-2B-sft-bf16`, `openbmb/MiniCPM-2B-dpo-bf16`, etc.)
74 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
75 - Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
76 - MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
docs/source/models/supported_models.rst
+4
@@ -83,6 +83,10 @@ Alongside each architecture, we include some popular models that use it.
83 - LLaMA, LLaMA-2, Vicuna, Alpaca, Yi
84 - :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
85 - ✅︎
86 + * - :code:`MiniCPMForCausalLM`
87 + - MiniCPM
88 + - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
89 + -
90 * - :code:`MistralForCausalLM`
91 - Mistral, Mistral-Instruct
92 - :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -41,6 +41,7 @@ _MODELS = {
41 # transformers's mpt class has lower case
42 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
43 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
44 + "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
45 "OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
46 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
47 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
vllm/model_executor/models/minicpm.py new
+537
@@ -0,0 +1,537 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only MiniCPM model compatible with HuggingFace weights."""
24 +import math
25 +from typing import Any, Dict, List, Optional, Tuple
26 +
27 +import torch
28 +from torch import nn
29 +
30 +from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import LoRAConfig
32 +from vllm.model_executor.layers.activation import SiluAndMul
33 +from vllm.model_executor.layers.fused_moe import fused_moe
34 +from vllm.model_executor.layers.layernorm import RMSNorm
35 +from vllm.model_executor.layers.linear import (LinearMethodBase,
36 + MergedColumnParallelLinear,
37 + QKVParallelLinear,
38 + ReplicatedLinear,
39 + RowParallelLinear)
40 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 +from vllm.model_executor.layers.rotary_embedding import get_rope
42 +from vllm.model_executor.layers.sampler import Sampler
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 +from vllm.model_executor.parallel_utils.communication_op import (
46 + tensor_model_parallel_all_reduce)
47 +from vllm.model_executor.parallel_utils.parallel_state import (
48 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49 +from vllm.model_executor.sampling_metadata import SamplingMetadata
50 +from vllm.model_executor.utils import set_weight_attrs
51 +from vllm.model_executor.weight_utils import (default_weight_loader,
52 + hf_model_weights_iterator)
53 +from vllm.sequence import SamplerOutput
54 +
55 +
56 +class MiniCPMMoE(nn.Module):
57 + """A tensor-parallel MoE implementation that shards each expert
58 + across all ranks.
59 +
60 + Each expert's weights are sharded across all ranks and a fused MoE
61 + kernel is used for the forward pass, and finally we reduce the outputs
62 + across ranks.
63 + """
64 +
65 + def __init__(
66 + self,
67 + num_experts: int,
68 + top_k: int,
69 + hidden_size: int,
70 + intermediate_size: int,
71 + params_dtype: Optional[torch.dtype] = None,
72 + tp_size: Optional[int] = None,
73 + ):
74 + super().__init__()
75 + self.tp_size = tp_size or get_tensor_model_parallel_world_size()
76 + self.num_total_experts = num_experts
77 + self.top_k = top_k
78 + self.hidden_size = hidden_size
79 + self.intermediate_size = intermediate_size // self.tp_size
80 +
81 + if params_dtype is None:
82 + params_dtype = torch.get_default_dtype()
83 + self.params_dtype = params_dtype
84 +
85 + self.gate = ReplicatedLinear(self.hidden_size,
86 + self.num_total_experts,
87 + bias=False,
88 + params_dtype=self.params_dtype,
89 + linear_method=None)
90 +
91 + self.ws = nn.Parameter(
92 + torch.empty(self.num_total_experts,
93 + 2 * self.intermediate_size,
94 + self.hidden_size,
95 + device="cuda",
96 + dtype=self.params_dtype))
97 + self.w2s = nn.Parameter(
98 + torch.empty(self.num_total_experts,
99 + self.hidden_size,
100 + self.intermediate_size,
101 + device="cuda",
102 + dtype=self.params_dtype))
103 +
104 + set_weight_attrs(self.ws, {
105 + "weight_loader": self.weight_loader,
106 + })
107 + set_weight_attrs(self.w2s, {
108 + "weight_loader": self.weight_loader,
109 + })
110 +
111 + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
112 + weight_name: str, expert_id: int):
113 + tp_rank = get_tensor_model_parallel_rank()
114 + param_data = param.data
115 + shard_size = self.intermediate_size
116 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
117 + if weight_name.endswith("w1.weight"):
118 + param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
119 + if weight_name.endswith("w3.weight"):
120 + param_data[expert_id,
121 + shard_size:2 * shard_size, :] = loaded_weight[shard, :]
122 + if weight_name.endswith("w2.weight"):
123 + param_data[expert_id, :, :] = loaded_weight[:, shard]
124 +
125 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
126 + num_tokens, hidden_size = hidden_states.shape
127 + hidden_states = hidden_states.view(-1, self.hidden_size)
128 + # router_logits: (num_tokens, n_experts)
129 + router_logits, _ = self.gate(hidden_states)
130 + final_hidden_states = fused_moe(hidden_states,
131 + self.ws,
132 + self.w2s,
133 + router_logits,
134 + self.top_k,
135 + renormalize=True,
136 + inplace=True)
137 +
138 + if self.tp_size > 1:
139 + final_hidden_states = tensor_model_parallel_all_reduce(
140 + final_hidden_states)
141 +
142 + return final_hidden_states.view(num_tokens, hidden_size)
143 +
144 +
145 +class MiniCPMMLP(nn.Module):
146 +
147 + def __init__(
148 + self,
149 + hidden_size: int,
150 + intermediate_size: int,
151 + hidden_act: str,
152 + linear_method: Optional[LinearMethodBase] = None,
153 + ) -> None:
154 + super().__init__()
155 + self.gate_up_proj = MergedColumnParallelLinear(
156 + hidden_size, [intermediate_size] * 2,
157 + bias=False,
158 + linear_method=linear_method)
159 + self.down_proj = RowParallelLinear(intermediate_size,
160 + hidden_size,
161 + bias=False,
162 + linear_method=linear_method)
163 + if hidden_act != "silu":
164 + raise ValueError(f"Unsupported activation: {hidden_act}. "
165 + "Only silu is supported for now.")
166 + self.act_fn = SiluAndMul()
167 +
168 + def forward(self, x):
169 + gate_up, _ = self.gate_up_proj(x)
170 + x = self.act_fn(gate_up)
171 + x, _ = self.down_proj(x)
172 + return x
173 +
174 +
175 +class MiniCPMAttention(nn.Module):
176 +
177 + def __init__(
178 + self,
179 + hidden_size: int,
180 + num_heads: int,
181 + num_kv_heads: int,
182 + rope_theta: float = 10000,
183 + rope_scaling: Optional[Dict[str, Any]] = None,
184 + max_position_embeddings: int = 8192,
185 + linear_method: Optional[LinearMethodBase] = None,
186 + ) -> None:
187 + super().__init__()
188 + self.hidden_size = hidden_size
189 + tp_size = get_tensor_model_parallel_world_size()
190 + self.total_num_heads = num_heads
191 + assert self.total_num_heads % tp_size == 0
192 + self.num_heads = self.total_num_heads // tp_size
193 + self.total_num_kv_heads = num_kv_heads
194 + if self.total_num_kv_heads >= tp_size:
195 + # Number of KV heads is greater than TP size, so we partition
196 + # the KV heads across multiple tensor parallel GPUs.
197 + assert self.total_num_kv_heads % tp_size == 0
198 + else:
199 + # Number of KV heads is less than TP size, so we replicate
200 + # the KV heads across multiple tensor parallel GPUs.
201 + assert tp_size % self.total_num_kv_heads == 0
202 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
203 + self.head_dim = hidden_size // self.total_num_heads
204 + self.q_size = self.num_heads * self.head_dim
205 + self.kv_size = self.num_kv_heads * self.head_dim
206 + self.scaling = self.head_dim**-0.5
207 + self.rope_theta = rope_theta
208 + self.max_position_embeddings = max_position_embeddings
209 +
210 + self.qkv_proj = QKVParallelLinear(
211 + hidden_size,
212 + self.head_dim,
213 + self.total_num_heads,
214 + self.total_num_kv_heads,
215 + bias=False,
216 + linear_method=linear_method,
217 + )
218 + self.o_proj = RowParallelLinear(
219 + self.total_num_heads * self.head_dim,
220 + hidden_size,
221 + bias=False,
222 + linear_method=linear_method,
223 + )
224 +
225 + self.rotary_emb = get_rope(
226 + self.head_dim,
227 + rotary_dim=self.head_dim,
228 + max_position=max_position_embeddings,
229 + base=rope_theta,
230 + rope_scaling=rope_scaling,
231 + )
232 + # set rope as fp32 instead of bf16
233 + self.rotary_emb.cos_sin_cache = self.rotary_emb._compute_cos_sin_cache(
234 + )
235 + self.attn = Attention(self.num_heads,
236 + self.head_dim,
237 + self.scaling,
238 + num_kv_heads=self.num_kv_heads)
239 +
240 + def forward(
241 + self,
242 + positions: torch.Tensor,
243 + hidden_states: torch.Tensor,
244 + kv_cache: torch.Tensor,
245 + attn_metadata: AttentionMetadata,
246 + ) -> torch.Tensor:
247 + qkv, _ = self.qkv_proj(hidden_states)
248 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
249 + orig_dtype = q.dtype
250 + q, k = q.float(), k.float()
251 + q, k = self.rotary_emb(positions, q, k)
252 + q, k = q.to(orig_dtype), k.to(orig_dtype)
253 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
254 + output, _ = self.o_proj(attn_output)
255 + return output
256 +
257 +
258 +class MiniCPMDecoderLayer(nn.Module):
259 +
260 + def __init__(
261 + self,
262 + config,
263 + linear_method: Optional[LinearMethodBase] = None,
264 + ) -> None:
265 + super().__init__()
266 + self.config = config
267 + self.hidden_size = config.hidden_size
268 + rope_theta = getattr(config, "rope_theta", 10000)
269 + rope_scaling = getattr(config, "rope_scaling", None)
270 + max_position_embeddings = getattr(config, "max_position_embeddings",
271 + 8192)
272 + self.self_attn = MiniCPMAttention(
273 + hidden_size=self.hidden_size,
274 + num_heads=config.num_attention_heads,
275 + num_kv_heads=config.num_key_value_heads,
276 + rope_theta=rope_theta,
277 + rope_scaling=rope_scaling,
278 + max_position_embeddings=max_position_embeddings,
279 + linear_method=linear_method,
280 + )
281 + self.num_experts = getattr(self.config, "num_experts", 0)
282 + if self.num_experts == 0:
283 + self.mlp = MiniCPMMLP(
284 + hidden_size=self.hidden_size,
285 + intermediate_size=config.intermediate_size,
286 + hidden_act=config.hidden_act,
287 + linear_method=linear_method,
288 + )
289 + else:
290 + self.mlp = MiniCPMMoE(num_experts=config.num_experts,
291 + top_k=config.num_experts_per_tok,
292 + hidden_size=config.hidden_size,
293 + intermediate_size=config.intermediate_size)
294 + self.input_layernorm = RMSNorm(config.hidden_size,
295 + eps=config.rms_norm_eps)
296 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
297 + eps=config.rms_norm_eps)
298 +
299 + def forward(
300 + self,
301 + positions: torch.Tensor,
302 + hidden_states: torch.Tensor,
303 + kv_cache: torch.Tensor,
304 + attn_metadata: AttentionMetadata,
305 + residual: Optional[torch.Tensor],
306 + ) -> Tuple[torch.Tensor, torch.Tensor]:
307 + # Self Attention
308 + residual = hidden_states
309 + hidden_states = self.input_layernorm(hidden_states)
310 + hidden_states = self.self_attn(
311 + positions=positions,
312 + hidden_states=hidden_states,
313 + kv_cache=kv_cache,
314 + attn_metadata=attn_metadata,
315 + )
316 + hidden_states = residual + hidden_states * \
317 + (self.config.scale_depth / math.sqrt(self.config.num_hidden_layers))
318 +
319 + # Fully Connected
320 + residual = hidden_states
321 + hidden_states = self.post_attention_layernorm(hidden_states)
322 + hidden_states = self.mlp(hidden_states)
323 + hidden_states = residual + hidden_states * \
324 + (self.config.scale_depth / math.sqrt(self.config.num_hidden_layers))
325 +
326 + return hidden_states, None
327 +
328 +
329 +class MiniCPMModel(nn.Module):
330 +
331 + def __init__(
332 + self,
333 + config,
334 + linear_method: Optional[LinearMethodBase] = None,
335 + lora_config: Optional[LoRAConfig] = None,
336 + ) -> None:
337 + super().__init__()
338 + self.config = config
339 + self.padding_idx = config.pad_token_id
340 + lora_vocab = (lora_config.lora_extra_vocab_size *
341 + (lora_config.max_loras or 1)) if lora_config else 0
342 + self.vocab_size = config.vocab_size + lora_vocab
343 + self.org_vocab_size = config.vocab_size
344 + self.embed_tokens = VocabParallelEmbedding(
345 + self.vocab_size,
346 + config.hidden_size,
347 + org_num_embeddings=config.vocab_size,
348 + )
349 + self.layers = nn.ModuleList([
350 + MiniCPMDecoderLayer(config, linear_method)
351 + for _ in range(config.num_hidden_layers)
352 + ])
353 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
354 +
355 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
356 + embedding = self.embed_tokens(input_ids)
357 + return embedding * self.config.scale_emb
358 +
359 + def forward(
360 + self,
361 + input_ids: torch.Tensor,
362 + positions: torch.Tensor,
363 + kv_caches: List[torch.Tensor],
364 + attn_metadata: AttentionMetadata,
365 + inputs_embeds: Optional[torch.Tensor] = None,
366 + ) -> torch.Tensor:
367 + if inputs_embeds is not None:
368 + hidden_states = inputs_embeds
369 + else:
370 + hidden_states = self.get_input_embeddings(input_ids)
371 + residual = None
372 +
373 + for i in range(len(self.layers)):
374 + layer = self.layers[i]
375 + hidden_states, residual = layer(
376 + positions,
377 + hidden_states,
378 + kv_caches[i],
379 + attn_metadata,
380 + residual,
381 + )
382 + hidden_states = self.norm(hidden_states)
383 + return hidden_states
384 +
385 +
386 +class MiniCPMForCausalLM(nn.Module):
387 + packed_modules_mapping = {
388 + "qkv_proj": [
389 + "q_proj",
390 + "k_proj",
391 + "v_proj",
392 + ],
393 + "gate_up_proj": [
394 + "gate_proj",
395 + "up_proj",
396 + ],
397 + }
398 +
399 + # LoRA specific attributes
400 + supported_lora_modules = [
401 + "qkv_proj",
402 + "o_proj",
403 + "gate_up_proj",
404 + "down_proj",
405 + "embed_tokens",
406 + "lm_head",
407 + ]
408 + embedding_modules = {
409 + "embed_tokens": "input_embeddings",
410 + "lm_head": "output_embeddings",
411 + }
412 + embedding_padding_modules = ["lm_head"]
413 +
414 + def __init__(
415 + self,
416 + config,
417 + linear_method: Optional[LinearMethodBase] = None,
418 + lora_config: Optional[LoRAConfig] = None,
419 + ) -> None:
420 + super().__init__()
421 + self.config = config
422 + self.num_experts = getattr(self.config, "num_experts", 0)
423 + self.linear_method = linear_method
424 + self.model = MiniCPMModel(config,
425 + linear_method,
426 + lora_config=lora_config)
427 + unpadded_vocab_size = config.vocab_size
428 + if lora_config:
429 + unpadded_vocab_size += lora_config.lora_extra_vocab_size
430 + if not self.config.tie_word_embeddings:
431 + self.lm_head = ParallelLMHead(
432 + unpadded_vocab_size,
433 + config.hidden_size,
434 + org_num_embeddings=config.vocab_size,
435 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
436 + # We need bigger padding if using lora for kernel
437 + # compatibility
438 + if not lora_config else lora_config.lora_vocab_padding_size,
439 + )
440 + self.scale_width = self.config.hidden_size / self.config.dim_model_base
441 +
442 + self.logits_processor = LogitsProcessor(unpadded_vocab_size,
443 + config.vocab_size)
444 + self.sampler = Sampler()
445 +
446 + def forward(
447 + self,
448 + input_ids: torch.Tensor,
449 + positions: torch.Tensor,
450 + kv_caches: List[torch.Tensor],
451 + attn_metadata: AttentionMetadata,
452 + ) -> torch.Tensor:
453 + hidden_states = self.model(input_ids, positions, kv_caches,
454 + attn_metadata)
455 + return hidden_states
456 +
457 + def compute_logits(self, hidden_states: torch.Tensor,
458 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
459 + hidden_states = hidden_states / self.scale_width
460 + if self.config.tie_word_embeddings:
461 + lm_head_weight = self.model.embed_tokens.weight
462 + else:
463 + lm_head_weight = self.lm_head.weight
464 + logits = self.logits_processor(lm_head_weight, hidden_states,
465 + sampling_metadata)
466 + return logits
467 +
468 + def sample(
469 + self,
470 + logits: torch.Tensor,
471 + sampling_metadata: SamplingMetadata,
472 + ) -> Optional[SamplerOutput]:
473 + next_tokens = self.sampler(logits, sampling_metadata)
474 + return next_tokens
475 +
476 + def load_weights(self,
477 + model_name_or_path: str,
478 + cache_dir: Optional[str] = None,
479 + load_format: str = "auto",
480 + revision: Optional[str] = None):
481 + stacked_params_mapping = [
482 + # (param_name, shard_name, shard_id)
483 + ("qkv_proj", "q_proj", "q"),
484 + ("qkv_proj", "k_proj", "k"),
485 + ("qkv_proj", "v_proj", "v"),
486 + ("gate_up_proj", "gate_proj", 0),
487 + ("gate_up_proj", "up_proj", 1),
488 + ]
489 + expert_params_mapping = [
490 + # (param_name, weight_name, expert_id)
491 + ("ws" if weight_name in ["w1", "w3"] else "w2s",
492 + f"experts.{expert_id}.{weight_name}.weight", expert_id)
493 + for expert_id in range(self.num_experts)
494 + for weight_name in ["w1", "w2", "w3"]
495 + ]
496 + params_dict = dict(self.named_parameters())
497 + for name, loaded_weight in hf_model_weights_iterator(
498 + model_name_or_path, cache_dir, load_format, revision):
499 + if "rotary_emb.inv_freq" in name:
500 + continue
501 + if ("rotary_emb.cos_cached" in name
502 + or "rotary_emb.sin_cached" in name):
503 + # Models trained using ColossalAI may include these tensors in
504 + # the checkpoint. Skip them.
505 + continue
506 +
507 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
508 + if weight_name not in name:
509 + continue
510 + name = name.replace(weight_name, param_name)
511 + # Skip loading extra bias for GPTQ models.
512 + if name.endswith(".bias") and name not in params_dict:
513 + continue
514 + param = params_dict[name]
515 + weight_loader = param.weight_loader
516 + weight_loader(param, loaded_weight, shard_id)
517 + break
518 + else:
519 + for param_name, weight_name, expert_id in expert_params_mapping:
520 + if weight_name not in name:
521 + continue
522 + name = name.replace(weight_name, param_name)
523 + param = params_dict[name]
524 + weight_loader = param.weight_loader
525 + weight_loader(param,
526 + loaded_weight,
527 + weight_name,
528 + expert_id=expert_id)
529 + break
530 + else:
531 + # Skip loading extra bias for GPTQ models.
532 + if name.endswith(".bias") and name not in params_dict:
533 + continue
534 + param = params_dict[name]
535 + weight_loader = getattr(param, "weight_loader",
536 + default_weight_loader)
537 + weight_loader(param, loaded_weight)