@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a0cf1ddc3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model] support minicpm3 (#8297)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

ywfang committed Sep 14, 2024 at 22:50 UTC 8a0cf1ddc323a571c9f46a85da067d44af5d2453
7 files changed +281 -37
.buildkite/run-cpu-test.sh
+1 -1
@@ -22,7 +22,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
22
23 # Run basic model test
24 docker exec cpu-test bash -c "
25 - pip install pytest matplotlib einops transformers_stream_generator
25 + pip install pytest matplotlib einops transformers_stream_generator datamodel_code_generator
26 pytest -v -s tests/models/decoder_only/language \
27 --ignore=tests/models/test_fp8.py \
28 --ignore=tests/models/decoder_only/language/test_jamba.py \
docs/source/models/supported_models.rst
+4
@@ -107,6 +107,10 @@ Decoder-only Language Models
107 - MiniCPM
108 - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
109 -
110 + * - :code:`MiniCPM3ForCausalLM`
111 + - MiniCPM3
112 + - :code:`openbmb/MiniCPM3-4B`, etc.
113 + -
114 * - :code:`MistralForCausalLM`
115 - Mistral, Mistral-Instruct
116 - :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
requirements-test.txt
+1
@@ -21,6 +21,7 @@ compressed-tensors==0.4.0 # required for compressed-tensors
21 timm # required for internvl test
22 transformers_stream_generator # required for qwen-vl test
23 matplotlib # required for qwen-vl test
24 +datamodel_code_generator # required for minicpm3 test
25
26 # TODO: Add this after fully implementing llava(mantis)
27 # git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
tests/models/decoder_only/language/test_big_models.py
+9 -6
@@ -5,7 +5,8 @@ This tests bigger models and use half precision.
5 Run `pytest tests/models/test_big_models.py`.
6 """
7 import pytest
8 -import torch
8 +
9 +from vllm.platforms import current_platform
10
11 from ...utils import check_outputs_equal
12
@@ -19,10 +20,12 @@ MODELS = [
20 # "Qwen/Qwen1.5-0.5B" # Broken,
21 ]
22
23 +if not current_platform.is_cpu():
24 + # MiniCPM requires fused_moe which is not supported by CPU
25 + MODELS.append("openbmb/MiniCPM3-4B")
26 +
27 #TODO: remove this after CPU float16 support ready
23 -target_dtype = "float"
24 -if torch.cuda.is_available():
25 - target_dtype = "half"
28 +target_dtype = "float" if current_platform.is_cpu() else "half"
29
30
31 @pytest.mark.parametrize("model", MODELS)
@@ -39,7 +42,7 @@ def test_models(
42 with hf_runner(model, dtype=dtype) as hf_model:
43 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
44
42 - with vllm_runner(model, dtype=dtype) as vllm_model:
45 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
46 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
47
48 check_outputs_equal(
@@ -57,7 +60,7 @@ def test_model_print(
60 model: str,
61 dtype: str,
62 ) -> None:
60 - with vllm_runner(model, dtype=dtype) as vllm_model:
63 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
64 # This test is for verifying whether the model's extra_repr
65 # can be printed correctly.
66 print(vllm_model.model.llm_engine.model_executor.driver_worker.
vllm/model_executor/models/__init__.py
+1
@@ -43,6 +43,7 @@ _GENERATION_MODELS = {
43 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
44 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
45 "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
46 + "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
47 "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
48 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
49 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
vllm/model_executor/models/minicpm.py
+49 -30
@@ -270,38 +270,47 @@ class MiniCPMDecoderLayer(nn.Module):
270 ) -> None:
271 super().__init__()
272 self.config = config
273 + self.cache_config = cache_config
274 + self.quant_config = quant_config
275 self.hidden_size = config.hidden_size
274 - rope_theta = getattr(config, "rope_theta", 10000)
275 - rope_scaling = getattr(config, "rope_scaling", None)
276 - max_position_embeddings = getattr(config, "max_position_embeddings",
277 - 8192)
276 + self.rope_theta = getattr(config, "rope_theta", 10000)
277 + self.rope_scaling = getattr(config, "rope_scaling", None)
278 + self.max_position_embeddings = getattr(config,
279 + "max_position_embeddings", 8192)
280 + self._init_attn_block()
281 + self._init_ffn_block()
282 +
283 + def _init_attn_block(self):
284 + self.input_layernorm = RMSNorm(self.config.hidden_size,
285 + eps=self.config.rms_norm_eps)
286 self.self_attn = MiniCPMAttention(
287 hidden_size=self.hidden_size,
280 - num_heads=config.num_attention_heads,
281 - num_kv_heads=config.num_key_value_heads,
282 - rope_theta=rope_theta,
283 - rope_scaling=rope_scaling,
284 - max_position_embeddings=max_position_embeddings,
285 - cache_config=cache_config,
286 - quant_config=quant_config,
288 + num_heads=self.config.num_attention_heads,
289 + num_kv_heads=self.config.num_key_value_heads,
290 + rope_theta=self.rope_theta,
291 + rope_scaling=self.rope_scaling,
292 + max_position_embeddings=self.max_position_embeddings,
293 + cache_config=self.cache_config,
294 + quant_config=self.quant_config,
295 )
296 +
297 + def _init_ffn_block(self):
298 + self.post_attention_layernorm = RMSNorm(self.config.hidden_size,
299 + eps=self.config.rms_norm_eps)
300 self.num_experts = getattr(self.config, "num_experts", 0)
301 if self.num_experts == 0:
302 self.mlp = MiniCPMMLP(
303 hidden_size=self.hidden_size,
292 - intermediate_size=config.intermediate_size,
293 - hidden_act=config.hidden_act,
294 - quant_config=quant_config,
304 + intermediate_size=self.config.intermediate_size,
305 + hidden_act=self.config.hidden_act,
306 + quant_config=self.quant_config,
307 )
308 else:
297 - self.mlp = MiniCPMMoE(num_experts=config.num_experts,
298 - top_k=config.num_experts_per_tok,
299 - hidden_size=config.hidden_size,
300 - intermediate_size=config.intermediate_size)
301 - self.input_layernorm = RMSNorm(config.hidden_size,
302 - eps=config.rms_norm_eps)
303 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
304 - eps=config.rms_norm_eps)
309 + self.mlp = MiniCPMMoE(
310 + num_experts=self.config.num_experts,
311 + top_k=self.config.num_experts_per_tok,
312 + hidden_size=self.config.hidden_size,
313 + intermediate_size=self.config.intermediate_size)
314
315 def forward(
316 self,
@@ -344,6 +353,8 @@ class MiniCPMModel(nn.Module):
353 ) -> None:
354 super().__init__()
355 self.config = config
356 + self.cache_config = cache_config
357 + self.quant_config = quant_config
358 self.padding_idx = config.pad_token_id
359 lora_vocab = (lora_config.lora_extra_vocab_size *
360 (lora_config.max_loras or 1)) if lora_config else 0
@@ -354,11 +365,15 @@ class MiniCPMModel(nn.Module):
365 config.hidden_size,
366 org_num_embeddings=config.vocab_size,
367 )
368 + self._init_layers()
369 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
370 +
371 + def _init_layers(self):
372 self.layers = nn.ModuleList([
358 - MiniCPMDecoderLayer(config, cache_config, quant_config)
359 - for _ in range(config.num_hidden_layers)
373 + MiniCPMDecoderLayer(self.config, self.cache_config,
374 + self.quant_config)
375 + for _ in range(self.config.num_hidden_layers)
376 ])
361 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
377
378 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
379 embedding = self.embed_tokens(input_ids)
@@ -431,13 +446,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
446
447 self.config = config
448 self.lora_config = lora_config
449 + self.cache_config = cache_config
450 + self.quant_config = quant_config
451
452 self.num_experts = getattr(self.config, "num_experts", 0)
436 - self.quant_config = quant_config
437 - self.model = MiniCPMModel(config,
438 - cache_config,
439 - quant_config,
440 - lora_config=lora_config)
453 + self._init_model()
454 unpadded_vocab_size = config.vocab_size
455 if lora_config:
456 unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -458,6 +471,12 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
471 config.vocab_size)
472 self.sampler = Sampler()
473
474 + def _init_model(self):
475 + self.model = MiniCPMModel(config=self.config,
476 + cache_config=self.cache_config,
477 + quant_config=self.quant_config,
478 + lora_config=self.lora_config)
479 +
480 def forward(
481 self,
482 input_ids: torch.Tensor,
vllm/model_executor/models/minicpm3.py new
+216
@@ -0,0 +1,216 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2024 The ModelBest team.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only MiniCPM3 model compatible with HuggingFace weights."""
25 +from typing import Any, Dict, Optional
26 +
27 +import torch
28 +from torch import nn
29 +
30 +from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig
32 +from vllm.distributed import get_tensor_model_parallel_world_size
33 +from vllm.model_executor.layers.layernorm import RMSNorm
34 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 + ReplicatedLinear,
36 + RowParallelLinear)
37 +from vllm.model_executor.layers.quantization.base_config import (
38 + QuantizationConfig)
39 +from vllm.model_executor.layers.rotary_embedding import get_rope
40 +from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer,
41 + MiniCPMForCausalLM,
42 + MiniCPMModel)
43 +
44 +
45 +class MiniCPM3Attention(nn.Module):
46 +
47 + def __init__(
48 + self,
49 + config,
50 + hidden_size: int,
51 + num_heads: int,
52 + qk_nope_head_dim: int,
53 + qk_rope_head_dim: int,
54 + v_head_dim: int,
55 + q_lora_rank: int,
56 + kv_lora_rank: int,
57 + rope_theta: float = 10000,
58 + rope_scaling: Optional[Dict[str, Any]] = None,
59 + max_position_embeddings: int = 8192,
60 + cache_config: Optional[CacheConfig] = None,
61 + quant_config: Optional[QuantizationConfig] = None,
62 + ) -> None:
63 + super().__init__()
64 + self.hidden_size = hidden_size
65 + self.qk_nope_head_dim = qk_nope_head_dim
66 + self.qk_rope_head_dim = qk_rope_head_dim
67 + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
68 + self.v_head_dim = v_head_dim
69 + self.q_lora_rank = q_lora_rank
70 + self.kv_lora_rank = kv_lora_rank
71 + self.num_heads = num_heads
72 +
73 + tp_size = get_tensor_model_parallel_world_size()
74 + assert self.num_heads % tp_size == 0
75 + self.num_local_heads = num_heads // tp_size
76 +
77 + self.scaling = self.qk_head_dim**-0.5
78 + self.rope_theta = rope_theta
79 + self.max_position_embeddings = max_position_embeddings
80 +
81 + self.q_a_proj = ReplicatedLinear(self.hidden_size,
82 + self.q_lora_rank,
83 + bias=False,
84 + quant_config=quant_config)
85 + self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
86 + self.q_b_proj = ColumnParallelLinear(q_lora_rank,
87 + self.num_heads * self.qk_head_dim,
88 + bias=False,
89 + quant_config=quant_config)
90 +
91 + self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size,
92 + self.kv_lora_rank +
93 + self.qk_rope_head_dim,
94 + bias=False,
95 + quant_config=quant_config)
96 + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
97 + eps=config.rms_norm_eps)
98 + self.kv_b_proj = ColumnParallelLinear(
99 + self.kv_lora_rank,
100 + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
101 + bias=False,
102 + quant_config=quant_config)
103 + # O projection.
104 + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
105 + self.hidden_size,
106 + bias=False,
107 + quant_config=quant_config)
108 +
109 + self.rotary_emb = get_rope(
110 + self.qk_rope_head_dim,
111 + rotary_dim=self.qk_rope_head_dim,
112 + max_position=max_position_embeddings,
113 + base=rope_theta,
114 + rope_scaling=rope_scaling,
115 + )
116 + self.attn = Attention(self.num_local_heads,
117 + self.qk_head_dim,
118 + self.scaling,
119 + num_kv_heads=self.num_local_heads,
120 + cache_config=cache_config,
121 + quant_config=quant_config)
122 +
123 + def forward(
124 + self,
125 + positions: torch.Tensor,
126 + hidden_states: torch.Tensor,
127 + kv_cache: torch.Tensor,
128 + attn_metadata: AttentionMetadata,
129 + ) -> torch.Tensor:
130 + q, _ = self.q_a_proj(hidden_states)
131 + q = self.q_a_layernorm(q)
132 + q, _ = self.q_b_proj(q)
133 + q = q.view(-1, self.num_local_heads, self.qk_head_dim)
134 + _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
135 + dim=-1)
136 + latent_cache, _ = self.kv_a_proj_with_mqa(hidden_states)
137 + kv_a, _ = latent_cache.split(
138 + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
139 + latent_cache = latent_cache.unsqueeze(1)
140 + kv_a = self.kv_a_layernorm(kv_a.contiguous())
141 + kv, _ = self.kv_b_proj(kv_a)
142 + kv = kv.view(-1, self.num_local_heads,
143 + self.qk_nope_head_dim + self.v_head_dim)
144 + k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
145 +
146 + k_pe = latent_cache[:, :, self.kv_lora_rank:]
147 +
148 + q_pe, k_pe = self.rotary_emb(
149 + positions,
150 + q_pe.reshape(-1, self.num_local_heads * self.qk_rope_head_dim),
151 + k_pe.reshape(-1, self.qk_rope_head_dim))
152 + q_pe = q_pe.view(-1, self.num_local_heads, self.qk_rope_head_dim)
153 + k_pe = k_pe.view(-1, 1, self.qk_rope_head_dim)
154 +
155 + q[..., self.qk_nope_head_dim:] = q_pe
156 +
157 + k = torch.empty_like(q)
158 +
159 + k[..., :self.qk_nope_head_dim] = k_nope
160 + k[..., self.qk_nope_head_dim:] = k_pe
161 +
162 + q = q.reshape(-1, self.num_local_heads * self.qk_head_dim)
163 + k = k.view(-1, self.num_local_heads * self.qk_head_dim)
164 + v = torch.nn.functional.pad(
165 + v, [0, self.qk_head_dim - self.v_head_dim],
166 + value=0).view(-1, self.num_local_heads * self.qk_head_dim)
167 +
168 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
169 + attn_output = attn_output.view(
170 + -1, self.num_local_heads,
171 + self.qk_head_dim)[..., :self.v_head_dim].reshape(
172 + -1, self.num_local_heads * self.v_head_dim)
173 +
174 + output, _ = self.o_proj(attn_output)
175 + return output
176 +
177 +
178 +class MiniCPM3DecoderLayer(MiniCPMDecoderLayer):
179 +
180 + def _init_attn_block(self):
181 + self.input_layernorm = RMSNorm(self.config.hidden_size,
182 + eps=self.config.rms_norm_eps)
183 + self.self_attn = MiniCPM3Attention(
184 + config=self.config,
185 + hidden_size=self.hidden_size,
186 + num_heads=self.config.num_attention_heads,
187 + qk_nope_head_dim=self.config.qk_nope_head_dim,
188 + qk_rope_head_dim=self.config.qk_rope_head_dim,
189 + v_head_dim=self.config.v_head_dim,
190 + q_lora_rank=self.config.q_lora_rank,
191 + kv_lora_rank=self.config.kv_lora_rank,
192 + rope_theta=self.rope_theta,
193 + rope_scaling=self.rope_scaling,
194 + max_position_embeddings=self.max_position_embeddings,
195 + cache_config=self.cache_config,
196 + quant_config=self.quant_config,
197 + )
198 +
199 +
200 +class MiniCPM3Model(MiniCPMModel):
201 +
202 + def _init_layers(self):
203 + self.layers = nn.ModuleList([
204 + MiniCPM3DecoderLayer(self.config, self.cache_config,
205 + self.quant_config)
206 + for _ in range(self.config.num_hidden_layers)
207 + ])
208 +
209 +
210 +class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
211 +
212 + def _init_model(self):
213 + self.model = MiniCPM3Model(config=self.config,
214 + cache_config=self.cache_config,
215 + quant_config=self.quant_config,
216 + lora_config=self.lora_config)