A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] support minicpm3 (#8297)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
ywfang committed
Sep 14, 2024 at 22:50 UTC
8a0cf1ddc323a571c9f46a85da067d44af5d2453
7 files changed
+281
-37
.buildkite/run-cpu-test.sh
+1
-1
@@ -22,7 +22,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
22
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
- pip install pytest matplotlib einops transformers_stream_generator
25
+ pip install pytest matplotlib einops transformers_stream_generator datamodel_code_generator
26
pytest -v -s tests/models/decoder_only/language \
27
--ignore=tests/models/test_fp8.py \
28
--ignore=tests/models/decoder_only/language/test_jamba.py \
docs/source/models/supported_models.rst
+4
@@ -107,6 +107,10 @@ Decoder-only Language Models
107
- MiniCPM
108
- :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
109
-
110
+ * - :code:`MiniCPM3ForCausalLM`
111
+ - MiniCPM3
112
+ - :code:`openbmb/MiniCPM3-4B`, etc.
113
+ -
114
* - :code:`MistralForCausalLM`
115
- Mistral, Mistral-Instruct
116
- :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
requirements-test.txt
+1
@@ -21,6 +21,7 @@ compressed-tensors==0.4.0 # required for compressed-tensors
21
timm # required for internvl test
22
transformers_stream_generator # required for qwen-vl test
23
matplotlib # required for qwen-vl test
24
+datamodel_code_generator # required for minicpm3 test
25
26
# TODO: Add this after fully implementing llava(mantis)
27
# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
tests/models/decoder_only/language/test_big_models.py
+9
-6
@@ -5,7 +5,8 @@ This tests bigger models and use half precision.
5
Run `pytest tests/models/test_big_models.py`.
6
"""
7
import pytest
8
-import torch
8
+
9
+from vllm.platforms import current_platform
10
11
from ...utils import check_outputs_equal
12
@@ -19,10 +20,12 @@ MODELS = [
20
# "Qwen/Qwen1.5-0.5B" # Broken,
21
]
22
23
+if not current_platform.is_cpu():
24
+ # MiniCPM requires fused_moe which is not supported by CPU
25
+ MODELS.append("openbmb/MiniCPM3-4B")
26
+
27
#TODO: remove this after CPU float16 support ready
23
-target_dtype = "float"
24
-if torch.cuda.is_available():
25
- target_dtype = "half"
28
+target_dtype = "float" if current_platform.is_cpu() else "half"
29
30
31
@pytest.mark.parametrize("model", MODELS)
@@ -39,7 +42,7 @@ def test_models(
42
with hf_runner(model, dtype=dtype) as hf_model:
43
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
44
42
- with vllm_runner(model, dtype=dtype) as vllm_model:
45
+ with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
46
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
47
48
check_outputs_equal(
@@ -57,7 +60,7 @@ def test_model_print(
60
model: str,
61
dtype: str,
62
) -> None:
60
- with vllm_runner(model, dtype=dtype) as vllm_model:
63
+ with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
64
# This test is for verifying whether the model's extra_repr
65
# can be printed correctly.
66
print(vllm_model.model.llm_engine.model_executor.driver_worker.
vllm/model_executor/models/__init__.py
+1
@@ -43,6 +43,7 @@ _GENERATION_MODELS = {
43
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
44
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
45
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
46
+ "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
47
"NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
48
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
49
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
vllm/model_executor/models/minicpm.py
+49
-30
@@ -270,38 +270,47 @@ class MiniCPMDecoderLayer(nn.Module):
270
) -> None:
271
super().__init__()
272
self.config = config
273
+ self.cache_config = cache_config
274
+ self.quant_config = quant_config
275
self.hidden_size = config.hidden_size
274
- rope_theta = getattr(config, "rope_theta", 10000)
275
- rope_scaling = getattr(config, "rope_scaling", None)
276
- max_position_embeddings = getattr(config, "max_position_embeddings",
277
- 8192)
276
+ self.rope_theta = getattr(config, "rope_theta", 10000)
277
+ self.rope_scaling = getattr(config, "rope_scaling", None)
278
+ self.max_position_embeddings = getattr(config,
279
+ "max_position_embeddings", 8192)
280
+ self._init_attn_block()
281
+ self._init_ffn_block()
282
+
283
+ def _init_attn_block(self):
284
+ self.input_layernorm = RMSNorm(self.config.hidden_size,
285
+ eps=self.config.rms_norm_eps)
286
self.self_attn = MiniCPMAttention(
287
hidden_size=self.hidden_size,
280
- num_heads=config.num_attention_heads,
281
- num_kv_heads=config.num_key_value_heads,
282
- rope_theta=rope_theta,
283
- rope_scaling=rope_scaling,
284
- max_position_embeddings=max_position_embeddings,
285
- cache_config=cache_config,
286
- quant_config=quant_config,
288
+ num_heads=self.config.num_attention_heads,
289
+ num_kv_heads=self.config.num_key_value_heads,
290
+ rope_theta=self.rope_theta,
291
+ rope_scaling=self.rope_scaling,
292
+ max_position_embeddings=self.max_position_embeddings,
293
+ cache_config=self.cache_config,
294
+ quant_config=self.quant_config,
295
)
296
+
297
+ def _init_ffn_block(self):
298
+ self.post_attention_layernorm = RMSNorm(self.config.hidden_size,
299
+ eps=self.config.rms_norm_eps)
300
self.num_experts = getattr(self.config, "num_experts", 0)
301
if self.num_experts == 0:
302
self.mlp = MiniCPMMLP(
303
hidden_size=self.hidden_size,
292
- intermediate_size=config.intermediate_size,
293
- hidden_act=config.hidden_act,
294
- quant_config=quant_config,
304
+ intermediate_size=self.config.intermediate_size,
305
+ hidden_act=self.config.hidden_act,
306
+ quant_config=self.quant_config,
307
)
308
else:
297
- self.mlp = MiniCPMMoE(num_experts=config.num_experts,
298
- top_k=config.num_experts_per_tok,
299
- hidden_size=config.hidden_size,
300
- intermediate_size=config.intermediate_size)
301
- self.input_layernorm = RMSNorm(config.hidden_size,
302
- eps=config.rms_norm_eps)
303
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
304
- eps=config.rms_norm_eps)
309
+ self.mlp = MiniCPMMoE(
310
+ num_experts=self.config.num_experts,
311
+ top_k=self.config.num_experts_per_tok,
312
+ hidden_size=self.config.hidden_size,
313
+ intermediate_size=self.config.intermediate_size)
314
315
def forward(
316
self,
@@ -344,6 +353,8 @@ class MiniCPMModel(nn.Module):
353
) -> None:
354
super().__init__()
355
self.config = config
356
+ self.cache_config = cache_config
357
+ self.quant_config = quant_config
358
self.padding_idx = config.pad_token_id
359
lora_vocab = (lora_config.lora_extra_vocab_size *
360
(lora_config.max_loras or 1)) if lora_config else 0
@@ -354,11 +365,15 @@ class MiniCPMModel(nn.Module):
365
config.hidden_size,
366
org_num_embeddings=config.vocab_size,
367
)
368
+ self._init_layers()
369
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
370
+
371
+ def _init_layers(self):
372
self.layers = nn.ModuleList([
358
- MiniCPMDecoderLayer(config, cache_config, quant_config)
359
- for _ in range(config.num_hidden_layers)
373
+ MiniCPMDecoderLayer(self.config, self.cache_config,
374
+ self.quant_config)
375
+ for _ in range(self.config.num_hidden_layers)
376
])
361
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
377
378
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
379
embedding = self.embed_tokens(input_ids)
@@ -431,13 +446,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
446
447
self.config = config
448
self.lora_config = lora_config
449
+ self.cache_config = cache_config
450
+ self.quant_config = quant_config
451
452
self.num_experts = getattr(self.config, "num_experts", 0)
436
- self.quant_config = quant_config
437
- self.model = MiniCPMModel(config,
438
- cache_config,
439
- quant_config,
440
- lora_config=lora_config)
453
+ self._init_model()
454
unpadded_vocab_size = config.vocab_size
455
if lora_config:
456
unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -458,6 +471,12 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
471
config.vocab_size)
472
self.sampler = Sampler()
473
474
+ def _init_model(self):
475
+ self.model = MiniCPMModel(config=self.config,
476
+ cache_config=self.cache_config,
477
+ quant_config=self.quant_config,
478
+ lora_config=self.lora_config)
479
+
480
def forward(
481
self,
482
input_ids: torch.Tensor,
vllm/model_executor/models/minicpm3.py
new
+216
@@ -0,0 +1,216 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2024 The ModelBest team.
5
+# Copyright 2023 The vLLM team.
6
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7
+#
8
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9
+# and OPT implementations in this library. It has been modified from its
10
+# original forms to accommodate minor architectural differences compared
11
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12
+#
13
+# Licensed under the Apache License, Version 2.0 (the "License");
14
+# you may not use this file except in compliance with the License.
15
+# You may obtain a copy of the License at
16
+#
17
+# http://www.apache.org/licenses/LICENSE-2.0
18
+#
19
+# Unless required by applicable law or agreed to in writing, software
20
+# distributed under the License is distributed on an "AS IS" BASIS,
21
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+# See the License for the specific language governing permissions and
23
+# limitations under the License.
24
+"""Inference-only MiniCPM3 model compatible with HuggingFace weights."""
25
+from typing import Any, Dict, Optional
26
+
27
+import torch
28
+from torch import nn
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig
32
+from vllm.distributed import get_tensor_model_parallel_world_size
33
+from vllm.model_executor.layers.layernorm import RMSNorm
34
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
+ ReplicatedLinear,
36
+ RowParallelLinear)
37
+from vllm.model_executor.layers.quantization.base_config import (
38
+ QuantizationConfig)
39
+from vllm.model_executor.layers.rotary_embedding import get_rope
40
+from vllm.model_executor.models.minicpm import (MiniCPMDecoderLayer,
41
+ MiniCPMForCausalLM,
42
+ MiniCPMModel)
43
+
44
+
45
+class MiniCPM3Attention(nn.Module):
46
+
47
+ def __init__(
48
+ self,
49
+ config,
50
+ hidden_size: int,
51
+ num_heads: int,
52
+ qk_nope_head_dim: int,
53
+ qk_rope_head_dim: int,
54
+ v_head_dim: int,
55
+ q_lora_rank: int,
56
+ kv_lora_rank: int,
57
+ rope_theta: float = 10000,
58
+ rope_scaling: Optional[Dict[str, Any]] = None,
59
+ max_position_embeddings: int = 8192,
60
+ cache_config: Optional[CacheConfig] = None,
61
+ quant_config: Optional[QuantizationConfig] = None,
62
+ ) -> None:
63
+ super().__init__()
64
+ self.hidden_size = hidden_size
65
+ self.qk_nope_head_dim = qk_nope_head_dim
66
+ self.qk_rope_head_dim = qk_rope_head_dim
67
+ self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
68
+ self.v_head_dim = v_head_dim
69
+ self.q_lora_rank = q_lora_rank
70
+ self.kv_lora_rank = kv_lora_rank
71
+ self.num_heads = num_heads
72
+
73
+ tp_size = get_tensor_model_parallel_world_size()
74
+ assert self.num_heads % tp_size == 0
75
+ self.num_local_heads = num_heads // tp_size
76
+
77
+ self.scaling = self.qk_head_dim**-0.5
78
+ self.rope_theta = rope_theta
79
+ self.max_position_embeddings = max_position_embeddings
80
+
81
+ self.q_a_proj = ReplicatedLinear(self.hidden_size,
82
+ self.q_lora_rank,
83
+ bias=False,
84
+ quant_config=quant_config)
85
+ self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
86
+ self.q_b_proj = ColumnParallelLinear(q_lora_rank,
87
+ self.num_heads * self.qk_head_dim,
88
+ bias=False,
89
+ quant_config=quant_config)
90
+
91
+ self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size,
92
+ self.kv_lora_rank +
93
+ self.qk_rope_head_dim,
94
+ bias=False,
95
+ quant_config=quant_config)
96
+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
97
+ eps=config.rms_norm_eps)
98
+ self.kv_b_proj = ColumnParallelLinear(
99
+ self.kv_lora_rank,
100
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
101
+ bias=False,
102
+ quant_config=quant_config)
103
+ # O projection.
104
+ self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
105
+ self.hidden_size,
106
+ bias=False,
107
+ quant_config=quant_config)
108
+
109
+ self.rotary_emb = get_rope(
110
+ self.qk_rope_head_dim,
111
+ rotary_dim=self.qk_rope_head_dim,
112
+ max_position=max_position_embeddings,
113
+ base=rope_theta,
114
+ rope_scaling=rope_scaling,
115
+ )
116
+ self.attn = Attention(self.num_local_heads,
117
+ self.qk_head_dim,
118
+ self.scaling,
119
+ num_kv_heads=self.num_local_heads,
120
+ cache_config=cache_config,
121
+ quant_config=quant_config)
122
+
123
+ def forward(
124
+ self,
125
+ positions: torch.Tensor,
126
+ hidden_states: torch.Tensor,
127
+ kv_cache: torch.Tensor,
128
+ attn_metadata: AttentionMetadata,
129
+ ) -> torch.Tensor:
130
+ q, _ = self.q_a_proj(hidden_states)
131
+ q = self.q_a_layernorm(q)
132
+ q, _ = self.q_b_proj(q)
133
+ q = q.view(-1, self.num_local_heads, self.qk_head_dim)
134
+ _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
135
+ dim=-1)
136
+ latent_cache, _ = self.kv_a_proj_with_mqa(hidden_states)
137
+ kv_a, _ = latent_cache.split(
138
+ [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
139
+ latent_cache = latent_cache.unsqueeze(1)
140
+ kv_a = self.kv_a_layernorm(kv_a.contiguous())
141
+ kv, _ = self.kv_b_proj(kv_a)
142
+ kv = kv.view(-1, self.num_local_heads,
143
+ self.qk_nope_head_dim + self.v_head_dim)
144
+ k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
145
+
146
+ k_pe = latent_cache[:, :, self.kv_lora_rank:]
147
+
148
+ q_pe, k_pe = self.rotary_emb(
149
+ positions,
150
+ q_pe.reshape(-1, self.num_local_heads * self.qk_rope_head_dim),
151
+ k_pe.reshape(-1, self.qk_rope_head_dim))
152
+ q_pe = q_pe.view(-1, self.num_local_heads, self.qk_rope_head_dim)
153
+ k_pe = k_pe.view(-1, 1, self.qk_rope_head_dim)
154
+
155
+ q[..., self.qk_nope_head_dim:] = q_pe
156
+
157
+ k = torch.empty_like(q)
158
+
159
+ k[..., :self.qk_nope_head_dim] = k_nope
160
+ k[..., self.qk_nope_head_dim:] = k_pe
161
+
162
+ q = q.reshape(-1, self.num_local_heads * self.qk_head_dim)
163
+ k = k.view(-1, self.num_local_heads * self.qk_head_dim)
164
+ v = torch.nn.functional.pad(
165
+ v, [0, self.qk_head_dim - self.v_head_dim],
166
+ value=0).view(-1, self.num_local_heads * self.qk_head_dim)
167
+
168
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
169
+ attn_output = attn_output.view(
170
+ -1, self.num_local_heads,
171
+ self.qk_head_dim)[..., :self.v_head_dim].reshape(
172
+ -1, self.num_local_heads * self.v_head_dim)
173
+
174
+ output, _ = self.o_proj(attn_output)
175
+ return output
176
+
177
+
178
+class MiniCPM3DecoderLayer(MiniCPMDecoderLayer):
179
+
180
+ def _init_attn_block(self):
181
+ self.input_layernorm = RMSNorm(self.config.hidden_size,
182
+ eps=self.config.rms_norm_eps)
183
+ self.self_attn = MiniCPM3Attention(
184
+ config=self.config,
185
+ hidden_size=self.hidden_size,
186
+ num_heads=self.config.num_attention_heads,
187
+ qk_nope_head_dim=self.config.qk_nope_head_dim,
188
+ qk_rope_head_dim=self.config.qk_rope_head_dim,
189
+ v_head_dim=self.config.v_head_dim,
190
+ q_lora_rank=self.config.q_lora_rank,
191
+ kv_lora_rank=self.config.kv_lora_rank,
192
+ rope_theta=self.rope_theta,
193
+ rope_scaling=self.rope_scaling,
194
+ max_position_embeddings=self.max_position_embeddings,
195
+ cache_config=self.cache_config,
196
+ quant_config=self.quant_config,
197
+ )
198
+
199
+
200
+class MiniCPM3Model(MiniCPMModel):
201
+
202
+ def _init_layers(self):
203
+ self.layers = nn.ModuleList([
204
+ MiniCPM3DecoderLayer(self.config, self.cache_config,
205
+ self.quant_config)
206
+ for _ in range(self.config.num_hidden_layers)
207
+ ])
208
+
209
+
210
+class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
211
+
212
+ def _init_model(self):
213
+ self.model = MiniCPM3Model(config=self.config,
214
+ cache_config=self.cache_config,
215
+ quant_config=self.quant_config,
216
+ lora_config=self.lora_config)