A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Adding Granite MoE. (#8206)
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Shawn Tan committed
Oct 2, 2024 at 21:33 UTC
19f0d2579695e518c9bfc166544cf23775772bf8
4 files changed
+492
-3
tests/models/decoder_only/language/test_granitemoe.py
new
+39
@@ -0,0 +1,39 @@
1
+"""Compare the outputs of HF and vLLM for Granite models using greedy sampling.
2
+
3
+Run `pytest tests/models/test_granite.py`.
4
+"""
5
+import pytest
6
+
7
+from ...utils import check_logprobs_close
8
+
9
+MODELS = [
10
+ "ibm/PowerMoE-3b",
11
+]
12
+
13
+
14
+@pytest.mark.parametrize("model", MODELS)
15
+@pytest.mark.parametrize("dtype", ["bfloat16"])
16
+@pytest.mark.parametrize("max_tokens", [64])
17
+@pytest.mark.parametrize("num_logprobs", [5])
18
+def test_models(
19
+ hf_runner,
20
+ vllm_runner,
21
+ example_prompts,
22
+ model: str,
23
+ dtype: str,
24
+ max_tokens: int,
25
+ num_logprobs: int,
26
+) -> None:
27
+ with hf_runner(model, dtype=dtype) as hf_model:
28
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
29
+ example_prompts, max_tokens, num_logprobs)
30
+
31
+ with vllm_runner(model, dtype=dtype) as vllm_model:
32
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
33
+ example_prompts, max_tokens, num_logprobs)
34
+ check_logprobs_close(
35
+ outputs_0_lst=hf_outputs,
36
+ outputs_1_lst=vllm_outputs,
37
+ name_0="hf",
38
+ name_1="vllm",
39
+ )
vllm/model_executor/models/__init__.py
+1
@@ -32,6 +32,7 @@ _GENERATION_MODELS = {
32
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
33
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
34
"GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
35
+ "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
36
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
37
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
38
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
vllm/model_executor/models/granite.py
+4
-3
@@ -404,9 +404,12 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA):
404
self.lm_head.weight = self.model.embed_tokens.weight
405
406
logit_scale = getattr(config, "logit_scale", 1.0)
407
+
408
+ if hasattr(config, "logits_scaling"):
409
+ logit_scale /= config.logits_scaling
410
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
411
config.vocab_size,
409
- logit_scale)
412
+ scale=logit_scale)
413
self.sampler = Sampler()
414
else:
415
self.lm_head = PPMissingLayer()
@@ -428,8 +431,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA):
431
sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
432
logits = self.logits_processor(self.lm_head, hidden_states,
433
sampling_metadata)
431
- if logits is not None:
432
- logits /= self.config.logits_scaling
434
return logits
435
436
def sample(
vllm/model_executor/models/granitemoe.py
new
+448
@@ -0,0 +1,448 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only GraniteMoe model."""
24
+from typing import Iterable, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+from transformers.models.granitemoe import GraniteMoeConfig
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
+from vllm.model_executor.layers.fused_moe import FusedMoE
34
+from vllm.model_executor.layers.layernorm import RMSNorm
35
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
36
+ ReplicatedLinear,
37
+ RowParallelLinear)
38
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
+from vllm.model_executor.layers.quantization.base_config import (
40
+ QuantizationConfig)
41
+from vllm.model_executor.layers.rotary_embedding import get_rope
42
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
+from vllm.model_executor.sampling_metadata import SamplingMetadata
46
+from vllm.sequence import IntermediateTensors
47
+
48
+from . import mixtral
49
+from .interfaces import SupportsLoRA
50
+from .utils import make_layers
51
+
52
+
53
+class GraniteMoeMoE(nn.Module):
54
+ """A tensor-parallel MoE implementation for GraniteMoe that shards each
55
+ expert across all ranks.
56
+ Each expert's weights are sharded across all ranks and a fused MoE
57
+ kernel is used for the forward pass, and finally we reduce the outputs
58
+ across ranks.
59
+ """
60
+
61
+ def __init__(self,
62
+ num_experts: int,
63
+ top_k: int,
64
+ hidden_size: int,
65
+ intermediate_size: int,
66
+ params_dtype: Optional[torch.dtype] = None,
67
+ quant_config: Optional[QuantizationConfig] = None,
68
+ tp_size: Optional[int] = None,
69
+ prefix: str = ""):
70
+ super().__init__()
71
+ self.hidden_size = hidden_size
72
+
73
+ # Gate always runs at half / full precision for now.
74
+ self.gate = ReplicatedLinear(hidden_size,
75
+ num_experts,
76
+ bias=False,
77
+ params_dtype=params_dtype,
78
+ quant_config=None,
79
+ prefix=f"{prefix}.gate")
80
+
81
+ self.experts = FusedMoE(num_experts=num_experts,
82
+ top_k=top_k,
83
+ hidden_size=hidden_size,
84
+ intermediate_size=intermediate_size,
85
+ params_dtype=params_dtype,
86
+ reduce_results=True,
87
+ renormalize=True,
88
+ quant_config=quant_config,
89
+ tp_size=tp_size,
90
+ prefix=f"{prefix}.experts")
91
+
92
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
93
+ # NOTE: hidden_states can have either 1D or 2D shape.
94
+ orig_shape = hidden_states.shape
95
+ hidden_states = hidden_states.view(-1, self.hidden_size)
96
+ # router_logits: (num_tokens, n_experts)
97
+ router_logits, _ = self.gate(hidden_states)
98
+ final_hidden_states = self.experts(hidden_states, router_logits)
99
+ return final_hidden_states.view(orig_shape)
100
+
101
+
102
+class GraniteMoeAttention(nn.Module):
103
+
104
+ def __init__(
105
+ self,
106
+ hidden_size: int,
107
+ num_heads: int,
108
+ num_kv_heads: int,
109
+ max_position: int = 4096 * 32,
110
+ rope_theta: float = 10000,
111
+ cache_config: Optional[CacheConfig] = None,
112
+ quant_config: Optional[QuantizationConfig] = None,
113
+ attention_multiplier: Optional[float] = None,
114
+ prefix: str = "",
115
+ ) -> None:
116
+ super().__init__()
117
+ self.hidden_size = hidden_size
118
+ tp_size = get_tensor_model_parallel_world_size()
119
+ self.total_num_heads = num_heads
120
+ assert self.total_num_heads % tp_size == 0
121
+ self.num_heads = self.total_num_heads // tp_size
122
+ self.total_num_kv_heads = num_kv_heads
123
+ if self.total_num_kv_heads >= tp_size:
124
+ # Number of KV heads is greater than TP size, so we partition
125
+ # the KV heads across multiple tensor parallel GPUs.
126
+ assert self.total_num_kv_heads % tp_size == 0
127
+ else:
128
+ # Number of KV heads is less than TP size, so we replicate
129
+ # the KV heads across multiple tensor parallel GPUs.
130
+ assert tp_size % self.total_num_kv_heads == 0
131
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
132
+ self.head_dim = hidden_size // self.total_num_heads
133
+ self.q_size = self.num_heads * self.head_dim
134
+ self.kv_size = self.num_kv_heads * self.head_dim
135
+ self.scaling = (attention_multiplier if attention_multiplier
136
+ is not None else self.head_dim**-1)
137
+ self.rope_theta = rope_theta
138
+
139
+ self.qkv_proj = QKVParallelLinear(
140
+ hidden_size,
141
+ self.head_dim,
142
+ self.total_num_heads,
143
+ self.total_num_kv_heads,
144
+ bias=False,
145
+ quant_config=quant_config,
146
+ prefix=f"{prefix}.qkv_proj",
147
+ )
148
+ self.o_proj = RowParallelLinear(
149
+ self.total_num_heads * self.head_dim,
150
+ hidden_size,
151
+ bias=False,
152
+ quant_config=quant_config,
153
+ prefix=f"{prefix}.o_proj",
154
+ )
155
+ self.rotary_emb = get_rope(
156
+ self.head_dim,
157
+ rotary_dim=self.head_dim,
158
+ max_position=max_position,
159
+ base=int(self.rope_theta),
160
+ is_neox_style=True,
161
+ )
162
+ self.attn = Attention(self.num_heads,
163
+ self.head_dim,
164
+ self.scaling,
165
+ num_kv_heads=self.num_kv_heads,
166
+ cache_config=cache_config,
167
+ quant_config=quant_config)
168
+
169
+ def forward(
170
+ self,
171
+ positions: torch.Tensor,
172
+ hidden_states: torch.Tensor,
173
+ kv_cache: torch.Tensor,
174
+ attn_metadata: AttentionMetadata,
175
+ ) -> torch.Tensor:
176
+ qkv, _ = self.qkv_proj(hidden_states)
177
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
178
+ q, k = self.rotary_emb(positions, q, k)
179
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
180
+ output, _ = self.o_proj(attn_output)
181
+ return output
182
+
183
+
184
+class GraniteMoeDecoderLayer(nn.Module):
185
+
186
+ def __init__(
187
+ self,
188
+ config: GraniteMoeConfig,
189
+ cache_config: Optional[CacheConfig] = None,
190
+ quant_config: Optional[QuantizationConfig] = None,
191
+ prefix: str = "",
192
+ ) -> None:
193
+ super().__init__()
194
+ self.hidden_size = config.hidden_size
195
+ # Requires transformers > 4.32.0
196
+ rope_theta = getattr(config, "rope_theta", 10000)
197
+ self.self_attn = GraniteMoeAttention(
198
+ hidden_size=self.hidden_size,
199
+ num_heads=config.num_attention_heads,
200
+ max_position=config.max_position_embeddings,
201
+ num_kv_heads=config.num_key_value_heads,
202
+ rope_theta=rope_theta,
203
+ cache_config=cache_config,
204
+ quant_config=quant_config,
205
+ prefix=f"{prefix}.self_attn",
206
+ attention_multiplier=config.attention_multiplier)
207
+ self.block_sparse_moe = GraniteMoeMoE(
208
+ num_experts=config.num_local_experts,
209
+ top_k=config.num_experts_per_tok,
210
+ hidden_size=config.hidden_size,
211
+ intermediate_size=config.intermediate_size,
212
+ quant_config=quant_config,
213
+ prefix=f"{prefix}.block_sparse_moe")
214
+
215
+ self.input_layernorm = RMSNorm(config.hidden_size,
216
+ eps=config.rms_norm_eps)
217
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
218
+ eps=config.rms_norm_eps)
219
+
220
+ self.residual_multiplier = config.residual_multiplier
221
+
222
+ def forward(
223
+ self,
224
+ positions: torch.Tensor,
225
+ hidden_states: torch.Tensor,
226
+ kv_cache: torch.Tensor,
227
+ attn_metadata: AttentionMetadata,
228
+ ) -> torch.Tensor:
229
+ # Self Attention
230
+ residual = hidden_states
231
+ hidden_states = self.input_layernorm(hidden_states)
232
+ hidden_states = self.self_attn(
233
+ positions=positions,
234
+ hidden_states=hidden_states,
235
+ kv_cache=kv_cache,
236
+ attn_metadata=attn_metadata,
237
+ )
238
+ hidden_states = residual + hidden_states * self.residual_multiplier
239
+ residual = hidden_states
240
+ hidden_states = self.post_attention_layernorm(hidden_states)
241
+ hidden_states = self.block_sparse_moe(hidden_states)
242
+ hidden_states = residual + hidden_states * self.residual_multiplier
243
+
244
+ return hidden_states
245
+
246
+
247
+class GraniteMoeModel(nn.Module):
248
+
249
+ def __init__(
250
+ self,
251
+ config: GraniteMoeConfig,
252
+ cache_config: Optional[CacheConfig] = None,
253
+ quant_config: Optional[QuantizationConfig] = None,
254
+ lora_config: Optional[LoRAConfig] = None,
255
+ prefix: str = "",
256
+ ) -> None:
257
+ super().__init__()
258
+ self.padding_idx = config.pad_token_id
259
+ lora_vocab = (lora_config.lora_extra_vocab_size *
260
+ (lora_config.max_loras or 1)) if lora_config else 0
261
+ self.vocab_size = config.vocab_size + lora_vocab
262
+ self.org_vocab_size = config.vocab_size
263
+
264
+ self.embed_tokens = VocabParallelEmbedding(
265
+ self.vocab_size,
266
+ config.hidden_size,
267
+ org_num_embeddings=config.vocab_size,
268
+ )
269
+ self.embedding_multiplier = config.embedding_multiplier
270
+
271
+ self.start_layer, self.end_layer, self.layers = make_layers(
272
+ config.num_hidden_layers,
273
+ lambda prefix: GraniteMoeDecoderLayer(
274
+ config, cache_config, quant_config=quant_config, prefix=prefix
275
+ ),
276
+ prefix=f"{prefix}.layers")
277
+
278
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
279
+
280
+ def forward(
281
+ self,
282
+ input_ids: torch.Tensor,
283
+ positions: torch.Tensor,
284
+ kv_caches: List[torch.Tensor],
285
+ attn_metadata: AttentionMetadata,
286
+ intermediate_tensors: Optional[IntermediateTensors],
287
+ ) -> torch.Tensor:
288
+ if get_pp_group().is_first_rank:
289
+ hidden_states = self.embed_tokens(input_ids)
290
+ hidden_states *= self.embedding_multiplier
291
+ residual = None
292
+ else:
293
+ assert intermediate_tensors is not None
294
+ hidden_states = intermediate_tensors["hidden_states"]
295
+ residual = intermediate_tensors["residual"]
296
+ for i in range(self.start_layer, self.end_layer):
297
+ layer = self.layers[i]
298
+ hidden_states = layer(positions, hidden_states,
299
+ kv_caches[i - self.start_layer],
300
+ attn_metadata)
301
+ if not get_pp_group().is_last_rank:
302
+ return IntermediateTensors({
303
+ "hidden_states": hidden_states,
304
+ "residual": residual
305
+ })
306
+ hidden_states = self.norm(hidden_states)
307
+ return hidden_states
308
+
309
+
310
+class GraniteMoeForCausalLM(nn.Module, SupportsLoRA):
311
+ fall_back_to_pt_during_load = False
312
+
313
+ packed_modules_mapping = {
314
+ "qkv_proj": [
315
+ "q_proj",
316
+ "k_proj",
317
+ "v_proj",
318
+ ],
319
+ }
320
+
321
+ # LoRA specific attributes
322
+ supported_lora_modules = [
323
+ "qkv_proj",
324
+ "o_proj",
325
+ "embed_tokens",
326
+ "lm_head",
327
+ ]
328
+ embedding_modules = {
329
+ "embed_tokens": "input_embeddings",
330
+ "lm_head": "output_embeddings",
331
+ }
332
+ embedding_padding_modules = ["lm_head"]
333
+
334
+ def __init__(
335
+ self,
336
+ config: GraniteMoeConfig,
337
+ cache_config: Optional[CacheConfig] = None,
338
+ quant_config: Optional[QuantizationConfig] = None,
339
+ lora_config: Optional[LoRAConfig] = None,
340
+ ) -> None:
341
+ super().__init__()
342
+
343
+ self.config = config
344
+ self.lora_config = lora_config
345
+
346
+ self.model = GraniteMoeModel(config,
347
+ cache_config,
348
+ quant_config,
349
+ lora_config=lora_config,
350
+ prefix="model")
351
+ self.unpadded_vocab_size = config.vocab_size
352
+ if lora_config:
353
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
354
+ self.lm_head = ParallelLMHead(
355
+ self.unpadded_vocab_size,
356
+ config.hidden_size,
357
+ org_num_embeddings=config.vocab_size,
358
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
359
+ # We need bigger padding if using lora for kernel
360
+ # compatibility
361
+ if not lora_config else lora_config.lora_vocab_padding_size,
362
+ quant_config=quant_config,
363
+ )
364
+ if config.tie_word_embeddings:
365
+ self.lm_head.weight = self.model.embed_tokens.weight
366
+
367
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
368
+ config.vocab_size,
369
+ scale=1 /
370
+ self.config.logits_scaling)
371
+
372
+ self.sampler = Sampler()
373
+
374
+ def forward(
375
+ self,
376
+ input_ids: torch.Tensor,
377
+ positions: torch.Tensor,
378
+ kv_caches: List[torch.Tensor],
379
+ attn_metadata: AttentionMetadata,
380
+ intermediate_tensors: Optional[IntermediateTensors] = None,
381
+ ) -> torch.Tensor:
382
+ hidden_states = self.model(input_ids, positions, kv_caches,
383
+ attn_metadata, intermediate_tensors)
384
+ return hidden_states
385
+
386
+ def compute_logits(
387
+ self, hidden_states: torch.Tensor,
388
+ sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
389
+ logits = self.logits_processor(self.lm_head, hidden_states,
390
+ sampling_metadata)
391
+ return logits
392
+
393
+ def make_empty_intermediate_tensors(
394
+ self, batch_size: int, dtype: torch.dtype,
395
+ device: torch.device) -> IntermediateTensors:
396
+ return IntermediateTensors({
397
+ "hidden_states":
398
+ torch.zeros((batch_size, self.config.hidden_size),
399
+ dtype=dtype,
400
+ device=device),
401
+ "residual":
402
+ torch.zeros((batch_size, self.config.hidden_size),
403
+ dtype=dtype,
404
+ device=device),
405
+ })
406
+
407
+ def sample(
408
+ self,
409
+ logits: Optional[torch.Tensor],
410
+ sampling_metadata: SamplingMetadata,
411
+ ) -> Optional[SamplerOutput]:
412
+ next_tokens = self.sampler(logits, sampling_metadata)
413
+ return next_tokens
414
+
415
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
416
+ new_weights = {}
417
+ for n, p in weights:
418
+ if n.endswith('.block_sparse_moe.input_linear.weight'):
419
+ for e in range(p.size(0)):
420
+ w1_name = n.replace(
421
+ '.block_sparse_moe.input_linear.weight',
422
+ ".block_sparse_moe.experts.%d.w1.weight" % e)
423
+ w3_name = n.replace(
424
+ '.block_sparse_moe.input_linear.weight',
425
+ ".block_sparse_moe.experts.%d.w3.weight" % e)
426
+ w1_param, w3_param = p[e].chunk(2, dim=0)
427
+ assert w1_name not in new_weights
428
+ assert w3_name not in new_weights
429
+ new_weights[w1_name] = w1_param
430
+ new_weights[w3_name] = w3_param
431
+ elif n.endswith('.block_sparse_moe.output_linear.weight'):
432
+ for e in range(p.size(0)):
433
+ w2_name = n.replace(
434
+ '.block_sparse_moe.output_linear.weight',
435
+ ".block_sparse_moe.experts.%d.w2.weight" % e)
436
+ w2_param = p[e]
437
+ assert w2_name not in new_weights
438
+ new_weights[w2_name] = w2_param
439
+ elif n.endswith('.block_sparse_moe.router.layer.weight'):
440
+ gate_name = n.replace('.block_sparse_moe.router.layer.weight',
441
+ ".block_sparse_moe.gate.weight")
442
+ assert gate_name not in new_weights
443
+ new_weights[gate_name] = p
444
+ elif n == 'lm_head.weight' and self.config.tie_word_embeddings:
445
+ pass
446
+ else:
447
+ new_weights[n] = p
448
+ mixtral.MixtralForCausalLM.load_weights(self, new_weights.items())