@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 19f0d25796
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Adding Granite MoE. (#8206)

Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Shawn Tan committed Oct 2, 2024 at 21:33 UTC 19f0d2579695e518c9bfc166544cf23775772bf8
4 files changed +492 -3
tests/models/decoder_only/language/test_granitemoe.py new
+39
@@ -0,0 +1,39 @@
1 +"""Compare the outputs of HF and vLLM for Granite models using greedy sampling.
2 +
3 +Run `pytest tests/models/test_granite.py`.
4 +"""
5 +import pytest
6 +
7 +from ...utils import check_logprobs_close
8 +
9 +MODELS = [
10 + "ibm/PowerMoE-3b",
11 +]
12 +
13 +
14 +@pytest.mark.parametrize("model", MODELS)
15 +@pytest.mark.parametrize("dtype", ["bfloat16"])
16 +@pytest.mark.parametrize("max_tokens", [64])
17 +@pytest.mark.parametrize("num_logprobs", [5])
18 +def test_models(
19 + hf_runner,
20 + vllm_runner,
21 + example_prompts,
22 + model: str,
23 + dtype: str,
24 + max_tokens: int,
25 + num_logprobs: int,
26 +) -> None:
27 + with hf_runner(model, dtype=dtype) as hf_model:
28 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
29 + example_prompts, max_tokens, num_logprobs)
30 +
31 + with vllm_runner(model, dtype=dtype) as vllm_model:
32 + vllm_outputs = vllm_model.generate_greedy_logprobs(
33 + example_prompts, max_tokens, num_logprobs)
34 + check_logprobs_close(
35 + outputs_0_lst=hf_outputs,
36 + outputs_1_lst=vllm_outputs,
37 + name_0="hf",
38 + name_1="vllm",
39 + )
vllm/model_executor/models/__init__.py
+1
@@ -32,6 +32,7 @@ _GENERATION_MODELS = {
32 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
33 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
34 "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
35 + "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
36 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
37 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
38 "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
vllm/model_executor/models/granite.py
+4 -3
@@ -404,9 +404,12 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA):
404 self.lm_head.weight = self.model.embed_tokens.weight
405
406 logit_scale = getattr(config, "logit_scale", 1.0)
407 +
408 + if hasattr(config, "logits_scaling"):
409 + logit_scale /= config.logits_scaling
410 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
411 config.vocab_size,
409 - logit_scale)
412 + scale=logit_scale)
413 self.sampler = Sampler()
414 else:
415 self.lm_head = PPMissingLayer()
@@ -428,8 +431,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA):
431 sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
432 logits = self.logits_processor(self.lm_head, hidden_states,
433 sampling_metadata)
431 - if logits is not None:
432 - logits /= self.config.logits_scaling
434 return logits
435
436 def sample(
vllm/model_executor/models/granitemoe.py new
+448
@@ -0,0 +1,448 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only GraniteMoe model."""
24 +from typing import Iterable, List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +from transformers.models.granitemoe import GraniteMoeConfig
29 +
30 +from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 +from vllm.model_executor.layers.fused_moe import FusedMoE
34 +from vllm.model_executor.layers.layernorm import RMSNorm
35 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
36 + ReplicatedLinear,
37 + RowParallelLinear)
38 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 +from vllm.model_executor.layers.quantization.base_config import (
40 + QuantizationConfig)
41 +from vllm.model_executor.layers.rotary_embedding import get_rope
42 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 +from vllm.model_executor.sampling_metadata import SamplingMetadata
46 +from vllm.sequence import IntermediateTensors
47 +
48 +from . import mixtral
49 +from .interfaces import SupportsLoRA
50 +from .utils import make_layers
51 +
52 +
53 +class GraniteMoeMoE(nn.Module):
54 + """A tensor-parallel MoE implementation for GraniteMoe that shards each
55 + expert across all ranks.
56 + Each expert's weights are sharded across all ranks and a fused MoE
57 + kernel is used for the forward pass, and finally we reduce the outputs
58 + across ranks.
59 + """
60 +
61 + def __init__(self,
62 + num_experts: int,
63 + top_k: int,
64 + hidden_size: int,
65 + intermediate_size: int,
66 + params_dtype: Optional[torch.dtype] = None,
67 + quant_config: Optional[QuantizationConfig] = None,
68 + tp_size: Optional[int] = None,
69 + prefix: str = ""):
70 + super().__init__()
71 + self.hidden_size = hidden_size
72 +
73 + # Gate always runs at half / full precision for now.
74 + self.gate = ReplicatedLinear(hidden_size,
75 + num_experts,
76 + bias=False,
77 + params_dtype=params_dtype,
78 + quant_config=None,
79 + prefix=f"{prefix}.gate")
80 +
81 + self.experts = FusedMoE(num_experts=num_experts,
82 + top_k=top_k,
83 + hidden_size=hidden_size,
84 + intermediate_size=intermediate_size,
85 + params_dtype=params_dtype,
86 + reduce_results=True,
87 + renormalize=True,
88 + quant_config=quant_config,
89 + tp_size=tp_size,
90 + prefix=f"{prefix}.experts")
91 +
92 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
93 + # NOTE: hidden_states can have either 1D or 2D shape.
94 + orig_shape = hidden_states.shape
95 + hidden_states = hidden_states.view(-1, self.hidden_size)
96 + # router_logits: (num_tokens, n_experts)
97 + router_logits, _ = self.gate(hidden_states)
98 + final_hidden_states = self.experts(hidden_states, router_logits)
99 + return final_hidden_states.view(orig_shape)
100 +
101 +
102 +class GraniteMoeAttention(nn.Module):
103 +
104 + def __init__(
105 + self,
106 + hidden_size: int,
107 + num_heads: int,
108 + num_kv_heads: int,
109 + max_position: int = 4096 * 32,
110 + rope_theta: float = 10000,
111 + cache_config: Optional[CacheConfig] = None,
112 + quant_config: Optional[QuantizationConfig] = None,
113 + attention_multiplier: Optional[float] = None,
114 + prefix: str = "",
115 + ) -> None:
116 + super().__init__()
117 + self.hidden_size = hidden_size
118 + tp_size = get_tensor_model_parallel_world_size()
119 + self.total_num_heads = num_heads
120 + assert self.total_num_heads % tp_size == 0
121 + self.num_heads = self.total_num_heads // tp_size
122 + self.total_num_kv_heads = num_kv_heads
123 + if self.total_num_kv_heads >= tp_size:
124 + # Number of KV heads is greater than TP size, so we partition
125 + # the KV heads across multiple tensor parallel GPUs.
126 + assert self.total_num_kv_heads % tp_size == 0
127 + else:
128 + # Number of KV heads is less than TP size, so we replicate
129 + # the KV heads across multiple tensor parallel GPUs.
130 + assert tp_size % self.total_num_kv_heads == 0
131 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
132 + self.head_dim = hidden_size // self.total_num_heads
133 + self.q_size = self.num_heads * self.head_dim
134 + self.kv_size = self.num_kv_heads * self.head_dim
135 + self.scaling = (attention_multiplier if attention_multiplier
136 + is not None else self.head_dim**-1)
137 + self.rope_theta = rope_theta
138 +
139 + self.qkv_proj = QKVParallelLinear(
140 + hidden_size,
141 + self.head_dim,
142 + self.total_num_heads,
143 + self.total_num_kv_heads,
144 + bias=False,
145 + quant_config=quant_config,
146 + prefix=f"{prefix}.qkv_proj",
147 + )
148 + self.o_proj = RowParallelLinear(
149 + self.total_num_heads * self.head_dim,
150 + hidden_size,
151 + bias=False,
152 + quant_config=quant_config,
153 + prefix=f"{prefix}.o_proj",
154 + )
155 + self.rotary_emb = get_rope(
156 + self.head_dim,
157 + rotary_dim=self.head_dim,
158 + max_position=max_position,
159 + base=int(self.rope_theta),
160 + is_neox_style=True,
161 + )
162 + self.attn = Attention(self.num_heads,
163 + self.head_dim,
164 + self.scaling,
165 + num_kv_heads=self.num_kv_heads,
166 + cache_config=cache_config,
167 + quant_config=quant_config)
168 +
169 + def forward(
170 + self,
171 + positions: torch.Tensor,
172 + hidden_states: torch.Tensor,
173 + kv_cache: torch.Tensor,
174 + attn_metadata: AttentionMetadata,
175 + ) -> torch.Tensor:
176 + qkv, _ = self.qkv_proj(hidden_states)
177 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
178 + q, k = self.rotary_emb(positions, q, k)
179 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
180 + output, _ = self.o_proj(attn_output)
181 + return output
182 +
183 +
184 +class GraniteMoeDecoderLayer(nn.Module):
185 +
186 + def __init__(
187 + self,
188 + config: GraniteMoeConfig,
189 + cache_config: Optional[CacheConfig] = None,
190 + quant_config: Optional[QuantizationConfig] = None,
191 + prefix: str = "",
192 + ) -> None:
193 + super().__init__()
194 + self.hidden_size = config.hidden_size
195 + # Requires transformers > 4.32.0
196 + rope_theta = getattr(config, "rope_theta", 10000)
197 + self.self_attn = GraniteMoeAttention(
198 + hidden_size=self.hidden_size,
199 + num_heads=config.num_attention_heads,
200 + max_position=config.max_position_embeddings,
201 + num_kv_heads=config.num_key_value_heads,
202 + rope_theta=rope_theta,
203 + cache_config=cache_config,
204 + quant_config=quant_config,
205 + prefix=f"{prefix}.self_attn",
206 + attention_multiplier=config.attention_multiplier)
207 + self.block_sparse_moe = GraniteMoeMoE(
208 + num_experts=config.num_local_experts,
209 + top_k=config.num_experts_per_tok,
210 + hidden_size=config.hidden_size,
211 + intermediate_size=config.intermediate_size,
212 + quant_config=quant_config,
213 + prefix=f"{prefix}.block_sparse_moe")
214 +
215 + self.input_layernorm = RMSNorm(config.hidden_size,
216 + eps=config.rms_norm_eps)
217 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
218 + eps=config.rms_norm_eps)
219 +
220 + self.residual_multiplier = config.residual_multiplier
221 +
222 + def forward(
223 + self,
224 + positions: torch.Tensor,
225 + hidden_states: torch.Tensor,
226 + kv_cache: torch.Tensor,
227 + attn_metadata: AttentionMetadata,
228 + ) -> torch.Tensor:
229 + # Self Attention
230 + residual = hidden_states
231 + hidden_states = self.input_layernorm(hidden_states)
232 + hidden_states = self.self_attn(
233 + positions=positions,
234 + hidden_states=hidden_states,
235 + kv_cache=kv_cache,
236 + attn_metadata=attn_metadata,
237 + )
238 + hidden_states = residual + hidden_states * self.residual_multiplier
239 + residual = hidden_states
240 + hidden_states = self.post_attention_layernorm(hidden_states)
241 + hidden_states = self.block_sparse_moe(hidden_states)
242 + hidden_states = residual + hidden_states * self.residual_multiplier
243 +
244 + return hidden_states
245 +
246 +
247 +class GraniteMoeModel(nn.Module):
248 +
249 + def __init__(
250 + self,
251 + config: GraniteMoeConfig,
252 + cache_config: Optional[CacheConfig] = None,
253 + quant_config: Optional[QuantizationConfig] = None,
254 + lora_config: Optional[LoRAConfig] = None,
255 + prefix: str = "",
256 + ) -> None:
257 + super().__init__()
258 + self.padding_idx = config.pad_token_id
259 + lora_vocab = (lora_config.lora_extra_vocab_size *
260 + (lora_config.max_loras or 1)) if lora_config else 0
261 + self.vocab_size = config.vocab_size + lora_vocab
262 + self.org_vocab_size = config.vocab_size
263 +
264 + self.embed_tokens = VocabParallelEmbedding(
265 + self.vocab_size,
266 + config.hidden_size,
267 + org_num_embeddings=config.vocab_size,
268 + )
269 + self.embedding_multiplier = config.embedding_multiplier
270 +
271 + self.start_layer, self.end_layer, self.layers = make_layers(
272 + config.num_hidden_layers,
273 + lambda prefix: GraniteMoeDecoderLayer(
274 + config, cache_config, quant_config=quant_config, prefix=prefix
275 + ),
276 + prefix=f"{prefix}.layers")
277 +
278 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
279 +
280 + def forward(
281 + self,
282 + input_ids: torch.Tensor,
283 + positions: torch.Tensor,
284 + kv_caches: List[torch.Tensor],
285 + attn_metadata: AttentionMetadata,
286 + intermediate_tensors: Optional[IntermediateTensors],
287 + ) -> torch.Tensor:
288 + if get_pp_group().is_first_rank:
289 + hidden_states = self.embed_tokens(input_ids)
290 + hidden_states *= self.embedding_multiplier
291 + residual = None
292 + else:
293 + assert intermediate_tensors is not None
294 + hidden_states = intermediate_tensors["hidden_states"]
295 + residual = intermediate_tensors["residual"]
296 + for i in range(self.start_layer, self.end_layer):
297 + layer = self.layers[i]
298 + hidden_states = layer(positions, hidden_states,
299 + kv_caches[i - self.start_layer],
300 + attn_metadata)
301 + if not get_pp_group().is_last_rank:
302 + return IntermediateTensors({
303 + "hidden_states": hidden_states,
304 + "residual": residual
305 + })
306 + hidden_states = self.norm(hidden_states)
307 + return hidden_states
308 +
309 +
310 +class GraniteMoeForCausalLM(nn.Module, SupportsLoRA):
311 + fall_back_to_pt_during_load = False
312 +
313 + packed_modules_mapping = {
314 + "qkv_proj": [
315 + "q_proj",
316 + "k_proj",
317 + "v_proj",
318 + ],
319 + }
320 +
321 + # LoRA specific attributes
322 + supported_lora_modules = [
323 + "qkv_proj",
324 + "o_proj",
325 + "embed_tokens",
326 + "lm_head",
327 + ]
328 + embedding_modules = {
329 + "embed_tokens": "input_embeddings",
330 + "lm_head": "output_embeddings",
331 + }
332 + embedding_padding_modules = ["lm_head"]
333 +
334 + def __init__(
335 + self,
336 + config: GraniteMoeConfig,
337 + cache_config: Optional[CacheConfig] = None,
338 + quant_config: Optional[QuantizationConfig] = None,
339 + lora_config: Optional[LoRAConfig] = None,
340 + ) -> None:
341 + super().__init__()
342 +
343 + self.config = config
344 + self.lora_config = lora_config
345 +
346 + self.model = GraniteMoeModel(config,
347 + cache_config,
348 + quant_config,
349 + lora_config=lora_config,
350 + prefix="model")
351 + self.unpadded_vocab_size = config.vocab_size
352 + if lora_config:
353 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
354 + self.lm_head = ParallelLMHead(
355 + self.unpadded_vocab_size,
356 + config.hidden_size,
357 + org_num_embeddings=config.vocab_size,
358 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
359 + # We need bigger padding if using lora for kernel
360 + # compatibility
361 + if not lora_config else lora_config.lora_vocab_padding_size,
362 + quant_config=quant_config,
363 + )
364 + if config.tie_word_embeddings:
365 + self.lm_head.weight = self.model.embed_tokens.weight
366 +
367 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
368 + config.vocab_size,
369 + scale=1 /
370 + self.config.logits_scaling)
371 +
372 + self.sampler = Sampler()
373 +
374 + def forward(
375 + self,
376 + input_ids: torch.Tensor,
377 + positions: torch.Tensor,
378 + kv_caches: List[torch.Tensor],
379 + attn_metadata: AttentionMetadata,
380 + intermediate_tensors: Optional[IntermediateTensors] = None,
381 + ) -> torch.Tensor:
382 + hidden_states = self.model(input_ids, positions, kv_caches,
383 + attn_metadata, intermediate_tensors)
384 + return hidden_states
385 +
386 + def compute_logits(
387 + self, hidden_states: torch.Tensor,
388 + sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
389 + logits = self.logits_processor(self.lm_head, hidden_states,
390 + sampling_metadata)
391 + return logits
392 +
393 + def make_empty_intermediate_tensors(
394 + self, batch_size: int, dtype: torch.dtype,
395 + device: torch.device) -> IntermediateTensors:
396 + return IntermediateTensors({
397 + "hidden_states":
398 + torch.zeros((batch_size, self.config.hidden_size),
399 + dtype=dtype,
400 + device=device),
401 + "residual":
402 + torch.zeros((batch_size, self.config.hidden_size),
403 + dtype=dtype,
404 + device=device),
405 + })
406 +
407 + def sample(
408 + self,
409 + logits: Optional[torch.Tensor],
410 + sampling_metadata: SamplingMetadata,
411 + ) -> Optional[SamplerOutput]:
412 + next_tokens = self.sampler(logits, sampling_metadata)
413 + return next_tokens
414 +
415 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
416 + new_weights = {}
417 + for n, p in weights:
418 + if n.endswith('.block_sparse_moe.input_linear.weight'):
419 + for e in range(p.size(0)):
420 + w1_name = n.replace(
421 + '.block_sparse_moe.input_linear.weight',
422 + ".block_sparse_moe.experts.%d.w1.weight" % e)
423 + w3_name = n.replace(
424 + '.block_sparse_moe.input_linear.weight',
425 + ".block_sparse_moe.experts.%d.w3.weight" % e)
426 + w1_param, w3_param = p[e].chunk(2, dim=0)
427 + assert w1_name not in new_weights
428 + assert w3_name not in new_weights
429 + new_weights[w1_name] = w1_param
430 + new_weights[w3_name] = w3_param
431 + elif n.endswith('.block_sparse_moe.output_linear.weight'):
432 + for e in range(p.size(0)):
433 + w2_name = n.replace(
434 + '.block_sparse_moe.output_linear.weight',
435 + ".block_sparse_moe.experts.%d.w2.weight" % e)
436 + w2_param = p[e]
437 + assert w2_name not in new_weights
438 + new_weights[w2_name] = w2_param
439 + elif n.endswith('.block_sparse_moe.router.layer.weight'):
440 + gate_name = n.replace('.block_sparse_moe.router.layer.weight',
441 + ".block_sparse_moe.gate.weight")
442 + assert gate_name not in new_weights
443 + new_weights[gate_name] = p
444 + elif n == 'lm_head.weight' and self.config.tie_word_embeddings:
445 + pass
446 + else:
447 + new_weights[n] = p
448 + mixtral.MixtralForCausalLM.load_weights(self, new_weights.items())