@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f8d60145b4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model] Add Granite model (#7436)

Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Shawn Tan committed Sep 1, 2024 at 21:37 UTC f8d60145b4d954b7a110073f77dc91842155a3d8
4 files changed +792
tests/models/test_granite.py new
+49
@@ -0,0 +1,49 @@
1 +"""Compare the outputs of HF and vLLM for Granite models using greedy sampling.
2 +
3 +Run `pytest tests/models/test_granite.py`.
4 +"""
5 +import importlib.metadata
6 +
7 +import pytest
8 +
9 +from .utils import check_logprobs_close
10 +
11 +TRANSFORMERS_VERSION = tuple(
12 + map(int,
13 + importlib.metadata.version("transformers").split(".")))
14 +
15 +MODELS = [
16 + "ibm/PowerLM-3b",
17 +]
18 +
19 +
20 +# GraniteForCausalLM will be in transformers >= 4.45
21 +@pytest.mark.skipif(TRANSFORMERS_VERSION < (4, 45),
22 + reason="granite model test requires transformers >= 4.45")
23 +@pytest.mark.parametrize("model", MODELS)
24 +@pytest.mark.parametrize("dtype", ["bfloat16"])
25 +@pytest.mark.parametrize("max_tokens", [64])
26 +@pytest.mark.parametrize("num_logprobs", [5])
27 +def test_models(
28 + hf_runner,
29 + vllm_runner,
30 + example_prompts,
31 + model: str,
32 + dtype: str,
33 + max_tokens: int,
34 + num_logprobs: int,
35 +) -> None:
36 + # TODO(sang): Sliding window should be tested separately.
37 + with hf_runner(model, dtype=dtype) as hf_model:
38 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
39 + example_prompts, max_tokens, num_logprobs)
40 +
41 + with vllm_runner(model, dtype=dtype) as vllm_model:
42 + vllm_outputs = vllm_model.generate_greedy_logprobs(
43 + example_prompts, max_tokens, num_logprobs)
44 + check_logprobs_close(
45 + outputs_0_lst=hf_outputs,
46 + outputs_1_lst=vllm_outputs,
47 + name_0="hf",
48 + name_1="vllm",
49 + )
vllm/model_executor/models/__init__.py
+1
@@ -65,6 +65,7 @@ _GENERATION_MODELS = {
65 "EAGLEModel": ("eagle", "EAGLE"),
66 "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
67 "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
68 + "GraniteForCausalLM": ("granite", "GraniteForCausalLM")
69 }
70
71 _EMBEDDING_MODELS = {
vllm/model_executor/models/granite.py new
+543
@@ -0,0 +1,543 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only IBM Granite model compatible with HuggingFace weights."""
24 +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25 +
26 +import torch
27 +from torch import nn
28 +
29 +from vllm.attention import Attention, AttentionMetadata
30 +from vllm.config import CacheConfig, LoRAConfig
31 +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32 + get_tensor_model_parallel_world_size)
33 +from vllm.model_executor.layers.activation import SiluAndMul
34 +from vllm.model_executor.layers.layernorm import RMSNorm
35 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
36 + QKVParallelLinear,
37 + RowParallelLinear)
38 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 +from vllm.model_executor.layers.quantization.base_config import (
40 + QuantizationConfig)
41 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
42 + get_compressed_tensors_cache_scale)
43 +from vllm.model_executor.layers.rotary_embedding import get_rope
44 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.vocab_parallel_embedding import (
46 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
47 +from vllm.model_executor.model_loader.weight_utils import (
48 + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
49 +from vllm.model_executor.sampling_metadata import SamplingMetadata
50 +from vllm.sequence import IntermediateTensors
51 +from vllm.transformers_utils.configs.granite import GraniteConfig
52 +from vllm.utils import is_hip
53 +
54 +from .interfaces import SupportsLoRA
55 +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
56 +
57 +
58 +class GraniteMLP(nn.Module):
59 +
60 + def __init__(
61 + self,
62 + hidden_size: int,
63 + intermediate_size: int,
64 + hidden_act: str,
65 + quant_config: Optional[QuantizationConfig] = None,
66 + bias: bool = False,
67 + prefix: str = "",
68 + ) -> None:
69 + super().__init__()
70 + self.gate_up_proj = MergedColumnParallelLinear(
71 + input_size=hidden_size,
72 + output_sizes=[intermediate_size] * 2,
73 + bias=bias,
74 + quant_config=quant_config,
75 + prefix=f"{prefix}.gate_up_proj")
76 + self.down_proj = RowParallelLinear(input_size=intermediate_size,
77 + output_size=hidden_size,
78 + bias=bias,
79 + quant_config=quant_config,
80 + prefix=f"{prefix}.down_proj")
81 + if hidden_act != "silu":
82 + raise ValueError(f"Unsupported activation: {hidden_act}. "
83 + "Only silu is supported for now.")
84 + self.act_fn = SiluAndMul()
85 +
86 + def forward(self, x):
87 + gate_up, _ = self.gate_up_proj(x)
88 + x = self.act_fn(gate_up)
89 + x, _ = self.down_proj(x)
90 + return x
91 +
92 +
93 +class GraniteAttention(nn.Module):
94 +
95 + def __init__(
96 + self,
97 + config: GraniteConfig,
98 + hidden_size: int,
99 + num_heads: int,
100 + num_kv_heads: int,
101 + rope_theta: float = 10000,
102 + rope_scaling: Optional[Dict[str, Any]] = None,
103 + max_position_embeddings: int = 8192,
104 + quant_config: Optional[QuantizationConfig] = None,
105 + bias: bool = False,
106 + cache_config: Optional[CacheConfig] = None,
107 + prefix: str = "",
108 + ) -> None:
109 + super().__init__()
110 + self.hidden_size = hidden_size
111 + tp_size = get_tensor_model_parallel_world_size()
112 + self.total_num_heads = num_heads
113 + assert self.total_num_heads % tp_size == 0
114 + self.num_heads = self.total_num_heads // tp_size
115 + self.total_num_kv_heads = num_kv_heads
116 + if self.total_num_kv_heads >= tp_size:
117 + # Number of KV heads is greater than TP size, so we partition
118 + # the KV heads across multiple tensor parallel GPUs.
119 + assert self.total_num_kv_heads % tp_size == 0
120 + else:
121 + # Number of KV heads is less than TP size, so we replicate
122 + # the KV heads across multiple tensor parallel GPUs.
123 + assert tp_size % self.total_num_kv_heads == 0
124 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
125 + # MistralConfig has an optional head_dim introduced by Mistral-Nemo
126 + self.head_dim = getattr(config, "head_dim",
127 + self.hidden_size // self.total_num_heads)
128 + self.q_size = self.num_heads * self.head_dim
129 + self.kv_size = self.num_kv_heads * self.head_dim
130 + self.scaling = config.attention_multiplier
131 + self.rope_theta = rope_theta
132 + self.max_position_embeddings = max_position_embeddings
133 +
134 + self.qkv_proj = QKVParallelLinear(
135 + hidden_size=hidden_size,
136 + head_size=self.head_dim,
137 + total_num_heads=self.total_num_heads,
138 + total_num_kv_heads=self.total_num_kv_heads,
139 + bias=bias,
140 + quant_config=quant_config,
141 + prefix=f"{prefix}.qkv_proj",
142 + )
143 + self.o_proj = RowParallelLinear(
144 + input_size=self.total_num_heads * self.head_dim,
145 + output_size=hidden_size,
146 + bias=bias,
147 + quant_config=quant_config,
148 + prefix=f"{prefix}.o_proj",
149 + )
150 +
151 + self.rotary_emb = get_rope(
152 + self.head_dim,
153 + rotary_dim=self.head_dim,
154 + max_position=max_position_embeddings,
155 + base=rope_theta,
156 + rope_scaling=rope_scaling,
157 + )
158 + self.attn = Attention(self.num_heads,
159 + self.head_dim,
160 + self.scaling,
161 + num_kv_heads=self.num_kv_heads,
162 + cache_config=cache_config,
163 + quant_config=quant_config)
164 +
165 + def forward(
166 + self,
167 + positions: torch.Tensor,
168 + hidden_states: torch.Tensor,
169 + kv_cache: torch.Tensor,
170 + attn_metadata: AttentionMetadata,
171 + ) -> torch.Tensor:
172 + qkv, _ = self.qkv_proj(hidden_states)
173 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
174 + q, k = self.rotary_emb(positions, q, k)
175 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
176 + output, _ = self.o_proj(attn_output)
177 + return output
178 +
179 +
180 +class GraniteDecoderLayer(nn.Module):
181 +
182 + def __init__(
183 + self,
184 + config: GraniteConfig,
185 + cache_config: Optional[CacheConfig] = None,
186 + quant_config: Optional[QuantizationConfig] = None,
187 + prefix: str = "",
188 + ) -> None:
189 + super().__init__()
190 + self.hidden_size = config.hidden_size
191 + self.residual_multiplier = config.residual_multiplier
192 + rope_theta = getattr(config, "rope_theta", 10000)
193 + rope_scaling = getattr(config, "rope_scaling", None)
194 + if rope_scaling is not None and getattr(
195 + config, "original_max_position_embeddings", None):
196 + rope_scaling["original_max_position_embeddings"] = (
197 + config.original_max_position_embeddings)
198 + max_position_embeddings = getattr(config, "max_position_embeddings",
199 + 8192)
200 + # Support abacusai/Smaug-72B-v0.1 with attention_bias
201 + # Support internlm/internlm-7b with bias
202 + attention_bias = getattr(config, "attention_bias", False) or getattr(
203 + config, "bias", False)
204 + self.self_attn = GraniteAttention(
205 + config=config,
206 + hidden_size=self.hidden_size,
207 + num_heads=config.num_attention_heads,
208 + num_kv_heads=getattr(config, "num_key_value_heads",
209 + config.num_attention_heads),
210 + rope_theta=rope_theta,
211 + rope_scaling=rope_scaling,
212 + max_position_embeddings=max_position_embeddings,
213 + quant_config=quant_config,
214 + bias=attention_bias,
215 + cache_config=cache_config,
216 + prefix=f"{prefix}.self_attn",
217 + )
218 +
219 + self.mlp = GraniteMLP(
220 + hidden_size=self.hidden_size,
221 + intermediate_size=config.intermediate_size,
222 + hidden_act=config.hidden_act,
223 + quant_config=quant_config,
224 + bias=getattr(config, "mlp_bias", False),
225 + prefix=f"{prefix}.mlp",
226 + )
227 + self.input_layernorm = RMSNorm(config.hidden_size,
228 + eps=config.rms_norm_eps)
229 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
230 + eps=config.rms_norm_eps)
231 +
232 + def forward(
233 + self,
234 + positions: torch.Tensor,
235 + hidden_states: torch.Tensor,
236 + kv_cache: torch.Tensor,
237 + attn_metadata: AttentionMetadata,
238 + ) -> Tuple[torch.Tensor, torch.Tensor]:
239 + # Self Attention
240 + residual = hidden_states
241 + hidden_states = self.input_layernorm(hidden_states)
242 + hidden_states = self.self_attn(
243 + positions=positions,
244 + hidden_states=hidden_states,
245 + kv_cache=kv_cache,
246 + attn_metadata=attn_metadata,
247 + )
248 + hidden_states = residual + hidden_states * self.residual_multiplier
249 + # Fully Connected
250 + residual = hidden_states
251 + hidden_states = self.post_attention_layernorm(hidden_states)
252 + hidden_states = self.mlp(hidden_states)
253 + hidden_states = residual + hidden_states * self.residual_multiplier
254 + return hidden_states
255 +
256 +
257 +class GraniteModel(nn.Module):
258 +
259 + def __init__(
260 + self,
261 + config: GraniteConfig,
262 + cache_config: Optional[CacheConfig] = None,
263 + quant_config: Optional[QuantizationConfig] = None,
264 + lora_config: Optional[LoRAConfig] = None,
265 + prefix: str = "",
266 + ) -> None:
267 + super().__init__()
268 + self.config = config
269 + self.padding_idx = config.pad_token_id
270 + lora_vocab = (lora_config.lora_extra_vocab_size *
271 + (lora_config.max_loras or 1)) if lora_config else 0
272 + self.vocab_size = config.vocab_size + lora_vocab
273 + self.org_vocab_size = config.vocab_size
274 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
275 + and get_pp_group().is_last_rank):
276 + self.embed_tokens = VocabParallelEmbedding(
277 + self.vocab_size,
278 + config.hidden_size,
279 + org_num_embeddings=config.vocab_size,
280 + quant_config=quant_config,
281 + )
282 + else:
283 + self.embed_tokens = PPMissingLayer()
284 + self.start_layer, self.end_layer, self.layers = make_layers(
285 + config.num_hidden_layers,
286 + lambda prefix: GraniteDecoderLayer(config=config,
287 + cache_config=cache_config,
288 + quant_config=quant_config,
289 + prefix=prefix),
290 + prefix=f"{prefix}.layers")
291 + if get_pp_group().is_last_rank:
292 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
293 + else:
294 + self.norm = PPMissingLayer()
295 +
296 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
297 + return self.embed_tokens(input_ids)
298 +
299 + def forward(
300 + self,
301 + input_ids: Optional[torch.Tensor],
302 + positions: torch.Tensor,
303 + kv_caches: List[torch.Tensor],
304 + attn_metadata: AttentionMetadata,
305 + intermediate_tensors: Optional[IntermediateTensors],
306 + inputs_embeds: Optional[torch.Tensor] = None,
307 + ) -> Union[torch.Tensor, IntermediateTensors]:
308 + if get_pp_group().is_first_rank:
309 + if inputs_embeds is not None:
310 + hidden_states = inputs_embeds
311 + else:
312 + hidden_states = self.get_input_embeddings(input_ids)
313 + residual = None
314 + else:
315 + assert intermediate_tensors is not None
316 + hidden_states = intermediate_tensors["hidden_states"]
317 + residual = intermediate_tensors["residual"]
318 +
319 + hidden_states *= self.config.embedding_multiplier
320 +
321 + for i in range(self.start_layer, self.end_layer):
322 + layer = self.layers[i]
323 + hidden_states = layer(
324 + positions,
325 + hidden_states,
326 + kv_caches[i - self.start_layer],
327 + attn_metadata,
328 + )
329 +
330 + if not get_pp_group().is_last_rank:
331 + return IntermediateTensors({
332 + "hidden_states": hidden_states,
333 + "residual": residual
334 + })
335 +
336 + hidden_states = self.norm(hidden_states)
337 + return hidden_states
338 +
339 +
340 +class GraniteForCausalLM(nn.Module, SupportsLoRA):
341 + packed_modules_mapping = {
342 + "qkv_proj": [
343 + "q_proj",
344 + "k_proj",
345 + "v_proj",
346 + ],
347 + "gate_up_proj": [
348 + "gate_proj",
349 + "up_proj",
350 + ],
351 + }
352 +
353 + # LoRA specific attributes
354 + supported_lora_modules = [
355 + "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
356 + "lm_head"
357 + ]
358 + embedding_modules = {
359 + "embed_tokens": "input_embeddings",
360 + "lm_head": "output_embeddings",
361 + }
362 + embedding_padding_modules = ["lm_head"]
363 + bitsandbytes_stacked_params_mapping = {
364 + # shard_name, weight_name, index
365 + "q_proj": ("qkv_proj", 0),
366 + "k_proj": ("qkv_proj", 1),
367 + "v_proj": ("qkv_proj", 2),
368 + "gate_proj": ("gate_up_proj", 0),
369 + "up_proj": ("gate_up_proj", 1),
370 + }
371 +
372 + def __init__(
373 + self,
374 + config: GraniteConfig,
375 + cache_config: Optional[CacheConfig] = None,
376 + quant_config: Optional[QuantizationConfig] = None,
377 + lora_config: Optional[LoRAConfig] = None,
378 + ) -> None:
379 + super().__init__()
380 +
381 + self.config = config
382 + self.lora_config = lora_config
383 +
384 + self.model = GraniteModel(config,
385 + cache_config,
386 + quant_config,
387 + lora_config=lora_config,
388 + prefix="model")
389 + if get_pp_group().is_last_rank:
390 + self.unpadded_vocab_size = config.vocab_size
391 + if lora_config:
392 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
393 + self.lm_head = ParallelLMHead(
394 + self.unpadded_vocab_size,
395 + config.hidden_size,
396 + org_num_embeddings=config.vocab_size,
397 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
398 + # We need bigger padding if using lora for kernel
399 + # compatibility
400 + if not lora_config else lora_config.lora_vocab_padding_size,
401 + quant_config=quant_config,
402 + )
403 + if config.tie_word_embeddings:
404 + self.lm_head.weight = self.model.embed_tokens.weight
405 +
406 + logit_scale = getattr(config, "logit_scale", 1.0)
407 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
408 + config.vocab_size,
409 + logit_scale)
410 + self.sampler = Sampler()
411 + else:
412 + self.lm_head = PPMissingLayer()
413 +
414 + def forward(
415 + self,
416 + input_ids: torch.Tensor,
417 + positions: torch.Tensor,
418 + kv_caches: List[torch.Tensor],
419 + attn_metadata: AttentionMetadata,
420 + intermediate_tensors: Optional[IntermediateTensors] = None,
421 + ) -> Union[torch.Tensor, IntermediateTensors]:
422 + model_output = self.model(input_ids, positions, kv_caches,
423 + attn_metadata, intermediate_tensors)
424 + return model_output
425 +
426 + def compute_logits(
427 + self, hidden_states: torch.Tensor,
428 + sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
429 + logits = self.logits_processor(self.lm_head, hidden_states,
430 + sampling_metadata)
431 + logits /= self.config.logits_scaling
432 + return logits
433 +
434 + def sample(
435 + self,
436 + logits: torch.Tensor,
437 + sampling_metadata: SamplingMetadata,
438 + ) -> Optional[SamplerOutput]:
439 + next_tokens = self.sampler(logits, sampling_metadata)
440 + return next_tokens
441 +
442 + def make_empty_intermediate_tensors(
443 + self, batch_size: int, dtype: torch.dtype,
444 + device: torch.device) -> IntermediateTensors:
445 + return IntermediateTensors({
446 + "hidden_states":
447 + torch.zeros((batch_size, self.config.hidden_size),
448 + dtype=dtype,
449 + device=device),
450 + "residual":
451 + torch.zeros((batch_size, self.config.hidden_size),
452 + dtype=dtype,
453 + device=device),
454 + })
455 +
456 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
457 + stacked_params_mapping = [
458 + # (param_name, shard_name, shard_id)
459 + (".qkv_proj", ".q_proj", "q"),
460 + (".qkv_proj", ".k_proj", "k"),
461 + (".qkv_proj", ".v_proj", "v"),
462 + (".gate_up_proj", ".gate_proj", 0),
463 + (".gate_up_proj", ".up_proj", 1),
464 + ]
465 + params_dict = dict(self.named_parameters())
466 + for name, loaded_weight in weights:
467 + if "rotary_emb.inv_freq" in name:
468 + continue
469 + if ("rotary_emb.cos_cached" in name
470 + or "rotary_emb.sin_cached" in name):
471 + # Models trained using ColossalAI may include these tensors in
472 + # the checkpoint. Skip them.
473 + continue
474 + # With tie_word_embeddings, we can skip lm_head.weight
475 + # The weight might appear unnecessarily in the files if the model is
476 + # processed with quantization, LoRA, fine-tuning, etc.
477 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
478 + continue
479 + if scale_name := get_compressed_tensors_cache_scale(name):
480 + # Loading kv cache scales for compressed-tensors quantization
481 + param = params_dict[scale_name]
482 + weight_loader = getattr(param, "weight_loader",
483 + default_weight_loader)
484 + loaded_weight = loaded_weight[0]
485 + weight_loader(param, loaded_weight)
486 + continue
487 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
488 + if weight_name not in name:
489 + continue
490 + name = name.replace(weight_name, param_name)
491 + # Skip loading extra bias for GPTQ models.
492 + if name.endswith(".bias") and name not in params_dict:
493 + continue
494 +
495 + if is_pp_missing_parameter(name, self):
496 + continue
497 +
498 + param = params_dict[name]
499 + weight_loader = param.weight_loader
500 + weight_loader(param, loaded_weight, shard_id)
501 +
502 + break
503 + else:
504 + # Skip loading extra bias for GPTQ models.
505 + if name.endswith(".bias") and name not in params_dict:
506 + continue
507 + # Remapping the name of FP8 kv-scale.
508 + name = maybe_remap_kv_scale_name(name, params_dict)
509 + if name is None:
510 + continue
511 +
512 + if is_pp_missing_parameter(name, self):
513 + continue
514 +
515 + param = params_dict[name]
516 + weight_loader = getattr(param, "weight_loader",
517 + default_weight_loader)
518 + weight_loader(param, loaded_weight)
519 +
520 + # If this function is called, it should always initialize KV cache scale
521 + # factors (or else raise an exception). Thus, handled exceptions should
522 + # make sure to leave KV cache scale factors in a known good (dummy) state
523 + def load_kv_cache_scales(self, quantization_param_path: str) -> None:
524 + tp_size = get_tensor_model_parallel_world_size()
525 + tp_rank = get_tensor_model_parallel_rank()
526 + for layer_idx, scaling_factor in kv_cache_scales_loader(
527 + quantization_param_path, tp_rank, tp_size,
528 + self.config.num_hidden_layers,
529 + self.config.__class__.model_type):
530 + if not isinstance(self.model.layers[layer_idx], nn.Identity):
531 + layer_self_attn = self.model.layers[layer_idx].self_attn
532 +
533 + if is_hip():
534 + # The scaling factor convention we are assuming is
535 + # quantized_value * scaling_factor ~= true_value
536 + # which is consistent with the practice of setting
537 + # scaling_factor = tensor_amax / FPtype_max
538 + scaling_factor *= 2
539 + if hasattr(layer_self_attn, "kv_scale"):
540 + layer_self_attn.attn._kv_scale = scaling_factor
541 + else:
542 + raise RuntimeError("Self attention has no KV cache scaling "
543 + "factor attribute!")
vllm/transformers_utils/configs/granite.py new
+199
@@ -0,0 +1,199 @@
1 +# coding=utf-8
2 +# Copyright 2024 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3 +#
4 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 +# and OPT implementations in this library. It has been modified from its
6 +# original forms to accommodate minor architectural differences compared
7 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +"""Granite model configuration"""
21 +
22 +from transformers.configuration_utils import PretrainedConfig
23 +from transformers.modeling_rope_utils import rope_config_validation
24 +from transformers.utils import logging
25 +
26 +logger = logging.get_logger(__name__)
27 +
28 +
29 +class GraniteConfig(PretrainedConfig):
30 + r"""
31 + This is the configuration class to store the configuration of
32 + a [`GraniteModel`]. It is used to instantiate an Granite
33 + model according to the specified arguments, defining the model architecture.
34 + Instantiating a configuration with the defaults will yield a similar
35 + configuration to that of the Granite-3B.
36 +
37 + Configuration objects inherit from [`PretrainedConfig`] and can be used to
38 + control the model outputs. Read the documentation from [`PretrainedConfig`]
39 + for more information.
40 +
41 +
42 + Args:
43 + vocab_size (`int`, *optional*, defaults to 32000):
44 + Vocabulary size of the Granite model. Defines the number of
45 + different tokens that can be represented by the `inputs_ids`
46 + passed when calling [`GraniteModel`]
47 + hidden_size (`int`, *optional*, defaults to 4096):
48 + Dimension of the hidden representations.
49 + intermediate_size (`int`, *optional*, defaults to 11008):
50 + Dimension of the MLP representations.
51 + num_hidden_layers (`int`, *optional*, defaults to 32):
52 + Number of hidden layers in the Transformer decoder.
53 + num_attention_heads (`int`, *optional*, defaults to 32):
54 + Number of attention heads for each attention layer in the
55 + Transformer decoder.
56 + num_key_value_heads (`int`, *optional*):
57 + This is the number of key_value heads that should be used to
58 + implement Grouped Query Attention. If
59 + `num_key_value_heads=num_attention_heads`, the model will use Multi
60 + Head Attention (MHA), if `num_key_value_heads=1` the model will use
61 + Multi Query Attention (MQA) otherwise GQA is used. When converting
62 + a multi-head checkpoint to a GQA checkpoint, each group key and
63 + value head should be constructed by meanpooling all the original
64 + heads within that group. For more details checkout
65 + [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
66 + specified, will default to `num_attention_heads`.
67 + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
68 + The non-linear activation function (function or string) in the
69 + decoder.
70 + max_position_embeddings (`int`, *optional*, defaults to 2048):
71 + The maximum sequence length that this model might ever be used with.
72 + initializer_range (`float`, *optional*, defaults to 0.02):
73 + The standard deviation of the truncated_normal_initializer for
74 + initializing all weight matrices.
75 + rms_norm_eps (`float`, *optional*, defaults to 1e-06):
76 + The epsilon used by the rms normalization layers.
77 + use_cache (`bool`, *optional*, defaults to `True`):
78 + Whether or not the model should return the last key/values
79 + attentions (not used by all models). Only relevant if
80 + `config.is_decoder=True`.
81 + pad_token_id (`int`, *optional*):
82 + Padding token id.
83 + bos_token_id (`int`, *optional*, defaults to 1):
84 + Beginning of stream token id.
85 + eos_token_id (`int`, *optional*, defaults to 2):
86 + End of stream token id.
87 + tie_word_embeddings (`bool`, *optional*, defaults to `False`):
88 + Whether to tie weight embeddings
89 + rope_theta (`float`, *optional*, defaults to 10000.0):
90 + The base period of the RoPE embeddings.
91 + rope_scaling (`Dict`, *optional*):
92 + Dictionary containing the scaling configuration for the RoPE
93 + embeddings. Currently supports two scaling strategies: linear and
94 + dynamic. Their scaling factor must be a float greater than 1. The
95 + expected format is
96 + `{"type": strategy name, "factor": scaling factor}`.
97 + When using this flag, don't update `max_position_embeddings` to
98 + the expected new maximum. See the following thread for more
99 + information on how these scaling strategies behave:
100 + https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/.
101 + This is an experimental feature, subject to breaking API changes
102 + in future versions.
103 + attention_bias (`bool`, *optional*, defaults to `False`):
104 + Whether to use a bias in the query, key, value and output
105 + projection layers during self-attention.
106 + attention_dropout (`float`, *optional*, defaults to 0.0):
107 + The dropout ratio for the attention probabilities.
108 + mlp_bias (`bool`, *optional*, defaults to `False`):
109 + Whether to use a bias in up_proj, down_proj and gate_proj layers
110 + in the MLP layers.
111 + embedding_multiplier (`float`, *optional*, defaults to 1.0):
112 + embedding multiplier
113 + logits_scaling (`float`, *optional*, defaults to 1.0):
114 + divisor for output logits
115 + residual_multiplier (`float`, *optional*, defaults to 1.0):
116 + residual multiplier
117 + attention_multiplier (`float`, *optional*, defaults to 1.0):
118 + attention multiplier
119 +
120 + ```python
121 + >>> from transformers import GraniteModel, GraniteConfig
122 +
123 + >>> # Initializing a Granite granite-3b style configuration
124 + >>> configuration = GraniteConfig()
125 +
126 + >>> # Initializing a model from the granite-7b style configuration
127 + >>> model = GraniteModel(configuration)
128 +
129 + >>> # Accessing the model configuration
130 + >>> configuration = model.config
131 + ```"""
132 +
133 + model_type = "granite"
134 + keys_to_ignore_at_inference = ["past_key_values"]
135 +
136 + def __init__(
137 + self,
138 + vocab_size=32000,
139 + hidden_size=4096,
140 + intermediate_size=11008,
141 + num_hidden_layers=32,
142 + num_attention_heads=32,
143 + num_key_value_heads=None,
144 + hidden_act="silu",
145 + max_position_embeddings=2048,
146 + initializer_range=0.02,
147 + rms_norm_eps=1e-6,
148 + use_cache=True,
149 + pad_token_id=None,
150 + bos_token_id=1,
151 + eos_token_id=2,
152 + tie_word_embeddings=False,
153 + rope_theta=10000.0,
154 + rope_scaling=None,
155 + attention_bias=False,
156 + attention_dropout=0.0,
157 + mlp_bias=False,
158 + embedding_multiplier=1.0,
159 + logits_scaling=1.0,
160 + residual_multiplier=1.0,
161 + attention_multiplier=1.0,
162 + **kwargs,
163 + ):
164 + self.vocab_size = vocab_size
165 + self.max_position_embeddings = max_position_embeddings
166 + self.hidden_size = hidden_size
167 + self.intermediate_size = intermediate_size
168 + self.num_hidden_layers = num_hidden_layers
169 + self.num_attention_heads = num_attention_heads
170 +
171 + # for backward compatibility
172 + if num_key_value_heads is None:
173 + num_key_value_heads = num_attention_heads
174 +
175 + self.num_key_value_heads = num_key_value_heads
176 + self.hidden_act = hidden_act
177 + self.initializer_range = initializer_range
178 + self.rms_norm_eps = rms_norm_eps
179 + self.use_cache = use_cache
180 + self.rope_theta = rope_theta
181 + self.rope_scaling = rope_scaling
182 + self.attention_bias = attention_bias
183 + self.attention_dropout = attention_dropout
184 + self.mlp_bias = mlp_bias
185 +
186 + self.embedding_multiplier = embedding_multiplier
187 + self.logits_scaling = logits_scaling
188 + self.residual_multiplier = residual_multiplier
189 + self.attention_multiplier = attention_multiplier
190 +
191 + super().__init__(
192 + pad_token_id=pad_token_id,
193 + bos_token_id=bos_token_id,
194 + eos_token_id=eos_token_id,
195 + tie_word_embeddings=tie_word_embeddings,
196 + **kwargs,
197 + )
198 +
199 + rope_config_validation(self)