A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] Add Granite model (#7436)
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Shawn Tan committed
Sep 1, 2024 at 21:37 UTC
f8d60145b4d954b7a110073f77dc91842155a3d8
4 files changed
+792
tests/models/test_granite.py
new
+49
@@ -0,0 +1,49 @@
1
+"""Compare the outputs of HF and vLLM for Granite models using greedy sampling.
2
+
3
+Run `pytest tests/models/test_granite.py`.
4
+"""
5
+import importlib.metadata
6
+
7
+import pytest
8
+
9
+from .utils import check_logprobs_close
10
+
11
+TRANSFORMERS_VERSION = tuple(
12
+ map(int,
13
+ importlib.metadata.version("transformers").split(".")))
14
+
15
+MODELS = [
16
+ "ibm/PowerLM-3b",
17
+]
18
+
19
+
20
+# GraniteForCausalLM will be in transformers >= 4.45
21
+@pytest.mark.skipif(TRANSFORMERS_VERSION < (4, 45),
22
+ reason="granite model test requires transformers >= 4.45")
23
+@pytest.mark.parametrize("model", MODELS)
24
+@pytest.mark.parametrize("dtype", ["bfloat16"])
25
+@pytest.mark.parametrize("max_tokens", [64])
26
+@pytest.mark.parametrize("num_logprobs", [5])
27
+def test_models(
28
+ hf_runner,
29
+ vllm_runner,
30
+ example_prompts,
31
+ model: str,
32
+ dtype: str,
33
+ max_tokens: int,
34
+ num_logprobs: int,
35
+) -> None:
36
+ # TODO(sang): Sliding window should be tested separately.
37
+ with hf_runner(model, dtype=dtype) as hf_model:
38
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
39
+ example_prompts, max_tokens, num_logprobs)
40
+
41
+ with vllm_runner(model, dtype=dtype) as vllm_model:
42
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
43
+ example_prompts, max_tokens, num_logprobs)
44
+ check_logprobs_close(
45
+ outputs_0_lst=hf_outputs,
46
+ outputs_1_lst=vllm_outputs,
47
+ name_0="hf",
48
+ name_1="vllm",
49
+ )
vllm/model_executor/models/__init__.py
+1
@@ -65,6 +65,7 @@ _GENERATION_MODELS = {
65
"EAGLEModel": ("eagle", "EAGLE"),
66
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
67
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
68
+ "GraniteForCausalLM": ("granite", "GraniteForCausalLM")
69
}
70
71
_EMBEDDING_MODELS = {
vllm/model_executor/models/granite.py
new
+543
@@ -0,0 +1,543 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only IBM Granite model compatible with HuggingFace weights."""
24
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
25
+
26
+import torch
27
+from torch import nn
28
+
29
+from vllm.attention import Attention, AttentionMetadata
30
+from vllm.config import CacheConfig, LoRAConfig
31
+from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32
+ get_tensor_model_parallel_world_size)
33
+from vllm.model_executor.layers.activation import SiluAndMul
34
+from vllm.model_executor.layers.layernorm import RMSNorm
35
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
36
+ QKVParallelLinear,
37
+ RowParallelLinear)
38
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
+from vllm.model_executor.layers.quantization.base_config import (
40
+ QuantizationConfig)
41
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
42
+ get_compressed_tensors_cache_scale)
43
+from vllm.model_executor.layers.rotary_embedding import get_rope
44
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.vocab_parallel_embedding import (
46
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
47
+from vllm.model_executor.model_loader.weight_utils import (
48
+ default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
49
+from vllm.model_executor.sampling_metadata import SamplingMetadata
50
+from vllm.sequence import IntermediateTensors
51
+from vllm.transformers_utils.configs.granite import GraniteConfig
52
+from vllm.utils import is_hip
53
+
54
+from .interfaces import SupportsLoRA
55
+from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
56
+
57
+
58
+class GraniteMLP(nn.Module):
59
+
60
+ def __init__(
61
+ self,
62
+ hidden_size: int,
63
+ intermediate_size: int,
64
+ hidden_act: str,
65
+ quant_config: Optional[QuantizationConfig] = None,
66
+ bias: bool = False,
67
+ prefix: str = "",
68
+ ) -> None:
69
+ super().__init__()
70
+ self.gate_up_proj = MergedColumnParallelLinear(
71
+ input_size=hidden_size,
72
+ output_sizes=[intermediate_size] * 2,
73
+ bias=bias,
74
+ quant_config=quant_config,
75
+ prefix=f"{prefix}.gate_up_proj")
76
+ self.down_proj = RowParallelLinear(input_size=intermediate_size,
77
+ output_size=hidden_size,
78
+ bias=bias,
79
+ quant_config=quant_config,
80
+ prefix=f"{prefix}.down_proj")
81
+ if hidden_act != "silu":
82
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
83
+ "Only silu is supported for now.")
84
+ self.act_fn = SiluAndMul()
85
+
86
+ def forward(self, x):
87
+ gate_up, _ = self.gate_up_proj(x)
88
+ x = self.act_fn(gate_up)
89
+ x, _ = self.down_proj(x)
90
+ return x
91
+
92
+
93
+class GraniteAttention(nn.Module):
94
+
95
+ def __init__(
96
+ self,
97
+ config: GraniteConfig,
98
+ hidden_size: int,
99
+ num_heads: int,
100
+ num_kv_heads: int,
101
+ rope_theta: float = 10000,
102
+ rope_scaling: Optional[Dict[str, Any]] = None,
103
+ max_position_embeddings: int = 8192,
104
+ quant_config: Optional[QuantizationConfig] = None,
105
+ bias: bool = False,
106
+ cache_config: Optional[CacheConfig] = None,
107
+ prefix: str = "",
108
+ ) -> None:
109
+ super().__init__()
110
+ self.hidden_size = hidden_size
111
+ tp_size = get_tensor_model_parallel_world_size()
112
+ self.total_num_heads = num_heads
113
+ assert self.total_num_heads % tp_size == 0
114
+ self.num_heads = self.total_num_heads // tp_size
115
+ self.total_num_kv_heads = num_kv_heads
116
+ if self.total_num_kv_heads >= tp_size:
117
+ # Number of KV heads is greater than TP size, so we partition
118
+ # the KV heads across multiple tensor parallel GPUs.
119
+ assert self.total_num_kv_heads % tp_size == 0
120
+ else:
121
+ # Number of KV heads is less than TP size, so we replicate
122
+ # the KV heads across multiple tensor parallel GPUs.
123
+ assert tp_size % self.total_num_kv_heads == 0
124
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
125
+ # MistralConfig has an optional head_dim introduced by Mistral-Nemo
126
+ self.head_dim = getattr(config, "head_dim",
127
+ self.hidden_size // self.total_num_heads)
128
+ self.q_size = self.num_heads * self.head_dim
129
+ self.kv_size = self.num_kv_heads * self.head_dim
130
+ self.scaling = config.attention_multiplier
131
+ self.rope_theta = rope_theta
132
+ self.max_position_embeddings = max_position_embeddings
133
+
134
+ self.qkv_proj = QKVParallelLinear(
135
+ hidden_size=hidden_size,
136
+ head_size=self.head_dim,
137
+ total_num_heads=self.total_num_heads,
138
+ total_num_kv_heads=self.total_num_kv_heads,
139
+ bias=bias,
140
+ quant_config=quant_config,
141
+ prefix=f"{prefix}.qkv_proj",
142
+ )
143
+ self.o_proj = RowParallelLinear(
144
+ input_size=self.total_num_heads * self.head_dim,
145
+ output_size=hidden_size,
146
+ bias=bias,
147
+ quant_config=quant_config,
148
+ prefix=f"{prefix}.o_proj",
149
+ )
150
+
151
+ self.rotary_emb = get_rope(
152
+ self.head_dim,
153
+ rotary_dim=self.head_dim,
154
+ max_position=max_position_embeddings,
155
+ base=rope_theta,
156
+ rope_scaling=rope_scaling,
157
+ )
158
+ self.attn = Attention(self.num_heads,
159
+ self.head_dim,
160
+ self.scaling,
161
+ num_kv_heads=self.num_kv_heads,
162
+ cache_config=cache_config,
163
+ quant_config=quant_config)
164
+
165
+ def forward(
166
+ self,
167
+ positions: torch.Tensor,
168
+ hidden_states: torch.Tensor,
169
+ kv_cache: torch.Tensor,
170
+ attn_metadata: AttentionMetadata,
171
+ ) -> torch.Tensor:
172
+ qkv, _ = self.qkv_proj(hidden_states)
173
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
174
+ q, k = self.rotary_emb(positions, q, k)
175
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
176
+ output, _ = self.o_proj(attn_output)
177
+ return output
178
+
179
+
180
+class GraniteDecoderLayer(nn.Module):
181
+
182
+ def __init__(
183
+ self,
184
+ config: GraniteConfig,
185
+ cache_config: Optional[CacheConfig] = None,
186
+ quant_config: Optional[QuantizationConfig] = None,
187
+ prefix: str = "",
188
+ ) -> None:
189
+ super().__init__()
190
+ self.hidden_size = config.hidden_size
191
+ self.residual_multiplier = config.residual_multiplier
192
+ rope_theta = getattr(config, "rope_theta", 10000)
193
+ rope_scaling = getattr(config, "rope_scaling", None)
194
+ if rope_scaling is not None and getattr(
195
+ config, "original_max_position_embeddings", None):
196
+ rope_scaling["original_max_position_embeddings"] = (
197
+ config.original_max_position_embeddings)
198
+ max_position_embeddings = getattr(config, "max_position_embeddings",
199
+ 8192)
200
+ # Support abacusai/Smaug-72B-v0.1 with attention_bias
201
+ # Support internlm/internlm-7b with bias
202
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
203
+ config, "bias", False)
204
+ self.self_attn = GraniteAttention(
205
+ config=config,
206
+ hidden_size=self.hidden_size,
207
+ num_heads=config.num_attention_heads,
208
+ num_kv_heads=getattr(config, "num_key_value_heads",
209
+ config.num_attention_heads),
210
+ rope_theta=rope_theta,
211
+ rope_scaling=rope_scaling,
212
+ max_position_embeddings=max_position_embeddings,
213
+ quant_config=quant_config,
214
+ bias=attention_bias,
215
+ cache_config=cache_config,
216
+ prefix=f"{prefix}.self_attn",
217
+ )
218
+
219
+ self.mlp = GraniteMLP(
220
+ hidden_size=self.hidden_size,
221
+ intermediate_size=config.intermediate_size,
222
+ hidden_act=config.hidden_act,
223
+ quant_config=quant_config,
224
+ bias=getattr(config, "mlp_bias", False),
225
+ prefix=f"{prefix}.mlp",
226
+ )
227
+ self.input_layernorm = RMSNorm(config.hidden_size,
228
+ eps=config.rms_norm_eps)
229
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
230
+ eps=config.rms_norm_eps)
231
+
232
+ def forward(
233
+ self,
234
+ positions: torch.Tensor,
235
+ hidden_states: torch.Tensor,
236
+ kv_cache: torch.Tensor,
237
+ attn_metadata: AttentionMetadata,
238
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
239
+ # Self Attention
240
+ residual = hidden_states
241
+ hidden_states = self.input_layernorm(hidden_states)
242
+ hidden_states = self.self_attn(
243
+ positions=positions,
244
+ hidden_states=hidden_states,
245
+ kv_cache=kv_cache,
246
+ attn_metadata=attn_metadata,
247
+ )
248
+ hidden_states = residual + hidden_states * self.residual_multiplier
249
+ # Fully Connected
250
+ residual = hidden_states
251
+ hidden_states = self.post_attention_layernorm(hidden_states)
252
+ hidden_states = self.mlp(hidden_states)
253
+ hidden_states = residual + hidden_states * self.residual_multiplier
254
+ return hidden_states
255
+
256
+
257
+class GraniteModel(nn.Module):
258
+
259
+ def __init__(
260
+ self,
261
+ config: GraniteConfig,
262
+ cache_config: Optional[CacheConfig] = None,
263
+ quant_config: Optional[QuantizationConfig] = None,
264
+ lora_config: Optional[LoRAConfig] = None,
265
+ prefix: str = "",
266
+ ) -> None:
267
+ super().__init__()
268
+ self.config = config
269
+ self.padding_idx = config.pad_token_id
270
+ lora_vocab = (lora_config.lora_extra_vocab_size *
271
+ (lora_config.max_loras or 1)) if lora_config else 0
272
+ self.vocab_size = config.vocab_size + lora_vocab
273
+ self.org_vocab_size = config.vocab_size
274
+ if get_pp_group().is_first_rank or (config.tie_word_embeddings
275
+ and get_pp_group().is_last_rank):
276
+ self.embed_tokens = VocabParallelEmbedding(
277
+ self.vocab_size,
278
+ config.hidden_size,
279
+ org_num_embeddings=config.vocab_size,
280
+ quant_config=quant_config,
281
+ )
282
+ else:
283
+ self.embed_tokens = PPMissingLayer()
284
+ self.start_layer, self.end_layer, self.layers = make_layers(
285
+ config.num_hidden_layers,
286
+ lambda prefix: GraniteDecoderLayer(config=config,
287
+ cache_config=cache_config,
288
+ quant_config=quant_config,
289
+ prefix=prefix),
290
+ prefix=f"{prefix}.layers")
291
+ if get_pp_group().is_last_rank:
292
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
293
+ else:
294
+ self.norm = PPMissingLayer()
295
+
296
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
297
+ return self.embed_tokens(input_ids)
298
+
299
+ def forward(
300
+ self,
301
+ input_ids: Optional[torch.Tensor],
302
+ positions: torch.Tensor,
303
+ kv_caches: List[torch.Tensor],
304
+ attn_metadata: AttentionMetadata,
305
+ intermediate_tensors: Optional[IntermediateTensors],
306
+ inputs_embeds: Optional[torch.Tensor] = None,
307
+ ) -> Union[torch.Tensor, IntermediateTensors]:
308
+ if get_pp_group().is_first_rank:
309
+ if inputs_embeds is not None:
310
+ hidden_states = inputs_embeds
311
+ else:
312
+ hidden_states = self.get_input_embeddings(input_ids)
313
+ residual = None
314
+ else:
315
+ assert intermediate_tensors is not None
316
+ hidden_states = intermediate_tensors["hidden_states"]
317
+ residual = intermediate_tensors["residual"]
318
+
319
+ hidden_states *= self.config.embedding_multiplier
320
+
321
+ for i in range(self.start_layer, self.end_layer):
322
+ layer = self.layers[i]
323
+ hidden_states = layer(
324
+ positions,
325
+ hidden_states,
326
+ kv_caches[i - self.start_layer],
327
+ attn_metadata,
328
+ )
329
+
330
+ if not get_pp_group().is_last_rank:
331
+ return IntermediateTensors({
332
+ "hidden_states": hidden_states,
333
+ "residual": residual
334
+ })
335
+
336
+ hidden_states = self.norm(hidden_states)
337
+ return hidden_states
338
+
339
+
340
+class GraniteForCausalLM(nn.Module, SupportsLoRA):
341
+ packed_modules_mapping = {
342
+ "qkv_proj": [
343
+ "q_proj",
344
+ "k_proj",
345
+ "v_proj",
346
+ ],
347
+ "gate_up_proj": [
348
+ "gate_proj",
349
+ "up_proj",
350
+ ],
351
+ }
352
+
353
+ # LoRA specific attributes
354
+ supported_lora_modules = [
355
+ "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
356
+ "lm_head"
357
+ ]
358
+ embedding_modules = {
359
+ "embed_tokens": "input_embeddings",
360
+ "lm_head": "output_embeddings",
361
+ }
362
+ embedding_padding_modules = ["lm_head"]
363
+ bitsandbytes_stacked_params_mapping = {
364
+ # shard_name, weight_name, index
365
+ "q_proj": ("qkv_proj", 0),
366
+ "k_proj": ("qkv_proj", 1),
367
+ "v_proj": ("qkv_proj", 2),
368
+ "gate_proj": ("gate_up_proj", 0),
369
+ "up_proj": ("gate_up_proj", 1),
370
+ }
371
+
372
+ def __init__(
373
+ self,
374
+ config: GraniteConfig,
375
+ cache_config: Optional[CacheConfig] = None,
376
+ quant_config: Optional[QuantizationConfig] = None,
377
+ lora_config: Optional[LoRAConfig] = None,
378
+ ) -> None:
379
+ super().__init__()
380
+
381
+ self.config = config
382
+ self.lora_config = lora_config
383
+
384
+ self.model = GraniteModel(config,
385
+ cache_config,
386
+ quant_config,
387
+ lora_config=lora_config,
388
+ prefix="model")
389
+ if get_pp_group().is_last_rank:
390
+ self.unpadded_vocab_size = config.vocab_size
391
+ if lora_config:
392
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
393
+ self.lm_head = ParallelLMHead(
394
+ self.unpadded_vocab_size,
395
+ config.hidden_size,
396
+ org_num_embeddings=config.vocab_size,
397
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
398
+ # We need bigger padding if using lora for kernel
399
+ # compatibility
400
+ if not lora_config else lora_config.lora_vocab_padding_size,
401
+ quant_config=quant_config,
402
+ )
403
+ if config.tie_word_embeddings:
404
+ self.lm_head.weight = self.model.embed_tokens.weight
405
+
406
+ logit_scale = getattr(config, "logit_scale", 1.0)
407
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
408
+ config.vocab_size,
409
+ logit_scale)
410
+ self.sampler = Sampler()
411
+ else:
412
+ self.lm_head = PPMissingLayer()
413
+
414
+ def forward(
415
+ self,
416
+ input_ids: torch.Tensor,
417
+ positions: torch.Tensor,
418
+ kv_caches: List[torch.Tensor],
419
+ attn_metadata: AttentionMetadata,
420
+ intermediate_tensors: Optional[IntermediateTensors] = None,
421
+ ) -> Union[torch.Tensor, IntermediateTensors]:
422
+ model_output = self.model(input_ids, positions, kv_caches,
423
+ attn_metadata, intermediate_tensors)
424
+ return model_output
425
+
426
+ def compute_logits(
427
+ self, hidden_states: torch.Tensor,
428
+ sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
429
+ logits = self.logits_processor(self.lm_head, hidden_states,
430
+ sampling_metadata)
431
+ logits /= self.config.logits_scaling
432
+ return logits
433
+
434
+ def sample(
435
+ self,
436
+ logits: torch.Tensor,
437
+ sampling_metadata: SamplingMetadata,
438
+ ) -> Optional[SamplerOutput]:
439
+ next_tokens = self.sampler(logits, sampling_metadata)
440
+ return next_tokens
441
+
442
+ def make_empty_intermediate_tensors(
443
+ self, batch_size: int, dtype: torch.dtype,
444
+ device: torch.device) -> IntermediateTensors:
445
+ return IntermediateTensors({
446
+ "hidden_states":
447
+ torch.zeros((batch_size, self.config.hidden_size),
448
+ dtype=dtype,
449
+ device=device),
450
+ "residual":
451
+ torch.zeros((batch_size, self.config.hidden_size),
452
+ dtype=dtype,
453
+ device=device),
454
+ })
455
+
456
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
457
+ stacked_params_mapping = [
458
+ # (param_name, shard_name, shard_id)
459
+ (".qkv_proj", ".q_proj", "q"),
460
+ (".qkv_proj", ".k_proj", "k"),
461
+ (".qkv_proj", ".v_proj", "v"),
462
+ (".gate_up_proj", ".gate_proj", 0),
463
+ (".gate_up_proj", ".up_proj", 1),
464
+ ]
465
+ params_dict = dict(self.named_parameters())
466
+ for name, loaded_weight in weights:
467
+ if "rotary_emb.inv_freq" in name:
468
+ continue
469
+ if ("rotary_emb.cos_cached" in name
470
+ or "rotary_emb.sin_cached" in name):
471
+ # Models trained using ColossalAI may include these tensors in
472
+ # the checkpoint. Skip them.
473
+ continue
474
+ # With tie_word_embeddings, we can skip lm_head.weight
475
+ # The weight might appear unnecessarily in the files if the model is
476
+ # processed with quantization, LoRA, fine-tuning, etc.
477
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
478
+ continue
479
+ if scale_name := get_compressed_tensors_cache_scale(name):
480
+ # Loading kv cache scales for compressed-tensors quantization
481
+ param = params_dict[scale_name]
482
+ weight_loader = getattr(param, "weight_loader",
483
+ default_weight_loader)
484
+ loaded_weight = loaded_weight[0]
485
+ weight_loader(param, loaded_weight)
486
+ continue
487
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
488
+ if weight_name not in name:
489
+ continue
490
+ name = name.replace(weight_name, param_name)
491
+ # Skip loading extra bias for GPTQ models.
492
+ if name.endswith(".bias") and name not in params_dict:
493
+ continue
494
+
495
+ if is_pp_missing_parameter(name, self):
496
+ continue
497
+
498
+ param = params_dict[name]
499
+ weight_loader = param.weight_loader
500
+ weight_loader(param, loaded_weight, shard_id)
501
+
502
+ break
503
+ else:
504
+ # Skip loading extra bias for GPTQ models.
505
+ if name.endswith(".bias") and name not in params_dict:
506
+ continue
507
+ # Remapping the name of FP8 kv-scale.
508
+ name = maybe_remap_kv_scale_name(name, params_dict)
509
+ if name is None:
510
+ continue
511
+
512
+ if is_pp_missing_parameter(name, self):
513
+ continue
514
+
515
+ param = params_dict[name]
516
+ weight_loader = getattr(param, "weight_loader",
517
+ default_weight_loader)
518
+ weight_loader(param, loaded_weight)
519
+
520
+ # If this function is called, it should always initialize KV cache scale
521
+ # factors (or else raise an exception). Thus, handled exceptions should
522
+ # make sure to leave KV cache scale factors in a known good (dummy) state
523
+ def load_kv_cache_scales(self, quantization_param_path: str) -> None:
524
+ tp_size = get_tensor_model_parallel_world_size()
525
+ tp_rank = get_tensor_model_parallel_rank()
526
+ for layer_idx, scaling_factor in kv_cache_scales_loader(
527
+ quantization_param_path, tp_rank, tp_size,
528
+ self.config.num_hidden_layers,
529
+ self.config.__class__.model_type):
530
+ if not isinstance(self.model.layers[layer_idx], nn.Identity):
531
+ layer_self_attn = self.model.layers[layer_idx].self_attn
532
+
533
+ if is_hip():
534
+ # The scaling factor convention we are assuming is
535
+ # quantized_value * scaling_factor ~= true_value
536
+ # which is consistent with the practice of setting
537
+ # scaling_factor = tensor_amax / FPtype_max
538
+ scaling_factor *= 2
539
+ if hasattr(layer_self_attn, "kv_scale"):
540
+ layer_self_attn.attn._kv_scale = scaling_factor
541
+ else:
542
+ raise RuntimeError("Self attention has no KV cache scaling "
543
+ "factor attribute!")
vllm/transformers_utils/configs/granite.py
new
+199
@@ -0,0 +1,199 @@
1
+# coding=utf-8
2
+# Copyright 2024 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+"""Granite model configuration"""
21
+
22
+from transformers.configuration_utils import PretrainedConfig
23
+from transformers.modeling_rope_utils import rope_config_validation
24
+from transformers.utils import logging
25
+
26
+logger = logging.get_logger(__name__)
27
+
28
+
29
+class GraniteConfig(PretrainedConfig):
30
+ r"""
31
+ This is the configuration class to store the configuration of
32
+ a [`GraniteModel`]. It is used to instantiate an Granite
33
+ model according to the specified arguments, defining the model architecture.
34
+ Instantiating a configuration with the defaults will yield a similar
35
+ configuration to that of the Granite-3B.
36
+
37
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to
38
+ control the model outputs. Read the documentation from [`PretrainedConfig`]
39
+ for more information.
40
+
41
+
42
+ Args:
43
+ vocab_size (`int`, *optional*, defaults to 32000):
44
+ Vocabulary size of the Granite model. Defines the number of
45
+ different tokens that can be represented by the `inputs_ids`
46
+ passed when calling [`GraniteModel`]
47
+ hidden_size (`int`, *optional*, defaults to 4096):
48
+ Dimension of the hidden representations.
49
+ intermediate_size (`int`, *optional*, defaults to 11008):
50
+ Dimension of the MLP representations.
51
+ num_hidden_layers (`int`, *optional*, defaults to 32):
52
+ Number of hidden layers in the Transformer decoder.
53
+ num_attention_heads (`int`, *optional*, defaults to 32):
54
+ Number of attention heads for each attention layer in the
55
+ Transformer decoder.
56
+ num_key_value_heads (`int`, *optional*):
57
+ This is the number of key_value heads that should be used to
58
+ implement Grouped Query Attention. If
59
+ `num_key_value_heads=num_attention_heads`, the model will use Multi
60
+ Head Attention (MHA), if `num_key_value_heads=1` the model will use
61
+ Multi Query Attention (MQA) otherwise GQA is used. When converting
62
+ a multi-head checkpoint to a GQA checkpoint, each group key and
63
+ value head should be constructed by meanpooling all the original
64
+ heads within that group. For more details checkout
65
+ [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
66
+ specified, will default to `num_attention_heads`.
67
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
68
+ The non-linear activation function (function or string) in the
69
+ decoder.
70
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
71
+ The maximum sequence length that this model might ever be used with.
72
+ initializer_range (`float`, *optional*, defaults to 0.02):
73
+ The standard deviation of the truncated_normal_initializer for
74
+ initializing all weight matrices.
75
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
76
+ The epsilon used by the rms normalization layers.
77
+ use_cache (`bool`, *optional*, defaults to `True`):
78
+ Whether or not the model should return the last key/values
79
+ attentions (not used by all models). Only relevant if
80
+ `config.is_decoder=True`.
81
+ pad_token_id (`int`, *optional*):
82
+ Padding token id.
83
+ bos_token_id (`int`, *optional*, defaults to 1):
84
+ Beginning of stream token id.
85
+ eos_token_id (`int`, *optional*, defaults to 2):
86
+ End of stream token id.
87
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
88
+ Whether to tie weight embeddings
89
+ rope_theta (`float`, *optional*, defaults to 10000.0):
90
+ The base period of the RoPE embeddings.
91
+ rope_scaling (`Dict`, *optional*):
92
+ Dictionary containing the scaling configuration for the RoPE
93
+ embeddings. Currently supports two scaling strategies: linear and
94
+ dynamic. Their scaling factor must be a float greater than 1. The
95
+ expected format is
96
+ `{"type": strategy name, "factor": scaling factor}`.
97
+ When using this flag, don't update `max_position_embeddings` to
98
+ the expected new maximum. See the following thread for more
99
+ information on how these scaling strategies behave:
100
+ https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/.
101
+ This is an experimental feature, subject to breaking API changes
102
+ in future versions.
103
+ attention_bias (`bool`, *optional*, defaults to `False`):
104
+ Whether to use a bias in the query, key, value and output
105
+ projection layers during self-attention.
106
+ attention_dropout (`float`, *optional*, defaults to 0.0):
107
+ The dropout ratio for the attention probabilities.
108
+ mlp_bias (`bool`, *optional*, defaults to `False`):
109
+ Whether to use a bias in up_proj, down_proj and gate_proj layers
110
+ in the MLP layers.
111
+ embedding_multiplier (`float`, *optional*, defaults to 1.0):
112
+ embedding multiplier
113
+ logits_scaling (`float`, *optional*, defaults to 1.0):
114
+ divisor for output logits
115
+ residual_multiplier (`float`, *optional*, defaults to 1.0):
116
+ residual multiplier
117
+ attention_multiplier (`float`, *optional*, defaults to 1.0):
118
+ attention multiplier
119
+
120
+ ```python
121
+ >>> from transformers import GraniteModel, GraniteConfig
122
+
123
+ >>> # Initializing a Granite granite-3b style configuration
124
+ >>> configuration = GraniteConfig()
125
+
126
+ >>> # Initializing a model from the granite-7b style configuration
127
+ >>> model = GraniteModel(configuration)
128
+
129
+ >>> # Accessing the model configuration
130
+ >>> configuration = model.config
131
+ ```"""
132
+
133
+ model_type = "granite"
134
+ keys_to_ignore_at_inference = ["past_key_values"]
135
+
136
+ def __init__(
137
+ self,
138
+ vocab_size=32000,
139
+ hidden_size=4096,
140
+ intermediate_size=11008,
141
+ num_hidden_layers=32,
142
+ num_attention_heads=32,
143
+ num_key_value_heads=None,
144
+ hidden_act="silu",
145
+ max_position_embeddings=2048,
146
+ initializer_range=0.02,
147
+ rms_norm_eps=1e-6,
148
+ use_cache=True,
149
+ pad_token_id=None,
150
+ bos_token_id=1,
151
+ eos_token_id=2,
152
+ tie_word_embeddings=False,
153
+ rope_theta=10000.0,
154
+ rope_scaling=None,
155
+ attention_bias=False,
156
+ attention_dropout=0.0,
157
+ mlp_bias=False,
158
+ embedding_multiplier=1.0,
159
+ logits_scaling=1.0,
160
+ residual_multiplier=1.0,
161
+ attention_multiplier=1.0,
162
+ **kwargs,
163
+ ):
164
+ self.vocab_size = vocab_size
165
+ self.max_position_embeddings = max_position_embeddings
166
+ self.hidden_size = hidden_size
167
+ self.intermediate_size = intermediate_size
168
+ self.num_hidden_layers = num_hidden_layers
169
+ self.num_attention_heads = num_attention_heads
170
+
171
+ # for backward compatibility
172
+ if num_key_value_heads is None:
173
+ num_key_value_heads = num_attention_heads
174
+
175
+ self.num_key_value_heads = num_key_value_heads
176
+ self.hidden_act = hidden_act
177
+ self.initializer_range = initializer_range
178
+ self.rms_norm_eps = rms_norm_eps
179
+ self.use_cache = use_cache
180
+ self.rope_theta = rope_theta
181
+ self.rope_scaling = rope_scaling
182
+ self.attention_bias = attention_bias
183
+ self.attention_dropout = attention_dropout
184
+ self.mlp_bias = mlp_bias
185
+
186
+ self.embedding_multiplier = embedding_multiplier
187
+ self.logits_scaling = logits_scaling
188
+ self.residual_multiplier = residual_multiplier
189
+ self.attention_multiplier = attention_multiplier
190
+
191
+ super().__init__(
192
+ pad_token_id=pad_token_id,
193
+ bos_token_id=bos_token_id,
194
+ eos_token_id=eos_token_id,
195
+ tie_word_embeddings=tie_word_embeddings,
196
+ **kwargs,
197
+ )
198
+
199
+ rope_config_validation(self)