A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model] Support Solar Model (#8386)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Geun, Lim committed
Sep 19, 2024 at 02:04 UTC
e18749ff09c277f7cdab278895ebdd9b1041b6e8
6 files changed
+834
-1
docs/source/models/supported_models.rst
+4
@@ -179,6 +179,10 @@ Decoder-only Language Models
179
- Starcoder2
180
- :code:`bigcode/starcoder2-3b`, :code:`bigcode/starcoder2-7b`, :code:`bigcode/starcoder2-15b`, etc.
181
-
182
+ * - :code:`SolarForCausalLM`
183
+ - EXAONE-3
184
+ - :code:`upstage/solar-pro-preview-instruct`, etc.
185
+ -
186
* - :code:`XverseForCausalLM`
187
- Xverse
188
- :code:`xverse/XVERSE-7B-Chat`, :code:`xverse/XVERSE-13B-Chat`, :code:`xverse/XVERSE-65B-Chat`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -60,6 +60,7 @@ _GENERATION_MODELS = {
60
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
61
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
62
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
63
+ "SolarForCausalLM": ("solar", "SolarForCausalLM"),
64
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
65
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
66
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
vllm/model_executor/models/solar.py
new
+580
@@ -0,0 +1,580 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Solar model compatible with HuggingFace weights."""
24
+
25
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
26
+
27
+import torch
28
+from torch import nn
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
+ get_tensor_model_parallel_world_size)
34
+from vllm.model_executor.layers.activation import SiluAndMul
35
+from vllm.model_executor.layers.layernorm import RMSNorm
36
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
+ QKVParallelLinear,
38
+ RowParallelLinear)
39
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
+from vllm.model_executor.layers.quantization.base_config import (
41
+ QuantizationConfig)
42
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
+ get_compressed_tensors_cache_scale)
44
+from vllm.model_executor.layers.rotary_embedding import get_rope
45
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46
+from vllm.model_executor.layers.vocab_parallel_embedding import (
47
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
+from vllm.model_executor.model_loader.weight_utils import (
49
+ default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
50
+from vllm.model_executor.models.interfaces import SupportsLoRA
51
+from vllm.model_executor.models.utils import (PPMissingLayer,
52
+ is_pp_missing_parameter,
53
+ make_layers)
54
+from vllm.model_executor.sampling_metadata import SamplingMetadata
55
+from vllm.sequence import IntermediateTensors
56
+from vllm.utils import is_hip
57
+
58
+
59
+class SolarMLP(nn.Module):
60
+
61
+ def __init__(
62
+ self,
63
+ hidden_size: int,
64
+ intermediate_size: int,
65
+ hidden_act: str,
66
+ quant_config: Optional[QuantizationConfig] = None,
67
+ bias: bool = False,
68
+ prefix: str = "",
69
+ ) -> None:
70
+ super().__init__()
71
+ self.gate_up_proj = MergedColumnParallelLinear(
72
+ input_size=hidden_size,
73
+ output_sizes=[intermediate_size] * 2,
74
+ bias=bias,
75
+ quant_config=quant_config,
76
+ prefix=f"{prefix}.gate_up_proj",
77
+ )
78
+ self.down_proj = RowParallelLinear(
79
+ input_size=intermediate_size,
80
+ output_size=hidden_size,
81
+ bias=bias,
82
+ quant_config=quant_config,
83
+ prefix=f"{prefix}.down_proj",
84
+ )
85
+ if hidden_act != "silu":
86
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
87
+ "Only silu is supported for now.")
88
+ self.act_fn = SiluAndMul()
89
+
90
+ def forward(self, x):
91
+ gate_up, _ = self.gate_up_proj(x)
92
+ x = self.act_fn(gate_up)
93
+ x, _ = self.down_proj(x)
94
+ return x
95
+
96
+
97
+class SolarAttention(nn.Module):
98
+
99
+ def __init__(
100
+ self,
101
+ config,
102
+ hidden_size: int,
103
+ num_heads: int,
104
+ num_kv_heads: int,
105
+ rope_theta: float = 10000,
106
+ rope_scaling: Optional[Dict[str, Any]] = None,
107
+ max_position_embeddings: int = 8192,
108
+ quant_config: Optional[QuantizationConfig] = None,
109
+ bias: bool = False,
110
+ cache_config: Optional[CacheConfig] = None,
111
+ prefix: str = "",
112
+ ) -> None:
113
+ super().__init__()
114
+ self.hidden_size = hidden_size
115
+ tp_size = get_tensor_model_parallel_world_size()
116
+ self.total_num_heads = num_heads
117
+ assert self.total_num_heads % tp_size == 0
118
+ self.num_heads = self.total_num_heads // tp_size
119
+ self.total_num_kv_heads = num_kv_heads
120
+ if self.total_num_kv_heads >= tp_size:
121
+ # Number of KV heads is greater than TP size, so we partition
122
+ # the KV heads across multiple tensor parallel GPUs.
123
+ assert self.total_num_kv_heads % tp_size == 0
124
+ else:
125
+ # Number of KV heads is less than TP size, so we replicate
126
+ # the KV heads across multiple tensor parallel GPUs.
127
+ assert tp_size % self.total_num_kv_heads == 0
128
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
129
+ # MistralConfig has an optional head_dim introduced by Mistral-Nemo
130
+ self.head_dim = getattr(config, "head_dim",
131
+ self.hidden_size // self.total_num_heads)
132
+ self.q_size = self.num_heads * self.head_dim
133
+ self.kv_size = self.num_kv_heads * self.head_dim
134
+ self.scaling = self.head_dim**-0.5
135
+ self.rope_theta = rope_theta
136
+ self.max_position_embeddings = max_position_embeddings
137
+
138
+ self.qkv_proj = QKVParallelLinear(
139
+ hidden_size=hidden_size,
140
+ head_size=self.head_dim,
141
+ total_num_heads=self.total_num_heads,
142
+ total_num_kv_heads=self.total_num_kv_heads,
143
+ bias=bias,
144
+ quant_config=quant_config,
145
+ prefix=f"{prefix}.qkv_proj",
146
+ )
147
+ self.o_proj = RowParallelLinear(
148
+ input_size=self.total_num_heads * self.head_dim,
149
+ output_size=hidden_size,
150
+ bias=bias,
151
+ quant_config=quant_config,
152
+ prefix=f"{prefix}.o_proj",
153
+ )
154
+
155
+ self.rotary_emb = get_rope(
156
+ self.head_dim,
157
+ rotary_dim=self.head_dim,
158
+ max_position=max_position_embeddings,
159
+ base=rope_theta,
160
+ rope_scaling=rope_scaling,
161
+ )
162
+ self.attn = Attention(
163
+ self.num_heads,
164
+ self.head_dim,
165
+ self.scaling,
166
+ num_kv_heads=self.num_kv_heads,
167
+ cache_config=cache_config,
168
+ quant_config=quant_config,
169
+ )
170
+
171
+ def forward(
172
+ self,
173
+ positions: torch.Tensor,
174
+ hidden_states: torch.Tensor,
175
+ kv_cache: torch.Tensor,
176
+ attn_metadata: AttentionMetadata,
177
+ ) -> torch.Tensor:
178
+ qkv, _ = self.qkv_proj(hidden_states)
179
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
180
+ q, k = self.rotary_emb(positions, q, k)
181
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
182
+ output, _ = self.o_proj(attn_output)
183
+ return output
184
+
185
+
186
+class SolarDecoderLayer(nn.Module):
187
+
188
+ def __init__(
189
+ self,
190
+ config,
191
+ cache_config: Optional[CacheConfig] = None,
192
+ quant_config: Optional[QuantizationConfig] = None,
193
+ prefix: str = "",
194
+ ) -> None:
195
+ super().__init__()
196
+ self.hidden_size = config.hidden_size
197
+ rope_theta = getattr(config, "rope_theta", 10000)
198
+ rope_scaling = getattr(config, "rope_scaling", None)
199
+
200
+ if rope_scaling is not None and getattr(
201
+ config, "original_max_position_embeddings", None):
202
+ rope_scaling["original_max_position_embeddings"] \
203
+ = config.original_max_position_embeddings
204
+ max_position_embeddings = getattr(config, "max_position_embeddings",
205
+ 8192)
206
+ # Support abacusai/Smaug-72B-v0.1 with attention_bias
207
+ # Support internlm/internlm-7b with bias
208
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
209
+ config, "bias", False)
210
+ self.self_attn = SolarAttention(
211
+ config=config,
212
+ hidden_size=self.hidden_size,
213
+ num_heads=config.num_attention_heads,
214
+ num_kv_heads=getattr(config, "num_key_value_heads",
215
+ config.num_attention_heads),
216
+ rope_theta=rope_theta,
217
+ rope_scaling=rope_scaling,
218
+ max_position_embeddings=max_position_embeddings,
219
+ quant_config=quant_config,
220
+ bias=attention_bias,
221
+ cache_config=cache_config,
222
+ prefix=f"{prefix}.self_attn",
223
+ )
224
+ self.mlp = SolarMLP(
225
+ hidden_size=self.hidden_size,
226
+ intermediate_size=config.intermediate_size,
227
+ hidden_act=config.hidden_act,
228
+ quant_config=quant_config,
229
+ bias=getattr(config, "mlp_bias", False),
230
+ prefix=f"{prefix}.mlp",
231
+ )
232
+ self.input_layernorm = RMSNorm(config.hidden_size,
233
+ eps=config.rms_norm_eps)
234
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
235
+ eps=config.rms_norm_eps)
236
+
237
+ def forward(
238
+ self,
239
+ positions: torch.Tensor,
240
+ hidden_states: torch.Tensor,
241
+ kv_cache: torch.Tensor,
242
+ attn_metadata: AttentionMetadata,
243
+ residual: Optional[torch.Tensor],
244
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
245
+ # Self Attention
246
+ if residual is None:
247
+ residual = hidden_states
248
+ hidden_states = self.input_layernorm(hidden_states)
249
+ else:
250
+ hidden_states, residual = self.input_layernorm(
251
+ hidden_states, residual)
252
+ hidden_states = self.self_attn(
253
+ positions=positions,
254
+ hidden_states=hidden_states,
255
+ kv_cache=kv_cache,
256
+ attn_metadata=attn_metadata,
257
+ )
258
+
259
+ # Fully Connected
260
+ hidden_states, residual = self.post_attention_layernorm(
261
+ hidden_states, residual)
262
+ hidden_states = self.mlp(hidden_states)
263
+ return hidden_states, residual
264
+
265
+
266
+class SolarModel(nn.Module):
267
+
268
+ def __init__(
269
+ self,
270
+ config,
271
+ cache_config: Optional[CacheConfig] = None,
272
+ quant_config: Optional[QuantizationConfig] = None,
273
+ lora_config: Optional[LoRAConfig] = None,
274
+ prefix: str = "",
275
+ ) -> None:
276
+ super().__init__()
277
+ self.config = config
278
+ self.padding_idx = config.pad_token_id
279
+ lora_vocab = ((lora_config.lora_extra_vocab_size *
280
+ (lora_config.max_loras or 1)) if lora_config else 0)
281
+ self.vocab_size = config.vocab_size + lora_vocab
282
+ self.org_vocab_size = config.vocab_size
283
+ if get_pp_group().is_first_rank or (config.tie_word_embeddings
284
+ and get_pp_group().is_last_rank):
285
+ self.embed_tokens = VocabParallelEmbedding(
286
+ self.vocab_size,
287
+ config.hidden_size,
288
+ org_num_embeddings=config.vocab_size,
289
+ )
290
+ else:
291
+ self.embed_tokens = PPMissingLayer()
292
+ self.start_layer, self.end_layer, self.layers = make_layers(
293
+ config.num_hidden_layers,
294
+ lambda prefix: SolarDecoderLayer(
295
+ config=config,
296
+ cache_config=cache_config,
297
+ quant_config=quant_config,
298
+ prefix=prefix,
299
+ ),
300
+ prefix=f"{prefix}.layers",
301
+ )
302
+ if get_pp_group().is_last_rank:
303
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
304
+ else:
305
+ self.norm = PPMissingLayer()
306
+
307
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
308
+ return self.embed_tokens(input_ids)
309
+
310
+ def forward(
311
+ self,
312
+ input_ids: Optional[torch.Tensor],
313
+ positions: torch.Tensor,
314
+ kv_caches: List[torch.Tensor],
315
+ attn_metadata: AttentionMetadata,
316
+ intermediate_tensors: Optional[IntermediateTensors],
317
+ inputs_embeds: Optional[torch.Tensor] = None,
318
+ ) -> Union[torch.Tensor, IntermediateTensors]:
319
+ if get_pp_group().is_first_rank:
320
+ if inputs_embeds is not None:
321
+ hidden_states = inputs_embeds
322
+ else:
323
+ hidden_states = self.get_input_embeddings(input_ids)
324
+ residual = None
325
+ else:
326
+ assert intermediate_tensors is not None
327
+ hidden_states = intermediate_tensors["hidden_states"]
328
+ residual = intermediate_tensors["residual"]
329
+
330
+ bskcn_h_1 = None
331
+ bskcn_h_2 = None
332
+ bskcn_r_1 = None
333
+ bskcn_r_2 = None
334
+ bskcn_tv = (self.config.bskcn_tv[0]
335
+ if self.training else self.config.bskcn_tv[1])
336
+
337
+ for i in range(self.start_layer, self.end_layer):
338
+ if i in self.config.bskcn_1:
339
+ bskcn_h_1 = hidden_states.clone()
340
+ bskcn_r_1 = residual.clone()
341
+ if i in self.config.bskcn_2:
342
+ bskcn_h_2 = hidden_states.clone()
343
+ bskcn_r_2 = residual.clone()
344
+ if i in self.config.bskcn_3:
345
+ hidden_states = bskcn_h_1 * bskcn_tv + hidden_states * (
346
+ 1 - bskcn_tv)
347
+ residual = bskcn_r_1 * bskcn_tv + residual * (1 - bskcn_tv)
348
+ if i in self.config.bskcn_4:
349
+ hidden_states = bskcn_h_2 * bskcn_tv + hidden_states * (
350
+ 1 - bskcn_tv)
351
+ residual = bskcn_r_2 * bskcn_tv + residual * (1 - bskcn_tv)
352
+ layer = self.layers[i]
353
+ hidden_states, residual = layer(
354
+ positions,
355
+ hidden_states,
356
+ kv_caches[i - self.start_layer],
357
+ attn_metadata,
358
+ residual,
359
+ )
360
+
361
+ if not get_pp_group().is_last_rank:
362
+ return IntermediateTensors({
363
+ "hidden_states": hidden_states,
364
+ "residual": residual
365
+ })
366
+
367
+ hidden_states, _ = self.norm(hidden_states, residual)
368
+ return hidden_states
369
+
370
+
371
+class SolarForCausalLM(nn.Module, SupportsLoRA):
372
+ packed_modules_mapping = {
373
+ "qkv_proj": [
374
+ "q_proj",
375
+ "k_proj",
376
+ "v_proj",
377
+ ],
378
+ "gate_up_proj": [
379
+ "gate_proj",
380
+ "up_proj",
381
+ ],
382
+ }
383
+
384
+ # LoRA specific attributes
385
+ supported_lora_modules = [
386
+ "qkv_proj",
387
+ "o_proj",
388
+ "gate_up_proj",
389
+ "down_proj",
390
+ "embed_tokens",
391
+ "lm_head",
392
+ ]
393
+ embedding_modules = {
394
+ "embed_tokens": "input_embeddings",
395
+ "lm_head": "output_embeddings",
396
+ }
397
+ embedding_padding_modules = ["lm_head"]
398
+ bitsandbytes_stacked_params_mapping = {
399
+ # shard_name, weight_name, index
400
+ "q_proj": ("qkv_proj", 0),
401
+ "k_proj": ("qkv_proj", 1),
402
+ "v_proj": ("qkv_proj", 2),
403
+ "gate_proj": ("gate_up_proj", 0),
404
+ "up_proj": ("gate_up_proj", 1),
405
+ }
406
+
407
+ def __init__(
408
+ self,
409
+ config,
410
+ cache_config: Optional[CacheConfig] = None,
411
+ quant_config: Optional[QuantizationConfig] = None,
412
+ lora_config: Optional[LoRAConfig] = None,
413
+ ) -> None:
414
+ super().__init__()
415
+
416
+ self.config = config
417
+ self.lora_config = lora_config
418
+
419
+ self.model = SolarModel(
420
+ config,
421
+ cache_config,
422
+ quant_config,
423
+ lora_config=lora_config,
424
+ prefix="model",
425
+ )
426
+ if get_pp_group().is_last_rank:
427
+ self.unpadded_vocab_size = config.vocab_size
428
+ if lora_config:
429
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
430
+ self.lm_head = ParallelLMHead(
431
+ self.unpadded_vocab_size,
432
+ config.hidden_size,
433
+ org_num_embeddings=config.vocab_size,
434
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
435
+ # We need bigger padding if using lora for kernel
436
+ # compatibility
437
+ if not lora_config else lora_config.lora_vocab_padding_size,
438
+ quant_config=quant_config,
439
+ )
440
+ if config.tie_word_embeddings:
441
+ self.lm_head.weight = self.model.embed_tokens.weight
442
+
443
+ logit_scale = getattr(config, "logit_scale", 1.0)
444
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
445
+ config.vocab_size,
446
+ logit_scale)
447
+ self.sampler = Sampler()
448
+ else:
449
+ self.lm_head = PPMissingLayer()
450
+
451
+ def forward(
452
+ self,
453
+ input_ids: torch.Tensor,
454
+ positions: torch.Tensor,
455
+ kv_caches: List[torch.Tensor],
456
+ attn_metadata: AttentionMetadata,
457
+ intermediate_tensors: Optional[IntermediateTensors] = None,
458
+ ) -> Union[torch.Tensor, IntermediateTensors]:
459
+ model_output = self.model(input_ids, positions, kv_caches,
460
+ attn_metadata, intermediate_tensors)
461
+ return model_output
462
+
463
+ def compute_logits(self, hidden_states: torch.Tensor,
464
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
465
+ logits = self.logits_processor(self.lm_head, hidden_states,
466
+ sampling_metadata)
467
+ return logits
468
+
469
+ def sample(
470
+ self,
471
+ logits: torch.Tensor,
472
+ sampling_metadata: SamplingMetadata,
473
+ ) -> Optional[SamplerOutput]:
474
+ next_tokens = self.sampler(logits, sampling_metadata)
475
+ return next_tokens
476
+
477
+ def make_empty_intermediate_tensors(
478
+ self, batch_size: int, dtype: torch.dtype,
479
+ device: torch.device) -> IntermediateTensors:
480
+ return IntermediateTensors({
481
+ "hidden_states":
482
+ torch.zeros(
483
+ (batch_size, self.config.hidden_size),
484
+ dtype=dtype,
485
+ device=device,
486
+ ),
487
+ "residual":
488
+ torch.zeros(
489
+ (batch_size, self.config.hidden_size),
490
+ dtype=dtype,
491
+ device=device,
492
+ ),
493
+ })
494
+
495
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
496
+ stacked_params_mapping = [
497
+ # (param_name, shard_name, shard_id)
498
+ (".qkv_proj", ".q_proj", "q"),
499
+ (".qkv_proj", ".k_proj", "k"),
500
+ (".qkv_proj", ".v_proj", "v"),
501
+ (".gate_up_proj", ".gate_proj", 0),
502
+ (".gate_up_proj", ".up_proj", 1),
503
+ ]
504
+ params_dict = dict(self.named_parameters())
505
+ for name, loaded_weight in weights:
506
+ if "rotary_emb.inv_freq" in name:
507
+ continue
508
+ if ("rotary_emb.cos_cached" in name
509
+ or "rotary_emb.sin_cached" in name):
510
+ # Models trained using ColossalAI may include these tensors in
511
+ # the checkpoint. Skip them.
512
+ continue
513
+ if scale_name := get_compressed_tensors_cache_scale(name):
514
+ # Loading kv cache scales for compressed-tensors quantization
515
+ param = params_dict[scale_name]
516
+ weight_loader = getattr(param, "weight_loader",
517
+ default_weight_loader)
518
+ loaded_weight = loaded_weight[0]
519
+ weight_loader(param, loaded_weight)
520
+ continue
521
+ for param_name, weight_name, shard_id in stacked_params_mapping:
522
+ if weight_name not in name:
523
+ continue
524
+ name = name.replace(weight_name, param_name)
525
+ # Skip loading extra bias for GPTQ models.
526
+ if name.endswith(".bias") and name not in params_dict:
527
+ continue
528
+
529
+ if is_pp_missing_parameter(name, self):
530
+ continue
531
+
532
+ param = params_dict[name]
533
+ weight_loader = param.weight_loader
534
+ weight_loader(param, loaded_weight, shard_id)
535
+
536
+ break
537
+ else:
538
+ # Skip loading extra bias for GPTQ models.
539
+ if name.endswith(".bias") and name not in params_dict:
540
+ continue
541
+ # Remapping the name of FP8 kv-scale.
542
+ name = maybe_remap_kv_scale_name(name, params_dict)
543
+ if name is None:
544
+ continue
545
+
546
+ if is_pp_missing_parameter(name, self):
547
+ continue
548
+
549
+ param = params_dict[name]
550
+ weight_loader = getattr(param, "weight_loader",
551
+ default_weight_loader)
552
+ weight_loader(param, loaded_weight)
553
+
554
+ # If this function is called, it should always initialize KV cache scale
555
+ # factors (or else raise an exception). Thus, handled exceptions should
556
+ # make sure to leave KV cache scale factors in a known good (dummy) state
557
+ def load_kv_cache_scales(self, quantization_param_path: str) -> None:
558
+ tp_size = get_tensor_model_parallel_world_size()
559
+ tp_rank = get_tensor_model_parallel_rank()
560
+ for layer_idx, scaling_factor in kv_cache_scales_loader(
561
+ quantization_param_path,
562
+ tp_rank,
563
+ tp_size,
564
+ self.config.num_hidden_layers,
565
+ self.config.__class__.model_type,
566
+ ):
567
+ if not isinstance(self.model.layers[layer_idx], nn.Identity):
568
+ layer_self_attn = self.model.layers[layer_idx].self_attn
569
+
570
+ if is_hip():
571
+ # The scaling factor convention we are assuming is
572
+ # quantized_value * scaling_factor ~= true_value
573
+ # which is consistent with the practice of setting
574
+ # scaling_factor = tensor_amax / FPtype_max
575
+ scaling_factor *= 2
576
+ if hasattr(layer_self_attn, "kv_scale"):
577
+ layer_self_attn.attn._kv_scale = scaling_factor
578
+ else:
579
+ raise RuntimeError("Self attention has no KV cache scaling "
580
+ "factor attribute!")
vllm/transformers_utils/config.py
+2
-1
@@ -24,7 +24,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
24
JAISConfig, MedusaConfig,
25
MLPSpeculatorConfig, MPTConfig,
26
NemotronConfig, RWConfig,
27
- UltravoxConfig)
27
+ SolarConfig, UltravoxConfig)
28
# yapf: enable
29
from vllm.transformers_utils.utils import check_gguf_file
30
@@ -50,6 +50,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
50
"exaone": ExaoneConfig,
51
"internvl_chat": InternVLChatConfig,
52
"nemotron": NemotronConfig,
53
+ "solar": SolarConfig,
54
"ultravox": UltravoxConfig,
55
# Granite can be removed from here once we have upgraded to
56
# transformers 4.45+
vllm/transformers_utils/configs/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.transformers_utils.configs.medusa import MedusaConfig
13
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
14
from vllm.transformers_utils.configs.mpt import MPTConfig
15
from vllm.transformers_utils.configs.nemotron import NemotronConfig
16
+from vllm.transformers_utils.configs.solar import SolarConfig
17
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
18
19
__all__ = [
@@ -27,6 +28,7 @@ __all__ = [
28
"ExaoneConfig",
29
"MLPSpeculatorConfig",
30
"NemotronConfig",
31
+ "SolarConfig",
32
"UltravoxConfig",
33
# Granite can be removed from here once we have upgraded to
34
# transformers 4.45+
vllm/transformers_utils/configs/solar.py
new
+245
@@ -0,0 +1,245 @@
1
+# coding=utf-8
2
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+"""Solar model configuration"""
21
+
22
+from transformers import PretrainedConfig
23
+from transformers.utils import logging
24
+
25
+logger = logging.get_logger(__name__)
26
+
27
+
28
+class SolarConfig(PretrainedConfig):
29
+ r"""
30
+ This is the configuration class to store
31
+ the configuration of a [`SolarModel`].
32
+ It is used to instantiate an LLaMA model
33
+ according to the specified arguments,
34
+ defining the model architecture.
35
+ Instantiating a configuration with the
36
+ defaults will yield a similar
37
+ configuration to that of the LLaMA-7B.
38
+ Configuration objects inherit from [`PretrainedConfig`]
39
+ and can be used to control the model outputs.
40
+ Read the documentation from [`PretrainedConfig`] for more information.
41
+ Args:
42
+ vocab_size (`int`, *optional*, defaults to 32000):
43
+ Vocabulary size of the LLaMA model.
44
+ Defines the number of different tokens
45
+ that can be represented by the `inputs_ids`
46
+ passed when calling [`SolarModel`]
47
+ hidden_size (`int`, *optional*, defaults to 4096):
48
+ Dimension of the hidden representations.
49
+ intermediate_size (`int`, *optional*, defaults to 11008):
50
+ Dimension of the MLP representations.
51
+ num_hidden_layers (`int`, *optional*, defaults to 32):
52
+ Number of hidden layers in the Transformer decoder.
53
+ num_attention_heads (`int`, *optional*, defaults to 32):
54
+ Number of attention heads for each attention layer
55
+ in the Transformer decoder.
56
+ num_key_value_heads (`int`, *optional*):
57
+ This is the number of key_value heads that
58
+ should be used to implement Grouped Query Attention. If
59
+ `num_key_value_heads=num_attention_heads`,
60
+ the model will use Multi Head Attention (MHA), if
61
+ `num_key_value_heads=1` the model
62
+ will use Multi Query Attention (MQA)
63
+ otherwise GQA is used. When
64
+ converting a multi-head checkpoint to a GQA checkpoint,
65
+ each group key and value head should be constructed
66
+ by meanpooling all the original heads within that group.
67
+ For more details checkout [this paper]
68
+ (https://arxiv.org/pdf/2305.13245.pdf).
69
+ If it is not specified, will default to
70
+ `num_attention_heads`.
71
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
72
+ The non-linear activation function (function or string)
73
+ in the decoder.
74
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
75
+ The maximum sequence length that this model might ever be used with.
76
+ Solar 1 supports up to 2048 tokens,
77
+ Solar 2 up to 4096, CodeSolar up to 16384.
78
+ initializer_range (`float`, *optional*, defaults to 0.02):
79
+ The standard deviation of
80
+ the truncated_normal_initializer for initializing
81
+ all weight matrices.
82
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
83
+ The epsilon used by the rms normalization layers.
84
+ use_cache (`bool`, *optional*, defaults to `True`):
85
+ Whether or not the model should return
86
+ the last key/values attentions (not used by all models). Only
87
+ relevant if `config.is_decoder=True`.
88
+ pad_token_id (`int`, *optional*):
89
+ Padding token id.
90
+ bos_token_id (`int`, *optional*, defaults to 1):
91
+ Beginning of stream token id.
92
+ eos_token_id (`int`, *optional*, defaults to 2):
93
+ End of stream token id.
94
+ pretraining_tp (`int`, *optional*, defaults to 1):
95
+ Experimental feature. Tensor parallelism rank
96
+ used during pretraining.
97
+ Please refer to [this
98
+ document](https://huggingface.co/docs/
99
+ transformers/main/
100
+ perf_train_gpu_many#tensor-parallelism)
101
+ to understand more about it. This value is
102
+ necessary to ensure exact reproducibility
103
+ of the pretraining results.
104
+ Please refer to [this
105
+ issue](https://github.com/pytorch/pytorch/issues/76232).
106
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
107
+ Whether to tie weight embeddings
108
+ rope_theta (`float`, *optional*, defaults to 10000.0):
109
+ The base period of the RoPE embeddings.
110
+ rope_scaling (`Dict`, *optional*):
111
+ Dictionary containing the scaling configuration for
112
+ the RoPE embeddings.
113
+ Currently supports two scaling
114
+ strategies: linear and dynamic.
115
+ Their scaling factor must be a float greater than 1.
116
+ The expected format is
117
+ `{"type": strategy name, "factor": scaling factor}`.
118
+ When using this flag, don't update
119
+ `max_position_embeddings` to the expected new maximum.
120
+ See the following thread for more information on how
121
+ these scaling strategies behave:
122
+ https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/
123
+ dynamically_scaled_rope_further_increases/. This is an
124
+ experimental feature, subject to breaking
125
+ API changes in future versions.
126
+ attention_bias (`bool`, *optional*, defaults to `False`):
127
+ Whether to use a bias in the query, key, value
128
+ and output projection layers during self-attention.
129
+ attention_dropout (`float`, *optional*, defaults to 0.0):
130
+ The dropout ratio for the attention probabilities.
131
+ mlp_bias (`bool`, *optional*, defaults to `False`):
132
+ Whether to use a bias in up_proj, down_proj and gate_proj
133
+ layers in the MLP layers.
134
+ sliding_window (`int`, *optional*, defaults to 2047):
135
+ Sliding window attention window size. If not specified,
136
+ will default to `2047`.
137
+ ```python
138
+ >>> from transformers import SolarModel, SolarConfig
139
+ >>> # Initializing a Solar-pro style configuration
140
+ >>> configuration = SolarConfig()
141
+ >>> # Initializing a model from the Solar-pro style configuration
142
+ >>> model = SolarModel(configuration)
143
+ >>> # Accessing the model configuration
144
+ >>> configuration = model.config
145
+ ```"""
146
+
147
+ model_type = "solar"
148
+ keys_to_ignore_at_inference = ["past_key_values"]
149
+
150
+ def __init__(
151
+ self,
152
+ vocab_size=32000,
153
+ hidden_size=4096,
154
+ intermediate_size=11008,
155
+ num_hidden_layers=32,
156
+ num_attention_heads=32,
157
+ num_key_value_heads=None,
158
+ hidden_act="silu",
159
+ max_position_embeddings=2048,
160
+ initializer_range=0.02,
161
+ rms_norm_eps=1e-6,
162
+ use_cache=True,
163
+ pad_token_id=None,
164
+ bos_token_id=1,
165
+ eos_token_id=2,
166
+ pretraining_tp=1,
167
+ tie_word_embeddings=False,
168
+ rope_theta=10000.0,
169
+ rope_scaling=None,
170
+ attention_bias=False,
171
+ attention_dropout=0.0,
172
+ mlp_bias=False,
173
+ sliding_window=2047,
174
+ bskcn_1=None,
175
+ bskcn_2=None,
176
+ bskcn_3=None,
177
+ bskcn_4=None,
178
+ bskcn_tv=None,
179
+ **kwargs,
180
+ ):
181
+ self.vocab_size = vocab_size
182
+ self.max_position_embeddings = max_position_embeddings
183
+ self.hidden_size = hidden_size
184
+ self.intermediate_size = intermediate_size
185
+ self.num_hidden_layers = num_hidden_layers
186
+ self.num_attention_heads = num_attention_heads
187
+
188
+ # for backward compatibility
189
+ if num_key_value_heads is None:
190
+ num_key_value_heads = num_attention_heads
191
+
192
+ self.num_key_value_heads = num_key_value_heads
193
+ self.hidden_act = hidden_act
194
+ self.initializer_range = initializer_range
195
+ self.rms_norm_eps = rms_norm_eps
196
+ self.pretraining_tp = pretraining_tp
197
+ self.use_cache = use_cache
198
+ self.rope_theta = rope_theta
199
+ self.rope_scaling = rope_scaling
200
+ self._rope_scaling_validation()
201
+ self.attention_bias = attention_bias
202
+ self.attention_dropout = attention_dropout
203
+ self.mlp_bias = mlp_bias
204
+ self.sliding_window = sliding_window
205
+ self.bskcn_1 = bskcn_1 if bskcn_1 is not None else [12, 20, 32, 44]
206
+ self.bskcn_2 = bskcn_2 if bskcn_2 is not None else [20, 32]
207
+ self.bskcn_3 = bskcn_3 if bskcn_3 is not None else [16, 24, 36, 48]
208
+ self.bskcn_4 = bskcn_4 if bskcn_4 is not None else [28, 40]
209
+ self.bskcn_tv = bskcn_tv if bskcn_tv is not None else [0.9, 0.8]
210
+
211
+ super().__init__(
212
+ pad_token_id=pad_token_id,
213
+ bos_token_id=bos_token_id,
214
+ eos_token_id=eos_token_id,
215
+ tie_word_embeddings=tie_word_embeddings,
216
+ **kwargs,
217
+ )
218
+
219
+ def _rope_scaling_validation(self):
220
+ """
221
+ Validate the `rope_scaling` configuration.
222
+ """
223
+ if self.rope_scaling is None:
224
+ return
225
+
226
+ if (not isinstance(self.rope_scaling, dict)
227
+ or len(self.rope_scaling) != 2):
228
+ raise ValueError(
229
+ "`rope_scaling` must be a dictionary with two fields,"
230
+ " `type` and `factor`, "
231
+ f"got {self.rope_scaling}")
232
+ rope_scaling_type = self.rope_scaling.get("type", None)
233
+ rope_scaling_factor = self.rope_scaling.get("factor", None)
234
+ if rope_scaling_type is None or rope_scaling_type not in [
235
+ "linear",
236
+ "dynamic",
237
+ ]:
238
+ raise ValueError(f"`rope_scaling`'s type field must be one of "
239
+ f"['linear', 'dynamic'], got {rope_scaling_type}")
240
+ if (rope_scaling_factor is None
241
+ or not isinstance(rope_scaling_factor, float)
242
+ or rope_scaling_factor <= 1.0):
243
+ raise ValueError(
244
+ f"`rope_scaling`'s factor field must be a float > 1,"
245
+ f" got {rope_scaling_factor}")