A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] Initial Support for Chameleon (#5770)
Roger Wang committed
Jul 21, 2024 at 17:37 UTC
c9eef37f329c952df92d4c8f4aa23e616f183d04
5 files changed
+665
-4
vllm/model_executor/models/__init__.py
+3
@@ -16,6 +16,9 @@ _GENERATION_MODELS = {
16
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19
+ "ChameleonForCausalLM":
20
+ ("chameleon", "ChameleonForConditionalGeneration"
21
+ ), #TODO(ywang96): fix model name when huggingface fixes it
22
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
23
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
24
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
vllm/model_executor/models/chameleon.py
new
+554
@@ -0,0 +1,554 @@
1
+from functools import cached_property
2
+from typing import Any, Dict, Iterable, List, Optional, Tuple
3
+
4
+import torch
5
+import torch.nn.functional as F
6
+from torch import nn
7
+
8
+from vllm.attention import Attention, AttentionMetadata
9
+from vllm.config import CacheConfig
10
+from vllm.distributed import get_tensor_model_parallel_world_size
11
+from vllm.model_executor.layers.activation import SiluAndMul
12
+from vllm.model_executor.layers.layernorm import RMSNorm
13
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
14
+ QKVParallelLinear,
15
+ RowParallelLinear)
16
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
+from vllm.model_executor.layers.quantization.base_config import (
18
+ QuantizationConfig)
19
+from vllm.model_executor.layers.rotary_embedding import get_rope
20
+from vllm.model_executor.layers.sampler import Sampler
21
+from vllm.model_executor.layers.vocab_parallel_embedding import (
22
+ ParallelLMHead, VocabParallelEmbedding)
23
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
+from vllm.model_executor.sampling_metadata import SamplingMetadata
25
+from vllm.sequence import IntermediateTensors, SamplerOutput
26
+from vllm.transformers_utils.configs import ChameleonConfig
27
+from vllm.utils import print_warning_once
28
+
29
+
30
+class ChameleonLayerNorm(nn.LayerNorm):
31
+
32
+ def __init__(self, hidden_size, *args, **kwargs):
33
+ super().__init__(hidden_size, *args, **kwargs)
34
+ self.normalized_shape = (hidden_size[-1], )
35
+
36
+ def forward(self, hidden_states):
37
+ hidden_states = F.layer_norm(hidden_states,
38
+ self.normalized_shape,
39
+ None,
40
+ None,
41
+ eps=1e-5)
42
+ hidden_states = hidden_states * self.weight + self.bias
43
+ return hidden_states
44
+
45
+
46
+# Copied from vllm.model_executor.models.llama.LlamaMLP -> ChameleonMLP
47
+class ChameleonMLP(nn.Module):
48
+
49
+ def __init__(
50
+ self,
51
+ hidden_size: int,
52
+ intermediate_size: int,
53
+ hidden_act: str,
54
+ quant_config: Optional[QuantizationConfig] = None,
55
+ bias: bool = False,
56
+ ) -> None:
57
+ super().__init__()
58
+ self.gate_up_proj = MergedColumnParallelLinear(
59
+ input_size=hidden_size,
60
+ output_sizes=[intermediate_size] * 2,
61
+ bias=bias,
62
+ quant_config=quant_config)
63
+ self.down_proj = RowParallelLinear(input_size=intermediate_size,
64
+ output_size=hidden_size,
65
+ bias=bias,
66
+ quant_config=quant_config)
67
+ if hidden_act != "silu":
68
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
69
+ "Only silu is supported for now.")
70
+ self.act_fn = SiluAndMul()
71
+
72
+ def forward(self, x):
73
+ gate_up, _ = self.gate_up_proj(x)
74
+ x = self.act_fn(gate_up)
75
+ x, _ = self.down_proj(x)
76
+ return x
77
+
78
+
79
+# Modified from vllm.model_executor.models.llama.LlamaAttention -> ChameleonAttention #noqa
80
+class ChameleonAttention(nn.Module):
81
+
82
+ def __init__(
83
+ self,
84
+ hidden_size: int,
85
+ num_heads: int,
86
+ num_kv_heads: int,
87
+ rope_theta: float = 10000,
88
+ rope_scaling: Optional[Dict[str, Any]] = None,
89
+ max_position_embeddings: int = 4096,
90
+ quant_config: Optional[QuantizationConfig] = None,
91
+ bias: bool = False,
92
+ cache_config: Optional[CacheConfig] = None,
93
+ ) -> None:
94
+ super().__init__()
95
+ self.hidden_size = hidden_size
96
+ tp_size = get_tensor_model_parallel_world_size()
97
+ self.total_num_heads = num_heads
98
+ assert self.total_num_heads % tp_size == 0
99
+ self.num_heads = self.total_num_heads // tp_size
100
+ self.total_num_kv_heads = num_kv_heads
101
+ if self.total_num_kv_heads >= tp_size:
102
+ # Number of KV heads is greater than TP size, so we partition
103
+ # the KV heads across multiple tensor parallel GPUs.
104
+ assert self.total_num_kv_heads % tp_size == 0
105
+ else:
106
+ # Number of KV heads is less than TP size, so we replicate
107
+ # the KV heads across multiple tensor parallel GPUs.
108
+ assert tp_size % self.total_num_kv_heads == 0
109
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110
+ self.head_dim = hidden_size // self.total_num_heads
111
+ self.q_size = self.num_heads * self.head_dim
112
+ self.kv_size = self.num_kv_heads * self.head_dim
113
+ self.scaling = self.head_dim**-0.5
114
+ self.rope_theta = rope_theta
115
+ self.max_position_embeddings = max_position_embeddings
116
+
117
+ self.qkv_proj = QKVParallelLinear(
118
+ hidden_size=hidden_size,
119
+ head_size=self.head_dim,
120
+ total_num_heads=self.total_num_heads,
121
+ total_num_kv_heads=self.total_num_kv_heads,
122
+ bias=bias,
123
+ quant_config=quant_config,
124
+ )
125
+ self.o_proj = RowParallelLinear(
126
+ input_size=self.total_num_heads * self.head_dim,
127
+ output_size=hidden_size,
128
+ bias=bias,
129
+ quant_config=quant_config,
130
+ )
131
+ self.q_norm = ChameleonLayerNorm((self.num_heads, self.head_dim))
132
+ self.k_norm = ChameleonLayerNorm((self.num_kv_heads, self.head_dim))
133
+ self.rotary_emb = get_rope(
134
+ self.head_dim,
135
+ rotary_dim=self.head_dim,
136
+ max_position=max_position_embeddings,
137
+ base=rope_theta,
138
+ rope_scaling=rope_scaling,
139
+ )
140
+
141
+ self.attn = Attention(self.num_heads,
142
+ self.head_dim,
143
+ self.scaling,
144
+ num_kv_heads=self.num_kv_heads,
145
+ cache_config=cache_config,
146
+ quant_config=quant_config)
147
+
148
+ def _apply_qk_norm(self, q: torch.Tensor,
149
+ k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
150
+ # reshape for layernorm
151
+ q = q.reshape(-1, self.num_heads, self.head_dim)
152
+ k = k.reshape(-1, self.num_kv_heads, self.head_dim)
153
+ q = self.q_norm(q)
154
+ k = self.k_norm(k)
155
+ q = q.view(*q.shape[:-2], -1)
156
+ k = k.view(*k.shape[:-2], -1)
157
+ return q, k
158
+
159
+ def forward(
160
+ self,
161
+ positions: torch.Tensor,
162
+ hidden_states: torch.Tensor,
163
+ kv_cache: torch.Tensor,
164
+ attn_metadata: AttentionMetadata,
165
+ ) -> torch.Tensor:
166
+ qkv, _ = self.qkv_proj(hidden_states)
167
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
168
+ q, k = self._apply_qk_norm(q, k)
169
+
170
+ q, k = self.rotary_emb(positions, q, k)
171
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
172
+ output, _ = self.o_proj(attn_output)
173
+ return output
174
+
175
+
176
+class ChameleonDecoderLayer(nn.Module):
177
+
178
+ def __init__(
179
+ self,
180
+ config: ChameleonConfig,
181
+ cache_config: Optional[CacheConfig] = None,
182
+ quant_config: Optional[QuantizationConfig] = None,
183
+ ) -> None:
184
+ super().__init__()
185
+ self.hidden_size = config.hidden_size
186
+ rope_theta = getattr(config, "rope_theta", 10000)
187
+ rope_scaling = getattr(config, "rope_scaling", None)
188
+ if rope_scaling is not None and getattr(
189
+ config, "original_max_position_embeddings", None):
190
+ rope_scaling["original_max_position_embeddings"] = (
191
+ config.original_max_position_embeddings)
192
+ max_position_embeddings = getattr(config, "max_position_embeddings",
193
+ 4096)
194
+
195
+ self.self_attn = ChameleonAttention(
196
+ hidden_size=self.hidden_size,
197
+ num_heads=config.num_attention_heads,
198
+ num_kv_heads=getattr(config, "num_key_value_heads",
199
+ config.num_attention_heads),
200
+ rope_theta=rope_theta,
201
+ rope_scaling=rope_scaling,
202
+ max_position_embeddings=max_position_embeddings,
203
+ quant_config=quant_config,
204
+ bias=False,
205
+ cache_config=cache_config,
206
+ )
207
+ self.mlp = ChameleonMLP(
208
+ hidden_size=self.hidden_size,
209
+ intermediate_size=config.intermediate_size,
210
+ hidden_act=config.hidden_act,
211
+ quant_config=quant_config,
212
+ bias=getattr(config, "mlp_bias", False),
213
+ )
214
+ self.input_layernorm = RMSNorm(config.hidden_size,
215
+ eps=config.rms_norm_eps)
216
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
217
+ eps=config.rms_norm_eps)
218
+
219
+ def forward(
220
+ self,
221
+ positions: torch.Tensor,
222
+ hidden_states: torch.Tensor,
223
+ kv_cache: torch.Tensor,
224
+ attn_metadata: AttentionMetadata,
225
+ residual: Optional[torch.Tensor],
226
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
227
+
228
+ if residual is None:
229
+ residual = hidden_states
230
+ hidden_states = self.input_layernorm(hidden_states)
231
+ else:
232
+ hidden_states, residual = self.input_layernorm(
233
+ hidden_states, residual)
234
+ hidden_states = self.self_attn(
235
+ positions=positions,
236
+ hidden_states=hidden_states,
237
+ kv_cache=kv_cache,
238
+ attn_metadata=attn_metadata,
239
+ )
240
+
241
+ # Fully Connected
242
+ hidden_states, residual = self.post_attention_layernorm(
243
+ hidden_states, residual)
244
+ hidden_states = self.mlp(hidden_states)
245
+
246
+ return hidden_states, residual
247
+
248
+
249
+class ChameleonSwinDecoderLayer(nn.Module):
250
+
251
+ def __init__(
252
+ self,
253
+ config: ChameleonConfig,
254
+ cache_config: Optional[CacheConfig] = None,
255
+ quant_config: Optional[QuantizationConfig] = None,
256
+ ) -> None:
257
+ super().__init__()
258
+ self.hidden_size = config.hidden_size
259
+ rope_theta = getattr(config, "rope_theta", 10000)
260
+ rope_scaling = getattr(config, "rope_scaling", None)
261
+ if rope_scaling is not None and getattr(
262
+ config, "original_max_position_embeddings", None):
263
+ rope_scaling["original_max_position_embeddings"] = (
264
+ config.original_max_position_embeddings)
265
+ max_position_embeddings = getattr(config, "max_position_embeddings",
266
+ 4096)
267
+
268
+ self.self_attn = ChameleonAttention(
269
+ hidden_size=self.hidden_size,
270
+ num_heads=config.num_attention_heads,
271
+ num_kv_heads=getattr(config, "num_key_value_heads",
272
+ config.num_attention_heads),
273
+ rope_theta=rope_theta,
274
+ rope_scaling=rope_scaling,
275
+ max_position_embeddings=max_position_embeddings,
276
+ quant_config=quant_config,
277
+ bias=False,
278
+ cache_config=cache_config,
279
+ )
280
+ self.mlp = ChameleonMLP(
281
+ hidden_size=self.hidden_size,
282
+ intermediate_size=config.intermediate_size,
283
+ hidden_act=config.hidden_act,
284
+ quant_config=quant_config,
285
+ bias=getattr(config, "mlp_bias", False),
286
+ )
287
+ self.input_layernorm = RMSNorm(config.hidden_size,
288
+ eps=config.rms_norm_eps)
289
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
290
+ eps=config.rms_norm_eps)
291
+
292
+ def forward(
293
+ self,
294
+ positions: torch.Tensor,
295
+ hidden_states: torch.Tensor,
296
+ kv_cache: torch.Tensor,
297
+ attn_metadata: AttentionMetadata,
298
+ residual: Optional[torch.Tensor],
299
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
300
+
301
+ residual = hidden_states
302
+ hidden_states = self.self_attn(
303
+ positions=positions,
304
+ hidden_states=hidden_states,
305
+ kv_cache=kv_cache,
306
+ attn_metadata=attn_metadata,
307
+ )
308
+
309
+ hidden_states = self.input_layernorm(hidden_states)
310
+ hidden_states = hidden_states + residual
311
+
312
+ # Fully Connected
313
+ residual = hidden_states
314
+ hidden_states = self.mlp(hidden_states)
315
+ hidden_states = self.post_attention_layernorm(hidden_states)
316
+ hidden_states = residual + hidden_states
317
+
318
+ return hidden_states, residual
319
+
320
+
321
+class ChameleonImageVocabularyMapping:
322
+ """
323
+ A class for mapping discrete image tokens from VQGAN to BPE tokens.
324
+ """
325
+
326
+ def __init__(self, vocab_map):
327
+ self.vocab_map = vocab_map
328
+ self.image_token_id = vocab_map.get("<image>")
329
+
330
+ @cached_property
331
+ def val2name(self):
332
+ return {v: k for k, v in self.vocab_map.items()}
333
+
334
+ @cached_property
335
+ def image_tokens(self):
336
+ return sorted([
337
+ val for name, val in self.vocab_map.items()
338
+ if name.startswith("IMGIMG")
339
+ ])
340
+
341
+ @cached_property
342
+ def bpe2img(self):
343
+ img_tkn_chr_mapping = {chr(ord("A") + i): str(i) for i in range(10)}
344
+
345
+ def remap(old_name: str) -> str:
346
+ return "".join(
347
+ img_tkn_chr_mapping.get(c, c)
348
+ for c in old_name[len("IMGIMG"):-1])
349
+
350
+ return {
351
+ tok: int(remap(self.val2name[tok]))
352
+ for tok in self.image_tokens
353
+ }
354
+
355
+ @cached_property
356
+ def img2bpe(self):
357
+ return {v: k for k, v in self.bpe2img.items()}
358
+
359
+ @cached_property
360
+ def bpe2img_search_tensors(self):
361
+ return torch.tensor(sorted(self.bpe2img.keys())), torch.tensor(
362
+ sorted(self.bpe2img.values()))
363
+
364
+ @cached_property
365
+ def img2bpe_mapping_tensor(self):
366
+ mapping = torch.zeros(max(self.img2bpe.keys()) + 1, dtype=torch.int)
367
+ for k, v in self.img2bpe.items():
368
+ mapping[k] = v
369
+ return mapping
370
+
371
+ def convert_img2bpe(self, img_batch: torch.Tensor) -> torch.Tensor:
372
+ device = img_batch.device
373
+ img_tokens = self.img2bpe_mapping_tensor[img_batch.to("cpu")]
374
+ return img_tokens.to(device)
375
+
376
+
377
+class ChameleonModel(nn.Module):
378
+
379
+ def __init__(
380
+ self,
381
+ config: ChameleonConfig,
382
+ cache_config: Optional[CacheConfig] = None,
383
+ quant_config: Optional[QuantizationConfig] = None,
384
+ ) -> None:
385
+ super().__init__()
386
+ self.config = config
387
+ self.padding_idx = config.pad_token_id
388
+ self.vocab_size = config.vocab_size
389
+ self.embed_tokens = VocabParallelEmbedding(
390
+ self.vocab_size,
391
+ config.hidden_size,
392
+ )
393
+ self.vocabulary_mapping = ChameleonImageVocabularyMapping(
394
+ config.vocabulary_map)
395
+ decoder_layer = ChameleonDecoderLayer if not self.config.swin_norm \
396
+ else ChameleonSwinDecoderLayer
397
+ self.layers = nn.ModuleList([
398
+ decoder_layer(config=config,
399
+ cache_config=cache_config,
400
+ quant_config=quant_config)
401
+ for _ in range(config.num_hidden_layers)
402
+ ])
403
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
404
+
405
+ # TODO: Support image input
406
+ # self.vqmodel = ChameleonVQModel(config.vq_config)
407
+
408
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
409
+ return self.embed_tokens(input_ids)
410
+
411
+ def forward(
412
+ self,
413
+ input_ids: Optional[torch.Tensor],
414
+ positions: torch.Tensor,
415
+ kv_caches: List[torch.Tensor],
416
+ attn_metadata: AttentionMetadata,
417
+ inputs_embeds: Optional[torch.Tensor] = None,
418
+ ) -> torch.Tensor:
419
+ if inputs_embeds is not None:
420
+ hidden_states = inputs_embeds
421
+ else:
422
+ hidden_states = self.get_input_embeddings(input_ids)
423
+ residual = None
424
+ for i in range(len(self.layers)):
425
+ layer = self.layers[i]
426
+ hidden_states, residual = layer(
427
+ positions,
428
+ hidden_states,
429
+ kv_caches[i],
430
+ attn_metadata,
431
+ residual,
432
+ )
433
+ hidden_states, _ = self.norm(hidden_states, residual)
434
+ return hidden_states
435
+
436
+
437
+class ChameleonForConditionalGeneration(nn.Module):
438
+
439
+ def __init__(
440
+ self,
441
+ config: ChameleonConfig,
442
+ cache_config: Optional[CacheConfig] = None,
443
+ quant_config: Optional[QuantizationConfig] = None,
444
+ ) -> None:
445
+ super().__init__()
446
+ self.config = config
447
+ self.model = ChameleonModel(config, cache_config, quant_config)
448
+ self.unpadded_vocab_size = config.vocab_size
449
+ self.lm_head = ParallelLMHead(
450
+ self.unpadded_vocab_size,
451
+ config.hidden_size,
452
+ )
453
+ if config.tie_word_embeddings:
454
+ self.lm_head.weight = self.model.embed_tokens.weight
455
+
456
+ logit_scale = getattr(config, "logit_scale", 1.0)
457
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
458
+ config.vocab_size, logit_scale)
459
+ self.sampler = Sampler()
460
+
461
+ def forward(
462
+ self,
463
+ input_ids: torch.Tensor,
464
+ positions: torch.Tensor,
465
+ kv_caches: List[torch.Tensor],
466
+ attn_metadata: AttentionMetadata,
467
+ intermediate_tensors: Optional[IntermediateTensors] = None,
468
+ **kwargs,
469
+ ) -> torch.Tensor:
470
+
471
+ # TODO (ywang96): Support image input
472
+ # image_tokens = self.process_image_input(**kwargs)
473
+ # image_mask = input_ids == self.vocabulary_mapping.image_token_id
474
+ # input_ids[special_image_mask] = image_tokens.flatten().to(input_ids.dtype) #noqa
475
+
476
+ hidden_states = self.model(input_ids, positions, kv_caches,
477
+ attn_metadata)
478
+ return hidden_states
479
+
480
+ def compute_logits(self, hidden_states: torch.Tensor,
481
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
482
+ logits = self.logits_processor(self.lm_head, hidden_states,
483
+ sampling_metadata)
484
+
485
+ # Disallow image tokens which does not include special
486
+ # begin-image and end-image tokens
487
+ image_tokens = self.model.vocabulary_mapping.image_tokens
488
+ logits[:, image_tokens] = torch.finfo(logits.dtype).min
489
+
490
+ return logits
491
+
492
+ def sample(
493
+ self,
494
+ logits: torch.Tensor,
495
+ sampling_metadata: SamplingMetadata,
496
+ ) -> Optional[SamplerOutput]:
497
+ next_tokens = self.sampler(logits, sampling_metadata)
498
+ return next_tokens
499
+
500
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
501
+ stacked_params_mapping = [
502
+ # (param_name, shard_name, shard_id)
503
+ (".qkv_proj", ".q_proj", "q"),
504
+ (".qkv_proj", ".k_proj", "k"),
505
+ (".qkv_proj", ".v_proj", "v"),
506
+ (".gate_up_proj", ".gate_proj", 0),
507
+ (".gate_up_proj", ".up_proj", 1),
508
+ ]
509
+ params_dict = dict(self.named_parameters())
510
+ for name, loaded_weight in weights:
511
+ if "rotary_emb.inv_freq" in name:
512
+ continue
513
+
514
+ # Skip loading vqgan
515
+ # TODO: add support for the vision model
516
+ if "vqmodel" in name:
517
+ continue
518
+ if ("rotary_emb.cos_cached" in name
519
+ or "rotary_emb.sin_cached" in name):
520
+ # Models trained using ColossalAI may include these tensors in
521
+ # the checkpoint. Skip them.
522
+ continue
523
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
524
+ if weight_name not in name:
525
+ continue
526
+ name = name.replace(weight_name, param_name)
527
+ # Skip loading extra bias for GPTQ models.
528
+ if name.endswith(".bias") and name not in params_dict:
529
+ continue
530
+ param = params_dict[name]
531
+ weight_loader = param.weight_loader
532
+ weight_loader(param, loaded_weight, shard_id)
533
+ break
534
+ else:
535
+ # Skip loading extra bias for GPTQ models.
536
+ if name.endswith(".bias") and name not in params_dict:
537
+ continue
538
+ # Remapping the name of FP8 kv-scale.
539
+ if name.endswith("kv_scale"):
540
+ remapped_kv_scale_name = name.replace(
541
+ ".kv_scale", ".attn.kv_scale")
542
+ if remapped_kv_scale_name not in params_dict:
543
+ print_warning_once(
544
+ f"Found kv scale in the checkpoint (e.g. {name}), "
545
+ "but not found the expected name in the model "
546
+ f"(e.g. {remapped_kv_scale_name}). kv-scale is "
547
+ "not loaded.")
548
+ continue
549
+ else:
550
+ name = remapped_kv_scale_name
551
+ param = params_dict[name]
552
+ weight_loader = getattr(param, "weight_loader",
553
+ default_weight_loader)
554
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+5
-4
@@ -5,10 +5,10 @@ from transformers import GenerationConfig, PretrainedConfig
5
6
from vllm.envs import VLLM_USE_MODELSCOPE
7
from vllm.logger import init_logger
8
-from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
- JAISConfig, MedusaConfig,
10
- MLPSpeculatorConfig, MPTConfig,
11
- RWConfig)
8
+from vllm.transformers_utils.configs import (ChameleonConfig, ChatGLMConfig,
9
+ DbrxConfig, JAISConfig,
10
+ MedusaConfig, MLPSpeculatorConfig,
11
+ MPTConfig, RWConfig)
12
13
if VLLM_USE_MODELSCOPE:
14
from modelscope import AutoConfig
@@ -18,6 +18,7 @@ else:
18
logger = init_logger(__name__)
19
20
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
21
+ "chameleon": ChameleonConfig,
22
"chatglm": ChatGLMConfig,
23
"dbrx": DbrxConfig,
24
"mpt": MPTConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,3 +1,4 @@
1
+from vllm.transformers_utils.configs.chameleon import ChameleonConfig
2
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
3
from vllm.transformers_utils.configs.dbrx import DbrxConfig
4
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -10,6 +11,7 @@ from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
11
from vllm.transformers_utils.configs.mpt import MPTConfig
12
13
__all__ = [
14
+ "ChameleonConfig",
15
"ChatGLMConfig",
16
"DbrxConfig",
17
"MPTConfig",
vllm/transformers_utils/configs/chameleon.py
new
+101
@@ -0,0 +1,101 @@
1
+from transformers import PretrainedConfig
2
+
3
+
4
+#TODO (ywang96): Remove this file and import it from
5
+# transformers once the new release with Chameleon support
6
+# is available.
7
+class ChameleonConfig(PretrainedConfig):
8
+
9
+ model_type = "chameleon"
10
+ is_composition = True
11
+ keys_to_ignore_at_inference = ["past_key_values"]
12
+
13
+ def __init__(
14
+ self,
15
+ vocab_size=65536,
16
+ hidden_size=4096,
17
+ intermediate_size=11008,
18
+ num_hidden_layers=32,
19
+ num_attention_heads=32,
20
+ num_key_value_heads=32,
21
+ hidden_act="silu",
22
+ max_position_embeddings=4096,
23
+ initializer_range=0.02,
24
+ rms_norm_eps=1e-05,
25
+ use_cache=True,
26
+ pad_token_id=None,
27
+ bos_token_id=1,
28
+ eos_token_id=2,
29
+ tie_word_embeddings=False,
30
+ rope_theta=10000.0,
31
+ rope_scaling=None,
32
+ attention_bias=False,
33
+ attention_dropout=0.0,
34
+ qk_layernorm=False,
35
+ swin_norm=False,
36
+ vq_config=None,
37
+ vocabulary_map=None,
38
+ mlp_bias=False,
39
+ **kwargs,
40
+ ):
41
+ self.vocab_size = vocab_size
42
+ self.max_position_embeddings = max_position_embeddings
43
+ self.hidden_size = hidden_size
44
+ self.intermediate_size = intermediate_size
45
+ self.num_hidden_layers = num_hidden_layers
46
+ self.num_attention_heads = num_attention_heads
47
+ self.mlp_bias = mlp_bias
48
+
49
+ # for backward compatibility
50
+ if num_key_value_heads is None:
51
+ num_key_value_heads = num_attention_heads
52
+
53
+ self.num_key_value_heads = num_key_value_heads
54
+ self.hidden_act = hidden_act
55
+ self.initializer_range = initializer_range
56
+ self.rms_norm_eps = rms_norm_eps
57
+ self.use_cache = use_cache
58
+ self.rope_theta = rope_theta
59
+ self.rope_scaling = rope_scaling
60
+ self._rope_scaling_validation()
61
+ self.attention_bias = attention_bias
62
+ self.attention_dropout = attention_dropout
63
+ self.qk_layernorm = qk_layernorm
64
+ self.swin_norm = swin_norm
65
+ # vq config is currently ignored
66
+ # self.vq_config = ChameleonVQConfig(**vq_config)
67
+ self.vocabulary_map = vocabulary_map
68
+
69
+ super().__init__(
70
+ pad_token_id=pad_token_id,
71
+ bos_token_id=bos_token_id,
72
+ eos_token_id=eos_token_id,
73
+ tie_word_embeddings=tie_word_embeddings,
74
+ **kwargs,
75
+ )
76
+
77
+ def _rope_scaling_validation(self):
78
+ """
79
+ Validate the `rope_scaling` configuration.
80
+ """
81
+ if self.rope_scaling is None:
82
+ return
83
+
84
+ if not isinstance(self.rope_scaling,
85
+ dict) or len(self.rope_scaling) != 2:
86
+ raise ValueError(
87
+ "`rope_scaling` must be a dictionary with with two fields, "
88
+ f"`type` and `factor`, got {self.rope_scaling}")
89
+ rope_scaling_type = self.rope_scaling.get("type", None)
90
+ rope_scaling_factor = self.rope_scaling.get("factor", None)
91
+ if rope_scaling_type is None or rope_scaling_type not in [
92
+ "linear", "dynamic"
93
+ ]:
94
+ raise ValueError(
95
+ "`rope_scaling`'s type field must be one of ['linear', "
96
+ f"'dynamic'], got {rope_scaling_type}")
97
+ if rope_scaling_factor is None or not isinstance(
98
+ rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
99
+ raise ValueError(
100
+ "`rope_scaling`'s factor field must be a float > 1, "
101
+ f"got {rope_scaling_factor}")