@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9eef37f32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Initial Support for Chameleon (#5770)

Roger Wang committed Jul 21, 2024 at 17:37 UTC c9eef37f329c952df92d4c8f4aa23e616f183d04
5 files changed +665 -4
vllm/model_executor/models/__init__.py
+3
@@ -16,6 +16,9 @@ _GENERATION_MODELS = {
16 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19 + "ChameleonForCausalLM":
20 + ("chameleon", "ChameleonForConditionalGeneration"
21 + ), #TODO(ywang96): fix model name when huggingface fixes it
22 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
23 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
24 "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
vllm/model_executor/models/chameleon.py new
+554
@@ -0,0 +1,554 @@
1 +from functools import cached_property
2 +from typing import Any, Dict, Iterable, List, Optional, Tuple
3 +
4 +import torch
5 +import torch.nn.functional as F
6 +from torch import nn
7 +
8 +from vllm.attention import Attention, AttentionMetadata
9 +from vllm.config import CacheConfig
10 +from vllm.distributed import get_tensor_model_parallel_world_size
11 +from vllm.model_executor.layers.activation import SiluAndMul
12 +from vllm.model_executor.layers.layernorm import RMSNorm
13 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
14 + QKVParallelLinear,
15 + RowParallelLinear)
16 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 +from vllm.model_executor.layers.quantization.base_config import (
18 + QuantizationConfig)
19 +from vllm.model_executor.layers.rotary_embedding import get_rope
20 +from vllm.model_executor.layers.sampler import Sampler
21 +from vllm.model_executor.layers.vocab_parallel_embedding import (
22 + ParallelLMHead, VocabParallelEmbedding)
23 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 +from vllm.model_executor.sampling_metadata import SamplingMetadata
25 +from vllm.sequence import IntermediateTensors, SamplerOutput
26 +from vllm.transformers_utils.configs import ChameleonConfig
27 +from vllm.utils import print_warning_once
28 +
29 +
30 +class ChameleonLayerNorm(nn.LayerNorm):
31 +
32 + def __init__(self, hidden_size, *args, **kwargs):
33 + super().__init__(hidden_size, *args, **kwargs)
34 + self.normalized_shape = (hidden_size[-1], )
35 +
36 + def forward(self, hidden_states):
37 + hidden_states = F.layer_norm(hidden_states,
38 + self.normalized_shape,
39 + None,
40 + None,
41 + eps=1e-5)
42 + hidden_states = hidden_states * self.weight + self.bias
43 + return hidden_states
44 +
45 +
46 +# Copied from vllm.model_executor.models.llama.LlamaMLP -> ChameleonMLP
47 +class ChameleonMLP(nn.Module):
48 +
49 + def __init__(
50 + self,
51 + hidden_size: int,
52 + intermediate_size: int,
53 + hidden_act: str,
54 + quant_config: Optional[QuantizationConfig] = None,
55 + bias: bool = False,
56 + ) -> None:
57 + super().__init__()
58 + self.gate_up_proj = MergedColumnParallelLinear(
59 + input_size=hidden_size,
60 + output_sizes=[intermediate_size] * 2,
61 + bias=bias,
62 + quant_config=quant_config)
63 + self.down_proj = RowParallelLinear(input_size=intermediate_size,
64 + output_size=hidden_size,
65 + bias=bias,
66 + quant_config=quant_config)
67 + if hidden_act != "silu":
68 + raise ValueError(f"Unsupported activation: {hidden_act}. "
69 + "Only silu is supported for now.")
70 + self.act_fn = SiluAndMul()
71 +
72 + def forward(self, x):
73 + gate_up, _ = self.gate_up_proj(x)
74 + x = self.act_fn(gate_up)
75 + x, _ = self.down_proj(x)
76 + return x
77 +
78 +
79 +# Modified from vllm.model_executor.models.llama.LlamaAttention -> ChameleonAttention #noqa
80 +class ChameleonAttention(nn.Module):
81 +
82 + def __init__(
83 + self,
84 + hidden_size: int,
85 + num_heads: int,
86 + num_kv_heads: int,
87 + rope_theta: float = 10000,
88 + rope_scaling: Optional[Dict[str, Any]] = None,
89 + max_position_embeddings: int = 4096,
90 + quant_config: Optional[QuantizationConfig] = None,
91 + bias: bool = False,
92 + cache_config: Optional[CacheConfig] = None,
93 + ) -> None:
94 + super().__init__()
95 + self.hidden_size = hidden_size
96 + tp_size = get_tensor_model_parallel_world_size()
97 + self.total_num_heads = num_heads
98 + assert self.total_num_heads % tp_size == 0
99 + self.num_heads = self.total_num_heads // tp_size
100 + self.total_num_kv_heads = num_kv_heads
101 + if self.total_num_kv_heads >= tp_size:
102 + # Number of KV heads is greater than TP size, so we partition
103 + # the KV heads across multiple tensor parallel GPUs.
104 + assert self.total_num_kv_heads % tp_size == 0
105 + else:
106 + # Number of KV heads is less than TP size, so we replicate
107 + # the KV heads across multiple tensor parallel GPUs.
108 + assert tp_size % self.total_num_kv_heads == 0
109 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110 + self.head_dim = hidden_size // self.total_num_heads
111 + self.q_size = self.num_heads * self.head_dim
112 + self.kv_size = self.num_kv_heads * self.head_dim
113 + self.scaling = self.head_dim**-0.5
114 + self.rope_theta = rope_theta
115 + self.max_position_embeddings = max_position_embeddings
116 +
117 + self.qkv_proj = QKVParallelLinear(
118 + hidden_size=hidden_size,
119 + head_size=self.head_dim,
120 + total_num_heads=self.total_num_heads,
121 + total_num_kv_heads=self.total_num_kv_heads,
122 + bias=bias,
123 + quant_config=quant_config,
124 + )
125 + self.o_proj = RowParallelLinear(
126 + input_size=self.total_num_heads * self.head_dim,
127 + output_size=hidden_size,
128 + bias=bias,
129 + quant_config=quant_config,
130 + )
131 + self.q_norm = ChameleonLayerNorm((self.num_heads, self.head_dim))
132 + self.k_norm = ChameleonLayerNorm((self.num_kv_heads, self.head_dim))
133 + self.rotary_emb = get_rope(
134 + self.head_dim,
135 + rotary_dim=self.head_dim,
136 + max_position=max_position_embeddings,
137 + base=rope_theta,
138 + rope_scaling=rope_scaling,
139 + )
140 +
141 + self.attn = Attention(self.num_heads,
142 + self.head_dim,
143 + self.scaling,
144 + num_kv_heads=self.num_kv_heads,
145 + cache_config=cache_config,
146 + quant_config=quant_config)
147 +
148 + def _apply_qk_norm(self, q: torch.Tensor,
149 + k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
150 + # reshape for layernorm
151 + q = q.reshape(-1, self.num_heads, self.head_dim)
152 + k = k.reshape(-1, self.num_kv_heads, self.head_dim)
153 + q = self.q_norm(q)
154 + k = self.k_norm(k)
155 + q = q.view(*q.shape[:-2], -1)
156 + k = k.view(*k.shape[:-2], -1)
157 + return q, k
158 +
159 + def forward(
160 + self,
161 + positions: torch.Tensor,
162 + hidden_states: torch.Tensor,
163 + kv_cache: torch.Tensor,
164 + attn_metadata: AttentionMetadata,
165 + ) -> torch.Tensor:
166 + qkv, _ = self.qkv_proj(hidden_states)
167 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
168 + q, k = self._apply_qk_norm(q, k)
169 +
170 + q, k = self.rotary_emb(positions, q, k)
171 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
172 + output, _ = self.o_proj(attn_output)
173 + return output
174 +
175 +
176 +class ChameleonDecoderLayer(nn.Module):
177 +
178 + def __init__(
179 + self,
180 + config: ChameleonConfig,
181 + cache_config: Optional[CacheConfig] = None,
182 + quant_config: Optional[QuantizationConfig] = None,
183 + ) -> None:
184 + super().__init__()
185 + self.hidden_size = config.hidden_size
186 + rope_theta = getattr(config, "rope_theta", 10000)
187 + rope_scaling = getattr(config, "rope_scaling", None)
188 + if rope_scaling is not None and getattr(
189 + config, "original_max_position_embeddings", None):
190 + rope_scaling["original_max_position_embeddings"] = (
191 + config.original_max_position_embeddings)
192 + max_position_embeddings = getattr(config, "max_position_embeddings",
193 + 4096)
194 +
195 + self.self_attn = ChameleonAttention(
196 + hidden_size=self.hidden_size,
197 + num_heads=config.num_attention_heads,
198 + num_kv_heads=getattr(config, "num_key_value_heads",
199 + config.num_attention_heads),
200 + rope_theta=rope_theta,
201 + rope_scaling=rope_scaling,
202 + max_position_embeddings=max_position_embeddings,
203 + quant_config=quant_config,
204 + bias=False,
205 + cache_config=cache_config,
206 + )
207 + self.mlp = ChameleonMLP(
208 + hidden_size=self.hidden_size,
209 + intermediate_size=config.intermediate_size,
210 + hidden_act=config.hidden_act,
211 + quant_config=quant_config,
212 + bias=getattr(config, "mlp_bias", False),
213 + )
214 + self.input_layernorm = RMSNorm(config.hidden_size,
215 + eps=config.rms_norm_eps)
216 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
217 + eps=config.rms_norm_eps)
218 +
219 + def forward(
220 + self,
221 + positions: torch.Tensor,
222 + hidden_states: torch.Tensor,
223 + kv_cache: torch.Tensor,
224 + attn_metadata: AttentionMetadata,
225 + residual: Optional[torch.Tensor],
226 + ) -> Tuple[torch.Tensor, torch.Tensor]:
227 +
228 + if residual is None:
229 + residual = hidden_states
230 + hidden_states = self.input_layernorm(hidden_states)
231 + else:
232 + hidden_states, residual = self.input_layernorm(
233 + hidden_states, residual)
234 + hidden_states = self.self_attn(
235 + positions=positions,
236 + hidden_states=hidden_states,
237 + kv_cache=kv_cache,
238 + attn_metadata=attn_metadata,
239 + )
240 +
241 + # Fully Connected
242 + hidden_states, residual = self.post_attention_layernorm(
243 + hidden_states, residual)
244 + hidden_states = self.mlp(hidden_states)
245 +
246 + return hidden_states, residual
247 +
248 +
249 +class ChameleonSwinDecoderLayer(nn.Module):
250 +
251 + def __init__(
252 + self,
253 + config: ChameleonConfig,
254 + cache_config: Optional[CacheConfig] = None,
255 + quant_config: Optional[QuantizationConfig] = None,
256 + ) -> None:
257 + super().__init__()
258 + self.hidden_size = config.hidden_size
259 + rope_theta = getattr(config, "rope_theta", 10000)
260 + rope_scaling = getattr(config, "rope_scaling", None)
261 + if rope_scaling is not None and getattr(
262 + config, "original_max_position_embeddings", None):
263 + rope_scaling["original_max_position_embeddings"] = (
264 + config.original_max_position_embeddings)
265 + max_position_embeddings = getattr(config, "max_position_embeddings",
266 + 4096)
267 +
268 + self.self_attn = ChameleonAttention(
269 + hidden_size=self.hidden_size,
270 + num_heads=config.num_attention_heads,
271 + num_kv_heads=getattr(config, "num_key_value_heads",
272 + config.num_attention_heads),
273 + rope_theta=rope_theta,
274 + rope_scaling=rope_scaling,
275 + max_position_embeddings=max_position_embeddings,
276 + quant_config=quant_config,
277 + bias=False,
278 + cache_config=cache_config,
279 + )
280 + self.mlp = ChameleonMLP(
281 + hidden_size=self.hidden_size,
282 + intermediate_size=config.intermediate_size,
283 + hidden_act=config.hidden_act,
284 + quant_config=quant_config,
285 + bias=getattr(config, "mlp_bias", False),
286 + )
287 + self.input_layernorm = RMSNorm(config.hidden_size,
288 + eps=config.rms_norm_eps)
289 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
290 + eps=config.rms_norm_eps)
291 +
292 + def forward(
293 + self,
294 + positions: torch.Tensor,
295 + hidden_states: torch.Tensor,
296 + kv_cache: torch.Tensor,
297 + attn_metadata: AttentionMetadata,
298 + residual: Optional[torch.Tensor],
299 + ) -> Tuple[torch.Tensor, torch.Tensor]:
300 +
301 + residual = hidden_states
302 + hidden_states = self.self_attn(
303 + positions=positions,
304 + hidden_states=hidden_states,
305 + kv_cache=kv_cache,
306 + attn_metadata=attn_metadata,
307 + )
308 +
309 + hidden_states = self.input_layernorm(hidden_states)
310 + hidden_states = hidden_states + residual
311 +
312 + # Fully Connected
313 + residual = hidden_states
314 + hidden_states = self.mlp(hidden_states)
315 + hidden_states = self.post_attention_layernorm(hidden_states)
316 + hidden_states = residual + hidden_states
317 +
318 + return hidden_states, residual
319 +
320 +
321 +class ChameleonImageVocabularyMapping:
322 + """
323 + A class for mapping discrete image tokens from VQGAN to BPE tokens.
324 + """
325 +
326 + def __init__(self, vocab_map):
327 + self.vocab_map = vocab_map
328 + self.image_token_id = vocab_map.get("<image>")
329 +
330 + @cached_property
331 + def val2name(self):
332 + return {v: k for k, v in self.vocab_map.items()}
333 +
334 + @cached_property
335 + def image_tokens(self):
336 + return sorted([
337 + val for name, val in self.vocab_map.items()
338 + if name.startswith("IMGIMG")
339 + ])
340 +
341 + @cached_property
342 + def bpe2img(self):
343 + img_tkn_chr_mapping = {chr(ord("A") + i): str(i) for i in range(10)}
344 +
345 + def remap(old_name: str) -> str:
346 + return "".join(
347 + img_tkn_chr_mapping.get(c, c)
348 + for c in old_name[len("IMGIMG"):-1])
349 +
350 + return {
351 + tok: int(remap(self.val2name[tok]))
352 + for tok in self.image_tokens
353 + }
354 +
355 + @cached_property
356 + def img2bpe(self):
357 + return {v: k for k, v in self.bpe2img.items()}
358 +
359 + @cached_property
360 + def bpe2img_search_tensors(self):
361 + return torch.tensor(sorted(self.bpe2img.keys())), torch.tensor(
362 + sorted(self.bpe2img.values()))
363 +
364 + @cached_property
365 + def img2bpe_mapping_tensor(self):
366 + mapping = torch.zeros(max(self.img2bpe.keys()) + 1, dtype=torch.int)
367 + for k, v in self.img2bpe.items():
368 + mapping[k] = v
369 + return mapping
370 +
371 + def convert_img2bpe(self, img_batch: torch.Tensor) -> torch.Tensor:
372 + device = img_batch.device
373 + img_tokens = self.img2bpe_mapping_tensor[img_batch.to("cpu")]
374 + return img_tokens.to(device)
375 +
376 +
377 +class ChameleonModel(nn.Module):
378 +
379 + def __init__(
380 + self,
381 + config: ChameleonConfig,
382 + cache_config: Optional[CacheConfig] = None,
383 + quant_config: Optional[QuantizationConfig] = None,
384 + ) -> None:
385 + super().__init__()
386 + self.config = config
387 + self.padding_idx = config.pad_token_id
388 + self.vocab_size = config.vocab_size
389 + self.embed_tokens = VocabParallelEmbedding(
390 + self.vocab_size,
391 + config.hidden_size,
392 + )
393 + self.vocabulary_mapping = ChameleonImageVocabularyMapping(
394 + config.vocabulary_map)
395 + decoder_layer = ChameleonDecoderLayer if not self.config.swin_norm \
396 + else ChameleonSwinDecoderLayer
397 + self.layers = nn.ModuleList([
398 + decoder_layer(config=config,
399 + cache_config=cache_config,
400 + quant_config=quant_config)
401 + for _ in range(config.num_hidden_layers)
402 + ])
403 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
404 +
405 + # TODO: Support image input
406 + # self.vqmodel = ChameleonVQModel(config.vq_config)
407 +
408 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
409 + return self.embed_tokens(input_ids)
410 +
411 + def forward(
412 + self,
413 + input_ids: Optional[torch.Tensor],
414 + positions: torch.Tensor,
415 + kv_caches: List[torch.Tensor],
416 + attn_metadata: AttentionMetadata,
417 + inputs_embeds: Optional[torch.Tensor] = None,
418 + ) -> torch.Tensor:
419 + if inputs_embeds is not None:
420 + hidden_states = inputs_embeds
421 + else:
422 + hidden_states = self.get_input_embeddings(input_ids)
423 + residual = None
424 + for i in range(len(self.layers)):
425 + layer = self.layers[i]
426 + hidden_states, residual = layer(
427 + positions,
428 + hidden_states,
429 + kv_caches[i],
430 + attn_metadata,
431 + residual,
432 + )
433 + hidden_states, _ = self.norm(hidden_states, residual)
434 + return hidden_states
435 +
436 +
437 +class ChameleonForConditionalGeneration(nn.Module):
438 +
439 + def __init__(
440 + self,
441 + config: ChameleonConfig,
442 + cache_config: Optional[CacheConfig] = None,
443 + quant_config: Optional[QuantizationConfig] = None,
444 + ) -> None:
445 + super().__init__()
446 + self.config = config
447 + self.model = ChameleonModel(config, cache_config, quant_config)
448 + self.unpadded_vocab_size = config.vocab_size
449 + self.lm_head = ParallelLMHead(
450 + self.unpadded_vocab_size,
451 + config.hidden_size,
452 + )
453 + if config.tie_word_embeddings:
454 + self.lm_head.weight = self.model.embed_tokens.weight
455 +
456 + logit_scale = getattr(config, "logit_scale", 1.0)
457 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
458 + config.vocab_size, logit_scale)
459 + self.sampler = Sampler()
460 +
461 + def forward(
462 + self,
463 + input_ids: torch.Tensor,
464 + positions: torch.Tensor,
465 + kv_caches: List[torch.Tensor],
466 + attn_metadata: AttentionMetadata,
467 + intermediate_tensors: Optional[IntermediateTensors] = None,
468 + **kwargs,
469 + ) -> torch.Tensor:
470 +
471 + # TODO (ywang96): Support image input
472 + # image_tokens = self.process_image_input(**kwargs)
473 + # image_mask = input_ids == self.vocabulary_mapping.image_token_id
474 + # input_ids[special_image_mask] = image_tokens.flatten().to(input_ids.dtype) #noqa
475 +
476 + hidden_states = self.model(input_ids, positions, kv_caches,
477 + attn_metadata)
478 + return hidden_states
479 +
480 + def compute_logits(self, hidden_states: torch.Tensor,
481 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
482 + logits = self.logits_processor(self.lm_head, hidden_states,
483 + sampling_metadata)
484 +
485 + # Disallow image tokens which does not include special
486 + # begin-image and end-image tokens
487 + image_tokens = self.model.vocabulary_mapping.image_tokens
488 + logits[:, image_tokens] = torch.finfo(logits.dtype).min
489 +
490 + return logits
491 +
492 + def sample(
493 + self,
494 + logits: torch.Tensor,
495 + sampling_metadata: SamplingMetadata,
496 + ) -> Optional[SamplerOutput]:
497 + next_tokens = self.sampler(logits, sampling_metadata)
498 + return next_tokens
499 +
500 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
501 + stacked_params_mapping = [
502 + # (param_name, shard_name, shard_id)
503 + (".qkv_proj", ".q_proj", "q"),
504 + (".qkv_proj", ".k_proj", "k"),
505 + (".qkv_proj", ".v_proj", "v"),
506 + (".gate_up_proj", ".gate_proj", 0),
507 + (".gate_up_proj", ".up_proj", 1),
508 + ]
509 + params_dict = dict(self.named_parameters())
510 + for name, loaded_weight in weights:
511 + if "rotary_emb.inv_freq" in name:
512 + continue
513 +
514 + # Skip loading vqgan
515 + # TODO: add support for the vision model
516 + if "vqmodel" in name:
517 + continue
518 + if ("rotary_emb.cos_cached" in name
519 + or "rotary_emb.sin_cached" in name):
520 + # Models trained using ColossalAI may include these tensors in
521 + # the checkpoint. Skip them.
522 + continue
523 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
524 + if weight_name not in name:
525 + continue
526 + name = name.replace(weight_name, param_name)
527 + # Skip loading extra bias for GPTQ models.
528 + if name.endswith(".bias") and name not in params_dict:
529 + continue
530 + param = params_dict[name]
531 + weight_loader = param.weight_loader
532 + weight_loader(param, loaded_weight, shard_id)
533 + break
534 + else:
535 + # Skip loading extra bias for GPTQ models.
536 + if name.endswith(".bias") and name not in params_dict:
537 + continue
538 + # Remapping the name of FP8 kv-scale.
539 + if name.endswith("kv_scale"):
540 + remapped_kv_scale_name = name.replace(
541 + ".kv_scale", ".attn.kv_scale")
542 + if remapped_kv_scale_name not in params_dict:
543 + print_warning_once(
544 + f"Found kv scale in the checkpoint (e.g. {name}), "
545 + "but not found the expected name in the model "
546 + f"(e.g. {remapped_kv_scale_name}). kv-scale is "
547 + "not loaded.")
548 + continue
549 + else:
550 + name = remapped_kv_scale_name
551 + param = params_dict[name]
552 + weight_loader = getattr(param, "weight_loader",
553 + default_weight_loader)
554 + weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+5 -4
@@ -5,10 +5,10 @@ from transformers import GenerationConfig, PretrainedConfig
5
6 from vllm.envs import VLLM_USE_MODELSCOPE
7 from vllm.logger import init_logger
8 -from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9 - JAISConfig, MedusaConfig,
10 - MLPSpeculatorConfig, MPTConfig,
11 - RWConfig)
8 +from vllm.transformers_utils.configs import (ChameleonConfig, ChatGLMConfig,
9 + DbrxConfig, JAISConfig,
10 + MedusaConfig, MLPSpeculatorConfig,
11 + MPTConfig, RWConfig)
12
13 if VLLM_USE_MODELSCOPE:
14 from modelscope import AutoConfig
@@ -18,6 +18,7 @@ else:
18 logger = init_logger(__name__)
19
20 _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
21 + "chameleon": ChameleonConfig,
22 "chatglm": ChatGLMConfig,
23 "dbrx": DbrxConfig,
24 "mpt": MPTConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,3 +1,4 @@
1 +from vllm.transformers_utils.configs.chameleon import ChameleonConfig
2 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
3 from vllm.transformers_utils.configs.dbrx import DbrxConfig
4 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -10,6 +11,7 @@ from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
11 from vllm.transformers_utils.configs.mpt import MPTConfig
12
13 __all__ = [
14 + "ChameleonConfig",
15 "ChatGLMConfig",
16 "DbrxConfig",
17 "MPTConfig",
vllm/transformers_utils/configs/chameleon.py new
+101
@@ -0,0 +1,101 @@
1 +from transformers import PretrainedConfig
2 +
3 +
4 +#TODO (ywang96): Remove this file and import it from
5 +# transformers once the new release with Chameleon support
6 +# is available.
7 +class ChameleonConfig(PretrainedConfig):
8 +
9 + model_type = "chameleon"
10 + is_composition = True
11 + keys_to_ignore_at_inference = ["past_key_values"]
12 +
13 + def __init__(
14 + self,
15 + vocab_size=65536,
16 + hidden_size=4096,
17 + intermediate_size=11008,
18 + num_hidden_layers=32,
19 + num_attention_heads=32,
20 + num_key_value_heads=32,
21 + hidden_act="silu",
22 + max_position_embeddings=4096,
23 + initializer_range=0.02,
24 + rms_norm_eps=1e-05,
25 + use_cache=True,
26 + pad_token_id=None,
27 + bos_token_id=1,
28 + eos_token_id=2,
29 + tie_word_embeddings=False,
30 + rope_theta=10000.0,
31 + rope_scaling=None,
32 + attention_bias=False,
33 + attention_dropout=0.0,
34 + qk_layernorm=False,
35 + swin_norm=False,
36 + vq_config=None,
37 + vocabulary_map=None,
38 + mlp_bias=False,
39 + **kwargs,
40 + ):
41 + self.vocab_size = vocab_size
42 + self.max_position_embeddings = max_position_embeddings
43 + self.hidden_size = hidden_size
44 + self.intermediate_size = intermediate_size
45 + self.num_hidden_layers = num_hidden_layers
46 + self.num_attention_heads = num_attention_heads
47 + self.mlp_bias = mlp_bias
48 +
49 + # for backward compatibility
50 + if num_key_value_heads is None:
51 + num_key_value_heads = num_attention_heads
52 +
53 + self.num_key_value_heads = num_key_value_heads
54 + self.hidden_act = hidden_act
55 + self.initializer_range = initializer_range
56 + self.rms_norm_eps = rms_norm_eps
57 + self.use_cache = use_cache
58 + self.rope_theta = rope_theta
59 + self.rope_scaling = rope_scaling
60 + self._rope_scaling_validation()
61 + self.attention_bias = attention_bias
62 + self.attention_dropout = attention_dropout
63 + self.qk_layernorm = qk_layernorm
64 + self.swin_norm = swin_norm
65 + # vq config is currently ignored
66 + # self.vq_config = ChameleonVQConfig(**vq_config)
67 + self.vocabulary_map = vocabulary_map
68 +
69 + super().__init__(
70 + pad_token_id=pad_token_id,
71 + bos_token_id=bos_token_id,
72 + eos_token_id=eos_token_id,
73 + tie_word_embeddings=tie_word_embeddings,
74 + **kwargs,
75 + )
76 +
77 + def _rope_scaling_validation(self):
78 + """
79 + Validate the `rope_scaling` configuration.
80 + """
81 + if self.rope_scaling is None:
82 + return
83 +
84 + if not isinstance(self.rope_scaling,
85 + dict) or len(self.rope_scaling) != 2:
86 + raise ValueError(
87 + "`rope_scaling` must be a dictionary with with two fields, "
88 + f"`type` and `factor`, got {self.rope_scaling}")
89 + rope_scaling_type = self.rope_scaling.get("type", None)
90 + rope_scaling_factor = self.rope_scaling.get("factor", None)
91 + if rope_scaling_type is None or rope_scaling_type not in [
92 + "linear", "dynamic"
93 + ]:
94 + raise ValueError(
95 + "`rope_scaling`'s type field must be one of ['linear', "
96 + f"'dynamic'], got {rope_scaling_type}")
97 + if rope_scaling_factor is None or not isinstance(
98 + rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
99 + raise ValueError(
100 + "`rope_scaling`'s factor field must be a float > 1, "
101 + f"got {rope_scaling_factor}")