A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[VLM][Model] Support image input for Chameleon (#6633)
Roger Wang committed
Jul 22, 2024 at 23:50 UTC
22fa2e35cbbb76fde94412f565572d290027a3c3
7 files changed
+696
-58
docs/source/models/supported_models.rst
+4
@@ -182,6 +182,10 @@ Vision Language Models
182
- Models
183
- Example HuggingFace Models
184
- :ref:`LoRA <lora>`
185
+ * - :code:`ChameleonForConditionalGeneration`
186
+ - Chameleon
187
+ - :code:`facebook/chameleon-7b` etc.
188
+ -
189
* - :code:`FuyuForCausalLM`
190
- Fuyu
191
- :code:`adept/fuyu-8b` etc.
tests/models/test_chameleon.py
new
+102
@@ -0,0 +1,102 @@
1
+import re
2
+from typing import List, Optional, Type
3
+
4
+import pytest
5
+
6
+from vllm.multimodal.utils import rescale_image_size
7
+
8
+from ..conftest import IMAGE_ASSETS, VllmRunner, _ImageAssets
9
+
10
+pytestmark = pytest.mark.vlm
11
+
12
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13
+ "stop_sign":
14
+ "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
15
+ "cherry_blossom":
16
+ "USER: <image>\nWhat is the season?\nASSISTANT:",
17
+})
18
+
19
+models = ["facebook/chameleon-7b"]
20
+
21
+
22
+#TODO (ywang96): Add correctness test when chameleon is
23
+# available on transformers.
24
+def run_test(
25
+ vllm_runner: Type[VllmRunner],
26
+ image_assets: _ImageAssets,
27
+ model: str,
28
+ *,
29
+ size_factors: List[float],
30
+ dtype: str,
31
+ max_tokens: int,
32
+ tensor_parallel_size: int,
33
+ distributed_executor_backend: Optional[str] = None,
34
+):
35
+ """Test if the model can generate text given
36
+ a batch of images and prompts.
37
+
38
+ """
39
+ images = [asset.pil_image for asset in image_assets]
40
+
41
+ inputs_per_image = [(
42
+ [prompt for _ in size_factors],
43
+ [rescale_image_size(image, factor) for factor in size_factors],
44
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
45
+
46
+ with vllm_runner(model,
47
+ max_model_len=4096,
48
+ dtype=dtype,
49
+ tensor_parallel_size=tensor_parallel_size,
50
+ distributed_executor_backend=distributed_executor_backend,
51
+ enforce_eager=True) as vllm_model:
52
+
53
+ for prompts, images in inputs_per_image:
54
+ vllm_outputs = vllm_model.generate_greedy(prompts,
55
+ max_tokens,
56
+ images=images)
57
+ for i in range(len(vllm_outputs)):
58
+
59
+ # format prompt back to original
60
+ replacements = {
61
+ "<racm3:break>": "",
62
+ "<eoss>": "",
63
+ "<reserved08706>": ""
64
+ }
65
+ pattern = '|'.join(replacements.keys())
66
+ vllm_result = re.sub(
67
+ pattern,
68
+ lambda match: replacements[match.group(0)], #noqa B023
69
+ vllm_outputs[i][1])
70
+ vllm_result = vllm_result.replace("<image>", "", 1023)
71
+ assert vllm_result[:len(prompts[i])] == prompts[i]
72
+
73
+ # assert at least 10 new characters are generated
74
+ # (to take stop token into account)
75
+ assert len(vllm_outputs[i][1]) - len(prompts[i]) > 10
76
+
77
+
78
+@pytest.mark.parametrize("model", models)
79
+@pytest.mark.parametrize(
80
+ "size_factors",
81
+ [
82
+ # Single-scale
83
+ [1.0],
84
+ # Single-scale, batched
85
+ [1.0, 1.0, 1.0],
86
+ # Multi-scale
87
+ [0.25, 0.5, 1.0],
88
+ ],
89
+)
90
+@pytest.mark.parametrize("dtype", ["bfloat16"])
91
+@pytest.mark.parametrize("max_tokens", [128])
92
+def test_models(vllm_runner, image_assets, model, size_factors, dtype: str,
93
+ max_tokens: int) -> None:
94
+ run_test(
95
+ vllm_runner,
96
+ image_assets,
97
+ model,
98
+ size_factors=size_factors,
99
+ dtype=dtype,
100
+ max_tokens=max_tokens,
101
+ tensor_parallel_size=1,
102
+ )
vllm/entrypoints/chat_utils.py
+2
-1
@@ -105,7 +105,8 @@ def _image_token_str(model_config: ModelConfig,
105
return None
106
if model_type.startswith("llava"):
107
return tokenizer.decode(model_config.hf_config.image_token_index)
108
-
108
+ if model_type == "chameleon":
109
+ return "<image>"
110
raise TypeError("Unknown model type: {model_type}")
111
112
vllm/model_executor/models/__init__.py
+4
-3
@@ -16,9 +16,10 @@ _GENERATION_MODELS = {
16
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19
- "ChameleonForCausalLM":
20
- ("chameleon", "ChameleonForConditionalGeneration"
21
- ), #TODO(ywang96): fix model name when huggingface fixes it
19
+ #TODO(ywang96): remove this when huggingface fixes the model repo
20
+ "ChameleonForCausalLM": ("chameleon", "ChameleonForConditionalGeneration"),
21
+ "ChameleonForConditionalGeneration":
22
+ ("chameleon", "ChameleonForConditionalGeneration"),
23
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
24
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
25
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
vllm/model_executor/models/chameleon.py
+534
-43
@@ -1,13 +1,17 @@
1
from functools import cached_property
2
-from typing import Any, Dict, Iterable, List, Optional, Tuple
2
+from typing import (Any, Dict, Iterable, List, Literal, Optional, Tuple,
3
+ TypedDict)
4
5
import torch
6
import torch.nn.functional as F
7
+from PIL import Image
8
from torch import nn
9
10
from vllm.attention import Attention, AttentionMetadata
9
-from vllm.config import CacheConfig
11
+from vllm.config import CacheConfig, MultiModalConfig
12
from vllm.distributed import get_tensor_model_parallel_world_size
13
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14
+from vllm.logger import init_logger
15
from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.model_executor.layers.layernorm import RMSNorm
17
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -22,10 +26,114 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
26
ParallelLMHead, VocabParallelEmbedding)
27
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
-from vllm.sequence import IntermediateTensors, SamplerOutput
26
-from vllm.transformers_utils.configs import ChameleonConfig
29
+from vllm.multimodal import MULTIMODAL_REGISTRY
30
+from vllm.multimodal.image import (cached_get_tokenizer,
31
+ repeat_and_pad_image_tokens)
32
+from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
33
+from vllm.transformers_utils.configs import (ChameleonConfig,
34
+ ChameleonVQVAEConfig)
35
from vllm.utils import print_warning_once
36
37
+from .interfaces import SupportsVision
38
+
39
+logger = init_logger(__name__)
40
+
41
+# These configs are not part of the model config but the preprocessor
42
+# and processor files, so we hardcode them in the model file for now.
43
+CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
44
+CHAMELEON_IMAGE_SEQ_LENGTH = 1024
45
+CHAMELEON_IMAGE_TOKEN_ID = 8711
46
+CHAMELEON_IMAGE_START_TOKEN_ID = 8197
47
+CHAMELEON_IMAGE_END_TOKEN_ID = 8196
48
+CHAMELEON_SEP_TOKEN_ID = 8710
49
+
50
+
51
+class ChameleonImagePixelInputs(TypedDict):
52
+ type: Literal["pixel_values"]
53
+ data: torch.Tensor
54
+ """Shape: `(batch_size, num_channels, height, width)`"""
55
+
56
+
57
+def get_max_chameleon_image_tokens(ctx: InputContext):
58
+ return CHAMELEON_IMAGE_SEQ_LENGTH
59
+
60
+
61
+def dummy_seq_data_for_chameleon(
62
+ seq_len: int,
63
+ *,
64
+ image_token_id: int,
65
+ image_feature_size_override: Optional[int] = None,
66
+):
67
+ if image_feature_size_override is None:
68
+ image_feature_size = CHAMELEON_IMAGE_SEQ_LENGTH
69
+ else:
70
+ image_feature_size = image_feature_size_override
71
+
72
+ token_ids = [image_token_id] * image_feature_size
73
+ token_ids += [0] * (seq_len - image_feature_size)
74
+ return SequenceData(token_ids)
75
+
76
+
77
+def dummy_image_for_chameleon(
78
+ image_width_override: Optional[int] = None,
79
+ image_height_override: Optional[int] = None,
80
+):
81
+ width = CHAMELEON_CROP_SIZE_WIDTH
82
+ height = CHAMELEON_CROP_SIZE_HEIGHT
83
+ if image_width_override is not None:
84
+ width = image_width_override
85
+ if image_height_override is not None:
86
+ height = image_height_override
87
+
88
+ image = Image.new("RGB", (width, height), color=0)
89
+ return {"image": image}
90
+
91
+
92
+def dummy_data_for_chameleon(ctx: InputContext, seq_len: int):
93
+
94
+ seq_data = dummy_seq_data_for_chameleon(
95
+ seq_len,
96
+ image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
97
+ )
98
+
99
+ mm_data = dummy_image_for_chameleon()
100
+ return seq_data, mm_data
101
+
102
+
103
+def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
104
+
105
+ """
106
+ Processing input prompt to insert required tokens for image placeholder.
107
+
108
+ See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58
109
+ """ # noqa
110
+
111
+ multi_modal_data = llm_inputs.get("multi_modal_data")
112
+ if multi_modal_data is None or "image" not in multi_modal_data:
113
+ return llm_inputs
114
+
115
+ model_config = ctx.model_config
116
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
117
+ new_prompt, new_token_ids = repeat_and_pad_image_tokens(
118
+ tokenizer,
119
+ llm_inputs.get("prompt"),
120
+ llm_inputs["prompt_token_ids"],
121
+ image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
122
+ repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
123
+ pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
124
+ pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID,
125
+ )
126
+
127
+ # Appending sep token for chat mode to follow default processor
128
+ # behavior
129
+ new_prompt += tokenizer.sep_token
130
+ new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
131
+
132
+ # NOTE: Create a defensive copy of the original inputs
133
+ return LLMInputs(prompt_token_ids=new_token_ids,
134
+ prompt=new_prompt,
135
+ multi_modal_data=multi_modal_data)
136
+
137
138
class ChameleonLayerNorm(nn.LayerNorm):
139
@@ -318,12 +426,333 @@ class ChameleonSwinDecoderLayer(nn.Module):
426
return hidden_states, residual
427
428
429
+# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEVectorQuantizer #noqa
430
+class ChameleonVQVAEVectorQuantizer(nn.Module):
431
+
432
+ def __init__(self, config: ChameleonVQVAEConfig):
433
+ super().__init__()
434
+ self.num_embeddings = config.num_embeddings
435
+ self.embedding_dim = config.embed_dim
436
+ self.beta = getattr(config, "beta", 0.25)
437
+
438
+ self.embedding = nn.Embedding(self.num_embeddings, self.embedding_dim)
439
+ self.re_embed = self.num_embeddings
440
+
441
+ def forward(self, hidden_state: torch.Tensor):
442
+ hidden_state = hidden_state.permute(0, 2, 3, 1).contiguous()
443
+ hidden_state_flattened = hidden_state.view(-1, self.embedding_dim)
444
+
445
+ # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
446
+ distances = (
447
+ torch.sum(hidden_state_flattened**2, dim=1, keepdim=True) +
448
+ torch.sum(self.embedding.weight**2, dim=1) -
449
+ 2 * torch.einsum("bd,dn->bn", hidden_state_flattened,
450
+ self.embedding.weight.transpose(0, 1)))
451
+
452
+ min_encoding_indices = torch.argmin(distances, dim=1)
453
+ hidden_state_quant = self.embedding(min_encoding_indices).view(
454
+ hidden_state.shape)
455
+
456
+ # compute loss for embedding
457
+ loss = torch.mean((hidden_state_quant.detach() - hidden_state)**
458
+ 2) + self.beta * torch.mean(
459
+ (hidden_state_quant - hidden_state.detach())**2)
460
+
461
+ # preserve gradients
462
+ hidden_state_quant = hidden_state + (hidden_state_quant -
463
+ hidden_state).detach()
464
+
465
+ # reshape back to match original input shape
466
+ hidden_state_quant = hidden_state_quant.permute(0, 3, 1,
467
+ 2).contiguous()
468
+
469
+ return hidden_state_quant, loss, min_encoding_indices
470
+
471
+
472
+# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderConvDownsample #noqa
473
+class ChameleonVQVAEEncoderConvDownsample(nn.Module):
474
+
475
+ def __init__(self, in_channels: int):
476
+ super().__init__()
477
+ self.conv = nn.Conv2d(in_channels,
478
+ in_channels,
479
+ kernel_size=3,
480
+ stride=2,
481
+ padding=0)
482
+
483
+ def forward(self, hidden_states: torch.Tensor):
484
+ # no asymmetric padding in torch conv, must do it ourselves
485
+ hidden_states = F.pad(hidden_states,
486
+ pad=(0, 1, 0, 1),
487
+ mode="constant",
488
+ value=0)
489
+ hidden_states = self.conv(hidden_states)
490
+ return hidden_states
491
+
492
+
493
+# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderResnetBlock #noqa
494
+class ChameleonVQVAEEncoderResnetBlock(nn.Module):
495
+
496
+ def __init__(
497
+ self,
498
+ config: ChameleonVQVAEConfig,
499
+ in_channels: int,
500
+ out_channels=None,
501
+ conv_shortcut=False,
502
+ ):
503
+ super().__init__()
504
+ self.in_channels = in_channels
505
+ self.out_channels = in_channels if out_channels is None \
506
+ else out_channels
507
+ self.use_conv_shortcut = conv_shortcut
508
+
509
+ self.norm1 = torch.nn.GroupNorm(num_groups=32,
510
+ num_channels=in_channels,
511
+ eps=1e-6,
512
+ affine=True)
513
+ self.conv1 = torch.nn.Conv2d(in_channels,
514
+ out_channels,
515
+ kernel_size=3,
516
+ stride=1,
517
+ padding=1)
518
+ self.norm2 = torch.nn.GroupNorm(num_groups=32,
519
+ num_channels=out_channels,
520
+ eps=1e-6,
521
+ affine=True)
522
+ self.dropout = torch.nn.Dropout(config.dropout)
523
+ self.conv2 = torch.nn.Conv2d(out_channels,
524
+ out_channels,
525
+ kernel_size=3,
526
+ stride=1,
527
+ padding=1)
528
+ if self.in_channels != self.out_channels:
529
+ if self.use_conv_shortcut:
530
+ self.conv_shortcut = torch.nn.Conv2d(in_channels,
531
+ out_channels,
532
+ kernel_size=3,
533
+ stride=1,
534
+ padding=1)
535
+ else:
536
+ self.nin_shortcut = torch.nn.Conv2d(in_channels,
537
+ out_channels,
538
+ kernel_size=1,
539
+ stride=1,
540
+ padding=0)
541
+
542
+ def forward(self, hidden_states: torch.Tensor):
543
+ residual = hidden_states
544
+ hidden_states = self.norm1(hidden_states)
545
+ hidden_states *= torch.sigmoid(hidden_states)
546
+ hidden_states = self.conv1(hidden_states)
547
+
548
+ hidden_states = self.norm2(hidden_states)
549
+ hidden_states *= torch.sigmoid(hidden_states)
550
+ hidden_states = self.dropout(hidden_states)
551
+ hidden_states = self.conv2(hidden_states)
552
+
553
+ if self.in_channels != self.out_channels:
554
+ if self.use_conv_shortcut:
555
+ residual = self.conv_shortcut(residual)
556
+ else:
557
+ residual = self.nin_shortcut(residual)
558
+
559
+ return residual + hidden_states
560
+
561
+
562
+# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderAttnBlock #noqa
563
+class ChameleonVQVAEEncoderAttnBlock(nn.Module):
564
+
565
+ def __init__(self, in_channels: int):
566
+ super().__init__()
567
+ self.in_channels = in_channels
568
+
569
+ self.norm = torch.nn.GroupNorm(num_groups=32,
570
+ num_channels=in_channels,
571
+ eps=1e-6,
572
+ affine=True)
573
+ self.q = torch.nn.Conv2d(in_channels,
574
+ in_channels,
575
+ kernel_size=1,
576
+ stride=1,
577
+ padding=0)
578
+ self.k = torch.nn.Conv2d(in_channels,
579
+ in_channels,
580
+ kernel_size=1,
581
+ stride=1,
582
+ padding=0)
583
+ self.v = torch.nn.Conv2d(in_channels,
584
+ in_channels,
585
+ kernel_size=1,
586
+ stride=1,
587
+ padding=0)
588
+ self.proj_out = torch.nn.Conv2d(in_channels,
589
+ in_channels,
590
+ kernel_size=1,
591
+ stride=1,
592
+ padding=0)
593
+
594
+ def forward(self, hidden_states: torch.Tensor):
595
+ residual = hidden_states
596
+ hidden_states = self.norm(hidden_states)
597
+ query_states = self.q(hidden_states)
598
+ key_states = self.k(hidden_states)
599
+ value_states = self.v(hidden_states)
600
+
601
+ # compute attention
602
+ batch_size, channels, height, width = query_states.shape
603
+ query_states = query_states.reshape(batch_size, channels,
604
+ height * width).permute(0, 2, 1)
605
+ key_states = key_states.reshape(batch_size, channels, height * width)
606
+ attn_weights = torch.bmm(query_states, key_states)
607
+ attn_weights = attn_weights * (int(channels)**(-0.5))
608
+ attn_weights = F.softmax(attn_weights, dim=2)
609
+
610
+ # attend to values
611
+ value_states = value_states.reshape(batch_size, channels,
612
+ height * width)
613
+ attn_weights = attn_weights.permute(0, 2, 1)
614
+ attn_output = torch.bmm(value_states,
615
+ attn_weights).reshape(batch_size, channels,
616
+ height, width)
617
+
618
+ attn_output = self.proj_out(attn_output)
619
+ return residual + attn_output
620
+
621
+
622
+# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoder #noqa
623
+class ChameleonVQVAEEncoder(nn.Module):
624
+
625
+ def __init__(self, config: ChameleonVQVAEConfig):
626
+ super().__init__()
627
+
628
+ self.num_resolutions = len(config.channel_multiplier)
629
+ self.num_res_blocks = config.num_res_blocks
630
+ base_channels = config.base_channels
631
+ resolution = config.resolution
632
+ in_channels = config.in_channels
633
+ double_latent = config.double_latent
634
+ latent_channels = config.latent_channels
635
+ channel_multiplier = config.channel_multiplier
636
+
637
+ self.conv_in = torch.nn.Conv2d(in_channels,
638
+ base_channels,
639
+ kernel_size=3,
640
+ stride=1,
641
+ padding=1)
642
+
643
+ curr_res = resolution
644
+ in_channel_multiplier = (1, ) + tuple(channel_multiplier)
645
+ self.in_channel_multiplier = in_channel_multiplier
646
+ self.down = nn.ModuleList()
647
+ for i_level in range(self.num_resolutions):
648
+ block = nn.ModuleList()
649
+ attn = nn.ModuleList()
650
+ block_in = base_channels * in_channel_multiplier[i_level]
651
+ block_out = base_channels * channel_multiplier[i_level]
652
+ for i_block in range(self.num_res_blocks):
653
+ block.append(
654
+ ChameleonVQVAEEncoderResnetBlock(
655
+ config=config,
656
+ in_channels=block_in,
657
+ out_channels=block_out,
658
+ ))
659
+ block_in = block_out
660
+ if (config.attn_resolutions is not None
661
+ and curr_res in config.attn_resolutions
662
+ and config.attn_type == "vanilla"):
663
+ attn.append(ChameleonVQVAEEncoderAttnBlock(block_in))
664
+
665
+ down = nn.Module()
666
+ down.block = block
667
+ down.attn = attn
668
+ if i_level != self.num_resolutions - 1:
669
+ down.downsample = ChameleonVQVAEEncoderConvDownsample(block_in)
670
+ curr_res = curr_res // 2
671
+ self.down.append(down)
672
+
673
+ self.mid = nn.Module()
674
+ self.mid.block_1 = ChameleonVQVAEEncoderResnetBlock(
675
+ config=config,
676
+ in_channels=block_in,
677
+ out_channels=block_in,
678
+ )
679
+ self.mid.attn_1 = ChameleonVQVAEEncoderAttnBlock(
680
+ block_in) if config.attn_type == "vanilla" else nn.Identity()
681
+ self.mid.block_2 = ChameleonVQVAEEncoderResnetBlock(
682
+ config=config,
683
+ in_channels=block_in,
684
+ out_channels=block_in,
685
+ )
686
+
687
+ self.norm_out = torch.nn.GroupNorm(num_groups=32,
688
+ num_channels=block_in,
689
+ eps=1e-6,
690
+ affine=True)
691
+ self.conv_out = torch.nn.Conv2d(
692
+ block_in,
693
+ 2 * latent_channels if double_latent else latent_channels,
694
+ kernel_size=3,
695
+ stride=1,
696
+ padding=1,
697
+ )
698
+
699
+ def forward(self, pixel_values: torch.Tensor):
700
+ # downsampling
701
+ hidden_states = [self.conv_in(pixel_values)]
702
+ for i_level in range(self.num_resolutions):
703
+ for i_block in range(self.num_res_blocks):
704
+ hidden_state = self.down[i_level].block[i_block](
705
+ hidden_states[-1], )
706
+ if len(self.down[i_level].attn) > 0:
707
+ hidden_state = self.down[i_level].attn[i_block](
708
+ hidden_state)
709
+ hidden_states.append(hidden_state)
710
+ if i_level != self.num_resolutions - 1:
711
+ hidden_states.append(self.down[i_level].downsample(
712
+ hidden_states[-1]))
713
+
714
+ # middle
715
+ last_hidden_state = hidden_states[-1]
716
+ last_hidden_state = self.mid.block_1(last_hidden_state)
717
+ last_hidden_state = self.mid.attn_1(last_hidden_state)
718
+ last_hidden_state = self.mid.block_2(last_hidden_state)
719
+
720
+ # end
721
+ last_hidden_state = self.norm_out(last_hidden_state)
722
+ last_hidden_state *= torch.sigmoid(last_hidden_state)
723
+ last_hidden_state = self.conv_out(last_hidden_state)
724
+ return last_hidden_state
725
+
726
+
727
+# Adapted from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAE #noqa
728
+class ChameleonVQVAE(nn.Module):
729
+
730
+ def __init__(self, config: ChameleonVQVAEConfig):
731
+ super().__init__()
732
+ self.encoder = ChameleonVQVAEEncoder(config)
733
+ self.quantize = ChameleonVQVAEVectorQuantizer(config)
734
+ self.quant_conv = torch.nn.Conv2d(config.latent_channels,
735
+ config.embed_dim, 1)
736
+ self.post_quant_conv = torch.nn.Conv2d(config.embed_dim,
737
+ config.latent_channels, 1)
738
+ self.eval() # Chameleon's VQ model is frozen
739
+
740
+ def encode(
741
+ self, pixel_values: torch.Tensor
742
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
743
+ hidden_states = self.encoder(pixel_values)
744
+ hidden_states = self.quant_conv(hidden_states)
745
+ quant, emb_loss, indices = self.quantize(hidden_states)
746
+ return quant, emb_loss, indices
747
+
748
+
749
+# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonImageVocabularyMapping #noqa
750
class ChameleonImageVocabularyMapping:
751
"""
752
A class for mapping discrete image tokens from VQGAN to BPE tokens.
753
"""
754
326
- def __init__(self, vocab_map):
755
+ def __init__(self, vocab_map: Dict[str, int]):
756
self.vocab_map = vocab_map
757
self.image_token_id = vocab_map.get("<image>")
758
@@ -401,13 +830,23 @@ class ChameleonModel(nn.Module):
830
for _ in range(config.num_hidden_layers)
831
])
832
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
404
-
405
- # TODO: Support image input
406
- # self.vqmodel = ChameleonVQModel(config.vq_config)
833
+ self.vqmodel = ChameleonVQVAE(config.vq_config)
834
835
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
836
return self.embed_tokens(input_ids)
837
838
+ def get_image_tokens(self, pixel_values: torch.Tensor) -> torch.Tensor:
839
+ """
840
+ Tokenizes images into discrete tokens with VQGAN module. Converts
841
+ obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
842
+ special tokens.
843
+ """
844
+ batch_size = pixel_values.shape[0]
845
+ _, _, image_toks = self.vqmodel.encode(pixel_values)
846
+ bpe_toks = self.vocabulary_mapping.convert_img2bpe(image_toks)
847
+ bpe_toks = bpe_toks.view(batch_size, -1)
848
+ return bpe_toks
849
+
850
def forward(
851
self,
852
input_ids: Optional[torch.Tensor],
@@ -434,16 +873,22 @@ class ChameleonModel(nn.Module):
873
return hidden_states
874
875
437
-class ChameleonForConditionalGeneration(nn.Module):
876
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
877
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
878
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon)
879
+@INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon)
880
+class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
881
882
def __init__(
883
self,
884
config: ChameleonConfig,
885
+ multimodal_config: MultiModalConfig,
886
cache_config: Optional[CacheConfig] = None,
887
quant_config: Optional[QuantizationConfig] = None,
888
) -> None:
889
super().__init__()
890
self.config = config
891
+ self.multimodal_config = multimodal_config
892
self.model = ChameleonModel(config, cache_config, quant_config)
893
self.unpadded_vocab_size = config.vocab_size
894
self.lm_head = ParallelLMHead(
@@ -458,6 +903,36 @@ class ChameleonForConditionalGeneration(nn.Module):
903
config.vocab_size, logit_scale)
904
self.sampler = Sampler()
905
906
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
907
+
908
+ expected_dims = (3, CHAMELEON_CROP_SIZE_HEIGHT,
909
+ CHAMELEON_CROP_SIZE_WIDTH)
910
+ actual_dims = tuple(data.shape[1:])
911
+
912
+ if actual_dims != expected_dims:
913
+ expected_expr = ("batch_size", *map(str, expected_dims))
914
+ raise ValueError(
915
+ f"The expected shape of pixel values is {expected_expr}. "
916
+ f"You supplied {tuple(data.shape)}.")
917
+
918
+ return data
919
+
920
+ def _parse_and_validate_image_input(
921
+ self, **kwargs: object) -> Optional[ChameleonImagePixelInputs]:
922
+ pixel_values = kwargs.pop("pixel_values", None)
923
+
924
+ if pixel_values is None:
925
+ return None
926
+
927
+ if not isinstance(pixel_values, torch.Tensor):
928
+ raise ValueError("Incorrect type of pixel values. "
929
+ f"Got type: {type(pixel_values)}")
930
+
931
+ return ChameleonImagePixelInputs(
932
+ type="pixel_values",
933
+ data=self._validate_pixel_values(pixel_values),
934
+ )
935
+
936
def forward(
937
self,
938
input_ids: torch.Tensor,
@@ -468,10 +943,17 @@ class ChameleonForConditionalGeneration(nn.Module):
943
**kwargs,
944
) -> torch.Tensor:
945
471
- # TODO (ywang96): Support image input
472
- # image_tokens = self.process_image_input(**kwargs)
473
- # image_mask = input_ids == self.vocabulary_mapping.image_token_id
474
- # input_ids[special_image_mask] = image_tokens.flatten().to(input_ids.dtype) #noqa
946
+ image_input = self._parse_and_validate_image_input(**kwargs)
947
+
948
+ if image_input is not None:
949
+ assert self.model.vqmodel is not None
950
+ image_tokens = self.model.get_image_tokens(image_input["data"].to(
951
+ self.config.torch_dtype))
952
+ image_token_id = self.model.vocabulary_mapping.image_token_id
953
+ special_image_mask = input_ids == image_token_id
954
+ image_tokens = image_tokens.to(input_ids.device, input_ids.dtype)
955
+ input_ids = input_ids.masked_scatter(special_image_mask,
956
+ image_tokens)
957
958
hidden_states = self.model(input_ids, positions, kv_caches,
959
attn_metadata)
@@ -511,43 +993,52 @@ class ChameleonForConditionalGeneration(nn.Module):
993
if "rotary_emb.inv_freq" in name:
994
continue
995
514
- # Skip loading vqgan
515
- # TODO: add support for the vision model
516
- if "vqmodel" in name:
517
- continue
996
if ("rotary_emb.cos_cached" in name
997
or "rotary_emb.sin_cached" in name):
998
# Models trained using ColossalAI may include these tensors in
999
# the checkpoint. Skip them.
1000
continue
523
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
524
- if weight_name not in name:
525
- continue
526
- name = name.replace(weight_name, param_name)
527
- # Skip loading extra bias for GPTQ models.
528
- if name.endswith(".bias") and name not in params_dict:
529
- continue
530
- param = params_dict[name]
531
- weight_loader = param.weight_loader
532
- weight_loader(param, loaded_weight, shard_id)
533
- break
1001
+ use_default_weight_loading = False
1002
+ if "vqmodel" in name:
1003
+ if self.model.vqmodel is not None:
1004
+ # We only do sharding for language model and
1005
+ # not vqvae for now.
1006
+ use_default_weight_loading = True
1007
else:
535
- # Skip loading extra bias for GPTQ models.
536
- if name.endswith(".bias") and name not in params_dict:
537
- continue
538
- # Remapping the name of FP8 kv-scale.
539
- if name.endswith("kv_scale"):
540
- remapped_kv_scale_name = name.replace(
541
- ".kv_scale", ".attn.kv_scale")
542
- if remapped_kv_scale_name not in params_dict:
543
- print_warning_once(
544
- f"Found kv scale in the checkpoint (e.g. {name}), "
545
- "but not found the expected name in the model "
546
- f"(e.g. {remapped_kv_scale_name}). kv-scale is "
547
- "not loaded.")
1008
+ for (param_name, weight_name,
1009
+ shard_id) in stacked_params_mapping:
1010
+ if weight_name not in name:
1011
+ continue
1012
+ name = name.replace(weight_name, param_name)
1013
+ # Skip loading extra bias for GPTQ models.
1014
+ if name.endswith(".bias") and name not in params_dict:
1015
+ continue
1016
+ param = params_dict[name]
1017
+ weight_loader = param.weight_loader
1018
+ weight_loader(param, loaded_weight, shard_id)
1019
+ break
1020
+ else:
1021
+ # Skip loading extra bias for GPTQ models.
1022
+ if name.endswith(".bias") and name not in params_dict:
1023
continue
549
- else:
550
- name = remapped_kv_scale_name
1024
+ # Remapping the name of FP8 kv-scale.
1025
+ if name.endswith("kv_scale"):
1026
+ remapped_kv_scale_name = name.replace(
1027
+ ".kv_scale", ".attn.kv_scale")
1028
+ if remapped_kv_scale_name not in params_dict:
1029
+ print_warning_once(
1030
+ "Found kv scale in the checkpoint (e.g. "
1031
+ f"{name}), but not found the expected name in "
1032
+ f"the model (e.g. {remapped_kv_scale_name}). "
1033
+ "kv-scale is not loaded.")
1034
+ continue
1035
+ else:
1036
+ name = remapped_kv_scale_name
1037
+ param = params_dict[name]
1038
+ weight_loader = getattr(param, "weight_loader",
1039
+ default_weight_loader)
1040
+ weight_loader(param, loaded_weight)
1041
+ if use_default_weight_loading and name in params_dict:
1042
param = params_dict[name]
1043
weight_loader = getattr(param, "weight_loader",
1044
default_weight_loader)
vllm/transformers_utils/configs/__init__.py
+3
-1
@@ -1,4 +1,5 @@
1
-from vllm.transformers_utils.configs.chameleon import ChameleonConfig
1
+from vllm.transformers_utils.configs.chameleon import (ChameleonConfig,
2
+ ChameleonVQVAEConfig)
3
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4
from vllm.transformers_utils.configs.dbrx import DbrxConfig
5
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -12,6 +13,7 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
13
14
__all__ = [
15
"ChameleonConfig",
16
+ "ChameleonVQVAEConfig",
17
"ChatGLMConfig",
18
"DbrxConfig",
19
"MPTConfig",
vllm/transformers_utils/configs/chameleon.py
+47
-10
@@ -1,3 +1,5 @@
1
+from typing import List, Optional
2
+
3
from transformers import PretrainedConfig
4
5
@@ -5,9 +7,7 @@ from transformers import PretrainedConfig
7
# transformers once the new release with Chameleon support
8
# is available.
9
class ChameleonConfig(PretrainedConfig):
8
-
10
model_type = "chameleon"
10
- is_composition = True
11
keys_to_ignore_at_inference = ["past_key_values"]
12
13
def __init__(
@@ -31,7 +31,7 @@ class ChameleonConfig(PretrainedConfig):
31
rope_scaling=None,
32
attention_bias=False,
33
attention_dropout=0.0,
34
- qk_layernorm=False,
34
+ model_parallel_size=1,
35
swin_norm=False,
36
vq_config=None,
37
vocabulary_map=None,
@@ -46,10 +46,6 @@ class ChameleonConfig(PretrainedConfig):
46
self.num_attention_heads = num_attention_heads
47
self.mlp_bias = mlp_bias
48
49
- # for backward compatibility
50
- if num_key_value_heads is None:
51
- num_key_value_heads = num_attention_heads
52
-
49
self.num_key_value_heads = num_key_value_heads
50
self.hidden_act = hidden_act
51
self.initializer_range = initializer_range
@@ -60,10 +56,14 @@ class ChameleonConfig(PretrainedConfig):
56
self._rope_scaling_validation()
57
self.attention_bias = attention_bias
58
self.attention_dropout = attention_dropout
63
- self.qk_layernorm = qk_layernorm
59
+ self.model_parallel_size = model_parallel_size
60
self.swin_norm = swin_norm
65
- # vq config is currently ignored
66
- # self.vq_config = ChameleonVQConfig(**vq_config)
61
+
62
+ if vq_config is None:
63
+ vq_config = {}
64
+
65
+ self.vq_config = ChameleonVQVAEConfig(**vq_config)
66
+
67
self.vocabulary_map = vocabulary_map
68
69
super().__init__(
@@ -99,3 +99,40 @@ class ChameleonConfig(PretrainedConfig):
99
raise ValueError(
100
"`rope_scaling`'s factor field must be a float > 1, "
101
f"got {rope_scaling_factor}")
102
+
103
+
104
+class ChameleonVQVAEConfig(PretrainedConfig):
105
+
106
+ model_type = "chameleon_vqgan"
107
+
108
+ def __init__(
109
+ self,
110
+ embed_dim: int = 256,
111
+ num_embeddings: int = 8192,
112
+ double_latent: bool = False,
113
+ latent_channels: int = 256,
114
+ resolution: int = 512,
115
+ in_channels: int = 3,
116
+ base_channels: int = 128,
117
+ channel_multiplier: List[int] = [1, 1, 2, 2, 4], #noqa
118
+ num_res_blocks: int = 2,
119
+ attn_resolutions: Optional[List[int]] = None,
120
+ dropout: float = 0.0,
121
+ attn_type: str = "vanilla",
122
+ initializer_range=0.02,
123
+ **kwargs,
124
+ ):
125
+ super().__init__(**kwargs)
126
+ self.embed_dim = embed_dim
127
+ self.num_embeddings = num_embeddings
128
+ self.double_latent = double_latent
129
+ self.latent_channels = latent_channels
130
+ self.resolution = resolution
131
+ self.in_channels = in_channels
132
+ self.base_channels = base_channels
133
+ self.channel_multiplier = channel_multiplier
134
+ self.num_res_blocks = num_res_blocks
135
+ self.attn_resolutions = attn_resolutions
136
+ self.dropout = dropout
137
+ self.attn_type = attn_type
138
+ self.initializer_range = initializer_range