@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 22fa2e35cb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[VLM][Model] Support image input for Chameleon (#6633)

Roger Wang committed Jul 22, 2024 at 23:50 UTC 22fa2e35cbbb76fde94412f565572d290027a3c3
7 files changed +696 -58
docs/source/models/supported_models.rst
+4
@@ -182,6 +182,10 @@ Vision Language Models
182 - Models
183 - Example HuggingFace Models
184 - :ref:`LoRA <lora>`
185 + * - :code:`ChameleonForConditionalGeneration`
186 + - Chameleon
187 + - :code:`facebook/chameleon-7b` etc.
188 + -
189 * - :code:`FuyuForCausalLM`
190 - Fuyu
191 - :code:`adept/fuyu-8b` etc.
tests/models/test_chameleon.py new
+102
@@ -0,0 +1,102 @@
1 +import re
2 +from typing import List, Optional, Type
3 +
4 +import pytest
5 +
6 +from vllm.multimodal.utils import rescale_image_size
7 +
8 +from ..conftest import IMAGE_ASSETS, VllmRunner, _ImageAssets
9 +
10 +pytestmark = pytest.mark.vlm
11 +
12 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13 + "stop_sign":
14 + "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
15 + "cherry_blossom":
16 + "USER: <image>\nWhat is the season?\nASSISTANT:",
17 +})
18 +
19 +models = ["facebook/chameleon-7b"]
20 +
21 +
22 +#TODO (ywang96): Add correctness test when chameleon is
23 +# available on transformers.
24 +def run_test(
25 + vllm_runner: Type[VllmRunner],
26 + image_assets: _ImageAssets,
27 + model: str,
28 + *,
29 + size_factors: List[float],
30 + dtype: str,
31 + max_tokens: int,
32 + tensor_parallel_size: int,
33 + distributed_executor_backend: Optional[str] = None,
34 +):
35 + """Test if the model can generate text given
36 + a batch of images and prompts.
37 +
38 + """
39 + images = [asset.pil_image for asset in image_assets]
40 +
41 + inputs_per_image = [(
42 + [prompt for _ in size_factors],
43 + [rescale_image_size(image, factor) for factor in size_factors],
44 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
45 +
46 + with vllm_runner(model,
47 + max_model_len=4096,
48 + dtype=dtype,
49 + tensor_parallel_size=tensor_parallel_size,
50 + distributed_executor_backend=distributed_executor_backend,
51 + enforce_eager=True) as vllm_model:
52 +
53 + for prompts, images in inputs_per_image:
54 + vllm_outputs = vllm_model.generate_greedy(prompts,
55 + max_tokens,
56 + images=images)
57 + for i in range(len(vllm_outputs)):
58 +
59 + # format prompt back to original
60 + replacements = {
61 + "<racm3:break>": "",
62 + "<eoss>": "",
63 + "<reserved08706>": ""
64 + }
65 + pattern = '|'.join(replacements.keys())
66 + vllm_result = re.sub(
67 + pattern,
68 + lambda match: replacements[match.group(0)], #noqa B023
69 + vllm_outputs[i][1])
70 + vllm_result = vllm_result.replace("<image>", "", 1023)
71 + assert vllm_result[:len(prompts[i])] == prompts[i]
72 +
73 + # assert at least 10 new characters are generated
74 + # (to take stop token into account)
75 + assert len(vllm_outputs[i][1]) - len(prompts[i]) > 10
76 +
77 +
78 +@pytest.mark.parametrize("model", models)
79 +@pytest.mark.parametrize(
80 + "size_factors",
81 + [
82 + # Single-scale
83 + [1.0],
84 + # Single-scale, batched
85 + [1.0, 1.0, 1.0],
86 + # Multi-scale
87 + [0.25, 0.5, 1.0],
88 + ],
89 +)
90 +@pytest.mark.parametrize("dtype", ["bfloat16"])
91 +@pytest.mark.parametrize("max_tokens", [128])
92 +def test_models(vllm_runner, image_assets, model, size_factors, dtype: str,
93 + max_tokens: int) -> None:
94 + run_test(
95 + vllm_runner,
96 + image_assets,
97 + model,
98 + size_factors=size_factors,
99 + dtype=dtype,
100 + max_tokens=max_tokens,
101 + tensor_parallel_size=1,
102 + )
vllm/entrypoints/chat_utils.py
+2 -1
@@ -105,7 +105,8 @@ def _image_token_str(model_config: ModelConfig,
105 return None
106 if model_type.startswith("llava"):
107 return tokenizer.decode(model_config.hf_config.image_token_index)
108 -
108 + if model_type == "chameleon":
109 + return "<image>"
110 raise TypeError("Unknown model type: {model_type}")
111
112
vllm/model_executor/models/__init__.py
+4 -3
@@ -16,9 +16,10 @@ _GENERATION_MODELS = {
16 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
17 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
18 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
19 - "ChameleonForCausalLM":
20 - ("chameleon", "ChameleonForConditionalGeneration"
21 - ), #TODO(ywang96): fix model name when huggingface fixes it
19 + #TODO(ywang96): remove this when huggingface fixes the model repo
20 + "ChameleonForCausalLM": ("chameleon", "ChameleonForConditionalGeneration"),
21 + "ChameleonForConditionalGeneration":
22 + ("chameleon", "ChameleonForConditionalGeneration"),
23 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
24 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
25 "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
vllm/model_executor/models/chameleon.py
+534 -43
@@ -1,13 +1,17 @@
1 from functools import cached_property
2 -from typing import Any, Dict, Iterable, List, Optional, Tuple
2 +from typing import (Any, Dict, Iterable, List, Literal, Optional, Tuple,
3 + TypedDict)
4
5 import torch
6 import torch.nn.functional as F
7 +from PIL import Image
8 from torch import nn
9
10 from vllm.attention import Attention, AttentionMetadata
9 -from vllm.config import CacheConfig
11 +from vllm.config import CacheConfig, MultiModalConfig
12 from vllm.distributed import get_tensor_model_parallel_world_size
13 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14 +from vllm.logger import init_logger
15 from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.model_executor.layers.layernorm import RMSNorm
17 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -22,10 +26,114 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
26 ParallelLMHead, VocabParallelEmbedding)
27 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 -from vllm.sequence import IntermediateTensors, SamplerOutput
26 -from vllm.transformers_utils.configs import ChameleonConfig
29 +from vllm.multimodal import MULTIMODAL_REGISTRY
30 +from vllm.multimodal.image import (cached_get_tokenizer,
31 + repeat_and_pad_image_tokens)
32 +from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
33 +from vllm.transformers_utils.configs import (ChameleonConfig,
34 + ChameleonVQVAEConfig)
35 from vllm.utils import print_warning_once
36
37 +from .interfaces import SupportsVision
38 +
39 +logger = init_logger(__name__)
40 +
41 +# These configs are not part of the model config but the preprocessor
42 +# and processor files, so we hardcode them in the model file for now.
43 +CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
44 +CHAMELEON_IMAGE_SEQ_LENGTH = 1024
45 +CHAMELEON_IMAGE_TOKEN_ID = 8711
46 +CHAMELEON_IMAGE_START_TOKEN_ID = 8197
47 +CHAMELEON_IMAGE_END_TOKEN_ID = 8196
48 +CHAMELEON_SEP_TOKEN_ID = 8710
49 +
50 +
51 +class ChameleonImagePixelInputs(TypedDict):
52 + type: Literal["pixel_values"]
53 + data: torch.Tensor
54 + """Shape: `(batch_size, num_channels, height, width)`"""
55 +
56 +
57 +def get_max_chameleon_image_tokens(ctx: InputContext):
58 + return CHAMELEON_IMAGE_SEQ_LENGTH
59 +
60 +
61 +def dummy_seq_data_for_chameleon(
62 + seq_len: int,
63 + *,
64 + image_token_id: int,
65 + image_feature_size_override: Optional[int] = None,
66 +):
67 + if image_feature_size_override is None:
68 + image_feature_size = CHAMELEON_IMAGE_SEQ_LENGTH
69 + else:
70 + image_feature_size = image_feature_size_override
71 +
72 + token_ids = [image_token_id] * image_feature_size
73 + token_ids += [0] * (seq_len - image_feature_size)
74 + return SequenceData(token_ids)
75 +
76 +
77 +def dummy_image_for_chameleon(
78 + image_width_override: Optional[int] = None,
79 + image_height_override: Optional[int] = None,
80 +):
81 + width = CHAMELEON_CROP_SIZE_WIDTH
82 + height = CHAMELEON_CROP_SIZE_HEIGHT
83 + if image_width_override is not None:
84 + width = image_width_override
85 + if image_height_override is not None:
86 + height = image_height_override
87 +
88 + image = Image.new("RGB", (width, height), color=0)
89 + return {"image": image}
90 +
91 +
92 +def dummy_data_for_chameleon(ctx: InputContext, seq_len: int):
93 +
94 + seq_data = dummy_seq_data_for_chameleon(
95 + seq_len,
96 + image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
97 + )
98 +
99 + mm_data = dummy_image_for_chameleon()
100 + return seq_data, mm_data
101 +
102 +
103 +def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
104 +
105 + """
106 + Processing input prompt to insert required tokens for image placeholder.
107 +
108 + See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58
109 + """ # noqa
110 +
111 + multi_modal_data = llm_inputs.get("multi_modal_data")
112 + if multi_modal_data is None or "image" not in multi_modal_data:
113 + return llm_inputs
114 +
115 + model_config = ctx.model_config
116 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
117 + new_prompt, new_token_ids = repeat_and_pad_image_tokens(
118 + tokenizer,
119 + llm_inputs.get("prompt"),
120 + llm_inputs["prompt_token_ids"],
121 + image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
122 + repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
123 + pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
124 + pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID,
125 + )
126 +
127 + # Appending sep token for chat mode to follow default processor
128 + # behavior
129 + new_prompt += tokenizer.sep_token
130 + new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
131 +
132 + # NOTE: Create a defensive copy of the original inputs
133 + return LLMInputs(prompt_token_ids=new_token_ids,
134 + prompt=new_prompt,
135 + multi_modal_data=multi_modal_data)
136 +
137
138 class ChameleonLayerNorm(nn.LayerNorm):
139
@@ -318,12 +426,333 @@ class ChameleonSwinDecoderLayer(nn.Module):
426 return hidden_states, residual
427
428
429 +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEVectorQuantizer #noqa
430 +class ChameleonVQVAEVectorQuantizer(nn.Module):
431 +
432 + def __init__(self, config: ChameleonVQVAEConfig):
433 + super().__init__()
434 + self.num_embeddings = config.num_embeddings
435 + self.embedding_dim = config.embed_dim
436 + self.beta = getattr(config, "beta", 0.25)
437 +
438 + self.embedding = nn.Embedding(self.num_embeddings, self.embedding_dim)
439 + self.re_embed = self.num_embeddings
440 +
441 + def forward(self, hidden_state: torch.Tensor):
442 + hidden_state = hidden_state.permute(0, 2, 3, 1).contiguous()
443 + hidden_state_flattened = hidden_state.view(-1, self.embedding_dim)
444 +
445 + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
446 + distances = (
447 + torch.sum(hidden_state_flattened**2, dim=1, keepdim=True) +
448 + torch.sum(self.embedding.weight**2, dim=1) -
449 + 2 * torch.einsum("bd,dn->bn", hidden_state_flattened,
450 + self.embedding.weight.transpose(0, 1)))
451 +
452 + min_encoding_indices = torch.argmin(distances, dim=1)
453 + hidden_state_quant = self.embedding(min_encoding_indices).view(
454 + hidden_state.shape)
455 +
456 + # compute loss for embedding
457 + loss = torch.mean((hidden_state_quant.detach() - hidden_state)**
458 + 2) + self.beta * torch.mean(
459 + (hidden_state_quant - hidden_state.detach())**2)
460 +
461 + # preserve gradients
462 + hidden_state_quant = hidden_state + (hidden_state_quant -
463 + hidden_state).detach()
464 +
465 + # reshape back to match original input shape
466 + hidden_state_quant = hidden_state_quant.permute(0, 3, 1,
467 + 2).contiguous()
468 +
469 + return hidden_state_quant, loss, min_encoding_indices
470 +
471 +
472 +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderConvDownsample #noqa
473 +class ChameleonVQVAEEncoderConvDownsample(nn.Module):
474 +
475 + def __init__(self, in_channels: int):
476 + super().__init__()
477 + self.conv = nn.Conv2d(in_channels,
478 + in_channels,
479 + kernel_size=3,
480 + stride=2,
481 + padding=0)
482 +
483 + def forward(self, hidden_states: torch.Tensor):
484 + # no asymmetric padding in torch conv, must do it ourselves
485 + hidden_states = F.pad(hidden_states,
486 + pad=(0, 1, 0, 1),
487 + mode="constant",
488 + value=0)
489 + hidden_states = self.conv(hidden_states)
490 + return hidden_states
491 +
492 +
493 +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderResnetBlock #noqa
494 +class ChameleonVQVAEEncoderResnetBlock(nn.Module):
495 +
496 + def __init__(
497 + self,
498 + config: ChameleonVQVAEConfig,
499 + in_channels: int,
500 + out_channels=None,
501 + conv_shortcut=False,
502 + ):
503 + super().__init__()
504 + self.in_channels = in_channels
505 + self.out_channels = in_channels if out_channels is None \
506 + else out_channels
507 + self.use_conv_shortcut = conv_shortcut
508 +
509 + self.norm1 = torch.nn.GroupNorm(num_groups=32,
510 + num_channels=in_channels,
511 + eps=1e-6,
512 + affine=True)
513 + self.conv1 = torch.nn.Conv2d(in_channels,
514 + out_channels,
515 + kernel_size=3,
516 + stride=1,
517 + padding=1)
518 + self.norm2 = torch.nn.GroupNorm(num_groups=32,
519 + num_channels=out_channels,
520 + eps=1e-6,
521 + affine=True)
522 + self.dropout = torch.nn.Dropout(config.dropout)
523 + self.conv2 = torch.nn.Conv2d(out_channels,
524 + out_channels,
525 + kernel_size=3,
526 + stride=1,
527 + padding=1)
528 + if self.in_channels != self.out_channels:
529 + if self.use_conv_shortcut:
530 + self.conv_shortcut = torch.nn.Conv2d(in_channels,
531 + out_channels,
532 + kernel_size=3,
533 + stride=1,
534 + padding=1)
535 + else:
536 + self.nin_shortcut = torch.nn.Conv2d(in_channels,
537 + out_channels,
538 + kernel_size=1,
539 + stride=1,
540 + padding=0)
541 +
542 + def forward(self, hidden_states: torch.Tensor):
543 + residual = hidden_states
544 + hidden_states = self.norm1(hidden_states)
545 + hidden_states *= torch.sigmoid(hidden_states)
546 + hidden_states = self.conv1(hidden_states)
547 +
548 + hidden_states = self.norm2(hidden_states)
549 + hidden_states *= torch.sigmoid(hidden_states)
550 + hidden_states = self.dropout(hidden_states)
551 + hidden_states = self.conv2(hidden_states)
552 +
553 + if self.in_channels != self.out_channels:
554 + if self.use_conv_shortcut:
555 + residual = self.conv_shortcut(residual)
556 + else:
557 + residual = self.nin_shortcut(residual)
558 +
559 + return residual + hidden_states
560 +
561 +
562 +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderAttnBlock #noqa
563 +class ChameleonVQVAEEncoderAttnBlock(nn.Module):
564 +
565 + def __init__(self, in_channels: int):
566 + super().__init__()
567 + self.in_channels = in_channels
568 +
569 + self.norm = torch.nn.GroupNorm(num_groups=32,
570 + num_channels=in_channels,
571 + eps=1e-6,
572 + affine=True)
573 + self.q = torch.nn.Conv2d(in_channels,
574 + in_channels,
575 + kernel_size=1,
576 + stride=1,
577 + padding=0)
578 + self.k = torch.nn.Conv2d(in_channels,
579 + in_channels,
580 + kernel_size=1,
581 + stride=1,
582 + padding=0)
583 + self.v = torch.nn.Conv2d(in_channels,
584 + in_channels,
585 + kernel_size=1,
586 + stride=1,
587 + padding=0)
588 + self.proj_out = torch.nn.Conv2d(in_channels,
589 + in_channels,
590 + kernel_size=1,
591 + stride=1,
592 + padding=0)
593 +
594 + def forward(self, hidden_states: torch.Tensor):
595 + residual = hidden_states
596 + hidden_states = self.norm(hidden_states)
597 + query_states = self.q(hidden_states)
598 + key_states = self.k(hidden_states)
599 + value_states = self.v(hidden_states)
600 +
601 + # compute attention
602 + batch_size, channels, height, width = query_states.shape
603 + query_states = query_states.reshape(batch_size, channels,
604 + height * width).permute(0, 2, 1)
605 + key_states = key_states.reshape(batch_size, channels, height * width)
606 + attn_weights = torch.bmm(query_states, key_states)
607 + attn_weights = attn_weights * (int(channels)**(-0.5))
608 + attn_weights = F.softmax(attn_weights, dim=2)
609 +
610 + # attend to values
611 + value_states = value_states.reshape(batch_size, channels,
612 + height * width)
613 + attn_weights = attn_weights.permute(0, 2, 1)
614 + attn_output = torch.bmm(value_states,
615 + attn_weights).reshape(batch_size, channels,
616 + height, width)
617 +
618 + attn_output = self.proj_out(attn_output)
619 + return residual + attn_output
620 +
621 +
622 +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoder #noqa
623 +class ChameleonVQVAEEncoder(nn.Module):
624 +
625 + def __init__(self, config: ChameleonVQVAEConfig):
626 + super().__init__()
627 +
628 + self.num_resolutions = len(config.channel_multiplier)
629 + self.num_res_blocks = config.num_res_blocks
630 + base_channels = config.base_channels
631 + resolution = config.resolution
632 + in_channels = config.in_channels
633 + double_latent = config.double_latent
634 + latent_channels = config.latent_channels
635 + channel_multiplier = config.channel_multiplier
636 +
637 + self.conv_in = torch.nn.Conv2d(in_channels,
638 + base_channels,
639 + kernel_size=3,
640 + stride=1,
641 + padding=1)
642 +
643 + curr_res = resolution
644 + in_channel_multiplier = (1, ) + tuple(channel_multiplier)
645 + self.in_channel_multiplier = in_channel_multiplier
646 + self.down = nn.ModuleList()
647 + for i_level in range(self.num_resolutions):
648 + block = nn.ModuleList()
649 + attn = nn.ModuleList()
650 + block_in = base_channels * in_channel_multiplier[i_level]
651 + block_out = base_channels * channel_multiplier[i_level]
652 + for i_block in range(self.num_res_blocks):
653 + block.append(
654 + ChameleonVQVAEEncoderResnetBlock(
655 + config=config,
656 + in_channels=block_in,
657 + out_channels=block_out,
658 + ))
659 + block_in = block_out
660 + if (config.attn_resolutions is not None
661 + and curr_res in config.attn_resolutions
662 + and config.attn_type == "vanilla"):
663 + attn.append(ChameleonVQVAEEncoderAttnBlock(block_in))
664 +
665 + down = nn.Module()
666 + down.block = block
667 + down.attn = attn
668 + if i_level != self.num_resolutions - 1:
669 + down.downsample = ChameleonVQVAEEncoderConvDownsample(block_in)
670 + curr_res = curr_res // 2
671 + self.down.append(down)
672 +
673 + self.mid = nn.Module()
674 + self.mid.block_1 = ChameleonVQVAEEncoderResnetBlock(
675 + config=config,
676 + in_channels=block_in,
677 + out_channels=block_in,
678 + )
679 + self.mid.attn_1 = ChameleonVQVAEEncoderAttnBlock(
680 + block_in) if config.attn_type == "vanilla" else nn.Identity()
681 + self.mid.block_2 = ChameleonVQVAEEncoderResnetBlock(
682 + config=config,
683 + in_channels=block_in,
684 + out_channels=block_in,
685 + )
686 +
687 + self.norm_out = torch.nn.GroupNorm(num_groups=32,
688 + num_channels=block_in,
689 + eps=1e-6,
690 + affine=True)
691 + self.conv_out = torch.nn.Conv2d(
692 + block_in,
693 + 2 * latent_channels if double_latent else latent_channels,
694 + kernel_size=3,
695 + stride=1,
696 + padding=1,
697 + )
698 +
699 + def forward(self, pixel_values: torch.Tensor):
700 + # downsampling
701 + hidden_states = [self.conv_in(pixel_values)]
702 + for i_level in range(self.num_resolutions):
703 + for i_block in range(self.num_res_blocks):
704 + hidden_state = self.down[i_level].block[i_block](
705 + hidden_states[-1], )
706 + if len(self.down[i_level].attn) > 0:
707 + hidden_state = self.down[i_level].attn[i_block](
708 + hidden_state)
709 + hidden_states.append(hidden_state)
710 + if i_level != self.num_resolutions - 1:
711 + hidden_states.append(self.down[i_level].downsample(
712 + hidden_states[-1]))
713 +
714 + # middle
715 + last_hidden_state = hidden_states[-1]
716 + last_hidden_state = self.mid.block_1(last_hidden_state)
717 + last_hidden_state = self.mid.attn_1(last_hidden_state)
718 + last_hidden_state = self.mid.block_2(last_hidden_state)
719 +
720 + # end
721 + last_hidden_state = self.norm_out(last_hidden_state)
722 + last_hidden_state *= torch.sigmoid(last_hidden_state)
723 + last_hidden_state = self.conv_out(last_hidden_state)
724 + return last_hidden_state
725 +
726 +
727 +# Adapted from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAE #noqa
728 +class ChameleonVQVAE(nn.Module):
729 +
730 + def __init__(self, config: ChameleonVQVAEConfig):
731 + super().__init__()
732 + self.encoder = ChameleonVQVAEEncoder(config)
733 + self.quantize = ChameleonVQVAEVectorQuantizer(config)
734 + self.quant_conv = torch.nn.Conv2d(config.latent_channels,
735 + config.embed_dim, 1)
736 + self.post_quant_conv = torch.nn.Conv2d(config.embed_dim,
737 + config.latent_channels, 1)
738 + self.eval() # Chameleon's VQ model is frozen
739 +
740 + def encode(
741 + self, pixel_values: torch.Tensor
742 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
743 + hidden_states = self.encoder(pixel_values)
744 + hidden_states = self.quant_conv(hidden_states)
745 + quant, emb_loss, indices = self.quantize(hidden_states)
746 + return quant, emb_loss, indices
747 +
748 +
749 +# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonImageVocabularyMapping #noqa
750 class ChameleonImageVocabularyMapping:
751 """
752 A class for mapping discrete image tokens from VQGAN to BPE tokens.
753 """
754
326 - def __init__(self, vocab_map):
755 + def __init__(self, vocab_map: Dict[str, int]):
756 self.vocab_map = vocab_map
757 self.image_token_id = vocab_map.get("<image>")
758
@@ -401,13 +830,23 @@ class ChameleonModel(nn.Module):
830 for _ in range(config.num_hidden_layers)
831 ])
832 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
404 -
405 - # TODO: Support image input
406 - # self.vqmodel = ChameleonVQModel(config.vq_config)
833 + self.vqmodel = ChameleonVQVAE(config.vq_config)
834
835 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
836 return self.embed_tokens(input_ids)
837
838 + def get_image_tokens(self, pixel_values: torch.Tensor) -> torch.Tensor:
839 + """
840 + Tokenizes images into discrete tokens with VQGAN module. Converts
841 + obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
842 + special tokens.
843 + """
844 + batch_size = pixel_values.shape[0]
845 + _, _, image_toks = self.vqmodel.encode(pixel_values)
846 + bpe_toks = self.vocabulary_mapping.convert_img2bpe(image_toks)
847 + bpe_toks = bpe_toks.view(batch_size, -1)
848 + return bpe_toks
849 +
850 def forward(
851 self,
852 input_ids: Optional[torch.Tensor],
@@ -434,16 +873,22 @@ class ChameleonModel(nn.Module):
873 return hidden_states
874
875
437 -class ChameleonForConditionalGeneration(nn.Module):
876 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
877 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
878 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon)
879 +@INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon)
880 +class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
881
882 def __init__(
883 self,
884 config: ChameleonConfig,
885 + multimodal_config: MultiModalConfig,
886 cache_config: Optional[CacheConfig] = None,
887 quant_config: Optional[QuantizationConfig] = None,
888 ) -> None:
889 super().__init__()
890 self.config = config
891 + self.multimodal_config = multimodal_config
892 self.model = ChameleonModel(config, cache_config, quant_config)
893 self.unpadded_vocab_size = config.vocab_size
894 self.lm_head = ParallelLMHead(
@@ -458,6 +903,36 @@ class ChameleonForConditionalGeneration(nn.Module):
903 config.vocab_size, logit_scale)
904 self.sampler = Sampler()
905
906 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
907 +
908 + expected_dims = (3, CHAMELEON_CROP_SIZE_HEIGHT,
909 + CHAMELEON_CROP_SIZE_WIDTH)
910 + actual_dims = tuple(data.shape[1:])
911 +
912 + if actual_dims != expected_dims:
913 + expected_expr = ("batch_size", *map(str, expected_dims))
914 + raise ValueError(
915 + f"The expected shape of pixel values is {expected_expr}. "
916 + f"You supplied {tuple(data.shape)}.")
917 +
918 + return data
919 +
920 + def _parse_and_validate_image_input(
921 + self, **kwargs: object) -> Optional[ChameleonImagePixelInputs]:
922 + pixel_values = kwargs.pop("pixel_values", None)
923 +
924 + if pixel_values is None:
925 + return None
926 +
927 + if not isinstance(pixel_values, torch.Tensor):
928 + raise ValueError("Incorrect type of pixel values. "
929 + f"Got type: {type(pixel_values)}")
930 +
931 + return ChameleonImagePixelInputs(
932 + type="pixel_values",
933 + data=self._validate_pixel_values(pixel_values),
934 + )
935 +
936 def forward(
937 self,
938 input_ids: torch.Tensor,
@@ -468,10 +943,17 @@ class ChameleonForConditionalGeneration(nn.Module):
943 **kwargs,
944 ) -> torch.Tensor:
945
471 - # TODO (ywang96): Support image input
472 - # image_tokens = self.process_image_input(**kwargs)
473 - # image_mask = input_ids == self.vocabulary_mapping.image_token_id
474 - # input_ids[special_image_mask] = image_tokens.flatten().to(input_ids.dtype) #noqa
946 + image_input = self._parse_and_validate_image_input(**kwargs)
947 +
948 + if image_input is not None:
949 + assert self.model.vqmodel is not None
950 + image_tokens = self.model.get_image_tokens(image_input["data"].to(
951 + self.config.torch_dtype))
952 + image_token_id = self.model.vocabulary_mapping.image_token_id
953 + special_image_mask = input_ids == image_token_id
954 + image_tokens = image_tokens.to(input_ids.device, input_ids.dtype)
955 + input_ids = input_ids.masked_scatter(special_image_mask,
956 + image_tokens)
957
958 hidden_states = self.model(input_ids, positions, kv_caches,
959 attn_metadata)
@@ -511,43 +993,52 @@ class ChameleonForConditionalGeneration(nn.Module):
993 if "rotary_emb.inv_freq" in name:
994 continue
995
514 - # Skip loading vqgan
515 - # TODO: add support for the vision model
516 - if "vqmodel" in name:
517 - continue
996 if ("rotary_emb.cos_cached" in name
997 or "rotary_emb.sin_cached" in name):
998 # Models trained using ColossalAI may include these tensors in
999 # the checkpoint. Skip them.
1000 continue
523 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
524 - if weight_name not in name:
525 - continue
526 - name = name.replace(weight_name, param_name)
527 - # Skip loading extra bias for GPTQ models.
528 - if name.endswith(".bias") and name not in params_dict:
529 - continue
530 - param = params_dict[name]
531 - weight_loader = param.weight_loader
532 - weight_loader(param, loaded_weight, shard_id)
533 - break
1001 + use_default_weight_loading = False
1002 + if "vqmodel" in name:
1003 + if self.model.vqmodel is not None:
1004 + # We only do sharding for language model and
1005 + # not vqvae for now.
1006 + use_default_weight_loading = True
1007 else:
535 - # Skip loading extra bias for GPTQ models.
536 - if name.endswith(".bias") and name not in params_dict:
537 - continue
538 - # Remapping the name of FP8 kv-scale.
539 - if name.endswith("kv_scale"):
540 - remapped_kv_scale_name = name.replace(
541 - ".kv_scale", ".attn.kv_scale")
542 - if remapped_kv_scale_name not in params_dict:
543 - print_warning_once(
544 - f"Found kv scale in the checkpoint (e.g. {name}), "
545 - "but not found the expected name in the model "
546 - f"(e.g. {remapped_kv_scale_name}). kv-scale is "
547 - "not loaded.")
1008 + for (param_name, weight_name,
1009 + shard_id) in stacked_params_mapping:
1010 + if weight_name not in name:
1011 + continue
1012 + name = name.replace(weight_name, param_name)
1013 + # Skip loading extra bias for GPTQ models.
1014 + if name.endswith(".bias") and name not in params_dict:
1015 + continue
1016 + param = params_dict[name]
1017 + weight_loader = param.weight_loader
1018 + weight_loader(param, loaded_weight, shard_id)
1019 + break
1020 + else:
1021 + # Skip loading extra bias for GPTQ models.
1022 + if name.endswith(".bias") and name not in params_dict:
1023 continue
549 - else:
550 - name = remapped_kv_scale_name
1024 + # Remapping the name of FP8 kv-scale.
1025 + if name.endswith("kv_scale"):
1026 + remapped_kv_scale_name = name.replace(
1027 + ".kv_scale", ".attn.kv_scale")
1028 + if remapped_kv_scale_name not in params_dict:
1029 + print_warning_once(
1030 + "Found kv scale in the checkpoint (e.g. "
1031 + f"{name}), but not found the expected name in "
1032 + f"the model (e.g. {remapped_kv_scale_name}). "
1033 + "kv-scale is not loaded.")
1034 + continue
1035 + else:
1036 + name = remapped_kv_scale_name
1037 + param = params_dict[name]
1038 + weight_loader = getattr(param, "weight_loader",
1039 + default_weight_loader)
1040 + weight_loader(param, loaded_weight)
1041 + if use_default_weight_loading and name in params_dict:
1042 param = params_dict[name]
1043 weight_loader = getattr(param, "weight_loader",
1044 default_weight_loader)
vllm/transformers_utils/configs/__init__.py
+3 -1
@@ -1,4 +1,5 @@
1 -from vllm.transformers_utils.configs.chameleon import ChameleonConfig
1 +from vllm.transformers_utils.configs.chameleon import (ChameleonConfig,
2 + ChameleonVQVAEConfig)
3 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4 from vllm.transformers_utils.configs.dbrx import DbrxConfig
5 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -12,6 +13,7 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
13
14 __all__ = [
15 "ChameleonConfig",
16 + "ChameleonVQVAEConfig",
17 "ChatGLMConfig",
18 "DbrxConfig",
19 "MPTConfig",
vllm/transformers_utils/configs/chameleon.py
+47 -10
@@ -1,3 +1,5 @@
1 +from typing import List, Optional
2 +
3 from transformers import PretrainedConfig
4
5
@@ -5,9 +7,7 @@ from transformers import PretrainedConfig
7 # transformers once the new release with Chameleon support
8 # is available.
9 class ChameleonConfig(PretrainedConfig):
8 -
10 model_type = "chameleon"
10 - is_composition = True
11 keys_to_ignore_at_inference = ["past_key_values"]
12
13 def __init__(
@@ -31,7 +31,7 @@ class ChameleonConfig(PretrainedConfig):
31 rope_scaling=None,
32 attention_bias=False,
33 attention_dropout=0.0,
34 - qk_layernorm=False,
34 + model_parallel_size=1,
35 swin_norm=False,
36 vq_config=None,
37 vocabulary_map=None,
@@ -46,10 +46,6 @@ class ChameleonConfig(PretrainedConfig):
46 self.num_attention_heads = num_attention_heads
47 self.mlp_bias = mlp_bias
48
49 - # for backward compatibility
50 - if num_key_value_heads is None:
51 - num_key_value_heads = num_attention_heads
52 -
49 self.num_key_value_heads = num_key_value_heads
50 self.hidden_act = hidden_act
51 self.initializer_range = initializer_range
@@ -60,10 +56,14 @@ class ChameleonConfig(PretrainedConfig):
56 self._rope_scaling_validation()
57 self.attention_bias = attention_bias
58 self.attention_dropout = attention_dropout
63 - self.qk_layernorm = qk_layernorm
59 + self.model_parallel_size = model_parallel_size
60 self.swin_norm = swin_norm
65 - # vq config is currently ignored
66 - # self.vq_config = ChameleonVQConfig(**vq_config)
61 +
62 + if vq_config is None:
63 + vq_config = {}
64 +
65 + self.vq_config = ChameleonVQVAEConfig(**vq_config)
66 +
67 self.vocabulary_map = vocabulary_map
68
69 super().__init__(
@@ -99,3 +99,40 @@ class ChameleonConfig(PretrainedConfig):
99 raise ValueError(
100 "`rope_scaling`'s factor field must be a float > 1, "
101 f"got {rope_scaling_factor}")
102 +
103 +
104 +class ChameleonVQVAEConfig(PretrainedConfig):
105 +
106 + model_type = "chameleon_vqgan"
107 +
108 + def __init__(
109 + self,
110 + embed_dim: int = 256,
111 + num_embeddings: int = 8192,
112 + double_latent: bool = False,
113 + latent_channels: int = 256,
114 + resolution: int = 512,
115 + in_channels: int = 3,
116 + base_channels: int = 128,
117 + channel_multiplier: List[int] = [1, 1, 2, 2, 4], #noqa
118 + num_res_blocks: int = 2,
119 + attn_resolutions: Optional[List[int]] = None,
120 + dropout: float = 0.0,
121 + attn_type: str = "vanilla",
122 + initializer_range=0.02,
123 + **kwargs,
124 + ):
125 + super().__init__(**kwargs)
126 + self.embed_dim = embed_dim
127 + self.num_embeddings = num_embeddings
128 + self.double_latent = double_latent
129 + self.latent_channels = latent_channels
130 + self.resolution = resolution
131 + self.in_channels = in_channels
132 + self.base_channels = base_channels
133 + self.channel_multiplier = channel_multiplier
134 + self.num_res_blocks = num_res_blocks
135 + self.attn_resolutions = attn_resolutions
136 + self.dropout = dropout
137 + self.attn_type = attn_type
138 + self.initializer_range = initializer_range