@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / edf309ebbe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[VLM] Support multimodal inputs for Florence-2 models (#13320)

Isotr0py committed Feb 27, 2025 at 18:06 UTC edf309ebbe25fcf55569c7fe94e3fe78428a8244
13 files changed +1078 -117
docs/source/models/supported_models.md
+7
@@ -715,6 +715,13 @@ See [this page](#generative-models) for more information on how to use generativ
715 *
716 * ✅︎
717 * ✅︎
718 +- * `Florence2ForConditionalGeneration`
719 + * Florence-2
720 + * T + I
721 + * `microsoft/Florence-2-base`, `microsoft/Florence-2-large` etc.
722 + *
723 + *
724 + *
725 - * `FuyuForCausalLM`
726 * Fuyu
727 * T + I
examples/offline_inference/florence2_inference.py
+23 -16
@@ -1,34 +1,45 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -'''
2 +"""
3 Demonstrate prompting of text-to-text
4 encoder/decoder models, specifically Florence-2
5 -'''
5 +"""
6 # TODO(Isotr0py):
7 # Move to offline_inference/vision_language.py
8 # after porting vision backbone
9 from vllm import LLM, SamplingParams
10 -
11 -dtype = "float"
10 +from vllm.assets.image import ImageAsset
11
12 # Create a Florence-2 encoder/decoder model instance
13 llm = LLM(
15 - model="microsoft/Florence-2-base",
16 - tokenizer="facebook/bart-base",
17 - dtype=dtype,
14 + model="microsoft/Florence-2-large",
15 + tokenizer="facebook/bart-large",
16 + max_num_seqs=8,
17 trust_remote_code=True,
18 )
19
20 prompts = [
22 - "<CAPTION>", "<DETAILED_CAPTION>", "<MORE_DETAILED_CAPTION>",
23 - "<CAPTION_TO_PHRASE_GROUNDING>", "<OD>", "<DENSE_REGION_CAPTION>",
24 - "<REGION_PROPOSAL>", "<OCR>", "<OCR_WITH_REGION>"
21 + { # implicit prompt with task token
22 + "prompt": "<DETAILED_CAPTION>",
23 + "multi_modal_data": {
24 + "image": ImageAsset("stop_sign").pil_image
25 + },
26 + },
27 + { # explicit encoder/decoder prompt
28 + "encoder_prompt": {
29 + "prompt": "Describe in detail what is shown in the image.",
30 + "multi_modal_data": {
31 + "image": ImageAsset("cherry_blossom").pil_image
32 + },
33 + },
34 + "decoder_prompt": "",
35 + },
36 ]
37 # Create a sampling params object.
38 sampling_params = SamplingParams(
39 temperature=0,
40 top_p=1.0,
41 min_tokens=0,
31 - max_tokens=20,
42 + max_tokens=128,
43 )
44
45 # Generate output tokens from the prompts. The output is a list of
@@ -38,9 +49,5 @@ outputs = llm.generate(prompts, sampling_params)
49
50 # Print the outputs.
51 for output in outputs:
41 - prompt = output.prompt
42 - encoder_prompt = output.encoder_prompt
52 generated_text = output.outputs[0].text
44 - print(f"Encoder prompt: {encoder_prompt!r}, "
45 - f"Decoder prompt: {prompt!r}, "
46 - f"Generated text: {generated_text!r}")
53 + print(f"Generated text: {generated_text!r}")
examples/offline_inference/vision_language.py
+17
@@ -82,6 +82,22 @@ def run_deepseek_vl2(question: str, modality: str):
82 return llm, prompt, stop_token_ids
83
84
85 +# Florence2
86 +def run_florence2(question: str, modality: str):
87 + assert modality == "image"
88 +
89 + llm = LLM(model="microsoft/Florence-2-large",
90 + tokenizer="facebook/bart-large",
91 + max_num_seqs=8,
92 + trust_remote_code=True,
93 + dtype="bfloat16",
94 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
95 +
96 + prompt = "<MORE_DETAILED_CAPTION>"
97 + stop_token_ids = None
98 + return llm, prompt, stop_token_ids
99 +
100 +
101 # Fuyu
102 def run_fuyu(question: str, modality: str):
103 assert modality == "image"
@@ -571,6 +587,7 @@ model_example_map = {
587 "blip-2": run_blip2,
588 "chameleon": run_chameleon,
589 "deepseek_vl_v2": run_deepseek_vl2,
590 + "florence2": run_florence2,
591 "fuyu": run_fuyu,
592 "glm4v": run_glm4v,
593 "h2ovl_chat": run_h2ovl,
tests/conftest.py
+3 -3
@@ -600,8 +600,8 @@ class HfRunner:
600 if images is not None and images[i] is not None:
601 processor_kwargs["images"] = images[i]
602
603 - encoder_input_ids = self.wrap_device(
604 - self.processor(**processor_kwargs).input_ids,
603 + encoder_inputs = self.wrap_device(
604 + self.processor(**processor_kwargs),
605 device=self.model.device.type,
606 )
607
@@ -615,13 +615,13 @@ class HfRunner:
615 )
616
617 output = self.model.generate(
618 - encoder_input_ids,
618 decoder_input_ids=decoder_input_ids,
619 use_cache=True,
620 do_sample=False,
621 max_new_tokens=max_tokens,
622 output_hidden_states=True,
623 return_dict_in_generate=True,
624 + **encoder_inputs,
625 **kwargs,
626 )
627
tests/models/decoder_only/audio_language/test_ultravox.py
+2 -2
@@ -15,7 +15,7 @@ from ....conftest import HfRunner, VllmRunner
15 from ....utils import RemoteOpenAIServer
16 from ...utils import check_logprobs_close
17
18 -MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
18 +MODEL_NAME = "fixie-ai/ultravox-v0_4"
19
20 AudioTuple = Tuple[np.ndarray, int]
21
@@ -187,7 +187,7 @@ def run_multi_audio_test(
187
188
189 @pytest.mark.core_model
190 -@pytest.mark.parametrize("dtype", ["half"])
190 +@pytest.mark.parametrize("dtype", ["bfloat16"])
191 @pytest.mark.parametrize("max_tokens", [128])
192 @pytest.mark.parametrize("num_logprobs", [5])
193 @pytest.mark.parametrize("vllm_kwargs", [
tests/models/encoder_decoder/vision_language/test_florence2.py
+88 -51
@@ -1,52 +1,59 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from functools import partial
4 -from typing import List, Optional, Tuple, Type
3 +from typing import Optional, Type
4
5 import pytest
6 from PIL import Image
7
9 -from vllm.inputs.data import ExplicitEncoderDecoderPrompt
8 +from vllm.inputs.data import ExplicitEncoderDecoderPrompt, TextPrompt
9 +from vllm.multimodal.image import rescale_image_size
10 from vllm.sequence import SampleLogprobs
11
12 -from ....conftest import HfRunner, VllmRunner
12 +from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13 from ...utils import check_logprobs_close
14
15 -Florence2Prompt = partial(ExplicitEncoderDecoderPrompt,
16 - decoder_prompt=None,
17 - mm_processor_kwargs=None)
18 -
15 MODELS = ["microsoft/Florence-2-base"]
16 # Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
17 # Therefore, we borrow the BartTokenizer from the original Bart model
18 TOKENIZER = "facebook/bart-base"
23 -PROMPTS = [
24 - Florence2Prompt(encoder_prompt="<CAPTION>"),
25 - Florence2Prompt(encoder_prompt="<DETAILED_CAPTION>"),
26 - Florence2Prompt(encoder_prompt="<MORE_DETAILED_CAPTION>"),
27 - Florence2Prompt(encoder_prompt="<CAPTION_TO_PHRASE_GROUNDING>"),
28 - Florence2Prompt(encoder_prompt="<DENSE_REGION_CAPTION>"),
29 - Florence2Prompt(encoder_prompt="<REGION_PROPOSAL>"),
30 - Florence2Prompt(encoder_prompt="<OCR_WITH_REGION>"),
31 - Florence2Prompt(encoder_prompt="<OCR>"),
32 - Florence2Prompt(encoder_prompt="<OD>"),
33 -]
19 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20 + "stop_sign":
21 + "<CAPTION>", # special task token
22 + "cherry_blossom":
23 + "Describe in detail what is shown in the image.",
24 +})
25 +
26
27 +def get_hf_images_prompts(
28 + prompts_: list[ExplicitEncoderDecoderPrompt[str, TextPrompt]],
29 +) -> tuple[list[ExplicitEncoderDecoderPrompt[str, str]], list[Image.Image]]:
30 + prompts, images = [], []
31 + for prompt in prompts_:
32 + encoder_prompt = prompt["encoder_prompt"]
33 + prompts.append(
34 + ExplicitEncoderDecoderPrompt(
35 + encoder_prompt=encoder_prompt["prompt"],
36 + decoder_prompt=None,
37 + ))
38 + images.append(encoder_prompt["multi_modal_data"]["image"])
39 + return prompts, images
40
36 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
37 - Optional[SampleLogprobs]], ):
38 - """Sanitize vllm output to be comparable with hf output."""
39 - output_ids, output_str, out_logprobs = vllm_output
41
41 - hf_output_str = "</s><s>" + output_str + "</s>"
42 +def hf_to_vllm_output(hf_output: tuple[list[int], str,
43 + Optional[SampleLogprobs]]):
44 + """Sanitize hf output to be comparable with vllm output."""
45 + output_ids, output_str, out_logprobs = hf_output
46
43 - return output_ids, hf_output_str, out_logprobs
47 + output_str = output_str.replace("</s>", "").replace("<s>", "")
48 + output_ids = [ids for ids in output_ids if ids not in [0, 2]]
49 +
50 + return output_ids, output_str, out_logprobs
51
52
53 def run_test(
54 hf_runner: Type[HfRunner],
55 vllm_runner: Type[VllmRunner],
49 - prompts: List[ExplicitEncoderDecoderPrompt],
56 + inputs: list[list[ExplicitEncoderDecoderPrompt]],
57 model: str,
58 *,
59 dtype: str,
@@ -56,46 +63,76 @@ def run_test(
63 distributed_executor_backend: Optional[str] = None,
64 ) -> None:
65 with vllm_runner(model,
66 + max_num_seqs=8,
67 tokenizer_name=TOKENIZER,
68 dtype=dtype,
69 tensor_parallel_size=tensor_parallel_size,
70 distributed_executor_backend=distributed_executor_backend,
71 enforce_eager=True) as vllm_model:
64 - vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
65 - prompts, max_tokens, num_logprobs)
72 + vllm_outputs_per_case = [
73 + vllm_model.generate_encoder_decoder_greedy_logprobs(
74 + prompts, max_tokens, num_logprobs=num_logprobs)
75 + for prompts in inputs
76 + ]
77 +
78 + hf_inputs = [get_hf_images_prompts(prompts) for prompts in inputs]
79
67 - # Florence-2 processors require image inputs
68 - dummy_image = Image.new(mode="RGB", size=(2, 2))
80 with hf_runner(model, dtype=dtype, skip_tokenizer_init=True) as hf_model:
81 hf_model.model.get_output_embeddings = lambda: \
82 hf_model.model.language_model.lm_head
72 - hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
73 - prompts,
74 - max_tokens,
75 - num_logprobs,
76 - images=[dummy_image] * len(prompts),
77 - ))
78 -
79 - check_logprobs_close(
80 - outputs_0_lst=hf_outputs,
81 - outputs_1_lst=[
82 - vllm_to_hf_output(vllm_output) for vllm_output in vllm_outputs
83 - ],
84 - name_0="hf",
85 - name_1="vllm",
86 - )
87 -
88 -
83 + hf_outputs_per_case = [
84 + hf_model.generate_encoder_decoder_greedy_logprobs_limit(
85 + prompts, max_tokens, num_logprobs=num_logprobs, images=images)
86 + for prompts, images in hf_inputs
87 + ]
88 +
89 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
90 + vllm_outputs_per_case):
91 + check_logprobs_close(
92 + outputs_0_lst=[hf_to_vllm_output(output) for output in hf_outputs],
93 + outputs_1_lst=vllm_outputs,
94 + name_0="hf",
95 + name_1="vllm",
96 + )
97 +
98 +
99 +@pytest.mark.core_model
100 @pytest.mark.parametrize("model", MODELS)
90 -@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
101 +@pytest.mark.parametrize(
102 + "size_factors",
103 + [
104 + # No image
105 + [],
106 + # Single-scale
107 + [1.0],
108 + # Single-scale, batched
109 + [1.0, 1.0, 1.0],
110 + # Multi-scale
111 + [0.25, 0.5, 1.0],
112 + ],
113 +)
114 +@pytest.mark.parametrize("dtype", ["float"])
115 @pytest.mark.parametrize("max_tokens", [64])
116 @pytest.mark.parametrize("num_logprobs", [5])
93 -def test_models(hf_runner, vllm_runner, model, dtype, max_tokens,
94 - num_logprobs) -> None:
117 +def test_models(hf_runner: Type[HfRunner], vllm_runner: Type[VllmRunner],
118 + image_assets: _ImageAssets, model: str,
119 + size_factors: list[int], dtype: str, max_tokens: int,
120 + num_logprobs: int) -> None:
121 + images = [asset.pil_image for asset in image_assets]
122 +
123 + inputs_per_image = [[
124 + ExplicitEncoderDecoderPrompt(
125 + encoder_prompt=TextPrompt(
126 + prompt=prompt,
127 + multi_modal_data={"image": rescale_image_size(image, factor)}),
128 + decoder_prompt=None,
129 + ) for factor in size_factors
130 + ] for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
131 +
132 run_test(
133 hf_runner,
134 vllm_runner,
98 - PROMPTS,
135 + inputs_per_image,
136 model,
137 dtype=dtype,
138 max_tokens=max_tokens,
tests/models/multimodal/processing/test_common.py
+3 -2
@@ -29,8 +29,8 @@ def _test_processing_correctness(
29 model_config = ModelConfig(
30 model_id,
31 task="auto",
32 - tokenizer=model_id,
33 - tokenizer_mode="auto",
32 + tokenizer=model_info.tokenizer or model_id,
33 + tokenizer_mode=model_info.tokenizer_mode,
34 trust_remote_code=model_info.trust_remote_code,
35 seed=0,
36 dtype="float16",
@@ -151,6 +151,7 @@ def _test_processing_correctness(
151 "Salesforce/blip2-opt-2.7b",
152 "facebook/chameleon-7b",
153 "deepseek-ai/deepseek-vl2-tiny",
154 + "microsoft/Florence-2-base",
155 "adept/fuyu-8b",
156 "THUDM/glm-4v-9b",
157 "h2oai/h2ovl-mississippi-800m",
tests/models/registry.py
+5 -5
@@ -193,11 +193,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
193 # [Encoder-decoder]
194 "BartModel": _HfExamplesInfo("facebook/bart-base"),
195 "BartForConditionalGeneration": _HfExamplesInfo("facebook/bart-large-cnn"),
196 - # Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
197 - # Therefore, we borrow the BartTokenizer from the original Bart model
198 - "Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
199 - tokenizer="facebook/bart-base",
200 - trust_remote_code=True), # noqa: E501
196 }
197
198 _EMBEDDING_EXAMPLE_MODELS = {
@@ -288,6 +283,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
283 extras={"v0.5": "fixie-ai/ultravox-v0_5-llama-3_2-1b"}, # noqa: E501
284 trust_remote_code=True),
285 # [Encoder-decoder]
286 + # Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
287 + # Therefore, we borrow the BartTokenizer from the original Bart model
288 + "Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
289 + tokenizer="facebook/bart-base",
290 + trust_remote_code=True), # noqa: E501
291 "MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
292 "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
293 }
vllm/model_executor/models/bart.py
+19 -8
@@ -588,8 +588,12 @@ class BartEncoder(nn.Module):
588
589 self.layernorm_embedding = nn.LayerNorm(embed_dim)
590
591 - def forward(self, input_ids: torch.Tensor,
592 - positions: torch.Tensor) -> torch.Tensor:
591 + def forward(
592 + self,
593 + input_ids: torch.Tensor,
594 + positions: torch.Tensor,
595 + inputs_embeds: Optional[torch.Tensor] = None,
596 + ) -> torch.Tensor:
597 r"""
598 Args:
599 input_ids
@@ -602,7 +606,8 @@ class BartEncoder(nn.Module):
606 Decoder output torch.Tensor
607 """
608 # retrieve input_ids and inputs_embeds
605 - inputs_embeds = self.embed_tokens(input_ids)
609 + if inputs_embeds is None:
610 + inputs_embeds = self.embed_tokens(input_ids)
611
612 embed_pos = self.embed_positions(positions)
613 embed_pos = embed_pos.to(inputs_embeds.device)
@@ -661,9 +666,13 @@ class BartDecoder(nn.Module):
666
667 self.layernorm_embedding = nn.LayerNorm(config.d_model)
668
664 - def forward(self, decoder_input_ids: torch.Tensor,
665 - decoder_positions: torch.Tensor,
666 - encoder_hidden_states: Optional[torch.Tensor]) -> torch.Tensor:
669 + def forward(
670 + self,
671 + decoder_input_ids: torch.Tensor,
672 + decoder_positions: torch.Tensor,
673 + encoder_hidden_states: Optional[torch.Tensor],
674 + inputs_embeds: Optional[torch.Tensor] = None,
675 + ) -> torch.Tensor:
676 r"""
677 Args:
678 decoder_input_ids
@@ -677,8 +686,10 @@ class BartDecoder(nn.Module):
686 Returns:
687 Decoder output torch.Tensor
688 """
680 -
681 - inputs_embeds = self.embed_tokens(decoder_input_ids)
689 + if inputs_embeds is None:
690 + inputs_embeds = self.embed_tokens(decoder_input_ids)
691 + else:
692 + decoder_positions = inputs_embeds[:, -1]
693
694 # embed positions
695 embed_pos = self.embed_positions(decoder_positions)
vllm/model_executor/models/florence2.py
+893 -20
@@ -1,10 +1,15 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 -from typing import Iterable, Optional, Set, Tuple
4 +from functools import cached_property
5 +from typing import (Iterable, List, Literal, Mapping, Optional, OrderedDict,
6 + Set, Tuple, TypedDict, Union)
7
8 import torch
9 import torch.nn as nn
10 +import torch.nn.functional as F
11 +from einops import rearrange
12 +from transformers import BatchFeature, PretrainedConfig
13
14 from vllm.config import VllmConfig
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -14,11 +19,567 @@ from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
19 BartParallelLMHead,
20 BartScaledWordEmbedding)
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 +from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
23 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
24 +from vllm.multimodal.parse import MultiModalDataDict, MultiModalDataItems
25 +from vllm.multimodal.processing import (BaseProcessingInfo,
26 + EncDecMultiModalProcessor,
27 + PromptReplacement,
28 + PromptReplacementDetails)
29 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30 from vllm.sequence import IntermediateTensors
31
19 -from .utils import AutoWeightsLoader
32 +from .interfaces import SupportsMultiModal
33 +from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
34
35
36 +class Florence2ImagePixelInputs(TypedDict):
37 + type: Literal["pixel_values"]
38 + data: torch.Tensor
39 + """Shape: (batch_size, num_channel, height, width)"""
40 +
41 +
42 +# ViT implementation are all copied from
43 +# https://huggingface.co/microsoft/Florence-2-base/blob/main/modeling_florence2.py
44 +class LearnedAbsolutePositionEmbedding2D(nn.Module):
45 + """
46 + This module learns positional embeddings up to a fixed maximum size.
47 + """
48 +
49 + def __init__(self, embedding_dim=256, num_pos=50):
50 + super().__init__()
51 + self.row_embeddings = nn.Embedding(num_pos, embedding_dim // 2)
52 + self.column_embeddings = nn.Embedding(
53 + num_pos, embedding_dim - (embedding_dim // 2))
54 +
55 + def forward(self, pixel_values):
56 + """
57 + pixel_values: (batch_size, height, width, num_channels)
58 + returns: (batch_size, height, width, embedding_dim * 2)
59 + """
60 + if len(pixel_values.shape) != 4:
61 + raise ValueError('pixel_values must be a 4D tensor')
62 + height, width = pixel_values.shape[1:3]
63 + width_values = torch.arange(width, device=pixel_values.device)
64 + height_values = torch.arange(height, device=pixel_values.device)
65 + x_emb = self.column_embeddings(width_values)
66 + y_emb = self.row_embeddings(height_values)
67 + # (height, width, embedding_dim * 2)
68 + pos = torch.cat([
69 + x_emb.unsqueeze(0).repeat(height, 1, 1),
70 + y_emb.unsqueeze(1).repeat(1, width, 1)
71 + ],
72 + dim=-1)
73 + # (embedding_dim * 2, height, width)
74 + pos = pos.permute(2, 0, 1)
75 + pos = pos.unsqueeze(0)
76 + # (batch_size, embedding_dim * 2, height, width)
77 + pos = pos.repeat(pixel_values.shape[0], 1, 1, 1)
78 + # (batch_size, height, width, embedding_dim * 2)
79 + pos = pos.permute(0, 2, 3, 1)
80 + return pos
81 +
82 +
83 +class PositionalEmbeddingCosine1D(nn.Module):
84 + """
85 + This class implements a very simple positional encoding. It follows closely
86 + the encoder from the link below:
87 + https://pytorch.org/tutorials/beginner/translation_transformer.html
88 + Args:
89 + embed_dim: The dimension of the embeddings.
90 + dropout_prob: The dropout probability.
91 + max_seq_len: The maximum length to precompute the positional encodings.
92 + """
93 +
94 + def __init__(self, embed_dim: int = 512, max_seq_len: int = 1024) -> None:
95 + super().__init__()
96 + self.embed_dim = embed_dim
97 + self.max_seq_len = max_seq_len
98 + # Generate the sinusoidal arrays.
99 + factor = math.log(10000)
100 + denominator = torch.exp(-factor * torch.arange(0, self.embed_dim, 2) /
101 + self.embed_dim)
102 + # Matrix where rows correspond to a positional embedding as a function
103 + # of the position index (i.e., the row index).
104 + frequencies = \
105 + torch.arange(0, self.max_seq_len) \
106 + .reshape(self.max_seq_len, 1) * denominator
107 + pos_idx_to_embed = torch.zeros((self.max_seq_len, self.embed_dim))
108 + # Populate uneven entries.
109 + pos_idx_to_embed[:, 0::2] = torch.sin(frequencies)
110 + pos_idx_to_embed[:, 1::2] = torch.cos(frequencies)
111 + # Save the positional embeddings in a constant buffer.
112 + # self.register_buffer("pos_idx_to_embed", pos_idx_to_embed)
113 + self.pos_idx_to_embed = nn.Parameter(pos_idx_to_embed,
114 + requires_grad=False)
115 +
116 + def forward(self, seq_embeds: torch.Tensor) -> torch.Tensor:
117 + """
118 + Args:
119 + seq_embeds: The sequence embeddings in order. Allowed size:
120 + 1. [T, D], where T is the length of the sequence, and D is the
121 + frame embedding dimension.
122 + 2. [B, T, D], where B is the batch size and T and D are the
123 + same as above.
124 + Returns a tensor of with the same dimensions as the input: i.e.,
125 + [1, T, D] or [T, D].
126 + """
127 + shape_len = len(seq_embeds.shape)
128 + assert 2 <= shape_len <= 3
129 + len_seq = seq_embeds.size(-2)
130 + assert len_seq <= self.max_seq_len
131 + pos_embeds = self.pos_idx_to_embed[0:seq_embeds.size(-2), :]
132 + # Adapt pre-computed positional embeddings to the input.
133 + if shape_len == 3:
134 + pos_embeds = pos_embeds.view(
135 + (1, pos_embeds.size(0), pos_embeds.size(1)))
136 + return pos_embeds
137 +
138 +
139 +class MySequential(nn.Sequential):
140 +
141 + def forward(self, *inputs):
142 + for module in self._modules.values():
143 + if isinstance(inputs, tuple):
144 + inputs = module(*inputs)
145 + else:
146 + inputs = module(inputs)
147 + return inputs
148 +
149 +
150 +class PreNorm(nn.Module):
151 +
152 + def __init__(self, norm, fn):
153 + super().__init__()
154 + self.norm = norm
155 + self.fn = fn
156 +
157 + def forward(self, x, *args, **kwargs):
158 + shortcut = x
159 + if self.norm is not None:
160 + x, size = self.fn(self.norm(x), *args, **kwargs)
161 + else:
162 + x, size = self.fn(x, *args, **kwargs)
163 +
164 + x = shortcut + x
165 +
166 + return x, size
167 +
168 +
169 +class Mlp(nn.Module):
170 +
171 + def __init__(
172 + self,
173 + in_features,
174 + hidden_features=None,
175 + out_features=None,
176 + act_layer=nn.GELU,
177 + ):
178 + super().__init__()
179 + out_features = out_features or in_features
180 + hidden_features = hidden_features or in_features
181 + self.net = nn.Sequential(
182 + OrderedDict([("fc1", nn.Linear(in_features, hidden_features)),
183 + ("act", act_layer()),
184 + ("fc2", nn.Linear(hidden_features, out_features))]))
185 +
186 + def forward(self, x, size):
187 + return self.net(x), size
188 +
189 +
190 +class DepthWiseConv2d(nn.Module):
191 +
192 + def __init__(
193 + self,
194 + dim_in,
195 + kernel_size,
196 + padding,
197 + stride,
198 + bias=True,
199 + ):
200 + super().__init__()
201 + self.dw = nn.Conv2d(dim_in,
202 + dim_in,
203 + kernel_size=kernel_size,
204 + padding=padding,
205 + groups=dim_in,
206 + stride=stride,
207 + bias=bias)
208 +
209 + def forward(self, x, size):
210 + B, N, C = x.shape
211 + H, W = size
212 + assert N == H * W
213 +
214 + x = self.dw(x.transpose(1, 2).view(B, C, H, W))
215 + size = (x.size(-2), x.size(-1))
216 + x = x.flatten(2).transpose(1, 2)
217 + return x, size
218 +
219 +
220 +class ConvEmbed(nn.Module):
221 + """ Image to Patch Embedding
222 + """
223 +
224 + def __init__(self,
225 + patch_size=7,
226 + in_chans=3,
227 + embed_dim=64,
228 + stride=4,
229 + padding=2,
230 + norm_layer=None,
231 + pre_norm=True):
232 + super().__init__()
233 + self.patch_size = patch_size
234 +
235 + self.proj = nn.Conv2d(in_chans,
236 + embed_dim,
237 + kernel_size=patch_size,
238 + stride=stride,
239 + padding=padding)
240 +
241 + dim_norm = in_chans if pre_norm else embed_dim
242 + self.norm = norm_layer(dim_norm) if norm_layer else None
243 +
244 + self.pre_norm = pre_norm
245 +
246 + def forward(self, x, size):
247 + H, W = size
248 + if len(x.size()) == 3:
249 + if self.norm and self.pre_norm:
250 + x = self.norm(x)
251 + x = rearrange(x, 'b (h w) c -> b c h w', h=H, w=W)
252 +
253 + x = self.proj(x)
254 +
255 + _, _, H, W = x.shape
256 + x = rearrange(x, 'b c h w -> b (h w) c')
257 + if self.norm and not self.pre_norm:
258 + x = self.norm(x)
259 +
260 + return x, (H, W)
261 +
262 +
263 +class ChannelAttention(nn.Module):
264 +
265 + def __init__(self, dim, groups=8, qkv_bias=True):
266 + super().__init__()
267 +
268 + self.groups = groups
269 + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
270 + self.proj = nn.Linear(dim, dim)
271 +
272 + def forward(self, x, size):
273 + B, N, C = x.shape
274 +
275 + qkv = self.qkv(x).reshape(B, N, 3, self.groups,
276 + C // self.groups).permute(2, 0, 3, 1, 4)
277 + q, k, v = qkv[0], qkv[1], qkv[2]
278 +
279 + q = q * (float(N)**-0.5)
280 + attention = q.transpose(-1, -2) @ k
281 + attention = attention.softmax(dim=-1)
282 + x = (attention @ v.transpose(-1, -2)).transpose(-1, -2)
283 + x = x.transpose(1, 2).reshape(B, N, C)
284 + x = self.proj(x)
285 + return x, size
286 +
287 +
288 +class ChannelBlock(nn.Module):
289 +
290 + def __init__(self,
291 + dim,
292 + groups,
293 + mlp_ratio=4.,
294 + qkv_bias=True,
295 + drop_path_rate=0.,
296 + act_layer=nn.GELU,
297 + norm_layer=nn.LayerNorm,
298 + conv_at_attn=True,
299 + conv_at_ffn=True):
300 + super().__init__()
301 +
302 + self.conv1 = PreNorm(None, DepthWiseConv2d(
303 + dim, 3, 1, 1)) if conv_at_attn else None
304 + self.channel_attn = PreNorm(
305 + norm_layer(dim),
306 + ChannelAttention(dim, groups=groups, qkv_bias=qkv_bias),
307 + )
308 + self.conv2 = PreNorm(None, DepthWiseConv2d(dim, 3, 1,
309 + 1)) if conv_at_ffn else None
310 + self.ffn = PreNorm(
311 + norm_layer(dim),
312 + Mlp(in_features=dim,
313 + hidden_features=int(dim * mlp_ratio),
314 + act_layer=act_layer),
315 + )
316 +
317 + def forward(self, x, size):
318 + if self.conv1:
319 + x, size = self.conv1(x, size)
320 + x, size = self.channel_attn(x, size)
321 +
322 + if self.conv2:
323 + x, size = self.conv2(x, size)
324 + x, size = self.ffn(x, size)
325 +
326 + return x, size
327 +
328 +
329 +def window_partition(x, window_size: int):
330 + B, H, W, C = x.shape
331 + x = x.view(B, H // window_size, window_size, W // window_size, window_size,
332 + C)
333 + windows = x.permute(0, 1, 3, 2, 4,
334 + 5).contiguous().view(-1, window_size, window_size, C)
335 + return windows
336 +
337 +
338 +def window_reverse(windows, batch_size: int, window_size: int, H: int, W: int):
339 + B = batch_size
340 +
341 + x = windows.view(B, H // window_size, W // window_size, window_size,
342 + window_size, -1)
343 + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
344 + return x
345 +
346 +
347 +class WindowAttention(nn.Module):
348 +
349 + def __init__(self, dim, num_heads, window_size, qkv_bias=True):
350 +
351 + super().__init__()
352 + self.dim = dim
353 + self.window_size = window_size
354 + self.num_heads = num_heads
355 + head_dim = dim // num_heads
356 + self.scale = float(head_dim)**-0.5
357 +
358 + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
359 + self.proj = nn.Linear(dim, dim)
360 +
361 + self.softmax = nn.Softmax(dim=-1)
362 +
363 + def forward(self, x, size):
364 +
365 + H, W = size
366 + B, L, C = x.shape
367 + assert L == H * W, "input feature has wrong size"
368 +
369 + x = x.view(B, H, W, C)
370 +
371 + pad_l = pad_t = 0
372 + pad_r = (self.window_size - W % self.window_size) % self.window_size
373 + pad_b = (self.window_size - H % self.window_size) % self.window_size
374 + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
375 + _, Hp, Wp, _ = x.shape
376 +
377 + x = window_partition(x, self.window_size)
378 + x = x.view(-1, self.window_size * self.window_size, C)
379 +
380 + # W-MSA/SW-MSA
381 + # attn_windows = self.attn(x_windows)
382 +
383 + B_, N, C = x.shape
384 + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads,
385 + C // self.num_heads).permute(2, 0, 3, 1, 4)
386 + q, k, v = qkv[0], qkv[1], qkv[2]
387 +
388 + q = q * self.scale
389 + attn = (q @ k.transpose(-2, -1))
390 + attn = self.softmax(attn)
391 +
392 + x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
393 + x = self.proj(x)
394 +
395 + # merge windows
396 + x = x.view(-1, self.window_size, self.window_size, C)
397 + x = window_reverse(x, B, self.window_size, Hp, Wp)
398 +
399 + if pad_r > 0 or pad_b > 0:
400 + x = x[:, :H, :W, :].contiguous()
401 +
402 + x = x.view(B, H * W, C)
403 +
404 + return x, size
405 +
406 +
407 +class SpatialBlock(nn.Module):
408 +
409 + def __init__(self,
410 + dim,
411 + num_heads,
412 + window_size,
413 + mlp_ratio=4.,
414 + qkv_bias=True,
415 + drop_path_rate=0.,
416 + act_layer=nn.GELU,
417 + norm_layer=nn.LayerNorm,
418 + conv_at_attn=True,
419 + conv_at_ffn=True):
420 + super().__init__()
421 +
422 + self.conv1 = PreNorm(None, DepthWiseConv2d(
423 + dim, 3, 1, 1)) if conv_at_attn else None
424 + self.window_attn = PreNorm(
425 + norm_layer(dim),
426 + WindowAttention(dim, num_heads, window_size, qkv_bias=qkv_bias),
427 + )
428 + self.conv2 = PreNorm(None, DepthWiseConv2d(dim, 3, 1,
429 + 1)) if conv_at_ffn else None
430 + self.ffn = PreNorm(
431 + norm_layer(dim),
432 + Mlp(in_features=dim,
433 + hidden_features=int(dim * mlp_ratio),
434 + act_layer=act_layer),
435 + )
436 +
437 + def forward(self, x, size):
438 + if self.conv1:
439 + x, size = self.conv1(x, size)
440 + x, size = self.window_attn(x, size)
441 +
442 + if self.conv2:
443 + x, size = self.conv2(x, size)
444 + x, size = self.ffn(x, size)
445 + return x, size
446 +
447 +
448 +class DaViT(nn.Module):
449 +
450 + def __init__(
451 + self,
452 + in_chans=3,
453 + num_classes=1000,
454 + depths=(1, 1, 3, 1),
455 + patch_size=(7, 2, 2, 2),
456 + patch_stride=(4, 2, 2, 2),
457 + patch_padding=(3, 0, 0, 0),
458 + patch_prenorm=(False, False, False, False),
459 + embed_dims=(64, 128, 192, 256),
460 + num_heads=(3, 6, 12, 24),
461 + num_groups=(3, 6, 12, 24),
462 + window_size=7,
463 + mlp_ratio=4.,
464 + qkv_bias=True,
465 + drop_path_rate=0.1,
466 + norm_layer=nn.LayerNorm,
467 + enable_checkpoint=False,
468 + conv_at_attn=True,
469 + conv_at_ffn=True,
470 + ):
471 + super().__init__()
472 +
473 + self.num_classes = num_classes
474 + self.embed_dims = embed_dims
475 + self.num_heads = num_heads
476 + self.num_groups = num_groups
477 + self.num_stages = len(self.embed_dims)
478 + self.enable_checkpoint = enable_checkpoint
479 + assert self.num_stages == len(self.num_heads) == len(self.num_groups)
480 +
481 + num_stages = len(embed_dims)
482 + dpr = [
483 + x.item() for x in torch.linspace(0, drop_path_rate,
484 + sum(depths) * 2)
485 + ]
486 +
487 + depth_offset = 0
488 + convs = []
489 + blocks = []
490 + for i in range(num_stages):
491 + conv_embed = ConvEmbed(
492 + patch_size=patch_size[i],
493 + stride=patch_stride[i],
494 + padding=patch_padding[i],
495 + in_chans=in_chans if i == 0 else self.embed_dims[i - 1],
496 + embed_dim=self.embed_dims[i],
497 + norm_layer=norm_layer,
498 + pre_norm=patch_prenorm[i])
499 + convs.append(conv_embed)
500 +
501 + block = MySequential(*[
502 + MySequential(
503 + OrderedDict([('spatial_block',
504 + SpatialBlock(
505 + embed_dims[i],
506 + num_heads[i],
507 + window_size,
508 + drop_path_rate=dpr[depth_offset + j * 2],
509 + qkv_bias=qkv_bias,
510 + mlp_ratio=mlp_ratio,
511 + conv_at_attn=conv_at_attn,
512 + conv_at_ffn=conv_at_ffn,
513 + )),
514 + ('channel_block',
515 + ChannelBlock(
516 + embed_dims[i],
517 + num_groups[i],
518 + drop_path_rate=dpr[depth_offset + j * 2 +
519 + 1],
520 + qkv_bias=qkv_bias,
521 + mlp_ratio=mlp_ratio,
522 + conv_at_attn=conv_at_attn,
523 + conv_at_ffn=conv_at_ffn,
524 + ))])) for j in range(depths[i])
525 + ])
526 + blocks.append(block)
527 + depth_offset += depths[i] * 2
528 +
529 + self.convs = nn.ModuleList(convs)
530 + self.blocks = nn.ModuleList(blocks)
531 +
532 + self.avgpool = nn.AdaptiveAvgPool1d(1)
533 +
534 + @property
535 + def dim_out(self):
536 + return self.embed_dims[-1]
537 +
538 + def forward_features_unpool(self, x):
539 + """
540 + forward until avg pooling
541 + Args:
542 + x (_type_): input image tensor
543 + """
544 + input_size = (x.size(2), x.size(3))
545 + for conv, block in zip(self.convs, self.blocks):
546 + x, input_size = conv(x, input_size)
547 + x, input_size = block(x, input_size)
548 + return x
549 +
550 + def forward_features(self, x):
551 + x = self.forward_features_unpool(x)
552 +
553 + # (batch_size, num_tokens, token_dim)
554 + x = self.avgpool(x.transpose(1, 2))
555 + # (batch_size, 1, num_tokens)
556 + x = torch.flatten(x, 1)
557 + x = self.norms(x)
558 +
559 + return x
560 +
561 + def forward(self, x):
562 + x = self.forward_features(x)
563 + x = self.head(x)
564 + return x
565 +
566 + @classmethod
567 + def from_config(cls, config):
568 + return cls(
569 + depths=config.depths,
570 + embed_dims=config.dim_embed,
571 + num_heads=config.num_heads,
572 + num_groups=config.num_groups,
573 + patch_size=config.patch_size,
574 + patch_stride=config.patch_stride,
575 + patch_padding=config.patch_padding,
576 + patch_prenorm=config.patch_prenorm,
577 + drop_path_rate=config.drop_path_rate,
578 + window_size=config.window_size,
579 + )
580 +
581 +
582 +# Language backbone and processor implementation
583 class Florence2LanguageModel(nn.Module):
584
585 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -47,9 +608,14 @@ class Florence2LanguageModel(nn.Module):
608 self.encoder.embed_tokens.weight = self.shared.weight
609 self.decoder.embed_tokens.weight = self.shared.weight
610
50 - def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
51 - encoder_input_ids: torch.Tensor,
52 - encoder_positions: torch.Tensor) -> torch.Tensor:
611 + def forward(
612 + self,
613 + input_ids: torch.Tensor,
614 + positions: torch.Tensor,
615 + encoder_input_ids: torch.Tensor,
616 + encoder_positions: torch.Tensor,
617 + inputs_embeds: Optional[torch.Tensor] = None,
618 + ) -> torch.Tensor:
619 r"""
620 Args:
621 input_ids
@@ -68,11 +634,12 @@ class Florence2LanguageModel(nn.Module):
634
635 encoder_hidden_states = None
636
71 - if encoder_input_ids.numel() > 0:
637 + if inputs_embeds is not None or encoder_input_ids.numel() > 0:
638 # Run encoder attention if a non-zero number of encoder tokens
639 # are provided as input
640 encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
75 - positions=encoder_positions)
641 + positions=encoder_positions,
642 + inputs_embeds=inputs_embeds)
643
644 # decoder outputs consists of
645 # (dec_features, past_key_value, dec_hidden, dec_attn)
@@ -112,6 +679,7 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
679 positions: torch.Tensor,
680 encoder_input_ids: torch.Tensor,
681 encoder_positions: torch.Tensor,
682 + inputs_embeds: Optional[torch.Tensor] = None,
683 **kwargs,
684 ) -> torch.Tensor:
685 r"""
@@ -127,8 +695,15 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
695 Returns:
696 Output torch.Tensor
697 """
130 - return self.model(input_ids, positions, encoder_input_ids,
131 - encoder_positions)
698 +
699 + return self.model(input_ids,
700 + positions,
701 + encoder_input_ids,
702 + encoder_positions,
703 + inputs_embeds=inputs_embeds)
704 +
705 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
706 + return self.model.encoder.embed_tokens(input_ids)
707
708 def compute_logits(
709 self,
@@ -177,21 +752,312 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
752 return loaded_params
753
754
180 -class Florence2ForConditionalGeneration(nn.Module):
755 +class Florence2ProcessingInfo(BaseProcessingInfo):
756 +
757 + def get_hf_config(self):
758 + return self.ctx.get_hf_config()
759 +
760 + def get_hf_processor(self):
761 + return self.ctx.get_hf_processor()
762 +
763 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
764 + return {"image": 1}
765 +
766 + def get_max_image_tokens(self) -> int:
767 + processor_config = self.ctx.get_hf_image_processor_config()
768 + return processor_config["image_seq_length"]
769 +
770 + def get_mm_max_tokens_per_item(
771 + self,
772 + seq_len: int,
773 + mm_counts: Mapping[str, int],
774 + ) -> Mapping[str, int]:
775 + return {"image": self.get_max_image_tokens()}
776 +
777 +
778 +class Florence2DummyInputsBuilder(
779 + BaseDummyInputsBuilder[Florence2ProcessingInfo]):
780 +
781 + def get_dummy_processor_inputs(
782 + self,
783 + seq_len: int,
784 + mm_counts: Mapping[str, int],
785 + ) -> ProcessorInputs:
786 + num_images = mm_counts.get("image", 0)
787 +
788 + target_width = target_height = self.info.get_hf_config().projection_dim
789 +
790 + mm_data = {
791 + "image":
792 + self._get_dummy_images(width=target_width,
793 + height=target_height,
794 + num_images=num_images)
795 + }
796 +
797 + return ProcessorInputs(
798 + prompt_text="",
799 + mm_data=mm_data,
800 + )
801 +
802 +
803 +class Florence2MultiModalProcessor(
804 + EncDecMultiModalProcessor[Florence2ProcessingInfo]):
805 +
806 + def _hf_processor_applies_repl(
807 + self,
808 + prompt_text: str,
809 + mm_items: MultiModalDataItems,
810 + hf_processor_mm_kwargs: Mapping[str, object],
811 + ) -> bool:
812 + return False
813 +
814 + def create_encoder_prompt(
815 + self,
816 + prompt: Union[str, list[int]],
817 + mm_data: MultiModalDataDict,
818 + ) -> Union[str, list[int]]:
819 + return prompt
820 +
821 + def create_decoder_prompt(
822 + self,
823 + prompt: Union[str, list[int]],
824 + mm_data: MultiModalDataDict,
825 + ) -> Union[str, list[int]]:
826 + return [self.info.get_hf_config().eos_token_id]
827 +
828 + def _call_hf_processor(
829 + self,
830 + prompt: str,
831 + mm_data: Mapping[str, object],
832 + mm_kwargs: Mapping[str, object],
833 + ) -> BatchFeature:
834 + if mm_data:
835 + processed_outputs = super()._call_hf_processor(
836 + prompt, mm_data, mm_kwargs)
837 + else:
838 + hf_processor = self.info.get_hf_processor()
839 + tokenizer = hf_processor.tokenizer
840 + prompt = hf_processor._construct_prompts([prompt])[0]
841 + processed_outputs = tokenizer(prompt,
842 + add_special_tokens=True,
843 + return_tensors="pt")
844 + return processed_outputs
845 +
846 + def _get_mm_fields_config(
847 + self,
848 + hf_inputs: BatchFeature,
849 + hf_processor_mm_kwargs: Mapping[str, object],
850 + ) -> Mapping[str, MultiModalFieldConfig]:
851 + return dict(pixel_values=MultiModalFieldConfig.batched("image"))
852 +
853 + def _get_prompt_replacements(
854 + self,
855 + mm_items: MultiModalDataItems,
856 + hf_processor_mm_kwargs: Mapping[str, object],
857 + out_mm_kwargs: MultiModalKwargs,
858 + ) -> list[PromptReplacement]:
859 + hf_config = self.info.get_hf_config()
860 + pad_token_id = hf_config.pad_token_id
861 + bos_token_id = hf_config.bos_token_id
862 + num_image_tokens = self.info.get_max_image_tokens()
863 + image_tokens = [pad_token_id] * num_image_tokens
864 +
865 + return [
866 + PromptReplacement(
867 + modality="image",
868 + target=[bos_token_id],
869 + replacement=PromptReplacementDetails(
870 + full=image_tokens + [bos_token_id],
871 + features=image_tokens,
872 + ),
873 + )
874 + ]
875 +
876 +
877 +@MULTIMODAL_REGISTRY.register_processor(
878 + Florence2MultiModalProcessor,
879 + info=Florence2ProcessingInfo,
880 + dummy_inputs=Florence2DummyInputsBuilder)
881 +class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal):
882
883 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
884 super().__init__()
885 config = vllm_config.model_config.hf_config
886 + processor_config = vllm_config.model_config.hf_image_processor_config
887
186 - # TODO(Isotr0py): Add vision backbone
888 + self.config = config
889 + self.vision_config = config.vision_config
890 + self.processor_config = processor_config
891 + assert config.vision_config.model_type == 'davit', (
892 + 'only DaViT is supported for now')
893 + self.vision_tower = DaViT.from_config(config=config.vision_config)
894 + self._build_image_projection_layers(config)
895 self.language_model = Florence2LanguageForConditionalGeneration(
896 vllm_config=vllm_config.with_hf_config(config.text_config),
897 prefix=f"{prefix}.language_model",
898 )
899 + self.pad_token_id = config.pad_token_id
900
192 - @property
901 + def _build_image_projection_layers(self, config: PretrainedConfig):
902 + image_dim_out = config.vision_config.dim_embed[-1]
903 + dim_projection = config.vision_config.projection_dim
904 + self.image_projection = nn.Parameter(
905 + torch.empty(image_dim_out, dim_projection))
906 + self.image_proj_norm = nn.LayerNorm(dim_projection)
907 + image_pos_embed_config = config.vision_config.image_pos_embed
908 + if image_pos_embed_config['type'] == 'learned_abs_2d':
909 + self.image_pos_embed = LearnedAbsolutePositionEmbedding2D(
910 + embedding_dim=image_dim_out,
911 + num_pos=image_pos_embed_config['max_pos_embeddings'])
912 + else:
913 + raise NotImplementedError("Florence2 only supports learned_abs_2d "
914 + "as image position embedding.")
915 +
916 + self.image_feature_source = config.vision_config.image_feature_source
917 +
918 + # temporal embedding
919 + visual_temporal_embedding_config = (
920 + self.vision_config.visual_temporal_embedding)
921 + if visual_temporal_embedding_config['type'] == 'COSINE':
922 + self.visual_temporal_embed = PositionalEmbeddingCosine1D(
923 + embed_dim=image_dim_out,
924 + max_seq_len=visual_temporal_embedding_config[
925 + 'max_temporal_embeddings'])
926 + else:
927 + raise NotImplementedError(
928 + 'Florence2 only supports COSINE as temporal embedding.')
929 +
930 + @cached_property
931 def sampler(self):
194 - return self.language_model.sampler
932 + if hasattr(self.language_model, "sampler"):
933 + return self.language_model.sampler
934 + return get_sampler()
935 +
936 + def _validate_pixel_values(
937 + self, data: Union[torch.Tensor, List[torch.Tensor]]
938 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
939 +
940 + size = self.processor_config["size"]
941 + h, w = size["height"], size["width"]
942 + expected_dims = (3, h, w)
943 +
944 + def _validate_shape(d: torch.Tensor):
945 + actual_dims = tuple(d.shape)
946 +
947 + if actual_dims != expected_dims:
948 + expected_expr = tuple(*map(str, expected_dims))
949 + raise ValueError(
950 + "The expected shape of pixel values per batch "
951 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
952 +
953 + for d in data:
954 + _validate_shape(d)
955 +
956 + return data
957 +
958 + def _parse_and_validate_image_input(self, **kwargs: object):
959 + pixel_values: Optional[Union[List[List[torch.Tensor]],
960 + List[torch.Tensor],
961 + torch.Tensor]] = kwargs.pop(
962 + "pixel_values", None)
963 + image_embeds: Optional[Union[List[List[torch.Tensor]],
964 + List[torch.Tensor],
965 + torch.Tensor]] = kwargs.pop(
966 + "image_embeds", None)
967 +
968 + if pixel_values is None and image_embeds is None:
969 + return None
970 +
971 + if pixel_values is not None and image_embeds is not None:
972 + raise ValueError(
973 + "Both pixel values and image embeds are provided.")
974 +
975 + if pixel_values is not None:
976 + return Florence2ImagePixelInputs(
977 + type="pixel_values",
978 + data=self._validate_pixel_values(
979 + flatten_bn(pixel_values, concat=True)),
980 + )
981 +
982 + if image_embeds is not None:
983 + raise NotImplementedError
984 +
985 + raise AssertionError("This line should be unreachable.")
986 +
987 + def _encode_image(self, pixel_values: torch.Tensor) -> torch.Tensor:
988 + dtype = next(self.vision_tower.parameters()).dtype
989 + pixel_values = pixel_values.to(dtype)
990 +
991 + batch_size, T = pixel_values.size(0), 1
992 + x = self.vision_tower.forward_features_unpool(pixel_values)
993 + if self.image_pos_embed is not None:
994 + x = x.view(batch_size * T, -1, x.shape[-1])
995 + num_tokens = x.shape[-2]
996 + h, w = int(num_tokens**0.5), int(num_tokens**0.5)
997 + assert h * w == num_tokens, (
998 + 'only support square feature maps for now')
999 + x = x.view(batch_size * T, h, w, x.shape[-1])
1000 + pos_embed = self.image_pos_embed(x)
1001 + x = x + pos_embed
1002 + x = x.view(batch_size, T * h * w, x.shape[-1])
1003 +
1004 + if self.visual_temporal_embed is not None:
1005 + visual_temporal_embed = self.visual_temporal_embed(
1006 + x.view(batch_size, T, -1, x.shape[-1])[:, :, 0])
1007 + x = x.view(batch_size, T, -1,
1008 + x.shape[-1]) + visual_temporal_embed.view(
1009 + 1, T, 1, x.shape[-1])
1010 +
1011 + x_feat_dict = {}
1012 +
1013 + spatial_avg_pool_x = x.view(batch_size, T, -1, x.shape[-1]).mean(dim=2)
1014 + x_feat_dict['spatial_avg_pool'] = spatial_avg_pool_x
1015 +
1016 + temporal_avg_pool_x = x.view(batch_size, T, -1,
1017 + x.shape[-1]).mean(dim=1)
1018 + x_feat_dict['temporal_avg_pool'] = temporal_avg_pool_x
1019 +
1020 + x = x.view(batch_size, T, -1, x.shape[-1])[:, -1]
1021 + x_feat_dict['last_frame'] = x
1022 +
1023 + new_x = []
1024 + for _image_feature_source in self.image_feature_source:
1025 + if _image_feature_source not in x_feat_dict:
1026 + raise ValueError('invalid image feature source: {}'.format(
1027 + _image_feature_source))
1028 + new_x.append(x_feat_dict[_image_feature_source])
1029 +
1030 + x = torch.cat(new_x, dim=1)
1031 +
1032 + x = x @ self.image_projection
1033 + x = self.image_proj_norm(x)
1034 +
1035 + return x
1036 +
1037 + def _process_image_input(
1038 + self, image_input: Florence2ImagePixelInputs) -> torch.Tensor:
1039 + assert image_input["type"] == "pixel_values"
1040 + pixel_values = image_input["data"]
1041 + return self._encode_image(pixel_values)
1042 +
1043 + def get_multimodal_embeddings(self, **kwargs: object) -> torch.Tensor:
1044 + image_input = self._parse_and_validate_image_input(**kwargs)
1045 + if image_input is None:
1046 + return None
1047 + vision_embeddings = self._process_image_input(image_input)
1048 + return vision_embeddings
1049 +
1050 + def get_input_embeddings(
1051 + self,
1052 + input_ids: torch.Tensor,
1053 + multimodal_embeddings: Optional[NestedTensors] = None,
1054 + ) -> torch.Tensor:
1055 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1056 + if multimodal_embeddings is not None:
1057 + inputs_embeds = merge_multimodal_embeddings(
1058 + input_ids, inputs_embeds, multimodal_embeddings,
1059 + self.pad_token_id)
1060 + return inputs_embeds
1061
1062 def forward(
1063 self,
@@ -216,8 +1082,19 @@ class Florence2ForConditionalGeneration(nn.Module):
1082 Returns:
1083 Output torch.Tensor
1084 """
219 - return self.language_model(input_ids, positions, encoder_input_ids,
220 - encoder_positions)
1085 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1086 + if encoder_input_ids.numel() > 0 or vision_embeddings is not None:
1087 + inputs_embeds = self.get_input_embeddings(encoder_input_ids,
1088 + vision_embeddings)
1089 + else:
1090 + inputs_embeds = None
1091 +
1092 + hidden_states = self.language_model(input_ids,
1093 + positions,
1094 + encoder_input_ids,
1095 + encoder_positions,
1096 + inputs_embeds=inputs_embeds)
1097 + return hidden_states
1098
1099 def compute_logits(
1100 self,
@@ -236,9 +1113,5 @@ class Florence2ForConditionalGeneration(nn.Module):
1113
1114 def load_weights(self, weights: Iterable[Tuple[str,
1115 torch.Tensor]]) -> Set[str]:
239 - skip_prefixes = [
240 - 'image_projection', "vision_tower", "image_proj_norm",
241 - "image_pos_embed", "visual_temporal_embed"
242 - ]
243 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1116 + loader = AutoWeightsLoader(self)
1117 return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1 -1
@@ -105,7 +105,6 @@ _TEXT_GENERATION_MODELS = {
105 # [Encoder-decoder]
106 "BartModel": ("bart", "BartForConditionalGeneration"),
107 "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
108 - "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
108 }
109
110 _EMBEDDING_MODELS = {
@@ -182,6 +181,7 @@ _MULTIMODAL_MODELS = {
181 "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
182 "UltravoxModel": ("ultravox", "UltravoxModel"),
183 # [Encoder-decoder]
184 + "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
185 "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
186 "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501
187 }
vllm/multimodal/processing.py
+13 -7
@@ -1303,6 +1303,14 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1303 """
1304 raise NotImplementedError
1305
1306 + def create_decoder_prompt(
1307 + self,
1308 + prompt: Union[str, list[int]],
1309 + mm_data: MultiModalDataDict,
1310 + ) -> Union[str, list[int]]:
1311 + """Create input prompt for the decoder."""
1312 + return prompt
1313 +
1314 def apply(
1315 self,
1316 prompt: Union[str, list[int]],
@@ -1323,17 +1331,15 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1331 hf_processor_mm_kwargs,
1332 )
1333
1326 - # We assumed the decoder prompt text is copied from
1327 - # the original encoder prompt without extra process
1334 tokenizer = self.info.get_tokenizer()
1329 - if isinstance(prompt, str):
1330 - decoder_prompt = prompt
1335 + decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
1336 + if isinstance(decoder_prompt, str):
1337 decoder_prompt_ids = encode_tokens(tokenizer,
1332 - prompt,
1338 + decoder_prompt,
1339 add_special_tokens=False)
1340 else:
1335 - decoder_prompt = decode_tokens(tokenizer, prompt)
1336 - decoder_prompt_ids = prompt
1341 + decoder_prompt_ids = decoder_prompt
1342 + decoder_prompt = decode_tokens(tokenizer, decoder_prompt)
1343
1344 mm_inputs = MultiModalEncDecInputs(
1345 encoder_prompt=encoder_inputs["prompt"],
vllm/multimodal/profiling.py
+4 -2
@@ -204,9 +204,11 @@ class MultiModalProfiler(Generic[_I]):
204 "and/or reduce `mm_counts`.", seq_len, total_len,
205 total_placeholders_by_modality)
206
207 + num_tokens_to_pad = max(total_len, seq_len) - total_len
208 + prompt_token_ids.extend([0] * num_tokens_to_pad)
209 +
210 return DummyData(
208 - seq_data=SequenceData.from_prompt_token_counts(
209 - (0, max(seq_len, total_len))),
211 + seq_data=SequenceData.from_seqs(prompt_token_ids),
212 multi_modal_data=None,
213 multi_modal_placeholders=None,
214 )