@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 831540cf04
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Support E5-V (#9576)

Cyrus Leung committed Oct 23, 2024 at 11:35 UTC 831540cf04b0b40cd1fe462356de4a30b831e4ea
12 files changed +532 -90
docs/source/models/supported_models.rst
+14
@@ -334,6 +334,14 @@ The following modalities are supported depending on the model:
334 - **V**\ ideo
335 - **A**\ udio
336
337 +Any combination of modalities joined by :code:`+` are supported.
338 +
339 +- e.g.: :code:`T + I` means that the model supports text-only, image-only, and text-with-image inputs.
340 +
341 +On the other hand, modalities separated by :code:`/` are mutually exclusive.
342 +
343 +- e.g.: :code:`T / I` means that the model supports text-only and image-only inputs, but not text-with-image inputs.
344 +
345 .. _supported_vlms:
346
347 Text Generation
@@ -484,6 +492,12 @@ Multimodal Embedding
492 - Example HF Models
493 - :ref:`LoRA <lora>`
494 - :ref:`PP <distributed_serving>`
495 + * - :code:`LlavaNextForConditionalGeneration`
496 + - LLaVA-NeXT-based
497 + - T / I
498 + - :code:`royokong/e5-v`
499 + -
500 + - ✅︎
501 * - :code:`Phi3VForCausalLM`
502 - Phi-3-Vision-based
503 - T + I
examples/offline_inference_vision_language.py
+3 -3
@@ -1,6 +1,6 @@
1 """
2 -This example shows how to use vLLM for running offline inference
3 -with the correct prompt format on vision language models.
2 +This example shows how to use vLLM for running offline inference with
3 +the correct prompt format on vision language models for text generation.
4
5 For most models, the prompt format should follow corresponding examples
6 on HuggingFace model repository.
@@ -450,7 +450,7 @@ def main(args):
450 if __name__ == "__main__":
451 parser = FlexibleArgumentParser(
452 description='Demo on using vLLM for offline inference with '
453 - 'vision language models')
453 + 'vision language models for text generation')
454 parser.add_argument('--model-type',
455 '-m',
456 type=str,
examples/offline_inference_vision_language_embedding.py
+169 -21
@@ -1,22 +1,170 @@
1 +"""
2 +This example shows how to use vLLM for running offline inference with
3 +the correct prompt format on vision language models for multimodal embedding.
4 +
5 +For most models, the prompt format should follow corresponding examples
6 +on HuggingFace model repository.
7 +"""
8 +from argparse import Namespace
9 +from typing import Literal, NamedTuple, Optional, TypedDict, Union, get_args
10 +
11 +from PIL.Image import Image
12 +
13 from vllm import LLM
2 -from vllm.assets.image import ImageAsset
3 -
4 -image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
5 -prompt = "<|image_1|> Represent the given image with the following question: What is in the image" # noqa: E501
6 -
7 -# Create an LLM.
8 -llm = LLM(
9 - model="TIGER-Lab/VLM2Vec-Full",
10 - task="embedding",
11 - trust_remote_code=True,
12 - max_model_len=4096,
13 - max_num_seqs=2,
14 - mm_processor_kwargs={"num_crops": 16},
15 -)
16 -
17 -# Generate embedding. The output is a list of EmbeddingRequestOutputs.
18 -outputs = llm.encode({"prompt": prompt, "multi_modal_data": {"image": image}})
19 -
20 -# Print the outputs.
21 -for output in outputs:
22 - print(output.outputs.embedding) # list of 3072 floats
14 +from vllm.multimodal.utils import fetch_image
15 +from vllm.utils import FlexibleArgumentParser
16 +
17 +
18 +class TextQuery(TypedDict):
19 + modality: Literal["text"]
20 + text: str
21 +
22 +
23 +class ImageQuery(TypedDict):
24 + modality: Literal["image"]
25 + image: Image
26 +
27 +
28 +class TextImageQuery(TypedDict):
29 + modality: Literal["text+image"]
30 + text: str
31 + image: Image
32 +
33 +
34 +QueryModality = Literal["text", "image", "text+image"]
35 +Query = Union[TextQuery, ImageQuery, TextImageQuery]
36 +
37 +
38 +class ModelRequestData(NamedTuple):
39 + llm: LLM
40 + prompt: str
41 + image: Optional[Image]
42 +
43 +
44 +def run_e5_v(query: Query):
45 + llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' # noqa: E501
46 +
47 + if query["modality"] == "text":
48 + text = query["text"]
49 + prompt = llama3_template.format(
50 + f"{text}\nSummary above sentence in one word: ")
51 + image = None
52 + elif query["modality"] == "image":
53 + prompt = llama3_template.format(
54 + "<image>\nSummary above image in one word: ")
55 + image = query["image"]
56 + else:
57 + modality = query['modality']
58 + raise ValueError(f"Unsupported query modality: '{modality}'")
59 +
60 + llm = LLM(
61 + model="royokong/e5-v",
62 + task="embedding",
63 + max_model_len=4096,
64 + )
65 +
66 + return ModelRequestData(
67 + llm=llm,
68 + prompt=prompt,
69 + image=image,
70 + )
71 +
72 +
73 +def run_vlm2vec(query: Query):
74 + if query["modality"] == "text":
75 + text = query["text"]
76 + prompt = f"Find me an everyday image that matches the given caption: {text}" # noqa: E501
77 + image = None
78 + elif query["modality"] == "image":
79 + prompt = "<|image_1|> Find a day-to-day image that looks similar to the provided image." # noqa: E501
80 + image = query["image"]
81 + elif query["modality"] == "text+image":
82 + text = query["text"]
83 + prompt = f"<|image_1|> Represent the given image with the following question: {text}" # noqa: E501
84 + image = query["image"]
85 + else:
86 + modality = query['modality']
87 + raise ValueError(f"Unsupported query modality: '{modality}'")
88 +
89 + llm = LLM(
90 + model="TIGER-Lab/VLM2Vec-Full",
91 + task="embedding",
92 + trust_remote_code=True,
93 + mm_processor_kwargs={"num_crops": 4},
94 + )
95 +
96 + return ModelRequestData(
97 + llm=llm,
98 + prompt=prompt,
99 + image=image,
100 + )
101 +
102 +
103 +def get_query(modality: QueryModality):
104 + if modality == "text":
105 + return TextQuery(modality="text", text="A dog sitting in the grass")
106 +
107 + if modality == "image":
108 + return ImageQuery(
109 + modality="image",
110 + image=fetch_image(
111 + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/American_Eskimo_Dog.jpg/360px-American_Eskimo_Dog.jpg" # noqa: E501
112 + ),
113 + )
114 +
115 + if modality == "text+image":
116 + return TextImageQuery(
117 + modality="text+image",
118 + text="A cat standing in the snow.",
119 + image=fetch_image(
120 + "https://upload.wikimedia.org/wikipedia/commons/thumb/b/b6/Felis_catus-cat_on_snow.jpg/179px-Felis_catus-cat_on_snow.jpg" # noqa: E501
121 + ),
122 + )
123 +
124 + msg = f"Modality {modality} is not supported."
125 + raise ValueError(msg)
126 +
127 +
128 +def run_encode(model: str, modality: QueryModality):
129 + query = get_query(modality)
130 + req_data = model_example_map[model](query)
131 +
132 + mm_data = {}
133 + if req_data.image is not None:
134 + mm_data["image"] = req_data.image
135 +
136 + outputs = req_data.llm.encode({
137 + "prompt": req_data.prompt,
138 + "multi_modal_data": mm_data,
139 + })
140 +
141 + for output in outputs:
142 + print(output.outputs.embedding)
143 +
144 +
145 +def main(args: Namespace):
146 + run_encode(args.model_name, args.modality)
147 +
148 +
149 +model_example_map = {
150 + "e5_v": run_e5_v,
151 + "vlm2vec": run_vlm2vec,
152 +}
153 +
154 +if __name__ == "__main__":
155 + parser = FlexibleArgumentParser(
156 + description='Demo on using vLLM for offline inference with '
157 + 'vision language models for multimodal embedding')
158 + parser.add_argument('--model-name',
159 + '-m',
160 + type=str,
161 + default="vlm2vec",
162 + choices=model_example_map.keys(),
163 + help='The name of the embedding model.')
164 + parser.add_argument('--modality',
165 + type=str,
166 + default="image",
167 + choices=get_args(QueryModality),
168 + help='Modality of the input.')
169 + args = parser.parse_args()
170 + main(args)
examples/offline_inference_vision_language_multi_image.py
+4 -3
@@ -1,7 +1,7 @@
1 """
2 This example shows how to use vLLM for running offline inference with
3 -multi-image input on vision language models, using the chat template defined
4 -by the model.
3 +multi-image input on vision language models for text generation,
4 +using the chat template defined by the model.
5 """
6 from argparse import Namespace
7 from typing import List, NamedTuple, Optional
@@ -334,7 +334,8 @@ def main(args: Namespace):
334 if __name__ == "__main__":
335 parser = FlexibleArgumentParser(
336 description='Demo on using vLLM for offline inference with '
337 - 'vision language models that support multi-image input')
337 + 'vision language models that support multi-image input for text '
338 + 'generation')
339 parser.add_argument('--model-type',
340 '-m',
341 type=str,
tests/conftest.py
+36 -24
@@ -43,10 +43,12 @@ _TEST_DIR = os.path.dirname(__file__)
43 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
44 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
45
46 -PromptImageInput = Union[List[Image.Image], List[List[Image.Image]]]
47 -PromptAudioInput = Union[List[Tuple[np.ndarray, int]],
48 - List[List[Tuple[np.ndarray, int]]]]
49 -PromptVideoInput = Union[List[np.ndarray], List[List[np.ndarray]]]
46 +_M = TypeVar("_M")
47 +_PromptMultiModalInput = Union[List[_M], List[List[_M]]]
48 +
49 +PromptImageInput = _PromptMultiModalInput[Image.Image]
50 +PromptAudioInput = _PromptMultiModalInput[Tuple[np.ndarray, int]]
51 +PromptVideoInput = _PromptMultiModalInput[np.ndarray]
52
53
54 def _read_prompts(filename: str) -> List[str]:
@@ -318,12 +320,12 @@ class HfRunner:
320 "text": prompt,
321 "return_tensors": "pt",
322 }
321 - if images is not None and images[i] is not None:
322 - processor_kwargs["images"] = images[i]
323 - if videos is not None and videos[i] is not None:
324 - processor_kwargs["videos"] = videos[i]
325 - if audios is not None and audios[i] is not None:
326 - audio, sr = audios[i]
323 + if images is not None and (image := images[i]) is not None:
324 + processor_kwargs["images"] = image
325 + if videos is not None and (video := videos[i]) is not None:
326 + processor_kwargs["videos"] = video
327 + if audios is not None and (audio_tuple := audios[i]) is not None:
328 + audio, sr = audio_tuple
329 processor_kwargs["audio"] = audio
330 processor_kwargs["sampling_rate"] = sr
331
@@ -338,7 +340,7 @@ class HfRunner:
340 self,
341 prompts: List[str],
342 images: Optional[PromptImageInput] = None,
341 - videos: Optional[List[np.ndarray]] = None,
343 + videos: Optional[PromptVideoInput] = None,
344 audios: Optional[PromptAudioInput] = None,
345 **kwargs: Any,
346 ) -> List[Tuple[List[List[int]], List[str]]]:
@@ -368,7 +370,7 @@ class HfRunner:
370 prompts: List[str],
371 max_tokens: int,
372 images: Optional[PromptImageInput] = None,
371 - videos: Optional[List[np.ndarray]] = None,
373 + videos: Optional[PromptVideoInput] = None,
374 audios: Optional[PromptAudioInput] = None,
375 **kwargs: Any,
376 ) -> List[Tuple[List[int], str]]:
@@ -409,7 +411,7 @@ class HfRunner:
411 prompts: List[str],
412 max_tokens: int,
413 images: Optional[PromptImageInput] = None,
412 - videos: Optional[List[np.ndarray]] = None,
414 + videos: Optional[PromptVideoInput] = None,
415 audios: Optional[PromptAudioInput] = None,
416 **kwargs: Any,
417 ) -> List[List[torch.Tensor]]:
@@ -488,7 +490,7 @@ class HfRunner:
490 num_logprobs: int,
491 images: Optional[PromptImageInput] = None,
492 audios: Optional[PromptAudioInput] = None,
491 - videos: Optional[List[np.ndarray]] = None,
493 + videos: Optional[PromptVideoInput] = None,
494 **kwargs: Any,
495 ) -> List[TokensTextLogprobs]:
496 all_inputs = self.get_inputs(prompts,
@@ -657,15 +659,18 @@ class VllmRunner:
659 inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
660 if images is not None:
661 for i, image in enumerate(images):
660 - inputs[i]["multi_modal_data"] = {"image": image}
662 + if image is not None:
663 + inputs[i]["multi_modal_data"] = {"image": image}
664
665 if videos is not None:
666 for i, video in enumerate(videos):
664 - inputs[i]["multi_modal_data"] = {"video": video}
667 + if video is not None:
668 + inputs[i]["multi_modal_data"] = {"video": video}
669
670 if audios is not None:
671 for i, audio in enumerate(audios):
668 - inputs[i]["multi_modal_data"] = {"audio": audio}
672 + if audio is not None:
673 + inputs[i]["multi_modal_data"] = {"audio": audio}
674
675 return inputs
676
@@ -837,13 +842,20 @@ class VllmRunner:
842 returned_outputs.append((token_ids, texts))
843 return returned_outputs
844
840 - def encode(self, prompts: List[str]) -> List[List[float]]:
841 - req_outputs = self.model.encode(prompts)
842 - outputs = []
843 - for req_output in req_outputs:
844 - embedding = req_output.outputs.embedding
845 - outputs.append(embedding)
846 - return outputs
845 + def encode(
846 + self,
847 + prompts: List[str],
848 + images: Optional[PromptImageInput] = None,
849 + videos: Optional[PromptVideoInput] = None,
850 + audios: Optional[PromptAudioInput] = None,
851 + ) -> List[List[float]]:
852 + inputs = self.get_inputs(prompts,
853 + images=images,
854 + videos=videos,
855 + audios=audios)
856 +
857 + req_outputs = self.model.encode(inputs)
858 + return [req_output.outputs.embedding for req_output in req_outputs]
859
860 def __enter__(self):
861 return self
tests/models/embedding/utils.py
+2 -1
@@ -16,7 +16,8 @@ def check_embeddings_close(
16
17 for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
18 zip(embeddings_0_lst, embeddings_1_lst)):
19 - assert len(embeddings_0) == len(embeddings_1)
19 + assert len(embeddings_0) == len(embeddings_1), (
20 + f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}")
21
22 sim = F.cosine_similarity(torch.tensor(embeddings_0),
23 torch.tensor(embeddings_1),
tests/models/embedding/vision_language/test_llava_next.py new
+135
@@ -0,0 +1,135 @@
1 +from typing import List, Type
2 +
3 +import pytest
4 +import torch.nn.functional as F
5 +from transformers import AutoModelForVision2Seq
6 +
7 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
8 +from ....utils import large_gpu_test
9 +from ..utils import check_embeddings_close
10 +
11 +llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' # noqa: E501
12 +
13 +HF_TEXT_PROMPTS = [
14 + # T -> X
15 + llama3_template.format(
16 + "The label of the object is stop sign\nSummary above sentence in one word: " # noqa: E501
17 + ),
18 + # T -> X
19 + llama3_template.format(
20 + "cherry blossom\nSummary above sentence in one word: "),
21 +]
22 +
23 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
24 + # I -> X
25 + "stop_sign":
26 + llama3_template.format("<image>\nSummary above image in one word: "),
27 + # I -> X
28 + "cherry_blossom":
29 + llama3_template.format("<image>\nSummary above image in one word: "),
30 +})
31 +
32 +MODELS = ["royokong/e5-v"]
33 +
34 +
35 +def _run_test(
36 + hf_runner: Type[HfRunner],
37 + vllm_runner: Type[VllmRunner],
38 + input_texts: List[str],
39 + input_images: PromptImageInput,
40 + model: str,
41 + *,
42 + dtype: str,
43 +) -> None:
44 + # NOTE: take care of the order. run vLLM first, and then run HF.
45 + # vLLM needs a fresh new process without cuda initialization.
46 + # if we run HF first, the cuda initialization will be done and it
47 + # will hurt multiprocessing backend with fork method (the default method).
48 + with vllm_runner(model,
49 + task="embedding",
50 + dtype=dtype,
51 + max_model_len=4096,
52 + enforce_eager=True) as vllm_model:
53 + vllm_outputs = vllm_model.encode(input_texts, images=input_images)
54 +
55 + with hf_runner(model, dtype=dtype,
56 + auto_cls=AutoModelForVision2Seq) as hf_model:
57 + # Patch the issue where image_token_id
58 + # exceeds the maximum allowed vocab size
59 + hf_model.model.resize_token_embeddings(
60 + hf_model.model.language_model.vocab_size + 1)
61 +
62 + all_inputs = hf_model.get_inputs(input_texts, images=input_images)
63 +
64 + all_outputs = []
65 + for inputs in all_inputs:
66 + # Based on: https://huggingface.co/royokong/e5-v
67 + outputs = hf_model.model(
68 + **hf_model.wrap_device(inputs,
69 + device=hf_model.model.device.type),
70 + return_dict=True,
71 + output_hidden_states=True,
72 + )
73 + pooled_output = F.normalize(outputs.hidden_states[-1][0, -1, :],
74 + dim=-1)
75 +
76 + all_outputs.append(pooled_output.tolist())
77 +
78 + hf_outputs = all_outputs
79 +
80 + check_embeddings_close(
81 + embeddings_0_lst=hf_outputs,
82 + embeddings_1_lst=vllm_outputs,
83 + name_0="hf",
84 + name_1="vllm",
85 + )
86 +
87 +
88 +@pytest.mark.parametrize("model", MODELS)
89 +@pytest.mark.parametrize("dtype", ["half"])
90 +def test_models_text(
91 + hf_runner,
92 + vllm_runner,
93 + image_assets,
94 + model: str,
95 + dtype: str,
96 +) -> None:
97 + input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
98 + input_texts = [text for text, _ in input_texts_images]
99 + input_images = [image for _, image in input_texts_images]
100 +
101 + _run_test(
102 + hf_runner,
103 + vllm_runner,
104 + input_texts,
105 + input_images, # type: ignore
106 + model,
107 + dtype=dtype,
108 + )
109 +
110 +
111 +@large_gpu_test(min_gb=48)
112 +@pytest.mark.parametrize("model", MODELS)
113 +@pytest.mark.parametrize("dtype", ["half"])
114 +def test_models_image(
115 + hf_runner,
116 + vllm_runner,
117 + image_assets,
118 + model: str,
119 + dtype: str,
120 +) -> None:
121 + input_texts_images = [
122 + (text, asset.pil_image)
123 + for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
124 + ]
125 + input_texts = [text for text, _ in input_texts_images]
126 + input_images = [image for _, image in input_texts_images]
127 +
128 + _run_test(
129 + hf_runner,
130 + vllm_runner,
131 + input_texts,
132 + input_images,
133 + model,
134 + dtype=dtype,
135 + )
tests/models/embedding/vision_language/test_phi3v.py
+77 -16
@@ -1,42 +1,53 @@
1 +from typing import List, Type
2 +
3 import pytest
4 import torch.nn.functional as F
5
4 -from ....conftest import IMAGE_ASSETS
6 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
7 +from ....utils import large_gpu_test
8 from ..utils import check_embeddings_close
9
10 +HF_TEXT_PROMPTS = [
11 + # T -> X
12 + "Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501
13 + # T -> X
14 + "Retrieve an image of this caption: cherry blossom",
15 +]
16 +
17 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 + # T + I -> X
19 "stop_sign":
20 "<|image_1|> Select the portion of the image that isolates the object of the given label: The label of the object is stop sign", # noqa: E501
21 + # I -> X
22 "cherry_blossom":
11 - "<|image_1|> Represent the given image with the following question: What is in the image", # noqa: E501
23 + "<|image_1|> Represent the given image for classification", # noqa: E501
24 })
25
26 MODELS = ["TIGER-Lab/VLM2Vec-Full"]
27
28
17 -@pytest.mark.parametrize("model", MODELS)
18 -@pytest.mark.parametrize("dtype", ["half"])
19 -def test_models(
20 - hf_runner,
21 - vllm_runner,
22 - example_prompts,
29 +def _run_test(
30 + hf_runner: Type[HfRunner],
31 + vllm_runner: Type[VllmRunner],
32 + input_texts: List[str],
33 + input_images: PromptImageInput,
34 model: str,
35 + *,
36 dtype: str,
37 ) -> None:
38 # NOTE: take care of the order. run vLLM first, and then run HF.
39 # vLLM needs a fresh new process without cuda initialization.
40 # if we run HF first, the cuda initialization will be done and it
41 # will hurt multiprocessing backend with fork method (the default method).
30 - with vllm_runner(model,
31 - task="embedding",
32 - max_model_len=4096,
33 - max_num_seqs=2,
34 - dtype=dtype,
42 + with vllm_runner(model, task="embedding", dtype=dtype,
43 enforce_eager=True) as vllm_model:
36 - vllm_outputs = vllm_model.encode(example_prompts)
44 + vllm_outputs = vllm_model.encode(input_texts, images=input_images)
45
38 - with hf_runner(model, dtype=dtype) as hf_model:
39 - all_inputs = hf_model.get_inputs(example_prompts)
46 + # use eager mode for hf runner, since phi3_v didn't work with flash_attn
47 + hf_model_kwargs = {"_attn_implementation": "eager"}
48 + with hf_runner(model, dtype=dtype,
49 + model_kwargs=hf_model_kwargs) as hf_model:
50 + all_inputs = hf_model.get_inputs(input_texts, images=input_images)
51
52 all_outputs = []
53 for inputs in all_inputs:
@@ -61,3 +72,53 @@ def test_models(
72 name_0="hf",
73 name_1="vllm",
74 )
75 +
76 +
77 +@pytest.mark.parametrize("model", MODELS)
78 +@pytest.mark.parametrize("dtype", ["half"])
79 +def test_models_text(
80 + hf_runner,
81 + vllm_runner,
82 + image_assets,
83 + model: str,
84 + dtype: str,
85 +) -> None:
86 + input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
87 + input_texts = [text for text, _ in input_texts_images]
88 + input_images = [image for _, image in input_texts_images]
89 +
90 + _run_test(
91 + hf_runner,
92 + vllm_runner,
93 + input_texts,
94 + input_images, # type: ignore
95 + model,
96 + dtype=dtype,
97 + )
98 +
99 +
100 +@large_gpu_test(min_gb=48)
101 +@pytest.mark.parametrize("model", MODELS)
102 +@pytest.mark.parametrize("dtype", ["half"])
103 +def test_models_image(
104 + hf_runner,
105 + vllm_runner,
106 + image_assets,
107 + model: str,
108 + dtype: str,
109 +) -> None:
110 + input_texts_images = [
111 + (text, asset.pil_image)
112 + for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
113 + ]
114 + input_texts = [text for text, _ in input_texts_images]
115 + input_images = [image for _, image in input_texts_images]
116 +
117 + _run_test(
118 + hf_runner,
119 + vllm_runner,
120 + input_texts,
121 + input_images,
122 + model,
123 + dtype=dtype,
124 + )
vllm/model_executor/models/llava_next.py
+22 -11
@@ -13,11 +13,13 @@ from typing_extensions import NotRequired
13 from vllm.attention import AttentionMetadata
14 from vllm.config import CacheConfig, MultiModalConfig
15 from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19 +from vllm.model_executor.pooling_metadata import PoolingMetadata
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
21 from vllm.multimodal import MULTIMODAL_REGISTRY
20 -from vllm.sequence import IntermediateTensors
22 +from vllm.sequence import IntermediateTensors, PoolerOutput
23 from vllm.utils import is_list_of
24
25 from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -28,8 +30,8 @@ from .llava import LlavaMultiModalProjector
30 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
32 get_siglip_patch_grid_length, input_processor_for_siglip)
31 -from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
32 - merge_multimodal_embeddings)
33 +from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
34 + init_vllm_registered_model)
35
36 # Result in the max possible feature size (2x2 grid of 336x336px tiles)
37 MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
@@ -312,6 +314,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
314 self.language_model = init_vllm_registered_model(
315 config.text_config, cache_config, quant_config)
316
317 + # The same model class supports both language generation and embedding
318 + # because the architecture name is the same
319 + self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
320 +
321 self.make_empty_intermediate_tensors = (
322 self.language_model.make_empty_intermediate_tensors)
323
@@ -605,14 +611,12 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
611 image_input = self._parse_and_validate_image_input(**kwargs)
612
613 if image_input is not None:
608 - vision_embeddings = self._process_image_input(image_input)
609 - inputs_embeds = self.language_model.model.get_input_embeddings(
610 - input_ids)
611 -
612 - inputs_embeds = merge_multimodal_embeddings(
613 - input_ids, inputs_embeds, vision_embeddings,
614 - self.config.image_token_index)
615 -
614 + inputs_embeds = embed_multimodal(
615 + input_ids,
616 + self.config.image_token_index,
617 + self.language_model.model.get_input_embeddings,
618 + lambda _: self._process_image_input(image_input),
619 + )
620 input_ids = None
621 else:
622 inputs_embeds = None
@@ -641,6 +645,13 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
645 ) -> Optional[SamplerOutput]:
646 return self.language_model.sample(logits, sampling_metadata)
647
648 + def pooler(
649 + self,
650 + hidden_states: torch.Tensor,
651 + pooling_metadata: PoolingMetadata,
652 + ) -> Optional[PoolerOutput]:
653 + return self._pooler(hidden_states, pooling_metadata)
654 +
655 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
656 loader = AutoWeightsLoader(self)
657 loader.load_weights(weights)
vllm/model_executor/models/phi3v.py
-2
@@ -467,8 +467,6 @@ def input_processor_for_phi3v(ctx: InputContext,
467
468 prompt_token_ids = inputs["prompt_token_ids"].copy()
469
470 - print("prompt_token_ids (old)", prompt_token_ids)
471 -
470 # masked placeholder with image token id
471 for idx in image_idx:
472 candidates = _get_image_placeholder_token_id_candidates(model_config,
vllm/model_executor/models/registry.py
+1
@@ -94,6 +94,7 @@ _EMBEDDING_MODELS = {
94 "MistralModel": ("llama", "LlamaEmbeddingModel"),
95 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
96 # [Multimodal]
97 + "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
98 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
99 }
100
vllm/model_executor/models/utils.py
+69 -9
@@ -1,7 +1,7 @@
1 import itertools
2 from dataclasses import dataclass, field
3 -from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
4 - Protocol, Tuple, Union, overload)
3 +from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
4 + Optional, Protocol, Tuple, Union, overload)
5
6 import torch
7 import torch.nn as nn
@@ -294,10 +294,11 @@ def _embedding_count_expression(embeddings: NestedTensors) -> str:
294 _embedding_count_expression(inner) for inner in embeddings)
295
296
297 -def merge_multimodal_embeddings(input_ids: torch.Tensor,
298 - inputs_embeds: torch.Tensor,
299 - multimodal_embeddings: NestedTensors,
300 - placeholder_token_id: int) -> torch.Tensor:
297 +def _merge_multimodal_embeddings(
298 + inputs_embeds: torch.Tensor,
299 + is_multimodal: torch.Tensor,
300 + multimodal_embeddings: NestedTensors,
301 +) -> torch.Tensor:
302 """
303 Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
304 positions in ``inputs_embeds`` corresponding to placeholder tokens in
@@ -306,8 +307,7 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
307 Note:
308 This updates ``inputs_embeds`` in place.
309 """
309 - mask = (input_ids == placeholder_token_id)
310 - num_expected_tokens = mask.sum().item()
310 + num_expected_tokens = is_multimodal.sum().item()
311 assert isinstance(num_expected_tokens, int)
312
313 flattened = _flatten_embeddings(multimodal_embeddings)
@@ -317,10 +317,70 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
317 f"Attempted to assign {expr} = {flattened.shape[0]} "
318 f"multimodal tokens to {num_expected_tokens} placeholders")
319
320 - inputs_embeds[mask] = flattened
320 + inputs_embeds[is_multimodal] = flattened
321 return inputs_embeds
322
323
324 +def embed_multimodal(
325 + input_ids: torch.Tensor,
326 + multimodal_token_id: int,
327 + get_text_embeds: Callable[[torch.Tensor], torch.Tensor],
328 + get_multimodal_embeds: Callable[[torch.Tensor], Union[torch.Tensor,
329 + List[torch.Tensor]]],
330 +) -> torch.Tensor:
331 + """
332 + Embed token IDs and multimodal inputs and combine their embeddings.
333 +
334 + ``multimodal_token_id`` is used to determine whether a token ID should
335 + be embedded using ``get_text_embeds`` or ``get_multimodal_embeds``.
336 +
337 + Compared to ``merge_multimodal_embeddings`, this avoids running
338 + ``get_text_embeds`` on ``input_ids[input_ids == multimodal_token_id]``
339 + which causes issues when the placeholder token ID exceeds the
340 + vocabulary size of the language model.
341 + """
342 + is_multimodal = input_ids == multimodal_token_id
343 + is_text = ~is_multimodal
344 +
345 + text_embeds = get_text_embeds(input_ids[is_text])
346 + multimodal_embeds = get_multimodal_embeds(input_ids[is_multimodal])
347 +
348 + merged_embeds = torch.empty(
349 + (input_ids.shape[0], text_embeds.shape[1]),
350 + dtype=text_embeds.dtype,
351 + device=text_embeds.device,
352 + )
353 +
354 + merged_embeds[is_text] = text_embeds
355 +
356 + return _merge_multimodal_embeddings(
357 + merged_embeds,
358 + is_multimodal,
359 + multimodal_embeds,
360 + )
361 +
362 +
363 +def merge_multimodal_embeddings(
364 + input_ids: torch.Tensor,
365 + inputs_embeds: torch.Tensor,
366 + multimodal_embeddings: NestedTensors,
367 + placeholder_token_id: int,
368 +) -> torch.Tensor:
369 + """
370 + Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
371 + positions in ``inputs_embeds`` corresponding to placeholder tokens in
372 + ``input_ids``.
373 +
374 + Note:
375 + This updates ``inputs_embeds`` in place.
376 + """
377 + return _merge_multimodal_embeddings(
378 + inputs_embeds,
379 + (input_ids == placeholder_token_id),
380 + multimodal_embeddings,
381 + )
382 +
383 +
384 class LayerFn(Protocol):
385
386 def __call__(self, prefix: str) -> torch.nn.Module: