A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Support E5-V (#9576)
Cyrus Leung committed
Oct 23, 2024 at 11:35 UTC
831540cf04b0b40cd1fe462356de4a30b831e4ea
12 files changed
+532
-90
docs/source/models/supported_models.rst
+14
@@ -334,6 +334,14 @@ The following modalities are supported depending on the model:
334
- **V**\ ideo
335
- **A**\ udio
336
337
+Any combination of modalities joined by :code:`+` are supported.
338
+
339
+- e.g.: :code:`T + I` means that the model supports text-only, image-only, and text-with-image inputs.
340
+
341
+On the other hand, modalities separated by :code:`/` are mutually exclusive.
342
+
343
+- e.g.: :code:`T / I` means that the model supports text-only and image-only inputs, but not text-with-image inputs.
344
+
345
.. _supported_vlms:
346
347
Text Generation
@@ -484,6 +492,12 @@ Multimodal Embedding
492
- Example HF Models
493
- :ref:`LoRA <lora>`
494
- :ref:`PP <distributed_serving>`
495
+ * - :code:`LlavaNextForConditionalGeneration`
496
+ - LLaVA-NeXT-based
497
+ - T / I
498
+ - :code:`royokong/e5-v`
499
+ -
500
+ - ✅︎
501
* - :code:`Phi3VForCausalLM`
502
- Phi-3-Vision-based
503
- T + I
examples/offline_inference_vision_language.py
+3
-3
@@ -1,6 +1,6 @@
1
"""
2
-This example shows how to use vLLM for running offline inference
3
-with the correct prompt format on vision language models.
2
+This example shows how to use vLLM for running offline inference with
3
+the correct prompt format on vision language models for text generation.
4
5
For most models, the prompt format should follow corresponding examples
6
on HuggingFace model repository.
@@ -450,7 +450,7 @@ def main(args):
450
if __name__ == "__main__":
451
parser = FlexibleArgumentParser(
452
description='Demo on using vLLM for offline inference with '
453
- 'vision language models')
453
+ 'vision language models for text generation')
454
parser.add_argument('--model-type',
455
'-m',
456
type=str,
examples/offline_inference_vision_language_embedding.py
+169
-21
@@ -1,22 +1,170 @@
1
+"""
2
+This example shows how to use vLLM for running offline inference with
3
+the correct prompt format on vision language models for multimodal embedding.
4
+
5
+For most models, the prompt format should follow corresponding examples
6
+on HuggingFace model repository.
7
+"""
8
+from argparse import Namespace
9
+from typing import Literal, NamedTuple, Optional, TypedDict, Union, get_args
10
+
11
+from PIL.Image import Image
12
+
13
from vllm import LLM
2
-from vllm.assets.image import ImageAsset
3
-
4
-image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
5
-prompt = "<|image_1|> Represent the given image with the following question: What is in the image" # noqa: E501
6
-
7
-# Create an LLM.
8
-llm = LLM(
9
- model="TIGER-Lab/VLM2Vec-Full",
10
- task="embedding",
11
- trust_remote_code=True,
12
- max_model_len=4096,
13
- max_num_seqs=2,
14
- mm_processor_kwargs={"num_crops": 16},
15
-)
16
-
17
-# Generate embedding. The output is a list of EmbeddingRequestOutputs.
18
-outputs = llm.encode({"prompt": prompt, "multi_modal_data": {"image": image}})
19
-
20
-# Print the outputs.
21
-for output in outputs:
22
- print(output.outputs.embedding) # list of 3072 floats
14
+from vllm.multimodal.utils import fetch_image
15
+from vllm.utils import FlexibleArgumentParser
16
+
17
+
18
+class TextQuery(TypedDict):
19
+ modality: Literal["text"]
20
+ text: str
21
+
22
+
23
+class ImageQuery(TypedDict):
24
+ modality: Literal["image"]
25
+ image: Image
26
+
27
+
28
+class TextImageQuery(TypedDict):
29
+ modality: Literal["text+image"]
30
+ text: str
31
+ image: Image
32
+
33
+
34
+QueryModality = Literal["text", "image", "text+image"]
35
+Query = Union[TextQuery, ImageQuery, TextImageQuery]
36
+
37
+
38
+class ModelRequestData(NamedTuple):
39
+ llm: LLM
40
+ prompt: str
41
+ image: Optional[Image]
42
+
43
+
44
+def run_e5_v(query: Query):
45
+ llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' # noqa: E501
46
+
47
+ if query["modality"] == "text":
48
+ text = query["text"]
49
+ prompt = llama3_template.format(
50
+ f"{text}\nSummary above sentence in one word: ")
51
+ image = None
52
+ elif query["modality"] == "image":
53
+ prompt = llama3_template.format(
54
+ "<image>\nSummary above image in one word: ")
55
+ image = query["image"]
56
+ else:
57
+ modality = query['modality']
58
+ raise ValueError(f"Unsupported query modality: '{modality}'")
59
+
60
+ llm = LLM(
61
+ model="royokong/e5-v",
62
+ task="embedding",
63
+ max_model_len=4096,
64
+ )
65
+
66
+ return ModelRequestData(
67
+ llm=llm,
68
+ prompt=prompt,
69
+ image=image,
70
+ )
71
+
72
+
73
+def run_vlm2vec(query: Query):
74
+ if query["modality"] == "text":
75
+ text = query["text"]
76
+ prompt = f"Find me an everyday image that matches the given caption: {text}" # noqa: E501
77
+ image = None
78
+ elif query["modality"] == "image":
79
+ prompt = "<|image_1|> Find a day-to-day image that looks similar to the provided image." # noqa: E501
80
+ image = query["image"]
81
+ elif query["modality"] == "text+image":
82
+ text = query["text"]
83
+ prompt = f"<|image_1|> Represent the given image with the following question: {text}" # noqa: E501
84
+ image = query["image"]
85
+ else:
86
+ modality = query['modality']
87
+ raise ValueError(f"Unsupported query modality: '{modality}'")
88
+
89
+ llm = LLM(
90
+ model="TIGER-Lab/VLM2Vec-Full",
91
+ task="embedding",
92
+ trust_remote_code=True,
93
+ mm_processor_kwargs={"num_crops": 4},
94
+ )
95
+
96
+ return ModelRequestData(
97
+ llm=llm,
98
+ prompt=prompt,
99
+ image=image,
100
+ )
101
+
102
+
103
+def get_query(modality: QueryModality):
104
+ if modality == "text":
105
+ return TextQuery(modality="text", text="A dog sitting in the grass")
106
+
107
+ if modality == "image":
108
+ return ImageQuery(
109
+ modality="image",
110
+ image=fetch_image(
111
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/American_Eskimo_Dog.jpg/360px-American_Eskimo_Dog.jpg" # noqa: E501
112
+ ),
113
+ )
114
+
115
+ if modality == "text+image":
116
+ return TextImageQuery(
117
+ modality="text+image",
118
+ text="A cat standing in the snow.",
119
+ image=fetch_image(
120
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/b/b6/Felis_catus-cat_on_snow.jpg/179px-Felis_catus-cat_on_snow.jpg" # noqa: E501
121
+ ),
122
+ )
123
+
124
+ msg = f"Modality {modality} is not supported."
125
+ raise ValueError(msg)
126
+
127
+
128
+def run_encode(model: str, modality: QueryModality):
129
+ query = get_query(modality)
130
+ req_data = model_example_map[model](query)
131
+
132
+ mm_data = {}
133
+ if req_data.image is not None:
134
+ mm_data["image"] = req_data.image
135
+
136
+ outputs = req_data.llm.encode({
137
+ "prompt": req_data.prompt,
138
+ "multi_modal_data": mm_data,
139
+ })
140
+
141
+ for output in outputs:
142
+ print(output.outputs.embedding)
143
+
144
+
145
+def main(args: Namespace):
146
+ run_encode(args.model_name, args.modality)
147
+
148
+
149
+model_example_map = {
150
+ "e5_v": run_e5_v,
151
+ "vlm2vec": run_vlm2vec,
152
+}
153
+
154
+if __name__ == "__main__":
155
+ parser = FlexibleArgumentParser(
156
+ description='Demo on using vLLM for offline inference with '
157
+ 'vision language models for multimodal embedding')
158
+ parser.add_argument('--model-name',
159
+ '-m',
160
+ type=str,
161
+ default="vlm2vec",
162
+ choices=model_example_map.keys(),
163
+ help='The name of the embedding model.')
164
+ parser.add_argument('--modality',
165
+ type=str,
166
+ default="image",
167
+ choices=get_args(QueryModality),
168
+ help='Modality of the input.')
169
+ args = parser.parse_args()
170
+ main(args)
examples/offline_inference_vision_language_multi_image.py
+4
-3
@@ -1,7 +1,7 @@
1
"""
2
This example shows how to use vLLM for running offline inference with
3
-multi-image input on vision language models, using the chat template defined
4
-by the model.
3
+multi-image input on vision language models for text generation,
4
+using the chat template defined by the model.
5
"""
6
from argparse import Namespace
7
from typing import List, NamedTuple, Optional
@@ -334,7 +334,8 @@ def main(args: Namespace):
334
if __name__ == "__main__":
335
parser = FlexibleArgumentParser(
336
description='Demo on using vLLM for offline inference with '
337
- 'vision language models that support multi-image input')
337
+ 'vision language models that support multi-image input for text '
338
+ 'generation')
339
parser.add_argument('--model-type',
340
'-m',
341
type=str,
tests/conftest.py
+36
-24
@@ -43,10 +43,12 @@ _TEST_DIR = os.path.dirname(__file__)
43
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
44
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
45
46
-PromptImageInput = Union[List[Image.Image], List[List[Image.Image]]]
47
-PromptAudioInput = Union[List[Tuple[np.ndarray, int]],
48
- List[List[Tuple[np.ndarray, int]]]]
49
-PromptVideoInput = Union[List[np.ndarray], List[List[np.ndarray]]]
46
+_M = TypeVar("_M")
47
+_PromptMultiModalInput = Union[List[_M], List[List[_M]]]
48
+
49
+PromptImageInput = _PromptMultiModalInput[Image.Image]
50
+PromptAudioInput = _PromptMultiModalInput[Tuple[np.ndarray, int]]
51
+PromptVideoInput = _PromptMultiModalInput[np.ndarray]
52
53
54
def _read_prompts(filename: str) -> List[str]:
@@ -318,12 +320,12 @@ class HfRunner:
320
"text": prompt,
321
"return_tensors": "pt",
322
}
321
- if images is not None and images[i] is not None:
322
- processor_kwargs["images"] = images[i]
323
- if videos is not None and videos[i] is not None:
324
- processor_kwargs["videos"] = videos[i]
325
- if audios is not None and audios[i] is not None:
326
- audio, sr = audios[i]
323
+ if images is not None and (image := images[i]) is not None:
324
+ processor_kwargs["images"] = image
325
+ if videos is not None and (video := videos[i]) is not None:
326
+ processor_kwargs["videos"] = video
327
+ if audios is not None and (audio_tuple := audios[i]) is not None:
328
+ audio, sr = audio_tuple
329
processor_kwargs["audio"] = audio
330
processor_kwargs["sampling_rate"] = sr
331
@@ -338,7 +340,7 @@ class HfRunner:
340
self,
341
prompts: List[str],
342
images: Optional[PromptImageInput] = None,
341
- videos: Optional[List[np.ndarray]] = None,
343
+ videos: Optional[PromptVideoInput] = None,
344
audios: Optional[PromptAudioInput] = None,
345
**kwargs: Any,
346
) -> List[Tuple[List[List[int]], List[str]]]:
@@ -368,7 +370,7 @@ class HfRunner:
370
prompts: List[str],
371
max_tokens: int,
372
images: Optional[PromptImageInput] = None,
371
- videos: Optional[List[np.ndarray]] = None,
373
+ videos: Optional[PromptVideoInput] = None,
374
audios: Optional[PromptAudioInput] = None,
375
**kwargs: Any,
376
) -> List[Tuple[List[int], str]]:
@@ -409,7 +411,7 @@ class HfRunner:
411
prompts: List[str],
412
max_tokens: int,
413
images: Optional[PromptImageInput] = None,
412
- videos: Optional[List[np.ndarray]] = None,
414
+ videos: Optional[PromptVideoInput] = None,
415
audios: Optional[PromptAudioInput] = None,
416
**kwargs: Any,
417
) -> List[List[torch.Tensor]]:
@@ -488,7 +490,7 @@ class HfRunner:
490
num_logprobs: int,
491
images: Optional[PromptImageInput] = None,
492
audios: Optional[PromptAudioInput] = None,
491
- videos: Optional[List[np.ndarray]] = None,
493
+ videos: Optional[PromptVideoInput] = None,
494
**kwargs: Any,
495
) -> List[TokensTextLogprobs]:
496
all_inputs = self.get_inputs(prompts,
@@ -657,15 +659,18 @@ class VllmRunner:
659
inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
660
if images is not None:
661
for i, image in enumerate(images):
660
- inputs[i]["multi_modal_data"] = {"image": image}
662
+ if image is not None:
663
+ inputs[i]["multi_modal_data"] = {"image": image}
664
665
if videos is not None:
666
for i, video in enumerate(videos):
664
- inputs[i]["multi_modal_data"] = {"video": video}
667
+ if video is not None:
668
+ inputs[i]["multi_modal_data"] = {"video": video}
669
670
if audios is not None:
671
for i, audio in enumerate(audios):
668
- inputs[i]["multi_modal_data"] = {"audio": audio}
672
+ if audio is not None:
673
+ inputs[i]["multi_modal_data"] = {"audio": audio}
674
675
return inputs
676
@@ -837,13 +842,20 @@ class VllmRunner:
842
returned_outputs.append((token_ids, texts))
843
return returned_outputs
844
840
- def encode(self, prompts: List[str]) -> List[List[float]]:
841
- req_outputs = self.model.encode(prompts)
842
- outputs = []
843
- for req_output in req_outputs:
844
- embedding = req_output.outputs.embedding
845
- outputs.append(embedding)
846
- return outputs
845
+ def encode(
846
+ self,
847
+ prompts: List[str],
848
+ images: Optional[PromptImageInput] = None,
849
+ videos: Optional[PromptVideoInput] = None,
850
+ audios: Optional[PromptAudioInput] = None,
851
+ ) -> List[List[float]]:
852
+ inputs = self.get_inputs(prompts,
853
+ images=images,
854
+ videos=videos,
855
+ audios=audios)
856
+
857
+ req_outputs = self.model.encode(inputs)
858
+ return [req_output.outputs.embedding for req_output in req_outputs]
859
860
def __enter__(self):
861
return self
tests/models/embedding/utils.py
+2
-1
@@ -16,7 +16,8 @@ def check_embeddings_close(
16
17
for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
18
zip(embeddings_0_lst, embeddings_1_lst)):
19
- assert len(embeddings_0) == len(embeddings_1)
19
+ assert len(embeddings_0) == len(embeddings_1), (
20
+ f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}")
21
22
sim = F.cosine_similarity(torch.tensor(embeddings_0),
23
torch.tensor(embeddings_1),
tests/models/embedding/vision_language/test_llava_next.py
new
+135
@@ -0,0 +1,135 @@
1
+from typing import List, Type
2
+
3
+import pytest
4
+import torch.nn.functional as F
5
+from transformers import AutoModelForVision2Seq
6
+
7
+from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
8
+from ....utils import large_gpu_test
9
+from ..utils import check_embeddings_close
10
+
11
+llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n' # noqa: E501
12
+
13
+HF_TEXT_PROMPTS = [
14
+ # T -> X
15
+ llama3_template.format(
16
+ "The label of the object is stop sign\nSummary above sentence in one word: " # noqa: E501
17
+ ),
18
+ # T -> X
19
+ llama3_template.format(
20
+ "cherry blossom\nSummary above sentence in one word: "),
21
+]
22
+
23
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
24
+ # I -> X
25
+ "stop_sign":
26
+ llama3_template.format("<image>\nSummary above image in one word: "),
27
+ # I -> X
28
+ "cherry_blossom":
29
+ llama3_template.format("<image>\nSummary above image in one word: "),
30
+})
31
+
32
+MODELS = ["royokong/e5-v"]
33
+
34
+
35
+def _run_test(
36
+ hf_runner: Type[HfRunner],
37
+ vllm_runner: Type[VllmRunner],
38
+ input_texts: List[str],
39
+ input_images: PromptImageInput,
40
+ model: str,
41
+ *,
42
+ dtype: str,
43
+) -> None:
44
+ # NOTE: take care of the order. run vLLM first, and then run HF.
45
+ # vLLM needs a fresh new process without cuda initialization.
46
+ # if we run HF first, the cuda initialization will be done and it
47
+ # will hurt multiprocessing backend with fork method (the default method).
48
+ with vllm_runner(model,
49
+ task="embedding",
50
+ dtype=dtype,
51
+ max_model_len=4096,
52
+ enforce_eager=True) as vllm_model:
53
+ vllm_outputs = vllm_model.encode(input_texts, images=input_images)
54
+
55
+ with hf_runner(model, dtype=dtype,
56
+ auto_cls=AutoModelForVision2Seq) as hf_model:
57
+ # Patch the issue where image_token_id
58
+ # exceeds the maximum allowed vocab size
59
+ hf_model.model.resize_token_embeddings(
60
+ hf_model.model.language_model.vocab_size + 1)
61
+
62
+ all_inputs = hf_model.get_inputs(input_texts, images=input_images)
63
+
64
+ all_outputs = []
65
+ for inputs in all_inputs:
66
+ # Based on: https://huggingface.co/royokong/e5-v
67
+ outputs = hf_model.model(
68
+ **hf_model.wrap_device(inputs,
69
+ device=hf_model.model.device.type),
70
+ return_dict=True,
71
+ output_hidden_states=True,
72
+ )
73
+ pooled_output = F.normalize(outputs.hidden_states[-1][0, -1, :],
74
+ dim=-1)
75
+
76
+ all_outputs.append(pooled_output.tolist())
77
+
78
+ hf_outputs = all_outputs
79
+
80
+ check_embeddings_close(
81
+ embeddings_0_lst=hf_outputs,
82
+ embeddings_1_lst=vllm_outputs,
83
+ name_0="hf",
84
+ name_1="vllm",
85
+ )
86
+
87
+
88
+@pytest.mark.parametrize("model", MODELS)
89
+@pytest.mark.parametrize("dtype", ["half"])
90
+def test_models_text(
91
+ hf_runner,
92
+ vllm_runner,
93
+ image_assets,
94
+ model: str,
95
+ dtype: str,
96
+) -> None:
97
+ input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
98
+ input_texts = [text for text, _ in input_texts_images]
99
+ input_images = [image for _, image in input_texts_images]
100
+
101
+ _run_test(
102
+ hf_runner,
103
+ vllm_runner,
104
+ input_texts,
105
+ input_images, # type: ignore
106
+ model,
107
+ dtype=dtype,
108
+ )
109
+
110
+
111
+@large_gpu_test(min_gb=48)
112
+@pytest.mark.parametrize("model", MODELS)
113
+@pytest.mark.parametrize("dtype", ["half"])
114
+def test_models_image(
115
+ hf_runner,
116
+ vllm_runner,
117
+ image_assets,
118
+ model: str,
119
+ dtype: str,
120
+) -> None:
121
+ input_texts_images = [
122
+ (text, asset.pil_image)
123
+ for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
124
+ ]
125
+ input_texts = [text for text, _ in input_texts_images]
126
+ input_images = [image for _, image in input_texts_images]
127
+
128
+ _run_test(
129
+ hf_runner,
130
+ vllm_runner,
131
+ input_texts,
132
+ input_images,
133
+ model,
134
+ dtype=dtype,
135
+ )
tests/models/embedding/vision_language/test_phi3v.py
+77
-16
@@ -1,42 +1,53 @@
1
+from typing import List, Type
2
+
3
import pytest
4
import torch.nn.functional as F
5
4
-from ....conftest import IMAGE_ASSETS
6
+from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
7
+from ....utils import large_gpu_test
8
from ..utils import check_embeddings_close
9
10
+HF_TEXT_PROMPTS = [
11
+ # T -> X
12
+ "Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501
13
+ # T -> X
14
+ "Retrieve an image of this caption: cherry blossom",
15
+]
16
+
17
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18
+ # T + I -> X
19
"stop_sign":
20
"<|image_1|> Select the portion of the image that isolates the object of the given label: The label of the object is stop sign", # noqa: E501
21
+ # I -> X
22
"cherry_blossom":
11
- "<|image_1|> Represent the given image with the following question: What is in the image", # noqa: E501
23
+ "<|image_1|> Represent the given image for classification", # noqa: E501
24
})
25
26
MODELS = ["TIGER-Lab/VLM2Vec-Full"]
27
28
17
-@pytest.mark.parametrize("model", MODELS)
18
-@pytest.mark.parametrize("dtype", ["half"])
19
-def test_models(
20
- hf_runner,
21
- vllm_runner,
22
- example_prompts,
29
+def _run_test(
30
+ hf_runner: Type[HfRunner],
31
+ vllm_runner: Type[VllmRunner],
32
+ input_texts: List[str],
33
+ input_images: PromptImageInput,
34
model: str,
35
+ *,
36
dtype: str,
37
) -> None:
38
# NOTE: take care of the order. run vLLM first, and then run HF.
39
# vLLM needs a fresh new process without cuda initialization.
40
# if we run HF first, the cuda initialization will be done and it
41
# will hurt multiprocessing backend with fork method (the default method).
30
- with vllm_runner(model,
31
- task="embedding",
32
- max_model_len=4096,
33
- max_num_seqs=2,
34
- dtype=dtype,
42
+ with vllm_runner(model, task="embedding", dtype=dtype,
43
enforce_eager=True) as vllm_model:
36
- vllm_outputs = vllm_model.encode(example_prompts)
44
+ vllm_outputs = vllm_model.encode(input_texts, images=input_images)
45
38
- with hf_runner(model, dtype=dtype) as hf_model:
39
- all_inputs = hf_model.get_inputs(example_prompts)
46
+ # use eager mode for hf runner, since phi3_v didn't work with flash_attn
47
+ hf_model_kwargs = {"_attn_implementation": "eager"}
48
+ with hf_runner(model, dtype=dtype,
49
+ model_kwargs=hf_model_kwargs) as hf_model:
50
+ all_inputs = hf_model.get_inputs(input_texts, images=input_images)
51
52
all_outputs = []
53
for inputs in all_inputs:
@@ -61,3 +72,53 @@ def test_models(
72
name_0="hf",
73
name_1="vllm",
74
)
75
+
76
+
77
+@pytest.mark.parametrize("model", MODELS)
78
+@pytest.mark.parametrize("dtype", ["half"])
79
+def test_models_text(
80
+ hf_runner,
81
+ vllm_runner,
82
+ image_assets,
83
+ model: str,
84
+ dtype: str,
85
+) -> None:
86
+ input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
87
+ input_texts = [text for text, _ in input_texts_images]
88
+ input_images = [image for _, image in input_texts_images]
89
+
90
+ _run_test(
91
+ hf_runner,
92
+ vllm_runner,
93
+ input_texts,
94
+ input_images, # type: ignore
95
+ model,
96
+ dtype=dtype,
97
+ )
98
+
99
+
100
+@large_gpu_test(min_gb=48)
101
+@pytest.mark.parametrize("model", MODELS)
102
+@pytest.mark.parametrize("dtype", ["half"])
103
+def test_models_image(
104
+ hf_runner,
105
+ vllm_runner,
106
+ image_assets,
107
+ model: str,
108
+ dtype: str,
109
+) -> None:
110
+ input_texts_images = [
111
+ (text, asset.pil_image)
112
+ for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
113
+ ]
114
+ input_texts = [text for text, _ in input_texts_images]
115
+ input_images = [image for _, image in input_texts_images]
116
+
117
+ _run_test(
118
+ hf_runner,
119
+ vllm_runner,
120
+ input_texts,
121
+ input_images,
122
+ model,
123
+ dtype=dtype,
124
+ )
vllm/model_executor/models/llava_next.py
+22
-11
@@ -13,11 +13,13 @@ from typing_extensions import NotRequired
13
from vllm.attention import AttentionMetadata
14
from vllm.config import CacheConfig, MultiModalConfig
15
from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19
+from vllm.model_executor.pooling_metadata import PoolingMetadata
20
from vllm.model_executor.sampling_metadata import SamplingMetadata
21
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.sequence import IntermediateTensors
22
+from vllm.sequence import IntermediateTensors, PoolerOutput
23
from vllm.utils import is_list_of
24
25
from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -28,8 +30,8 @@ from .llava import LlavaMultiModalProjector
30
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
32
get_siglip_patch_grid_length, input_processor_for_siglip)
31
-from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
32
- merge_multimodal_embeddings)
33
+from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
34
+ init_vllm_registered_model)
35
36
# Result in the max possible feature size (2x2 grid of 336x336px tiles)
37
MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
@@ -312,6 +314,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
314
self.language_model = init_vllm_registered_model(
315
config.text_config, cache_config, quant_config)
316
317
+ # The same model class supports both language generation and embedding
318
+ # because the architecture name is the same
319
+ self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
320
+
321
self.make_empty_intermediate_tensors = (
322
self.language_model.make_empty_intermediate_tensors)
323
@@ -605,14 +611,12 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
611
image_input = self._parse_and_validate_image_input(**kwargs)
612
613
if image_input is not None:
608
- vision_embeddings = self._process_image_input(image_input)
609
- inputs_embeds = self.language_model.model.get_input_embeddings(
610
- input_ids)
611
-
612
- inputs_embeds = merge_multimodal_embeddings(
613
- input_ids, inputs_embeds, vision_embeddings,
614
- self.config.image_token_index)
615
-
614
+ inputs_embeds = embed_multimodal(
615
+ input_ids,
616
+ self.config.image_token_index,
617
+ self.language_model.model.get_input_embeddings,
618
+ lambda _: self._process_image_input(image_input),
619
+ )
620
input_ids = None
621
else:
622
inputs_embeds = None
@@ -641,6 +645,13 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
645
) -> Optional[SamplerOutput]:
646
return self.language_model.sample(logits, sampling_metadata)
647
648
+ def pooler(
649
+ self,
650
+ hidden_states: torch.Tensor,
651
+ pooling_metadata: PoolingMetadata,
652
+ ) -> Optional[PoolerOutput]:
653
+ return self._pooler(hidden_states, pooling_metadata)
654
+
655
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
656
loader = AutoWeightsLoader(self)
657
loader.load_weights(weights)
vllm/model_executor/models/phi3v.py
-2
@@ -467,8 +467,6 @@ def input_processor_for_phi3v(ctx: InputContext,
467
468
prompt_token_ids = inputs["prompt_token_ids"].copy()
469
470
- print("prompt_token_ids (old)", prompt_token_ids)
471
-
470
# masked placeholder with image token id
471
for idx in image_idx:
472
candidates = _get_image_placeholder_token_id_candidates(model_config,
vllm/model_executor/models/registry.py
+1
@@ -94,6 +94,7 @@ _EMBEDDING_MODELS = {
94
"MistralModel": ("llama", "LlamaEmbeddingModel"),
95
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
96
# [Multimodal]
97
+ "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
98
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
99
}
100
vllm/model_executor/models/utils.py
+69
-9
@@ -1,7 +1,7 @@
1
import itertools
2
from dataclasses import dataclass, field
3
-from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
4
- Protocol, Tuple, Union, overload)
3
+from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
4
+ Optional, Protocol, Tuple, Union, overload)
5
6
import torch
7
import torch.nn as nn
@@ -294,10 +294,11 @@ def _embedding_count_expression(embeddings: NestedTensors) -> str:
294
_embedding_count_expression(inner) for inner in embeddings)
295
296
297
-def merge_multimodal_embeddings(input_ids: torch.Tensor,
298
- inputs_embeds: torch.Tensor,
299
- multimodal_embeddings: NestedTensors,
300
- placeholder_token_id: int) -> torch.Tensor:
297
+def _merge_multimodal_embeddings(
298
+ inputs_embeds: torch.Tensor,
299
+ is_multimodal: torch.Tensor,
300
+ multimodal_embeddings: NestedTensors,
301
+) -> torch.Tensor:
302
"""
303
Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
304
positions in ``inputs_embeds`` corresponding to placeholder tokens in
@@ -306,8 +307,7 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
307
Note:
308
This updates ``inputs_embeds`` in place.
309
"""
309
- mask = (input_ids == placeholder_token_id)
310
- num_expected_tokens = mask.sum().item()
310
+ num_expected_tokens = is_multimodal.sum().item()
311
assert isinstance(num_expected_tokens, int)
312
313
flattened = _flatten_embeddings(multimodal_embeddings)
@@ -317,10 +317,70 @@ def merge_multimodal_embeddings(input_ids: torch.Tensor,
317
f"Attempted to assign {expr} = {flattened.shape[0]} "
318
f"multimodal tokens to {num_expected_tokens} placeholders")
319
320
- inputs_embeds[mask] = flattened
320
+ inputs_embeds[is_multimodal] = flattened
321
return inputs_embeds
322
323
324
+def embed_multimodal(
325
+ input_ids: torch.Tensor,
326
+ multimodal_token_id: int,
327
+ get_text_embeds: Callable[[torch.Tensor], torch.Tensor],
328
+ get_multimodal_embeds: Callable[[torch.Tensor], Union[torch.Tensor,
329
+ List[torch.Tensor]]],
330
+) -> torch.Tensor:
331
+ """
332
+ Embed token IDs and multimodal inputs and combine their embeddings.
333
+
334
+ ``multimodal_token_id`` is used to determine whether a token ID should
335
+ be embedded using ``get_text_embeds`` or ``get_multimodal_embeds``.
336
+
337
+ Compared to ``merge_multimodal_embeddings`, this avoids running
338
+ ``get_text_embeds`` on ``input_ids[input_ids == multimodal_token_id]``
339
+ which causes issues when the placeholder token ID exceeds the
340
+ vocabulary size of the language model.
341
+ """
342
+ is_multimodal = input_ids == multimodal_token_id
343
+ is_text = ~is_multimodal
344
+
345
+ text_embeds = get_text_embeds(input_ids[is_text])
346
+ multimodal_embeds = get_multimodal_embeds(input_ids[is_multimodal])
347
+
348
+ merged_embeds = torch.empty(
349
+ (input_ids.shape[0], text_embeds.shape[1]),
350
+ dtype=text_embeds.dtype,
351
+ device=text_embeds.device,
352
+ )
353
+
354
+ merged_embeds[is_text] = text_embeds
355
+
356
+ return _merge_multimodal_embeddings(
357
+ merged_embeds,
358
+ is_multimodal,
359
+ multimodal_embeds,
360
+ )
361
+
362
+
363
+def merge_multimodal_embeddings(
364
+ input_ids: torch.Tensor,
365
+ inputs_embeds: torch.Tensor,
366
+ multimodal_embeddings: NestedTensors,
367
+ placeholder_token_id: int,
368
+) -> torch.Tensor:
369
+ """
370
+ Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
371
+ positions in ``inputs_embeds`` corresponding to placeholder tokens in
372
+ ``input_ids``.
373
+
374
+ Note:
375
+ This updates ``inputs_embeds`` in place.
376
+ """
377
+ return _merge_multimodal_embeddings(
378
+ inputs_embeds,
379
+ (input_ids == placeholder_token_id),
380
+ multimodal_embeddings,
381
+ )
382
+
383
+
384
class LayerFn(Protocol):
385
386
def __call__(self, prefix: str) -> torch.nn.Module: