| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | |
| 6 | import pytest |
| 7 | import requests |
| 8 | from transformers import AutoProcessor |
| 9 | |
| 10 | from tests.utils import VLLM_PATH, RemoteOpenAIServer |
| 11 | from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse |
| 12 | from vllm.multimodal.media import MediaWithBytes |
| 13 | from vllm.multimodal.utils import encode_image_url, fetch_image |
| 14 | |
| 15 | MODEL_NAME = "TIGER-Lab/VLM2Vec-Full" |
| 16 | MAXIMUM_IMAGES = 2 |
| 17 | |
| 18 | vlm2vec_jinja_path = VLLM_PATH / "examples/pooling/embed/template/vlm2vec_phi3v.jinja" |
| 19 | assert vlm2vec_jinja_path.exists() |
| 20 | |
| 21 | # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA) |
| 22 | TEST_IMAGE_ASSETS = [ |
| 23 | "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" |
| 24 | "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png", |
| 25 | "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png", |
| 26 | "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png", |
| 27 | ] |
| 28 | |
| 29 | input_text = "The best thing about vLLM is that it supports many different models" |
| 30 | image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg" |
| 31 | image_base64 = {"url": encode_image_url(fetch_image(image_url))} |
| 32 | |
| 33 | |
| 34 | @pytest.fixture(scope="module") |
| 35 | def server(): |
| 36 | args = [ |
| 37 | "--runner", |
| 38 | "pooling", |
| 39 | "--max-model-len", |
| 40 | "2048", |
| 41 | "--max-num-seqs", |
| 42 | "5", |
| 43 | "--enforce-eager", |
| 44 | "--trust-remote-code", |
| 45 | "--limit-mm-per-prompt", |
| 46 | json.dumps({"image": MAXIMUM_IMAGES}), |
| 47 | "--chat-template", |
| 48 | str(vlm2vec_jinja_path), |
| 49 | "--trust-request-mm-kwargs", |
| 50 | ] |
| 51 | |
| 52 | with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: |
| 53 | yield remote_server |
| 54 | |
| 55 | |
| 56 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 57 | def test_chat_text_request(server: RemoteOpenAIServer, model_name: str): |
| 58 | messages = [ |
| 59 | { |
| 60 | "role": "user", |
| 61 | "content": input_text, |
| 62 | }, |
| 63 | ] |
| 64 | |
| 65 | # note: vlm2vec_phi3v.jinja |
| 66 | # Embedding models should only embed one message at a time. |
| 67 | |
| 68 | response = requests.post( |
| 69 | server.url_for("v1/embeddings"), |
| 70 | json={"model": model_name, "messages": messages}, |
| 71 | ) |
| 72 | response.raise_for_status() |
| 73 | |
| 74 | output = EmbeddingResponse.model_validate(response.json()) |
| 75 | assert len(output.data) == 1 |
| 76 | assert output.model == MODEL_NAME |
| 77 | assert len(output.data[0].embedding) == 3072 |
| 78 | assert output.usage.prompt_tokens == 14 |
| 79 | |
| 80 | |
| 81 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 82 | def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str): |
| 83 | messages = [ |
| 84 | { |
| 85 | "role": "user", |
| 86 | "content": [ |
| 87 | {"type": "text", "text": "Represent the user's input."}, |
| 88 | {"type": "image_url", "image_url": {"url": image_url}}, |
| 89 | ], |
| 90 | } |
| 91 | ] |
| 92 | |
| 93 | response = requests.post( |
| 94 | server.url_for("v1/embeddings"), |
| 95 | json={"model": model_name, "messages": messages}, |
| 96 | ) |
| 97 | response.raise_for_status() |
| 98 | |
| 99 | output = EmbeddingResponse.model_validate(response.json()) |
| 100 | assert len(output.data) == 1 |
| 101 | assert output.model == MODEL_NAME |
| 102 | assert len(output.data[0].embedding) == 3072 |
| 103 | assert output.usage.prompt_tokens == 767 |
| 104 | |
| 105 | |
| 106 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 107 | def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str): |
| 108 | messages = [ |
| 109 | { |
| 110 | "role": "user", |
| 111 | "content": [ |
| 112 | {"type": "text", "text": "Represent the user's input."}, |
| 113 | {"type": "image_url", "image_url": image_base64}, |
| 114 | ], |
| 115 | } |
| 116 | ] |
| 117 | |
| 118 | response = requests.post( |
| 119 | server.url_for("v1/embeddings"), |
| 120 | json={"model": model_name, "messages": messages}, |
| 121 | ) |
| 122 | response.raise_for_status() |
| 123 | |
| 124 | output = EmbeddingResponse.model_validate(response.json()) |
| 125 | assert len(output.data) == 1 |
| 126 | assert output.model == MODEL_NAME |
| 127 | assert len(output.data[0].embedding) == 3072 |
| 128 | assert output.usage.prompt_tokens == 767 |
| 129 | |
| 130 | |
| 131 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 132 | def test_chat_image_with_media_io_kwargs(server: RemoteOpenAIServer, model_name: str): |
| 133 | rgba_image_url = ( |
| 134 | "https://vllm-public-assets.s3.us-west-2.amazonaws.com" |
| 135 | "/vision_model_images/RGBA_comp.png" |
| 136 | ) |
| 137 | messages = [ |
| 138 | { |
| 139 | "role": "user", |
| 140 | "content": [ |
| 141 | {"type": "text", "text": "Represent the user's input."}, |
| 142 | {"type": "image_url", "image_url": {"url": rgba_image_url}}, |
| 143 | ], |
| 144 | } |
| 145 | ] |
| 146 | |
| 147 | response = requests.post( |
| 148 | server.url_for("v1/embeddings"), |
| 149 | json={ |
| 150 | "model": model_name, |
| 151 | "messages": messages, |
| 152 | "media_io_kwargs": { |
| 153 | "image": {"rgba_background_color": [0, 0, 0]}, |
| 154 | }, |
| 155 | }, |
| 156 | ) |
| 157 | response.raise_for_status() |
| 158 | |
| 159 | output = EmbeddingResponse.model_validate(response.json()) |
| 160 | assert len(output.data) == 1 |
| 161 | assert len(output.data[0].embedding) == 3072 |
| 162 | |
| 163 | |
| 164 | def get_hf_prompt_tokens(model_name, content, image_url): |
| 165 | processor = AutoProcessor.from_pretrained( |
| 166 | model_name, trust_remote_code=True, num_crops=4 |
| 167 | ) |
| 168 | |
| 169 | placeholder = "<|image_1|> " |
| 170 | prompt = f"{placeholder}{content}" |
| 171 | image = fetch_image(image_url) |
| 172 | # Unwrap MediaWithBytes if present |
| 173 | if isinstance(image, MediaWithBytes): |
| 174 | image = image.media |
| 175 | images = [image] |
| 176 | inputs = processor(prompt, images, return_tensors="pt") |
| 177 | return inputs.input_ids.shape[1] |
| 178 | |
| 179 | |
| 180 | @pytest.mark.asyncio |
| 181 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 182 | @pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True) |
| 183 | async def test_image_embedding( |
| 184 | server: RemoteOpenAIServer, model_name: str, image_url: str |
| 185 | ): |
| 186 | content_text = "Represent the given image." |
| 187 | messages = [ |
| 188 | { |
| 189 | "role": "user", |
| 190 | "content": [ |
| 191 | {"type": "image_url", "image_url": {"url": image_url}}, |
| 192 | {"type": "text", "text": content_text}, |
| 193 | ], |
| 194 | } |
| 195 | ] |
| 196 | |
| 197 | response = requests.post( |
| 198 | server.url_for("v1/embeddings"), |
| 199 | json={"model": model_name, "messages": messages, "encoding_format": "float"}, |
| 200 | ) |
| 201 | response.raise_for_status() |
| 202 | embeddings = EmbeddingResponse.model_validate(response.json()) |
| 203 | |
| 204 | hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url) |
| 205 | |
| 206 | assert embeddings.id is not None |
| 207 | assert len(embeddings.data) == 1 |
| 208 | assert len(embeddings.data[0].embedding) == 3072 |
| 209 | assert embeddings.usage.completion_tokens == 0 |
| 210 | assert embeddings.usage.prompt_tokens == hf_prompt_tokens |
| 211 | assert embeddings.usage.total_tokens == hf_prompt_tokens |