| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | |
| 6 | import openai |
| 7 | import pytest |
| 8 | import pytest_asyncio |
| 9 | |
| 10 | from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS |
| 11 | from tests.utils import RemoteOpenAIServer |
| 12 | from vllm.multimodal.utils import encode_image_url |
| 13 | |
| 14 | # Use a small vision model for testing |
| 15 | MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct" |
| 16 | MAXIMUM_IMAGES = 2 |
| 17 | |
| 18 | |
| 19 | @pytest.fixture(scope="module") |
| 20 | def default_image_server_args(): |
| 21 | return [ |
| 22 | "--enforce-eager", |
| 23 | "--max-model-len", |
| 24 | "6000", |
| 25 | "--max-num-seqs", |
| 26 | "128", |
| 27 | "--limit-mm-per-prompt", |
| 28 | json.dumps({"image": MAXIMUM_IMAGES}), |
| 29 | ] |
| 30 | |
| 31 | |
| 32 | @pytest.fixture(scope="module") |
| 33 | def image_server(default_image_server_args): |
| 34 | with RemoteOpenAIServer( |
| 35 | MODEL_NAME, |
| 36 | default_image_server_args, |
| 37 | env_dict={"VLLM_ENABLE_RESPONSES_API_STORE": "1"}, |
| 38 | ) as remote_server: |
| 39 | yield remote_server |
| 40 | |
| 41 | |
| 42 | @pytest_asyncio.fixture |
| 43 | async def client(image_server): |
| 44 | async with image_server.get_async_client() as async_client: |
| 45 | yield async_client |
| 46 | |
| 47 | |
| 48 | @pytest.fixture(scope="session") |
| 49 | def url_encoded_image(local_asset_server) -> dict[str, str]: |
| 50 | return { |
| 51 | image_url: encode_image_url(local_asset_server.get_image_asset(image_url)) |
| 52 | for image_url in TEST_IMAGE_ASSETS |
| 53 | } |
| 54 | |
| 55 | |
| 56 | @pytest.mark.asyncio |
| 57 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 58 | @pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True) |
| 59 | async def test_single_chat_session_image( |
| 60 | client: openai.AsyncOpenAI, model_name: str, image_url: str |
| 61 | ): |
| 62 | content_text = "What's in this image?" |
| 63 | messages = [ |
| 64 | { |
| 65 | "role": "user", |
| 66 | "content": [ |
| 67 | { |
| 68 | "type": "input_image", |
| 69 | "image_url": image_url, |
| 70 | "detail": "auto", |
| 71 | }, |
| 72 | {"type": "input_text", "text": content_text}, |
| 73 | ], |
| 74 | } |
| 75 | ] |
| 76 | |
| 77 | # test image url |
| 78 | response = await client.responses.create( |
| 79 | model=model_name, |
| 80 | input=messages, |
| 81 | ) |
| 82 | assert len(response.output_text) > 0 |
| 83 | |
| 84 | |
| 85 | @pytest.mark.asyncio |
| 86 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 87 | @pytest.mark.parametrize("raw_image_url", TEST_IMAGE_ASSETS) |
| 88 | async def test_single_chat_session_image_base64encoded( |
| 89 | client: openai.AsyncOpenAI, |
| 90 | model_name: str, |
| 91 | raw_image_url: str, |
| 92 | url_encoded_image: dict[str, str], |
| 93 | ): |
| 94 | content_text = "What's in this image?" |
| 95 | messages = [ |
| 96 | { |
| 97 | "role": "user", |
| 98 | "content": [ |
| 99 | { |
| 100 | "type": "input_image", |
| 101 | "image_url": url_encoded_image[raw_image_url], |
| 102 | "detail": "auto", |
| 103 | }, |
| 104 | {"type": "input_text", "text": content_text}, |
| 105 | ], |
| 106 | } |
| 107 | ] |
| 108 | # test image base64 |
| 109 | response = await client.responses.create( |
| 110 | model=model_name, |
| 111 | input=messages, |
| 112 | ) |
| 113 | assert len(response.output_text) > 0 |
| 114 | |
| 115 | |
| 116 | @pytest.mark.asyncio |
| 117 | @pytest.mark.parametrize("model_name", [MODEL_NAME]) |
| 118 | @pytest.mark.parametrize( |
| 119 | "image_urls", |
| 120 | [TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))], |
| 121 | indirect=True, |
| 122 | ) |
| 123 | async def test_multi_image_input( |
| 124 | client: openai.AsyncOpenAI, model_name: str, image_urls: list[str] |
| 125 | ): |
| 126 | messages = [ |
| 127 | { |
| 128 | "role": "user", |
| 129 | "content": [ |
| 130 | *( |
| 131 | { |
| 132 | "type": "input_image", |
| 133 | "image_url": image_url, |
| 134 | "detail": "auto", |
| 135 | } |
| 136 | for image_url in image_urls |
| 137 | ), |
| 138 | {"type": "input_text", "text": "What's in this image?"}, |
| 139 | ], |
| 140 | } |
| 141 | ] |
| 142 | |
| 143 | if len(image_urls) > MAXIMUM_IMAGES: |
| 144 | with pytest.raises(openai.BadRequestError): # test multi-image input |
| 145 | await client.responses.create( |
| 146 | model=model_name, |
| 147 | input=messages, |
| 148 | ) |
| 149 | # the server should still work afterwards |
| 150 | response = await client.responses.create( |
| 151 | model=model_name, |
| 152 | input=[ |
| 153 | { |
| 154 | "role": "user", |
| 155 | "content": "What's the weather like in Paris today?", |
| 156 | } |
| 157 | ], |
| 158 | ) |
| 159 | assert len(response.output_text) > 0 |
| 160 | else: |
| 161 | response = await client.responses.create( |
| 162 | model=model_name, |
| 163 | input=messages, |
| 164 | ) |
| 165 | assert len(response.output_text) > 0 |