@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb62dda2c3

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 00:00 UTC

[Misc] Introduce `encode_*_url` utility function (#31208)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 23, 2025 at 21:45 UTC bb62dda2c32ab2be79bea0d0e1a96b4926b6c63f

14 files changed +134 -96

tests/entrypoints/openai/test_audio.py

+11 -5

@@ -8,7 +8,7 @@ import pytest

8 import pytest_asyncio

10 from vllm.assets.audio import AudioAsset

11 -from vllm.multimodal.utils import encode_audio_base64, fetch_audio

11 +from vllm.multimodal.utils import encode_audio_base64, encode_audio_url, fetch_audio

13 from ...utils import RemoteOpenAIServer

@@ -53,6 +53,14 @@ def base64_encoded_audio() -> dict[str, str]:

53 }

56 +@pytest.fixture(scope="session")

57 +def url_encoded_audio() -> dict[str, str]:

58 + return {

59 + audio_url: encode_audio_url(*fetch_audio(audio_url))

60 + for audio_url in TEST_AUDIO_URLS

61 + }

62 +

63 +

64 def dummy_messages_from_audio_url(

65 audio_urls: str | list[str],

66 content_text: str = "What's happening in this audio?",

@@ -149,11 +157,9 @@ async def test_single_chat_session_audio_base64encoded(

157 client: openai.AsyncOpenAI,

158 model_name: str,

159 audio_url: str,

152 - base64_encoded_audio: dict[str, str],

160 + url_encoded_audio: dict[str, str],

161 ):

154 - messages = dummy_messages_from_audio_url(

155 - f"data:audio/wav;base64,{base64_encoded_audio[audio_url]}"

156 - )

162 + messages = dummy_messages_from_audio_url(url_encoded_audio[audio_url])

163

164 # test single completion

165 chat_completion = await client.chat.completions.create(

tests/entrypoints/openai/test_video.py

+7 -11

@@ -7,7 +7,7 @@ import openai

7 import pytest

8 import pytest_asyncio

10 -from vllm.multimodal.utils import encode_video_base64, fetch_video

10 +from vllm.multimodal.utils import encode_video_url, fetch_video

12 from ...utils import RemoteOpenAIServer

@@ -48,9 +48,9 @@ async def client(server):

50 @pytest.fixture(scope="session")

51 -def base64_encoded_video() -> dict[str, str]:

51 +def url_encoded_video() -> dict[str, str]:

52 return {

53 - video_url: encode_video_base64(fetch_video(video_url)[0])

53 + video_url: encode_video_url(fetch_video(video_url)[0])

54 for video_url in TEST_VIDEO_URLS

55 }

@@ -175,11 +175,9 @@ async def test_single_chat_session_video_base64encoded(

175 client: openai.AsyncOpenAI,

176 model_name: str,

177 video_url: str,

178 - base64_encoded_video: dict[str, str],

178 + url_encoded_video: dict[str, str],

179 ):

180 - messages = dummy_messages_from_video_url(

181 - f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"

182 - )

180 + messages = dummy_messages_from_video_url(url_encoded_video[video_url])

181

182 # test single completion

183 chat_completion = await client.chat.completions.create(

@@ -223,11 +221,9 @@ async def test_single_chat_session_video_base64encoded_beamsearch(

221 client: openai.AsyncOpenAI,

222 model_name: str,

223 video_url: str,

226 - base64_encoded_video: dict[str, str],

224 + url_encoded_video: dict[str, str],

225 ):

228 - messages = dummy_messages_from_video_url(

229 - f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"

230 - )

226 + messages = dummy_messages_from_video_url(url_encoded_video[video_url])

227

228 chat_completion = await client.chat.completions.create(

229 model=model_name,

tests/entrypoints/openai/test_vision.py

+8 -12

@@ -9,7 +9,7 @@ import pytest_asyncio

9 from transformers import AutoProcessor

11 from vllm.multimodal.base import MediaWithBytes

12 -from vllm.multimodal.utils import encode_image_base64, fetch_image

12 +from vllm.multimodal.utils import encode_image_url, fetch_image

14 from ...utils import RemoteOpenAIServer

@@ -35,7 +35,7 @@ EXPECTED_MM_BEAM_SEARCH_RES = [

35 ],

36 [

37 "The image shows a Venn diagram with three over",

38 - "The image shows a colorful Venn diagram with",

38 + "The image displays a Venn diagram with three over",

39 ],

40 [

41 "This image displays a gradient of colors ranging from",

@@ -70,11 +70,9 @@ async def client(server):

72 @pytest.fixture(scope="session")

73 -def base64_encoded_image(local_asset_server) -> dict[str, str]:

73 +def url_encoded_image(local_asset_server) -> dict[str, str]:

74 return {

75 - image_asset: encode_image_base64(

76 - local_asset_server.get_image_asset(image_asset)

77 - )

75 + image_asset: encode_image_url(local_asset_server.get_image_asset(image_asset))

76 for image_asset in TEST_IMAGE_ASSETS

77 }

@@ -234,11 +232,11 @@ async def test_single_chat_session_image_base64encoded(

232 model_name: str,

233 raw_image_url: str,

234 image_url: str,

237 - base64_encoded_image: dict[str, str],

235 + url_encoded_image: dict[str, str],

236 ):

237 content_text = "What's in this image?"

238 messages = dummy_messages_from_image_url(

241 - f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}",

239 + url_encoded_image[raw_image_url],

240 content_text,

241 )

242

@@ -288,15 +286,13 @@ async def test_single_chat_session_image_base64encoded_beamsearch(

286 client: openai.AsyncOpenAI,

287 model_name: str,

288 image_idx: int,

291 - base64_encoded_image: dict[str, str],

289 + url_encoded_image: dict[str, str],

290 ):

291 # NOTE: This test also validates that we pass MM data through beam search

292 raw_image_url = TEST_IMAGE_ASSETS[image_idx]

293 expected_res = EXPECTED_MM_BEAM_SEARCH_RES[image_idx]

294

297 - messages = dummy_messages_from_image_url(

298 - f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}"

299 - )

295 + messages = dummy_messages_from_image_url(url_encoded_image[raw_image_url])

296

297 chat_completion = await client.chat.completions.create(

298 model=model_name,

tests/entrypoints/pooling/embed/test_online_vision.py

+1 -9

@@ -10,7 +10,7 @@ from transformers import AutoProcessor

10 from tests.utils import VLLM_PATH, RemoteOpenAIServer

11 from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse

12 from vllm.multimodal.base import MediaWithBytes

13 -from vllm.multimodal.utils import encode_image_base64, fetch_image

13 +from vllm.multimodal.utils import fetch_image

15 MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"

16 MAXIMUM_IMAGES = 2

@@ -48,14 +48,6 @@ def server():

48 yield remote_server

51 -@pytest.fixture(scope="session")

52 -def base64_encoded_image(local_asset_server) -> dict[str, str]:

53 - return {

54 - image_url: encode_image_base64(local_asset_server.get_image_asset(image_url))

55 - for image_url in TEST_IMAGE_ASSETS

56 - }

57 -

58 -

51 def get_hf_prompt_tokens(model_name, content, image_url):

52 processor = AutoProcessor.from_pretrained(

53 model_name, trust_remote_code=True, num_crops=4

tests/entrypoints/test_chat_utils.py

+6 -9

@@ -25,9 +25,9 @@ from vllm.entrypoints.chat_utils import (

25 )

26 from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict

27 from vllm.multimodal.utils import (

28 - encode_audio_base64,

29 - encode_image_base64,

30 - encode_video_base64,

28 + encode_audio_url,

29 + encode_image_url,

30 + encode_video_url,

31 )

32 from vllm.tokenizers import get_tokenizer

33 from vllm.tokenizers.mistral import MistralTokenizer

@@ -141,22 +141,19 @@ def mistral_model_config():

141 @pytest.fixture(scope="module")

142 def image_url():

143 image = ImageAsset("cherry_blossom")

144 - base64 = encode_image_base64(image.pil_image)

145 - return f"data:image/jpeg;base64,{base64}"

144 + return encode_image_url(image.pil_image)

145

146

147 @pytest.fixture(scope="module")

148 def video_url():

149 video = VideoAsset("baby_reading", 1)

151 - base64 = encode_video_base64(video.np_ndarrays)

152 - return f"data:video/jpeg;base64,{base64}"

150 + return encode_video_url(video.np_ndarrays)

151

152

153 @pytest.fixture(scope="module")

154 def audio_url():

155 audio = AudioAsset("mary_had_lamb")

158 - base64 = encode_audio_base64(*audio.audio_and_sample_rate)

159 - return f"data:audio/ogg;base64,{base64}"

156 + return encode_audio_url(*audio.audio_and_sample_rate)

157

158

159 def _assert_mm_data_is_image_input(

tests/models/multimodal/generation/test_keye.py

+2 -5

@@ -8,7 +8,7 @@ from PIL.Image import Image

8 from transformers import AutoProcessor

10 from vllm import LLM, EngineArgs, SamplingParams

11 -from vllm.multimodal.utils import encode_image_base64

11 +from vllm.multimodal.utils import encode_image_url

13 MODEL_NAME = "Kwai-Keye/Keye-VL-8B-Preview"

@@ -31,10 +31,7 @@ def test_keye_vl(

31 question: str,

32 ):

33 images = [asset.pil_image for asset in image_assets]

34 -

35 - image_urls = [

36 - f"data:image/jpeg;base64,{encode_image_base64(image)}" for image in images

37 - ]

34 + image_urls = [encode_image_url(image) for image in images]

36 engine_args = EngineArgs(

37 model=MODEL_NAME,

tests/models/multimodal/generation/test_vit_backend_functionality.py

+4 -9

@@ -15,7 +15,7 @@ from transformers import AutoProcessor

16 from vllm import LLM, EngineArgs, SamplingParams

17 from vllm.attention.backends.registry import AttentionBackendEnum

18 -from vllm.multimodal.utils import encode_image_base64

18 +from vllm.multimodal.utils import encode_image_url

19 from vllm.multimodal.video import sample_frames_from_video

20 from vllm.platforms import current_platform

@@ -178,8 +178,7 @@ def build_dots_ocr_prompt(images, config):

178 """Build Dots.OCR specific prompt with OCR instructions."""

179 # Use only stop_sign image for Dots.OCR

180 image = images[0] # Already filtered to stop_sign

181 -

182 - image_url = f"data:image/jpeg;base64,{encode_image_base64(image)}"

181 + image_url = encode_image_url(image)

182

183 placeholders = [{"type": "image_url", "image_url": {"url": image_url}}]

184 messages = [

@@ -204,9 +203,7 @@ def build_processor_prompt(images, config):

203 config["model_name"], trust_remote_code=True

204 )

205

207 - image_urls = [

208 - f"data:image/jpeg;base64,{encode_image_base64(img)}" for img in images

209 - ]

206 + image_urls = [encode_image_url(img) for img in images]

207 placeholders = [{"type": "image", "image": url} for url in image_urls]

208 messages = [

209 {

@@ -225,9 +222,7 @@ def build_processor_prompt(images, config):

222

223 def build_ovis_prompt(images, config):

224 """Build Ovis2.5 specific prompt with custom format."""

228 - image_urls = [

229 - f"data:image/jpeg;base64,{encode_image_base64(img)}" for img in images

230 - ]

225 + image_urls = [encode_image_url(img) for img in images]

226

227 placeholders = "\n".join(

228 f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)

tests/v1/ec_connector/integration/test_epd_correctness.py

+3 -7

@@ -31,7 +31,7 @@ import openai

31 import requests

33 from vllm.assets.image import ImageAsset

34 -from vllm.multimodal.utils import encode_image_base64

34 +from vllm.multimodal.utils import encode_image_url

36 MAX_OUTPUT_LEN = 256

@@ -49,9 +49,7 @@ SAMPLE_PROMPTS_MM: list[dict] = [

49 "content": [

50 {

51 "type": "image_url",

52 - "image_url": {

53 - "url": f"data:image;base64,{encode_image_base64(image_1)}"

54 - },

52 + "image_url": {"url": encode_image_url(image_1)},

53 },

54 {"type": "text", "text": "What's in this image?"},

55 ],

@@ -66,9 +64,7 @@ SAMPLE_PROMPTS_MM: list[dict] = [

64 "content": [

65 {

66 "type": "image_url",

69 - "image_url": {

70 - "url": f"data:image;base64,{encode_image_base64(image_2)}"

71 - },

67 + "image_url": {"url": encode_image_url(image_2)},

68 },

69 {

70 "type": "image_url",

tests/v1/entrypoints/openai/serving_responses/test_image.py

+5 -5

@@ -8,7 +8,7 @@ import pytest

8 import pytest_asyncio

10 from tests.utils import RemoteOpenAIServer

11 -from vllm.multimodal.utils import encode_image_base64

11 +from vllm.multimodal.utils import encode_image_url

13 # Use a small vision model for testing

14 MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct"

@@ -52,9 +52,9 @@ async def client(image_server):

54 @pytest.fixture(scope="session")

55 -def base64_encoded_image(local_asset_server) -> dict[str, str]:

55 +def url_encoded_image(local_asset_server) -> dict[str, str]:

56 return {

57 - image_url: encode_image_base64(local_asset_server.get_image_asset(image_url))

57 + image_url: encode_image_url(local_asset_server.get_image_asset(image_url))

58 for image_url in TEST_IMAGE_ASSETS

59 }

@@ -95,7 +95,7 @@ async def test_single_chat_session_image_base64encoded(

95 client: openai.AsyncOpenAI,

96 model_name: str,

97 raw_image_url: str,

98 - base64_encoded_image: dict[str, str],

98 + url_encoded_image: dict[str, str],

99 ):

100 content_text = "What's in this image?"

101 messages = [

@@ -104,7 +104,7 @@ async def test_single_chat_session_image_base64encoded(

104 "content": [

105 {

106 "type": "input_image",

107 - "image_url": f"data:image/jpeg;base64,{base64_encoded_image[raw_image_url]}", # noqa: E501

107 + "image_url": url_encoded_image[raw_image_url],

108 "detail": "auto",

109 },

110 {"type": "input_text", "text": content_text},

tests/v1/kv_connector/unit/test_example_connector.py

+2 -2

@@ -9,7 +9,7 @@ from PIL import Image

9 from vllm import LLM, EngineArgs, SamplingParams

10 from vllm.assets.image import ImageAsset

11 from vllm.config import KVTransferConfig

12 -from vllm.multimodal.utils import encode_image_base64

12 +from vllm.multimodal.utils import encode_image_url

13 from vllm.platforms import current_platform

15 MODEL_NAME = "RedHatAI/Qwen2.5-VL-3B-Instruct-quantized.w8a8"

@@ -74,7 +74,7 @@ def process_prompt(processor, llm: LLM, question: str, image_urls: list[Image]):

74 placeholders = [

75 {

76 "type": "image_url",

77 - "image_url": {"url": f"data:image;base64,{encode_image_base64(image_pil)}"},

77 + "image_url": {"url": encode_image_url(image_pil)},

78 }

79 for image_pil in image_urls

80 ]

tests/v1/tpu/test_multimodal.py

+10 -15

@@ -4,7 +4,7 @@

4 import openai

5 import pytest

7 -from vllm.multimodal.utils import encode_image_base64

7 +from vllm.multimodal.utils import encode_image_url

8 from vllm.platforms import current_platform

10 from ...entrypoints.openai.test_vision import TEST_IMAGE_ASSETS

@@ -12,11 +12,9 @@ from ...utils import RemoteOpenAIServer

14 @pytest.fixture(scope="session")

15 -def base64_encoded_image(local_asset_server) -> dict[str, str]:

15 +def url_encoded_image(local_asset_server) -> dict[str, str]:

16 return {

17 - image_asset: encode_image_base64(

18 - local_asset_server.get_image_asset(image_asset)

19 - )

17 + image_asset: encode_image_url(local_asset_server.get_image_asset(image_asset))

18 for image_asset in TEST_IMAGE_ASSETS

19 }

@@ -24,19 +22,16 @@ def base64_encoded_image(local_asset_server) -> dict[str, str]:

22 @pytest.mark.asyncio

23 @pytest.mark.skipif(not current_platform.is_tpu(), reason="This test needs a TPU")

24 @pytest.mark.parametrize("model_name", ["llava-hf/llava-1.5-7b-hf"])

27 -async def test_basic_vision(model_name: str, base64_encoded_image: dict[str, str]):

25 +async def test_basic_vision(model_name: str, url_encoded_image: dict[str, str]):

26 pytest.skip("Skip this test until it's fixed.")

30 - def whats_in_this_image_msg(b64):

28 + def whats_in_this_image_msg(url):

29 return [

30 {

31 "role": "user",

32 "content": [

33 {"type": "text", "text": "What's in this image?"},

36 - {

37 - "type": "image_url",

38 - "image_url": {"url": f"data:image/jpeg;base64,{b64}"},

39 - },

34 + {"type": "image_url", "image_url": {"url": url}},

35 ],

36 }

37 ]

@@ -63,14 +58,14 @@ async def test_basic_vision(model_name: str, base64_encoded_image: dict[str, str

59 # Other requests now should be much faster

60 for image_url in TEST_IMAGE_ASSETS:

66 - image_base64 = base64_encoded_image[image_url]

67 - chat_completion_from_base64 = await client.chat.completions.create(

61 + image_url = url_encoded_image[image_url]

62 + chat_completion_from_url = await client.chat.completions.create(

63 model=model_name,

69 - messages=whats_in_this_image_msg(image_base64),

64 + messages=whats_in_this_image_msg(image_url),

65 max_completion_tokens=24,

66 temperature=0.0,

67 )

73 - result = chat_completion_from_base64

68 + result = chat_completion_from_url

69 assert result

70 choice = result.choices[0]

71 assert choice.finish_reason == "length"

vllm/multimodal/audio.py

+7 -2

@@ -111,11 +111,16 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):

111 def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:

112 return librosa.load(filepath, sr=None)

113

114 - def encode_base64(self, media: tuple[npt.NDArray, int]) -> str:

114 + def encode_base64(

115 + self,

116 + media: tuple[npt.NDArray, int],

117 + *,

118 + audio_format: str = "WAV",

119 + ) -> str:

120 audio, sr = media

121

122 with BytesIO() as buffer:

118 - soundfile.write(buffer, audio, sr, format="WAV")

123 + soundfile.write(buffer, audio, sr, format=audio_format)

124 data = buffer.getvalue()

125

126 return base64.b64encode(data).decode("utf-8")

vllm/multimodal/image.py

+14 -1

@@ -8,8 +8,12 @@ import pybase64

8 import torch

9 from PIL import Image

11 +from vllm.logger import init_logger

12 +

13 from .base import MediaIO, MediaWithBytes

15 +logger = init_logger(__file__)

16 +

18 def rescale_image_size(

19 image: Image.Image, size_factor: float, transpose: int = -1

@@ -104,8 +108,17 @@ class ImageMediaIO(MediaIO[Image.Image]):

108 self,

109 media: Image.Image,

110 *,

107 - image_format: str = "JPEG",

111 + image_format: str | None = None,

112 ) -> str:

113 + if image_format is None:

114 + logger.warning_once(

115 + "The default format of `ImageMediaIO.encode_base64` will be changed "

116 + 'from "JPEG" to "PNG" in v0.15 to avoid lossy compression. '

117 + "To continue using the old default, "

118 + 'pass `format="JPEG"` explicitly to silence this warning.'

119 + )

120 + image_format = "JPEG"

121 +

122 image = media

123

124 with BytesIO() as buffer:

vllm/multimodal/utils.py

+54 -4

@@ -3,6 +3,7 @@

4 import asyncio

5 import atexit

6 +import mimetypes

7 from collections.abc import Generator, Set

8 from concurrent.futures import ThreadPoolExecutor

9 from itertools import groupby

@@ -357,17 +358,31 @@ class MediaConnector:

358 def encode_audio_base64(

359 audio: np.ndarray,

360 sampling_rate: int,

361 + *,

362 + format: str = "WAV",

363 ) -> str:

364 """Encode audio as base64."""

365 audio_io = AudioMediaIO()

363 - return audio_io.encode_base64((audio, sampling_rate))

366 + return audio_io.encode_base64((audio, sampling_rate), audio_format=format)

367 +

368 +

369 +def encode_audio_url(

370 + audio: np.ndarray,

371 + sampling_rate: int,

372 + *,

373 + format: str = "WAV",

374 +) -> str:

375 + """Encode audio as a data URL."""

376 + audio_b64 = encode_audio_base64(audio, sampling_rate, format=format)

377 + mimetype = mimetypes.types_map.get("." + format.lower(), "audio")

378 + return f"data:{mimetype};base64,{audio_b64}"

379

380

381 def encode_image_base64(

382 image: Image.Image,

383 *,

384 image_mode: str = "RGB",

370 - format: str = "JPEG",

385 + format: str | None = None,

386 ) -> str:

387 """

388 Encode a pillow image to base64 format.

@@ -378,10 +393,45 @@ def encode_image_base64(

393 return image_io.encode_base64(image, image_format=format)

394

395

381 -def encode_video_base64(frames: npt.NDArray) -> str:

396 +def encode_image_url(

397 + image: Image.Image,

398 + *,

399 + image_mode: str = "RGB",

400 + format: str = "PNG",

401 +) -> str:

402 + """

403 + Encode a pillow image as a data URL.

404 +

405 + By default, the image is converted into RGB format before being encoded.

406 + """

407 + image_b64 = encode_image_base64(image, image_mode=image_mode, format=format)

408 + mimetype = mimetypes.types_map.get("." + format.lower(), "image")

409 + return f"data:{mimetype};base64,{image_b64}"

410 +

411 +

412 +def encode_video_base64(

413 + frames: npt.NDArray,

414 + *,

415 + format: str = "JPEG",

416 +) -> str:

417 image_io = ImageMediaIO()

418 video_io = VideoMediaIO(image_io)

384 - return video_io.encode_base64(frames)

419 + return video_io.encode_base64(frames, video_format=format)

420 +

421 +

422 +def encode_video_url(

423 + frames: npt.NDArray,

424 + *,

425 + format: str = "JPEG",

426 +) -> str:

427 + video_b64 = encode_video_base64(frames, format=format)

428 +

429 + if format.lower() == "jpeg":

430 + mimetype = "video/jpeg"

431 + else:

432 + mimetype = mimetypes.types_map.get("." + format.lower(), "video")

433 +

434 + return f"data:{mimetype};base64,{video_b64}"

435

436

437 def argsort_mm_positions(