@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 00c3d68e45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend][Core] Add plumbing to support audio language models (#7446)

Peter Salas committed Aug 13, 2024 at 10:39 UTC 00c3d68e45bad901989c1afe3c223225dc9a5d6d
24 files changed +600 -121
docs/source/conf.py
+2
@@ -112,6 +112,8 @@ autodoc_mock_imports = [
112 "tensorizer",
113 "pynvml",
114 "outlines",
115 + "librosa",
116 + "soundfile",
117 "gguf",
118 "lark",
119 ]
docs/source/models/enabling_multimodal_inputs.rst
+11 -11
@@ -15,14 +15,14 @@ This document walks you through the steps to extend a vLLM model so that it acce
15 It is assumed that you have already implemented the model in vLLM according to :ref:`these steps <adding_a_new_model>`.
16 Further update the model as follows:
17
18 -- Implement the :class:`~vllm.model_executor.models.interfaces.SupportsVision` interface.
18 +- Implement the :class:`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
19
20 .. code-block:: diff
21
22 - + from vllm.model_executor.models.interfaces import SupportsVision
22 + + from vllm.model_executor.models.interfaces import SupportsMultiModal
23
24 - class YourModelForImage2Seq(nn.Module):
25 - + class YourModelForImage2Seq(nn.Module, SupportsVision):
25 + + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
26
27 .. note::
28 The model class does not have to be named :code:`*ForCausalLM`.
@@ -51,11 +51,11 @@ This decorator accepts a function that maps multi-modal inputs to the keyword ar
51
52 .. code-block:: diff
53
54 - from vllm.model_executor.models.interfaces import SupportsVision
54 + from vllm.model_executor.models.interfaces import SupportsMultiModal
55 + from vllm.multimodal import MULTIMODAL_REGISTRY
56
57 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
58 - class YourModelForImage2Seq(nn.Module, SupportsVision):
58 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
59
60 A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
61
@@ -72,13 +72,13 @@ and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.regis
72 .. code-block:: diff
73
74 from vllm.inputs import INPUT_REGISTRY
75 - from vllm.model_executor.models.interfaces import SupportsVision
75 + from vllm.model_executor.models.interfaces import SupportsMultiModal
76 from vllm.multimodal import MULTIMODAL_REGISTRY
77
78 @MULTIMODAL_REGISTRY.register_image_input_mapper()
79 + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
80 @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
81 - class YourModelForImage2Seq(nn.Module, SupportsVision):
81 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
82
83 Here are some examples:
84
@@ -98,13 +98,13 @@ In such cases, you can define your own dummy data by registering a factory metho
98 .. code-block:: diff
99
100 from vllm.inputs import INPUT_REGISTRY
101 - from vllm.model_executor.models.interfaces import SupportsVision
101 + from vllm.model_executor.models.interfaces import SupportsMultiModal
102 from vllm.multimodal import MULTIMODAL_REGISTRY
103
104 @MULTIMODAL_REGISTRY.register_image_input_mapper()
105 @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
106 + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
107 - class YourModelForImage2Seq(nn.Module, SupportsVision):
107 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
108
109 .. note::
110 The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
@@ -128,14 +128,14 @@ You can register input processors via :meth:`INPUT_REGISTRY.register_input_proce
128 .. code-block:: diff
129
130 from vllm.inputs import INPUT_REGISTRY
131 - from vllm.model_executor.models.interfaces import SupportsVision
131 + from vllm.model_executor.models.interfaces import SupportsMultiModal
132 from vllm.multimodal import MULTIMODAL_REGISTRY
133
134 @MULTIMODAL_REGISTRY.register_image_input_mapper()
135 @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
136 @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
137 + @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
138 - class YourModelForImage2Seq(nn.Module, SupportsVision):
138 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
139
140 A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
141 Here are some examples:
requirements-common.txt
+2
@@ -20,4 +20,6 @@ outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
20 typing_extensions >= 4.10
21 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
22 pyzmq
23 +librosa # Required for audio processing
24 +soundfile # Required for audio processing
25 gguf == 0.9.1
tests/entrypoints/openai/test_audio.py new
+351
@@ -0,0 +1,351 @@
1 +import math
2 +import sys
3 +import time
4 +from typing import Dict, List, Optional, Tuple, Union, cast
5 +from unittest.mock import patch
6 +
7 +import librosa
8 +import numpy as np
9 +import openai
10 +import pytest
11 +import requests
12 +import torch
13 +
14 +from vllm import ModelRegistry
15 +from vllm.config import MultiModalConfig
16 +from vllm.inputs import INPUT_REGISTRY
17 +from vllm.inputs.data import LLMInputs
18 +from vllm.inputs.registry import InputContext
19 +from vllm.model_executor.models.interfaces import SupportsMultiModal
20 +from vllm.model_executor.models.opt import OPTForCausalLM
21 +from vllm.multimodal import MULTIMODAL_REGISTRY
22 +from vllm.multimodal.base import MultiModalInputs
23 +from vllm.multimodal.image import (cached_get_tokenizer,
24 + repeat_and_pad_image_tokens)
25 +from vllm.multimodal.utils import encode_audio_base64, fetch_audio
26 +from vllm.utils import get_open_port
27 +
28 +from ...utils import VLLM_PATH
29 +
30 +chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
31 +assert chatml_jinja_path.exists()
32 +
33 +MODEL_NAME = "facebook/opt-125m"
34 +TEST_AUDIO_URLS = [
35 + "https://upload.wikimedia.org/wikipedia/en/b/bf/Dave_Niehaus_Winning_Call_1995_AL_Division_Series.ogg",
36 +]
37 +
38 +
39 +def server_function(port):
40 +
41 + def fake_input_mapper(ctx: InputContext, data: object):
42 + assert isinstance(data, tuple)
43 + (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
44 +
45 + # Resample it to 1 sample per second
46 + audio = librosa.resample(audio, orig_sr=sr, target_sr=1)
47 + return MultiModalInputs({"processed_audio": torch.from_numpy(audio)})
48 +
49 + def fake_input_processor(ctx: InputContext, llm_inputs: LLMInputs):
50 + multi_modal_data = llm_inputs.get("multi_modal_data")
51 + if multi_modal_data is None or "audio" not in multi_modal_data:
52 + return llm_inputs
53 +
54 + audio, sr = multi_modal_data.get("audio")
55 + audio_duration = math.ceil(len(audio) / sr)
56 +
57 + new_prompt, new_token_ids = repeat_and_pad_image_tokens(
58 + cached_get_tokenizer(ctx.model_config.tokenizer),
59 + llm_inputs.get("prompt"),
60 + llm_inputs["prompt_token_ids"],
61 + image_token_id=62, # "_"
62 + repeat_count=audio_duration)
63 +
64 + return LLMInputs(prompt_token_ids=new_token_ids,
65 + prompt=new_prompt,
66 + multi_modal_data=multi_modal_data)
67 +
68 + @MULTIMODAL_REGISTRY.register_input_mapper("audio", fake_input_mapper)
69 + @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
70 + "audio", lambda *_, **__: 100)
71 + @INPUT_REGISTRY.register_input_processor(fake_input_processor)
72 + class FakeAudioModel(OPTForCausalLM, SupportsMultiModal):
73 +
74 + def __init__(self, *args, multimodal_config: MultiModalConfig,
75 + **kwargs):
76 + assert multimodal_config is not None
77 + super().__init__(*args, **kwargs)
78 +
79 + def forward(
80 + self,
81 + *args,
82 + processed_audio: Optional[torch.Tensor] = None,
83 + **kwargs,
84 + ) -> torch.Tensor:
85 + return super().forward(*args, **kwargs)
86 +
87 + ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
88 +
89 + with patch("vllm.entrypoints.chat_utils._mm_token_str",
90 + lambda *_, **__: "_"):
91 + sys.argv = ["placeholder.py"] + \
92 + (f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
93 + "--dtype bfloat16 --enforce-eager --api-key token-abc123 "
94 + f"--port {port} --chat-template {chatml_jinja_path} "
95 + "--disable-frontend-multiprocessing").split()
96 + import runpy
97 + runpy.run_module('vllm.entrypoints.openai.api_server',
98 + run_name='__main__')
99 +
100 +
101 +@pytest.fixture(scope="module")
102 +def client():
103 + port = get_open_port()
104 + ctx = torch.multiprocessing.get_context("spawn")
105 + server = ctx.Process(target=server_function, args=(port, ))
106 + server.start()
107 + MAX_SERVER_START_WAIT_S = 60
108 + client = openai.AsyncOpenAI(
109 + base_url=f"http://localhost:{port}/v1",
110 + api_key="token-abc123",
111 + )
112 + # run health check
113 + health_url = f"http://localhost:{port}/health"
114 + start = time.time()
115 + while True:
116 + try:
117 + if requests.get(health_url).status_code == 200:
118 + break
119 + except Exception as err:
120 + result = server.exitcode
121 + if result is not None:
122 + raise RuntimeError("Server exited unexpectedly.") from err
123 +
124 + time.sleep(0.5)
125 + if time.time() - start > MAX_SERVER_START_WAIT_S:
126 + raise RuntimeError("Server failed to start in time.") from err
127 +
128 + try:
129 + yield client
130 + finally:
131 + server.kill()
132 +
133 +
134 +@pytest.fixture(scope="session")
135 +def base64_encoded_audio() -> Dict[str, str]:
136 + return {
137 + audio_url: encode_audio_base64(*fetch_audio(audio_url))
138 + for audio_url in TEST_AUDIO_URLS
139 + }
140 +
141 +
142 +@pytest.mark.asyncio
143 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
144 +@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
145 +async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
146 + model_name: str, audio_url: str):
147 + messages = [{
148 + "role":
149 + "user",
150 + "content": [
151 + {
152 + "type": "audio_url",
153 + "audio_url": {
154 + "url": audio_url
155 + }
156 + },
157 + {
158 + "type": "text",
159 + "text": "What's happening in this audio?"
160 + },
161 + ],
162 + }]
163 +
164 + # test single completion
165 + chat_completion = await client.chat.completions.create(model=model_name,
166 + messages=messages,
167 + max_tokens=10,
168 + logprobs=True,
169 + top_logprobs=5)
170 + assert len(chat_completion.choices) == 1
171 +
172 + choice = chat_completion.choices[0]
173 + assert choice.finish_reason == "length"
174 + assert chat_completion.usage == openai.types.CompletionUsage(
175 + completion_tokens=10, prompt_tokens=36, total_tokens=46)
176 +
177 + message = choice.message
178 + message = chat_completion.choices[0].message
179 + assert message.content is not None and len(message.content) >= 10
180 + assert message.role == "assistant"
181 + messages.append({"role": "assistant", "content": message.content})
182 +
183 + # test multi-turn dialogue
184 + messages.append({"role": "user", "content": "express your result in json"})
185 + chat_completion = await client.chat.completions.create(
186 + model=model_name,
187 + messages=messages,
188 + max_tokens=10,
189 + )
190 + message = chat_completion.choices[0].message
191 + assert message.content is not None and len(message.content) >= 0
192 +
193 +
194 +@pytest.mark.asyncio
195 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
196 +@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
197 +async def test_single_chat_session_audio_base64encoded(
198 + client: openai.AsyncOpenAI, model_name: str, audio_url: str,
199 + base64_encoded_audio: Dict[str, str]):
200 +
201 + messages = [{
202 + "role":
203 + "user",
204 + "content": [
205 + {
206 + "type": "audio_url",
207 + "audio_url": {
208 + "url":
209 + f"data:audio/wav;base64,{base64_encoded_audio[audio_url]}"
210 + }
211 + },
212 + {
213 + "type": "text",
214 + "text": "What's happening in this audio?"
215 + },
216 + ],
217 + }]
218 +
219 + # test single completion
220 + chat_completion = await client.chat.completions.create(model=model_name,
221 + messages=messages,
222 + max_tokens=10,
223 + logprobs=True,
224 + top_logprobs=5)
225 + assert len(chat_completion.choices) == 1
226 +
227 + choice = chat_completion.choices[0]
228 + assert choice.finish_reason == "length"
229 + assert chat_completion.usage == openai.types.CompletionUsage(
230 + completion_tokens=10, prompt_tokens=36, total_tokens=46)
231 +
232 + message = choice.message
233 + message = chat_completion.choices[0].message
234 + assert message.content is not None and len(message.content) >= 10
235 + assert message.role == "assistant"
236 + messages.append({"role": "assistant", "content": message.content})
237 +
238 + # test multi-turn dialogue
239 + messages.append({"role": "user", "content": "express your result in json"})
240 + chat_completion = await client.chat.completions.create(
241 + model=model_name,
242 + messages=messages,
243 + max_tokens=10,
244 + )
245 + message = chat_completion.choices[0].message
246 + assert message.content is not None and len(message.content) >= 0
247 +
248 +
249 +@pytest.mark.asyncio
250 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
251 +@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
252 +async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
253 + model_name: str, audio_url: str):
254 + messages = [{
255 + "role":
256 + "user",
257 + "content": [
258 + {
259 + "type": "audio_url",
260 + "audio_url": {
261 + "url": audio_url
262 + }
263 + },
264 + {
265 + "type": "text",
266 + "text": "What's happening in this audio?"
267 + },
268 + ],
269 + }]
270 +
271 + # test single completion
272 + chat_completion = await client.chat.completions.create(
273 + model=model_name,
274 + messages=messages,
275 + max_tokens=10,
276 + temperature=0.0,
277 + )
278 + output = chat_completion.choices[0].message.content
279 + stop_reason = chat_completion.choices[0].finish_reason
280 +
281 + # test streaming
282 + stream = await client.chat.completions.create(
283 + model=model_name,
284 + messages=messages,
285 + max_tokens=10,
286 + temperature=0.0,
287 + stream=True,
288 + )
289 + chunks: List[str] = []
290 + finish_reason_count = 0
291 + async for chunk in stream:
292 + delta = chunk.choices[0].delta
293 + if delta.role:
294 + assert delta.role == "assistant"
295 + if delta.content:
296 + chunks.append(delta.content)
297 + if chunk.choices[0].finish_reason is not None:
298 + finish_reason_count += 1
299 + # finish reason should only return in last block
300 + assert finish_reason_count == 1
301 + assert chunk.choices[0].finish_reason == stop_reason
302 + assert delta.content
303 + assert "".join(chunks) == output
304 +
305 +
306 +@pytest.mark.asyncio
307 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
308 +@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
309 +async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
310 + audio_url: str):
311 +
312 + messages = [{
313 + "role":
314 + "user",
315 + "content": [
316 + {
317 + "type": "audio_url",
318 + "audio_url": {
319 + "url": audio_url
320 + }
321 + },
322 + {
323 + "type": "audio_url",
324 + "audio_url": {
325 + "url": audio_url
326 + }
327 + },
328 + {
329 + "type": "text",
330 + "text": "What's happening in this audio?"
331 + },
332 + ],
333 + }]
334 +
335 + with pytest.raises(openai.BadRequestError): # test multi-audio input
336 + await client.chat.completions.create(
337 + model=model_name,
338 + messages=messages,
339 + max_tokens=10,
340 + temperature=0.0,
341 + )
342 +
343 + # the server should still work afterwards
344 + completion = await client.completions.create(
345 + model=model_name,
346 + prompt=[0, 0, 0, 0, 0],
347 + max_tokens=5,
348 + temperature=0.0,
349 + )
350 + completion = completion.choices[0].text
351 + assert completion is not None and len(completion) >= 0
vllm/entrypoints/chat_utils.py
+69 -32
@@ -2,7 +2,8 @@ import codecs
2 from dataclasses import dataclass
3 from functools import lru_cache
4 from pathlib import Path
5 -from typing import Any, Awaitable, Iterable, List, Optional, Tuple, Union, cast
5 +from typing import (Any, Awaitable, Iterable, List, Literal, Optional, Tuple,
6 + Union, cast)
7
8 # yapf conflicts with isort for this block
9 # yapf: disable
@@ -21,12 +22,27 @@ from typing_extensions import Required, TypedDict
22 from vllm.config import ModelConfig
23 from vllm.logger import init_logger
24 from vllm.multimodal import MultiModalDataDict
24 -from vllm.multimodal.utils import async_get_and_parse_image
25 +from vllm.multimodal.utils import (async_get_and_parse_audio,
26 + async_get_and_parse_image)
27 from vllm.transformers_utils.tokenizer import AnyTokenizer
28
29 logger = init_logger(__name__)
30
31
32 +class AudioURL(TypedDict, total=False):
33 + url: Required[str]
34 + """
35 + Either a URL of the audio or a data URL with base64 encoded audio data.
36 + """
37 +
38 +
39 +class ChatCompletionContentPartAudioParam(TypedDict, total=False):
40 + audio_url: Required[AudioURL]
41 +
42 + type: Required[Literal["audio_url"]]
43 + """The type of the content part."""
44 +
45 +
46 class CustomChatCompletionContentPartParam(TypedDict, total=False):
47 __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
48
@@ -35,6 +51,7 @@ class CustomChatCompletionContentPartParam(TypedDict, total=False):
51
52
53 ChatCompletionContentPartParam = Union[OpenAIChatCompletionContentPartParam,
54 + ChatCompletionContentPartAudioParam,
55 CustomChatCompletionContentPartParam]
56
57
@@ -97,34 +114,41 @@ def load_chat_template(
114
115
116 @lru_cache(maxsize=None)
100 -def _image_token_str(model_config: ModelConfig,
101 - tokenizer: PreTrainedTokenizer) -> Optional[str]:
117 +def _mm_token_str(model_config: ModelConfig, tokenizer: PreTrainedTokenizer,
118 + modality: Literal["image", "audio"]) -> Optional[str]:
119 # TODO: Let user specify how to insert image tokens into prompt
120 # (similar to chat template)
104 - model_type = model_config.hf_config.model_type
105 - if model_type == "phi3_v":
106 - # Workaround since this token is not defined in the tokenizer
107 - return "<|image_1|>"
108 - if model_type == "minicpmv":
109 - return "(<image>./</image>)"
110 - if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
111 - # These models do not use image tokens in the prompt
112 - return None
113 - if model_type.startswith("llava"):
114 - return tokenizer.decode(model_config.hf_config.image_token_index)
115 - if model_type in ("chameleon", "internvl_chat"):
116 - return "<image>"
117 - raise TypeError(f"Unknown model type: {model_type}")
118 -
119 -
120 -# TODO: Let user specify how to insert image tokens into prompt
121 + if modality == "image":
122 + model_type = model_config.hf_config.model_type
123 + if model_type == "phi3_v":
124 + # Workaround since this token is not defined in the tokenizer
125 + return "<|image_1|>"
126 + if model_type == "minicpmv":
127 + return "(<image>./</image>)"
128 + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
129 + # These models do not use image tokens in the prompt
130 + return None
131 + if model_type.startswith("llava"):
132 + return tokenizer.decode(model_config.hf_config.image_token_index)
133 + if model_type in ("chameleon", "internvl_chat"):
134 + return "<image>"
135 +
136 + raise TypeError(f"Unknown model type: {model_type}")
137 + elif modality == "audio":
138 + raise TypeError("No audio models are supported yet.")
139 + else:
140 + raise TypeError(f"Unknown modality: {modality}")
141 +
142 +
143 +# TODO: Let user specify how to insert multimodal tokens into prompt
144 # (similar to chat template)
122 -def _get_full_image_text_prompt(image_token_str: str, text_prompt: str) -> str:
123 - """Combine image and text prompts for vision language model"""
145 +def _get_full_multimodal_text_prompt(placeholder_token_str: str,
146 + text_prompt: str) -> str:
147 + """Combine multimodal prompts for a multimodal language model"""
148
149 # NOTE: For now we assume all model architectures use the same
126 - # image + text prompt format. This may change in the future.
127 - return f"{image_token_str}\n{text_prompt}"
150 + # placeholder + text prompt format. This may change in the future.
151 + return f"{placeholder_token_str}\n{text_prompt}"
152
153
154 def _parse_chat_message_content_parts(
@@ -135,6 +159,7 @@ def _parse_chat_message_content_parts(
159 ) -> ChatMessageParseResult:
160 texts: List[str] = []
161 mm_futures: List[Awaitable[MultiModalDataDict]] = []
162 + modality: Literal["image", "audio"] = "image"
163
164 for part in parts:
165 part_type = part["type"]
@@ -142,9 +167,10 @@ def _parse_chat_message_content_parts(
167 text = cast(ChatCompletionContentPartTextParam, part)["text"]
168 texts.append(text)
169 elif part_type == "image_url":
170 + modality = "image"
171 if len(mm_futures) > 0:
172 raise NotImplementedError(
147 - "Multiple 'image_url' input is currently not supported.")
173 + "Multiple multimodal inputs is currently not supported.")
174
175 image_url = cast(ChatCompletionContentPartImageParam,
176 part)["image_url"]
@@ -156,21 +182,32 @@ def _parse_chat_message_content_parts(
182
183 image_future = async_get_and_parse_image(image_url["url"])
184 mm_futures.append(image_future)
185 + elif part_type == "audio_url":
186 + modality = "audio"
187 + if len(mm_futures) > 0:
188 + raise NotImplementedError(
189 + "Multiple multimodal inputs is currently not supported.")
190 +
191 + audio_url = cast(ChatCompletionContentPartAudioParam,
192 + part)["audio_url"]
193 + audio_future = async_get_and_parse_audio(audio_url["url"])
194 + mm_futures.append(audio_future)
195 else:
196 raise NotImplementedError(f"Unknown part type: {part_type}")
197
198 text_prompt = "\n".join(texts)
199
200 if mm_futures:
165 - image_token_str = _image_token_str(model_config, tokenizer)
166 - if image_token_str is not None:
167 - if image_token_str in text_prompt:
201 + placeholder_token_str = _mm_token_str(model_config, tokenizer,
202 + modality)
203 + if placeholder_token_str is not None:
204 + if placeholder_token_str in text_prompt:
205 logger.warning(
169 - "Detected image token string in the text prompt. "
206 + "Detected multi-modal token string in the text prompt. "
207 "Skipping prompt formatting.")
208 else:
172 - text_prompt = _get_full_image_text_prompt(
173 - image_token_str=image_token_str,
209 + text_prompt = _get_full_multimodal_text_prompt(
210 + placeholder_token_str=placeholder_token_str,
211 text_prompt=text_prompt,
212 )
213
vllm/envs.py
+6
@@ -44,6 +44,7 @@ if TYPE_CHECKING:
44 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
45 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
46 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
47 + VLLM_AUDIO_FETCH_TIMEOUT: int = 5
48 VLLM_TARGET_DEVICE: str = "cuda"
49 MAX_JOBS: Optional[str] = None
50 NVCC_THREADS: Optional[str] = None
@@ -321,6 +322,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
322 "VLLM_IMAGE_FETCH_TIMEOUT":
323 lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
324
325 + # Timeout for fetching audio when serving multimodal models
326 + # Default is 5 seconds
327 + "VLLM_AUDIO_FETCH_TIMEOUT":
328 + lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "5")),
329 +
330 # Path to the XLA persistent cache directory.
331 # Only used for XLA devices such as TPUs.
332 "VLLM_XLA_CACHE_PATH":
vllm/model_executor/model_loader/loader.py
+2 -2
@@ -38,7 +38,7 @@ from vllm.model_executor.model_loader.weight_utils import (
38 safetensors_weights_iterator)
39 from vllm.model_executor.models.interfaces import (has_inner_state,
40 supports_lora,
41 - supports_vision)
41 + supports_multimodal)
42 from vllm.model_executor.utils import set_weight_attrs
43 from vllm.platforms import current_platform
44 from vllm.utils import is_pin_memory_available
@@ -131,7 +131,7 @@ def _get_model_initialization_kwargs(
131 "be added in the future. If this is important to you, "
132 "please open an issue on github.")
133
134 - if supports_vision(model_class):
134 + if supports_multimodal(model_class):
135 if multimodal_config is None:
136 raise ValueError("Provide vision related configurations "
137 "through LLM entrypoint or engine arguments.")
vllm/model_executor/models/blip2.py
+6 -6
@@ -20,8 +20,8 @@ from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
20
21 from .blip import (BlipVisionModel, dummy_image_for_blip,
22 get_max_blip_image_tokens)
23 -from .interfaces import SupportsVision
24 -from .utils import merge_vision_embeddings
23 +from .interfaces import SupportsMultiModal
24 +from .utils import merge_multimodal_embeddings
25
26 _KEYS_TO_MODIFY_MAPPING = {
27 "language_model.lm_head": "lm_head",
@@ -457,7 +457,7 @@ def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
457 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
458 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2)
459 @INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
460 -class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
460 +class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
461
462 def __init__(self,
463 config: Blip2Config,
@@ -621,9 +621,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
621 vision_embeddings = self._process_image_input(image_input)
622 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
623
624 - inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
625 - vision_embeddings,
626 - BLIP2_IMAGE_TOKEN_ID)
624 + inputs_embeds = merge_multimodal_embeddings(
625 + input_ids, inputs_embeds, vision_embeddings,
626 + BLIP2_IMAGE_TOKEN_ID)
627
628 input_ids = None
629 else:
vllm/model_executor/models/chameleon.py
+2 -2
@@ -35,7 +35,7 @@ from vllm.multimodal.image import (cached_get_tokenizer,
35 from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
36 from vllm.utils import print_warning_once
37
38 -from .interfaces import SupportsVision
38 +from .interfaces import SupportsMultiModal
39
40 logger = init_logger(__name__)
41
@@ -886,7 +886,7 @@ class ChameleonModel(nn.Module):
886 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
887 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon)
888 @INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon)
889 -class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
889 +class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal):
890
891 def __init__(
892 self,
vllm/model_executor/models/fuyu.py
+6 -6
@@ -40,8 +40,8 @@ from vllm.multimodal.image import (cached_get_image_processor,
40 cached_get_tokenizer)
41 from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
42
43 -from .interfaces import SupportsVision
44 -from .utils import merge_vision_embeddings
43 +from .interfaces import SupportsMultiModal
44 +from .utils import merge_multimodal_embeddings
45
46 logger = init_logger(__name__)
47
@@ -209,7 +209,7 @@ def input_mapper_for_fuyu(ctx: InputContext, data: object):
209 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
210 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu)
211 @INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
212 -class FuyuForCausalLM(nn.Module, SupportsVision):
212 +class FuyuForCausalLM(nn.Module, SupportsMultiModal):
213
214 def __init__(self,
215 config: FuyuConfig,
@@ -271,9 +271,9 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
271 if image_input is not None:
272 vision_embeddings = self._process_image_input(image_input)
273 inputs_embeds = self.language_model.model.embed_tokens(input_ids)
274 - inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
275 - vision_embeddings,
276 - self.image_token_id)
274 + inputs_embeds = merge_multimodal_embeddings(
275 + input_ids, inputs_embeds, vision_embeddings,
276 + self.image_token_id)
277
278 else:
279 inputs_embeds = None
vllm/model_executor/models/interfaces.py
+16 -12
@@ -10,12 +10,15 @@ logger = init_logger(__name__)
10
11
12 @runtime_checkable
13 -class SupportsVision(Protocol):
14 - """The interface required for all vision language models (VLMs)."""
13 +class SupportsMultiModal(Protocol):
14 + """
15 + The interface required for all multimodal (vision or audio) language
16 + models.
17 + """
18
16 - supports_vision: ClassVar[Literal[True]] = True
19 + supports_multimodal: ClassVar[Literal[True]] = True
20 """
18 - A flag that indicates this model supports vision inputs.
21 + A flag that indicates this model supports multimodal inputs.
22
23 Note:
24 There is no need to redefine this flag if this class is in the
@@ -29,30 +32,31 @@ class SupportsVision(Protocol):
32 # We can't use runtime_checkable with ClassVar for issubclass checks
33 # so we need to treat the class as an instance and use isinstance instead
34 @runtime_checkable
32 -class _SupportsVisionType(Protocol):
33 - supports_vision: Literal[True]
35 +class _SupportsMultiModalType(Protocol):
36 + supports_multimodal: Literal[True]
37
38 def __call__(self, *, multimodal_config: MultiModalConfig) -> None:
39 ...
40
41
42 @overload
40 -def supports_vision(model: Type[object]) -> TypeIs[Type[SupportsVision]]:
43 +def supports_multimodal(
44 + model: Type[object]) -> TypeIs[Type[SupportsMultiModal]]:
45 ...
46
47
48 @overload
45 -def supports_vision(model: object) -> TypeIs[SupportsVision]:
49 +def supports_multimodal(model: object) -> TypeIs[SupportsMultiModal]:
50 ...
51
52
49 -def supports_vision(
53 +def supports_multimodal(
54 model: Union[Type[object], object],
51 -) -> Union[TypeIs[Type[SupportsVision]], TypeIs[SupportsVision]]:
55 +) -> Union[TypeIs[Type[SupportsMultiModal]], TypeIs[SupportsMultiModal]]:
56 if isinstance(model, type):
53 - return isinstance(model, _SupportsVisionType)
57 + return isinstance(model, _SupportsMultiModalType)
58
55 - return isinstance(model, SupportsVision)
59 + return isinstance(model, SupportsMultiModal)
60
61
62 @runtime_checkable
vllm/model_executor/models/internvl.py
+6 -6
@@ -27,9 +27,9 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
27
28 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
29 get_clip_num_patches)
30 -from .interfaces import SupportsVision
30 +from .interfaces import SupportsMultiModal
31 from .utils import (filter_weights, init_vllm_registered_model,
32 - merge_vision_embeddings)
32 + merge_multimodal_embeddings)
33
34 IMG_START = '<img>'
35 IMG_END = '</img>'
@@ -292,7 +292,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
292 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
293 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_internvl)
294 @INPUT_REGISTRY.register_input_processor(input_processor_for_internvl)
295 -class InternVLChatModel(nn.Module, SupportsVision):
295 +class InternVLChatModel(nn.Module, SupportsMultiModal):
296
297 def __init__(self,
298 config: PretrainedConfig,
@@ -451,9 +451,9 @@ class InternVLChatModel(nn.Module, SupportsVision):
451 inputs_embeds = self.language_model.model.get_input_embeddings(
452 input_ids)
453 vision_embeddings = self._process_image_input(image_input)
454 - inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
455 - vision_embeddings,
456 - self.img_context_token_id)
454 + inputs_embeds = merge_multimodal_embeddings(
455 + input_ids, inputs_embeds, vision_embeddings,
456 + self.img_context_token_id)
457 input_ids = None
458 else:
459 inputs_embeds = None
vllm/model_executor/models/llava.py
+4 -4
@@ -19,12 +19,12 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
19 from .clip import (CLIPVisionModel, dummy_image_for_clip,
20 dummy_seq_data_for_clip, get_max_clip_image_tokens,
21 input_processor_for_clip)
22 -from .interfaces import SupportsVision
22 +from .interfaces import SupportsMultiModal
23 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24 dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
25 input_processor_for_siglip)
26 from .utils import (filter_weights, init_vllm_registered_model,
27 - merge_vision_embeddings)
27 + merge_multimodal_embeddings)
28
29
30 class LlavaImagePixelInputs(TypedDict):
@@ -181,7 +181,7 @@ def _init_vision_tower(hf_config: LlavaConfig):
181 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
182 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
183 @INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
184 -class LlavaForConditionalGeneration(nn.Module, SupportsVision):
184 +class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
185
186 def __init__(self,
187 config: LlavaConfig,
@@ -338,7 +338,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
338 inputs_embeds = self.language_model.model.get_input_embeddings(
339 input_ids)
340
341 - inputs_embeds = merge_vision_embeddings(
341 + inputs_embeds = merge_multimodal_embeddings(
342 input_ids, inputs_embeds, vision_embeddings,
343 self.config.image_token_index)
344
vllm/model_executor/models/llava_next.py
+4 -4
@@ -23,13 +23,13 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
23 from .clip import (CLIPVisionModel, dummy_image_for_clip,
24 dummy_seq_data_for_clip, get_clip_image_feature_size,
25 get_clip_patch_grid_length, input_processor_for_clip)
26 -from .interfaces import SupportsVision
26 +from .interfaces import SupportsMultiModal
27 from .llava import LlavaMultiModalProjector
28 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
30 get_siglip_patch_grid_length, input_processor_for_siglip)
31 from .utils import (filter_weights, init_vllm_registered_model,
32 - merge_vision_embeddings)
32 + merge_multimodal_embeddings)
33
34 logger = init_logger(__name__)
35
@@ -275,7 +275,7 @@ def _init_vision_tower(hf_config: LlavaNextConfig):
275 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
276 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
277 @INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next)
278 -class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
278 +class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
279
280 def __init__(self,
281 config: LlavaNextConfig,
@@ -571,7 +571,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
571 inputs_embeds = self.language_model.model.get_input_embeddings(
572 input_ids)
573
574 - inputs_embeds = merge_vision_embeddings(
574 + inputs_embeds = merge_multimodal_embeddings(
575 input_ids, inputs_embeds, vision_embeddings,
576 self.config.image_token_index)
577
vllm/model_executor/models/minicpmv.py
+2 -2
@@ -48,7 +48,7 @@ from vllm.model_executor.layers.sampler import Sampler
48 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 -from vllm.model_executor.models.interfaces import SupportsVision
51 +from vllm.model_executor.models.interfaces import SupportsMultiModal
52 from vllm.model_executor.models.llama import LlamaModel
53 from vllm.model_executor.models.minicpm import MiniCPMModel
54 from vllm.model_executor.models.qwen2 import Qwen2Model
@@ -479,7 +479,7 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
479 return llm_inputs
480
481
482 -class MiniCPMVBaseModel(nn.Module, SupportsVision):
482 +class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
483 """
484 The abstract class of MiniCPMV can only be inherited, but cannot be
485 instantiated.
vllm/model_executor/models/paligemma.py
+4 -4
@@ -19,10 +19,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
19 from vllm.multimodal.image import cached_get_tokenizer
20 from vllm.sequence import IntermediateTensors, SamplerOutput
21
22 -from .interfaces import SupportsVision
22 +from .interfaces import SupportsMultiModal
23 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
25 -from .utils import merge_vision_embeddings
25 +from .utils import merge_multimodal_embeddings
26
27 logger = init_logger(__name__)
28
@@ -130,7 +130,7 @@ class PaliGemmaMultiModalProjector(nn.Module):
130 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens)
131 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)
132 @INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)
133 -class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
133 +class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
134
135 def __init__(self,
136 config: PaliGemmaConfig,
@@ -244,7 +244,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
244
245 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
246
247 - inputs_embeds = merge_vision_embeddings(
247 + inputs_embeds = merge_multimodal_embeddings(
248 input_ids, inputs_embeds, vision_embeddings,
249 self.config.image_token_index)
250
vllm/model_executor/models/phi3v.py
+6 -6
@@ -42,8 +42,8 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
42
43 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
44 input_processor_for_clip)
45 -from .interfaces import SupportsVision
46 -from .utils import merge_vision_embeddings
45 +from .interfaces import SupportsMultiModal
46 +from .utils import merge_multimodal_embeddings
47
48 logger = init_logger(__name__)
49
@@ -453,7 +453,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
453 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
454 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
455 @INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
456 -class Phi3VForCausalLM(nn.Module, SupportsVision):
456 +class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
457
458 def __init__(self,
459 config: PretrainedConfig,
@@ -568,9 +568,9 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
568 if image_input is not None:
569 vision_embeddings = self._process_image_input(image_input)
570 inputs_embeds = self.model.get_input_embeddings(input_ids)
571 - inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
572 - vision_embeddings,
573 - self.image_token_id)
571 + inputs_embeds = merge_multimodal_embeddings(
572 + input_ids, inputs_embeds, vision_embeddings,
573 + self.image_token_id)
574 input_ids = None
575 else:
576 inputs_embeds = None
vllm/model_executor/models/utils.py
+15 -14
@@ -54,41 +54,42 @@ def init_vllm_registered_model(
54 )
55
56
57 -def merge_vision_embeddings(input_ids: torch.Tensor,
58 - inputs_embeds: torch.Tensor,
59 - vision_embeddings: BatchedTensors,
60 - image_token_id: int) -> torch.Tensor:
57 +def merge_multimodal_embeddings(input_ids: torch.Tensor,
58 + inputs_embeds: torch.Tensor,
59 + multimodal_embeddings: BatchedTensors,
60 + placeholder_token_id: int) -> torch.Tensor:
61 """
62 - Merge ``vision_embeddings`` into ``inputs_embeds`` by overwriting the
63 - positions in ``inputs_embeds`` corresponding to placeholder image tokens in
62 + Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
63 + positions in ``inputs_embeds`` corresponding to placeholder tokens in
64 ``input_ids``.
65
66 Note:
67 This updates ``inputs_embeds`` in place.
68 """
69 - mask = (input_ids == image_token_id)
69 + mask = (input_ids == placeholder_token_id)
70 num_expected_tokens = mask.sum()
71
72 - if isinstance(vision_embeddings, torch.Tensor):
73 - batch_size, batch_tokens, *_, embed_dim = vision_embeddings.shape
72 + if isinstance(multimodal_embeddings, torch.Tensor):
73 + batch_size, batch_tokens, *_, embed_dim = multimodal_embeddings.shape
74 total_tokens = batch_size * batch_tokens
75 if num_expected_tokens != total_tokens:
76 expr = f"{batch_size} x {batch_tokens}"
77 raise ValueError(
78 f"Attempted to assign {expr} = {total_tokens} "
79 - f"image tokens to {num_expected_tokens} placeholders")
79 + f"multimodal tokens to {num_expected_tokens} placeholders")
80
81 - inputs_embeds[mask] = vision_embeddings.view(total_tokens, embed_dim)
81 + inputs_embeds[mask] = multimodal_embeddings.view(
82 + total_tokens, embed_dim)
83 else:
83 - size_per_batch = [t.shape[0] for t in vision_embeddings]
84 + size_per_batch = [t.shape[0] for t in multimodal_embeddings]
85 total_tokens = sum(size_per_batch)
86 if num_expected_tokens != total_tokens:
87 expr = ' + '.join(map(str, size_per_batch))
88 raise ValueError(
89 f"Attempted to assign {expr} = {total_tokens} "
89 - f"image tokens to {num_expected_tokens} placeholders")
90 + f"multimodal tokens to {num_expected_tokens} placeholders")
91
91 - inputs_embeds[mask] = torch.cat(vision_embeddings)
92 + inputs_embeds[mask] = torch.cat(multimodal_embeddings)
93
94 return inputs_embeds
95
vllm/multimodal/audio.py new
+17
@@ -0,0 +1,17 @@
1 +from vllm.inputs.registry import InputContext
2 +from vllm.multimodal.base import MultiModalInputs, MultiModalPlugin
3 +
4 +
5 +class AudioPlugin(MultiModalPlugin):
6 + """Plugin for audio data."""
7 +
8 + def get_data_key(self) -> str:
9 + return "audio"
10 +
11 + def _default_input_mapper(self, ctx: InputContext,
12 + data: object) -> MultiModalInputs:
13 + raise NotImplementedError("There is no default audio input mapper")
14 +
15 + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
16 + raise NotImplementedError(
17 + "There is no default maximum multimodal tokens")
vllm/multimodal/base.py
+5 -1
@@ -3,8 +3,9 @@ from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 from typing import Any, Callable, Dict, List, Optional
5 from typing import Sequence as GenericSequence
6 -from typing import Type, TypedDict, TypeVar, Union, cast
6 +from typing import Tuple, Type, TypedDict, TypeVar, Union, cast
7
8 +import numpy as np
9 import torch
10 import torch.types
11 from PIL import Image
@@ -121,6 +122,9 @@ class MultiModalDataBuiltins(TypedDict, total=False):
122 image: Image.Image
123 """The input image."""
124
125 + audio: Tuple[np.ndarray, Union[int, float]]
126 + """The input audio and its sampling rate."""
127 +
128
129 MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
130 """
vllm/multimodal/registry.py
+2 -1
@@ -6,6 +6,7 @@ import torch
6 from vllm.config import ModelConfig
7 from vllm.logger import init_logger
8
9 +from .audio import AudioPlugin
10 from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
11 MultiModalPlugin, MultiModalTokensCalc)
12 from .image import ImagePlugin
@@ -19,7 +20,7 @@ class MultiModalRegistry:
20 :class:`~vllm.multimodal.MultiModalPlugin` for each modality.
21 """
22
22 - DEFAULT_PLUGINS = (ImagePlugin(), )
23 + DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin())
24
25 def __init__(
26 self,
vllm/multimodal/utils.py
+56 -2
@@ -1,11 +1,14 @@
1 import base64
2 from io import BytesIO
3 -from typing import Union
3 +from typing import Tuple, Union
4
5 +import librosa
6 +import numpy as np
7 +import soundfile
8 from PIL import Image
9
10 from vllm.connections import global_http_connection
8 -from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
11 +from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
12 from vllm.multimodal.base import MultiModalDataDict
13
14
@@ -63,11 +66,62 @@ async def async_fetch_image(image_url: str,
66 return image.convert(image_mode)
67
68
69 +def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
70 + """
71 + Load audio from a URL.
72 + """
73 + if audio_url.startswith("http"):
74 + audio_bytes = global_http_connection.get_bytes(
75 + audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
76 + elif audio_url.startswith("data:audio"):
77 + _, audio_base64 = audio_url.split(",", 1)
78 + audio_bytes = base64.b64decode(audio_base64)
79 + else:
80 + raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
81 + "with either 'data:audio' or 'http'.")
82 +
83 + return librosa.load(BytesIO(audio_bytes), sr=None)
84 +
85 +
86 +async def async_fetch_audio(
87 + audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
88 + """
89 + Asynchronously fetch audio from a URL.
90 + """
91 + if audio_url.startswith("http"):
92 + audio_bytes = await global_http_connection.async_get_bytes(
93 + audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
94 + elif audio_url.startswith("data:audio"):
95 + _, audio_base64 = audio_url.split(",", 1)
96 + audio_bytes = base64.b64decode(audio_base64)
97 + else:
98 + raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
99 + "with either 'data:audio' or 'http'.")
100 +
101 + return librosa.load(BytesIO(audio_bytes), sr=None)
102 +
103 +
104 +async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
105 + audio, sr = await async_fetch_audio(audio_url)
106 + return {"audio": (audio, sr)}
107 +
108 +
109 async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
110 image = await async_fetch_image(image_url)
111 return {"image": image}
112
113
114 +def encode_audio_base64(
115 + audio: np.ndarray,
116 + sampling_rate: int,
117 +) -> str:
118 + """Encode audio as base64."""
119 + buffered = BytesIO()
120 + soundfile.write(buffered, audio, sampling_rate, format="WAV")
121 +
122 + return base64.b64encode(buffered.getvalue()).decode('utf-8')
123 +
124 +
125 def encode_image_base64(
126 image: Image.Image,
127 *,
vllm/worker/model_runner.py
+4 -4
@@ -40,7 +40,7 @@ from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
40 from vllm.model_executor.model_loader import get_model
41 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
42 from vllm.model_executor.models.interfaces import (supports_lora,
43 - supports_vision)
43 + supports_multimodal)
44 from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
45 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
46 MultiModalInputs)
@@ -900,9 +900,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
900
901 if self.lora_config:
902 assert supports_lora(self.model), "Model does not support LoRA"
903 - assert not supports_vision(
903 + assert not supports_multimodal(
904 self.model
905 - ), "To be tested: vision language model with LoRA settings."
905 + ), "To be tested: multimodal language model with LoRA settings."
906
907 self.lora_manager = LRUCacheWorkerLoRAManager(
908 self.scheduler_config.max_num_seqs,
@@ -1054,7 +1054,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1054 # of images processed.
1055 model_config = self.model_config
1056
1057 - if supports_vision(self.model):
1057 + if supports_multimodal(self.model):
1058 max_mm_tokens = MULTIMODAL_REGISTRY \
1059 .get_max_multimodal_tokens(model_config)
1060 max_num_seqs_orig = max_num_seqs
vllm/worker/xpu_model_runner.py
+2 -2
@@ -12,7 +12,7 @@ from vllm.distributed import broadcast_tensor_dict
12 from vllm.inputs import INPUT_REGISTRY
13 from vllm.logger import init_logger
14 from vllm.model_executor.model_loader import get_model
15 -from vllm.model_executor.models.interfaces import supports_vision
15 +from vllm.model_executor.models.interfaces import supports_multimodal
16 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17 MultiModalInputs)
18 from vllm.sampling_params import SamplingParams
@@ -165,7 +165,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
165 # of images processed.
166 model_config = self.model_config
167
168 - if supports_vision(self.model):
168 + if supports_multimodal(self.model):
169 max_mm_tokens = MULTIMODAL_REGISTRY \
170 .get_max_multimodal_tokens(model_config)
171 max_num_seqs_orig = max_num_seqs