A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend][Core] Add plumbing to support audio language models (#7446)
Peter Salas committed
Aug 13, 2024 at 10:39 UTC
00c3d68e45bad901989c1afe3c223225dc9a5d6d
24 files changed
+600
-121
docs/source/conf.py
+2
@@ -112,6 +112,8 @@ autodoc_mock_imports = [
112
"tensorizer",
113
"pynvml",
114
"outlines",
115
+ "librosa",
116
+ "soundfile",
117
"gguf",
118
"lark",
119
]
docs/source/models/enabling_multimodal_inputs.rst
+11
-11
@@ -15,14 +15,14 @@ This document walks you through the steps to extend a vLLM model so that it acce
15
It is assumed that you have already implemented the model in vLLM according to :ref:`these steps <adding_a_new_model>`.
16
Further update the model as follows:
17
18
-- Implement the :class:`~vllm.model_executor.models.interfaces.SupportsVision` interface.
18
+- Implement the :class:`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
19
20
.. code-block:: diff
21
22
- + from vllm.model_executor.models.interfaces import SupportsVision
22
+ + from vllm.model_executor.models.interfaces import SupportsMultiModal
23
24
- class YourModelForImage2Seq(nn.Module):
25
- + class YourModelForImage2Seq(nn.Module, SupportsVision):
25
+ + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
26
27
.. note::
28
The model class does not have to be named :code:`*ForCausalLM`.
@@ -51,11 +51,11 @@ This decorator accepts a function that maps multi-modal inputs to the keyword ar
51
52
.. code-block:: diff
53
54
- from vllm.model_executor.models.interfaces import SupportsVision
54
+ from vllm.model_executor.models.interfaces import SupportsMultiModal
55
+ from vllm.multimodal import MULTIMODAL_REGISTRY
56
57
+ @MULTIMODAL_REGISTRY.register_image_input_mapper()
58
- class YourModelForImage2Seq(nn.Module, SupportsVision):
58
+ class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
59
60
A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
61
@@ -72,13 +72,13 @@ and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.regis
72
.. code-block:: diff
73
74
from vllm.inputs import INPUT_REGISTRY
75
- from vllm.model_executor.models.interfaces import SupportsVision
75
+ from vllm.model_executor.models.interfaces import SupportsMultiModal
76
from vllm.multimodal import MULTIMODAL_REGISTRY
77
78
@MULTIMODAL_REGISTRY.register_image_input_mapper()
79
+ @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
80
@INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
81
- class YourModelForImage2Seq(nn.Module, SupportsVision):
81
+ class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
82
83
Here are some examples:
84
@@ -98,13 +98,13 @@ In such cases, you can define your own dummy data by registering a factory metho
98
.. code-block:: diff
99
100
from vllm.inputs import INPUT_REGISTRY
101
- from vllm.model_executor.models.interfaces import SupportsVision
101
+ from vllm.model_executor.models.interfaces import SupportsMultiModal
102
from vllm.multimodal import MULTIMODAL_REGISTRY
103
104
@MULTIMODAL_REGISTRY.register_image_input_mapper()
105
@MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
106
+ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
107
- class YourModelForImage2Seq(nn.Module, SupportsVision):
107
+ class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
108
109
.. note::
110
The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
@@ -128,14 +128,14 @@ You can register input processors via :meth:`INPUT_REGISTRY.register_input_proce
128
.. code-block:: diff
129
130
from vllm.inputs import INPUT_REGISTRY
131
- from vllm.model_executor.models.interfaces import SupportsVision
131
+ from vllm.model_executor.models.interfaces import SupportsMultiModal
132
from vllm.multimodal import MULTIMODAL_REGISTRY
133
134
@MULTIMODAL_REGISTRY.register_image_input_mapper()
135
@MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
136
@INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
137
+ @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
138
- class YourModelForImage2Seq(nn.Module, SupportsVision):
138
+ class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
139
140
A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
141
Here are some examples:
requirements-common.txt
+2
@@ -20,4 +20,6 @@ outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
20
typing_extensions >= 4.10
21
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
22
pyzmq
23
+librosa # Required for audio processing
24
+soundfile # Required for audio processing
25
gguf == 0.9.1
tests/entrypoints/openai/test_audio.py
new
+351
@@ -0,0 +1,351 @@
1
+import math
2
+import sys
3
+import time
4
+from typing import Dict, List, Optional, Tuple, Union, cast
5
+from unittest.mock import patch
6
+
7
+import librosa
8
+import numpy as np
9
+import openai
10
+import pytest
11
+import requests
12
+import torch
13
+
14
+from vllm import ModelRegistry
15
+from vllm.config import MultiModalConfig
16
+from vllm.inputs import INPUT_REGISTRY
17
+from vllm.inputs.data import LLMInputs
18
+from vllm.inputs.registry import InputContext
19
+from vllm.model_executor.models.interfaces import SupportsMultiModal
20
+from vllm.model_executor.models.opt import OPTForCausalLM
21
+from vllm.multimodal import MULTIMODAL_REGISTRY
22
+from vllm.multimodal.base import MultiModalInputs
23
+from vllm.multimodal.image import (cached_get_tokenizer,
24
+ repeat_and_pad_image_tokens)
25
+from vllm.multimodal.utils import encode_audio_base64, fetch_audio
26
+from vllm.utils import get_open_port
27
+
28
+from ...utils import VLLM_PATH
29
+
30
+chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
31
+assert chatml_jinja_path.exists()
32
+
33
+MODEL_NAME = "facebook/opt-125m"
34
+TEST_AUDIO_URLS = [
35
+ "https://upload.wikimedia.org/wikipedia/en/b/bf/Dave_Niehaus_Winning_Call_1995_AL_Division_Series.ogg",
36
+]
37
+
38
+
39
+def server_function(port):
40
+
41
+ def fake_input_mapper(ctx: InputContext, data: object):
42
+ assert isinstance(data, tuple)
43
+ (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
44
+
45
+ # Resample it to 1 sample per second
46
+ audio = librosa.resample(audio, orig_sr=sr, target_sr=1)
47
+ return MultiModalInputs({"processed_audio": torch.from_numpy(audio)})
48
+
49
+ def fake_input_processor(ctx: InputContext, llm_inputs: LLMInputs):
50
+ multi_modal_data = llm_inputs.get("multi_modal_data")
51
+ if multi_modal_data is None or "audio" not in multi_modal_data:
52
+ return llm_inputs
53
+
54
+ audio, sr = multi_modal_data.get("audio")
55
+ audio_duration = math.ceil(len(audio) / sr)
56
+
57
+ new_prompt, new_token_ids = repeat_and_pad_image_tokens(
58
+ cached_get_tokenizer(ctx.model_config.tokenizer),
59
+ llm_inputs.get("prompt"),
60
+ llm_inputs["prompt_token_ids"],
61
+ image_token_id=62, # "_"
62
+ repeat_count=audio_duration)
63
+
64
+ return LLMInputs(prompt_token_ids=new_token_ids,
65
+ prompt=new_prompt,
66
+ multi_modal_data=multi_modal_data)
67
+
68
+ @MULTIMODAL_REGISTRY.register_input_mapper("audio", fake_input_mapper)
69
+ @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
70
+ "audio", lambda *_, **__: 100)
71
+ @INPUT_REGISTRY.register_input_processor(fake_input_processor)
72
+ class FakeAudioModel(OPTForCausalLM, SupportsMultiModal):
73
+
74
+ def __init__(self, *args, multimodal_config: MultiModalConfig,
75
+ **kwargs):
76
+ assert multimodal_config is not None
77
+ super().__init__(*args, **kwargs)
78
+
79
+ def forward(
80
+ self,
81
+ *args,
82
+ processed_audio: Optional[torch.Tensor] = None,
83
+ **kwargs,
84
+ ) -> torch.Tensor:
85
+ return super().forward(*args, **kwargs)
86
+
87
+ ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
88
+
89
+ with patch("vllm.entrypoints.chat_utils._mm_token_str",
90
+ lambda *_, **__: "_"):
91
+ sys.argv = ["placeholder.py"] + \
92
+ (f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
93
+ "--dtype bfloat16 --enforce-eager --api-key token-abc123 "
94
+ f"--port {port} --chat-template {chatml_jinja_path} "
95
+ "--disable-frontend-multiprocessing").split()
96
+ import runpy
97
+ runpy.run_module('vllm.entrypoints.openai.api_server',
98
+ run_name='__main__')
99
+
100
+
101
+@pytest.fixture(scope="module")
102
+def client():
103
+ port = get_open_port()
104
+ ctx = torch.multiprocessing.get_context("spawn")
105
+ server = ctx.Process(target=server_function, args=(port, ))
106
+ server.start()
107
+ MAX_SERVER_START_WAIT_S = 60
108
+ client = openai.AsyncOpenAI(
109
+ base_url=f"http://localhost:{port}/v1",
110
+ api_key="token-abc123",
111
+ )
112
+ # run health check
113
+ health_url = f"http://localhost:{port}/health"
114
+ start = time.time()
115
+ while True:
116
+ try:
117
+ if requests.get(health_url).status_code == 200:
118
+ break
119
+ except Exception as err:
120
+ result = server.exitcode
121
+ if result is not None:
122
+ raise RuntimeError("Server exited unexpectedly.") from err
123
+
124
+ time.sleep(0.5)
125
+ if time.time() - start > MAX_SERVER_START_WAIT_S:
126
+ raise RuntimeError("Server failed to start in time.") from err
127
+
128
+ try:
129
+ yield client
130
+ finally:
131
+ server.kill()
132
+
133
+
134
+@pytest.fixture(scope="session")
135
+def base64_encoded_audio() -> Dict[str, str]:
136
+ return {
137
+ audio_url: encode_audio_base64(*fetch_audio(audio_url))
138
+ for audio_url in TEST_AUDIO_URLS
139
+ }
140
+
141
+
142
+@pytest.mark.asyncio
143
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
144
+@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
145
+async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
146
+ model_name: str, audio_url: str):
147
+ messages = [{
148
+ "role":
149
+ "user",
150
+ "content": [
151
+ {
152
+ "type": "audio_url",
153
+ "audio_url": {
154
+ "url": audio_url
155
+ }
156
+ },
157
+ {
158
+ "type": "text",
159
+ "text": "What's happening in this audio?"
160
+ },
161
+ ],
162
+ }]
163
+
164
+ # test single completion
165
+ chat_completion = await client.chat.completions.create(model=model_name,
166
+ messages=messages,
167
+ max_tokens=10,
168
+ logprobs=True,
169
+ top_logprobs=5)
170
+ assert len(chat_completion.choices) == 1
171
+
172
+ choice = chat_completion.choices[0]
173
+ assert choice.finish_reason == "length"
174
+ assert chat_completion.usage == openai.types.CompletionUsage(
175
+ completion_tokens=10, prompt_tokens=36, total_tokens=46)
176
+
177
+ message = choice.message
178
+ message = chat_completion.choices[0].message
179
+ assert message.content is not None and len(message.content) >= 10
180
+ assert message.role == "assistant"
181
+ messages.append({"role": "assistant", "content": message.content})
182
+
183
+ # test multi-turn dialogue
184
+ messages.append({"role": "user", "content": "express your result in json"})
185
+ chat_completion = await client.chat.completions.create(
186
+ model=model_name,
187
+ messages=messages,
188
+ max_tokens=10,
189
+ )
190
+ message = chat_completion.choices[0].message
191
+ assert message.content is not None and len(message.content) >= 0
192
+
193
+
194
+@pytest.mark.asyncio
195
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
196
+@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
197
+async def test_single_chat_session_audio_base64encoded(
198
+ client: openai.AsyncOpenAI, model_name: str, audio_url: str,
199
+ base64_encoded_audio: Dict[str, str]):
200
+
201
+ messages = [{
202
+ "role":
203
+ "user",
204
+ "content": [
205
+ {
206
+ "type": "audio_url",
207
+ "audio_url": {
208
+ "url":
209
+ f"data:audio/wav;base64,{base64_encoded_audio[audio_url]}"
210
+ }
211
+ },
212
+ {
213
+ "type": "text",
214
+ "text": "What's happening in this audio?"
215
+ },
216
+ ],
217
+ }]
218
+
219
+ # test single completion
220
+ chat_completion = await client.chat.completions.create(model=model_name,
221
+ messages=messages,
222
+ max_tokens=10,
223
+ logprobs=True,
224
+ top_logprobs=5)
225
+ assert len(chat_completion.choices) == 1
226
+
227
+ choice = chat_completion.choices[0]
228
+ assert choice.finish_reason == "length"
229
+ assert chat_completion.usage == openai.types.CompletionUsage(
230
+ completion_tokens=10, prompt_tokens=36, total_tokens=46)
231
+
232
+ message = choice.message
233
+ message = chat_completion.choices[0].message
234
+ assert message.content is not None and len(message.content) >= 10
235
+ assert message.role == "assistant"
236
+ messages.append({"role": "assistant", "content": message.content})
237
+
238
+ # test multi-turn dialogue
239
+ messages.append({"role": "user", "content": "express your result in json"})
240
+ chat_completion = await client.chat.completions.create(
241
+ model=model_name,
242
+ messages=messages,
243
+ max_tokens=10,
244
+ )
245
+ message = chat_completion.choices[0].message
246
+ assert message.content is not None and len(message.content) >= 0
247
+
248
+
249
+@pytest.mark.asyncio
250
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
251
+@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
252
+async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
253
+ model_name: str, audio_url: str):
254
+ messages = [{
255
+ "role":
256
+ "user",
257
+ "content": [
258
+ {
259
+ "type": "audio_url",
260
+ "audio_url": {
261
+ "url": audio_url
262
+ }
263
+ },
264
+ {
265
+ "type": "text",
266
+ "text": "What's happening in this audio?"
267
+ },
268
+ ],
269
+ }]
270
+
271
+ # test single completion
272
+ chat_completion = await client.chat.completions.create(
273
+ model=model_name,
274
+ messages=messages,
275
+ max_tokens=10,
276
+ temperature=0.0,
277
+ )
278
+ output = chat_completion.choices[0].message.content
279
+ stop_reason = chat_completion.choices[0].finish_reason
280
+
281
+ # test streaming
282
+ stream = await client.chat.completions.create(
283
+ model=model_name,
284
+ messages=messages,
285
+ max_tokens=10,
286
+ temperature=0.0,
287
+ stream=True,
288
+ )
289
+ chunks: List[str] = []
290
+ finish_reason_count = 0
291
+ async for chunk in stream:
292
+ delta = chunk.choices[0].delta
293
+ if delta.role:
294
+ assert delta.role == "assistant"
295
+ if delta.content:
296
+ chunks.append(delta.content)
297
+ if chunk.choices[0].finish_reason is not None:
298
+ finish_reason_count += 1
299
+ # finish reason should only return in last block
300
+ assert finish_reason_count == 1
301
+ assert chunk.choices[0].finish_reason == stop_reason
302
+ assert delta.content
303
+ assert "".join(chunks) == output
304
+
305
+
306
+@pytest.mark.asyncio
307
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
308
+@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
309
+async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
310
+ audio_url: str):
311
+
312
+ messages = [{
313
+ "role":
314
+ "user",
315
+ "content": [
316
+ {
317
+ "type": "audio_url",
318
+ "audio_url": {
319
+ "url": audio_url
320
+ }
321
+ },
322
+ {
323
+ "type": "audio_url",
324
+ "audio_url": {
325
+ "url": audio_url
326
+ }
327
+ },
328
+ {
329
+ "type": "text",
330
+ "text": "What's happening in this audio?"
331
+ },
332
+ ],
333
+ }]
334
+
335
+ with pytest.raises(openai.BadRequestError): # test multi-audio input
336
+ await client.chat.completions.create(
337
+ model=model_name,
338
+ messages=messages,
339
+ max_tokens=10,
340
+ temperature=0.0,
341
+ )
342
+
343
+ # the server should still work afterwards
344
+ completion = await client.completions.create(
345
+ model=model_name,
346
+ prompt=[0, 0, 0, 0, 0],
347
+ max_tokens=5,
348
+ temperature=0.0,
349
+ )
350
+ completion = completion.choices[0].text
351
+ assert completion is not None and len(completion) >= 0
vllm/entrypoints/chat_utils.py
+69
-32
@@ -2,7 +2,8 @@ import codecs
2
from dataclasses import dataclass
3
from functools import lru_cache
4
from pathlib import Path
5
-from typing import Any, Awaitable, Iterable, List, Optional, Tuple, Union, cast
5
+from typing import (Any, Awaitable, Iterable, List, Literal, Optional, Tuple,
6
+ Union, cast)
7
8
# yapf conflicts with isort for this block
9
# yapf: disable
@@ -21,12 +22,27 @@ from typing_extensions import Required, TypedDict
22
from vllm.config import ModelConfig
23
from vllm.logger import init_logger
24
from vllm.multimodal import MultiModalDataDict
24
-from vllm.multimodal.utils import async_get_and_parse_image
25
+from vllm.multimodal.utils import (async_get_and_parse_audio,
26
+ async_get_and_parse_image)
27
from vllm.transformers_utils.tokenizer import AnyTokenizer
28
29
logger = init_logger(__name__)
30
31
32
+class AudioURL(TypedDict, total=False):
33
+ url: Required[str]
34
+ """
35
+ Either a URL of the audio or a data URL with base64 encoded audio data.
36
+ """
37
+
38
+
39
+class ChatCompletionContentPartAudioParam(TypedDict, total=False):
40
+ audio_url: Required[AudioURL]
41
+
42
+ type: Required[Literal["audio_url"]]
43
+ """The type of the content part."""
44
+
45
+
46
class CustomChatCompletionContentPartParam(TypedDict, total=False):
47
__pydantic_config__ = ConfigDict(extra="allow") # type: ignore
48
@@ -35,6 +51,7 @@ class CustomChatCompletionContentPartParam(TypedDict, total=False):
51
52
53
ChatCompletionContentPartParam = Union[OpenAIChatCompletionContentPartParam,
54
+ ChatCompletionContentPartAudioParam,
55
CustomChatCompletionContentPartParam]
56
57
@@ -97,34 +114,41 @@ def load_chat_template(
114
115
116
@lru_cache(maxsize=None)
100
-def _image_token_str(model_config: ModelConfig,
101
- tokenizer: PreTrainedTokenizer) -> Optional[str]:
117
+def _mm_token_str(model_config: ModelConfig, tokenizer: PreTrainedTokenizer,
118
+ modality: Literal["image", "audio"]) -> Optional[str]:
119
# TODO: Let user specify how to insert image tokens into prompt
120
# (similar to chat template)
104
- model_type = model_config.hf_config.model_type
105
- if model_type == "phi3_v":
106
- # Workaround since this token is not defined in the tokenizer
107
- return "<|image_1|>"
108
- if model_type == "minicpmv":
109
- return "(<image>./</image>)"
110
- if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
111
- # These models do not use image tokens in the prompt
112
- return None
113
- if model_type.startswith("llava"):
114
- return tokenizer.decode(model_config.hf_config.image_token_index)
115
- if model_type in ("chameleon", "internvl_chat"):
116
- return "<image>"
117
- raise TypeError(f"Unknown model type: {model_type}")
118
-
119
-
120
-# TODO: Let user specify how to insert image tokens into prompt
121
+ if modality == "image":
122
+ model_type = model_config.hf_config.model_type
123
+ if model_type == "phi3_v":
124
+ # Workaround since this token is not defined in the tokenizer
125
+ return "<|image_1|>"
126
+ if model_type == "minicpmv":
127
+ return "(<image>./</image>)"
128
+ if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
129
+ # These models do not use image tokens in the prompt
130
+ return None
131
+ if model_type.startswith("llava"):
132
+ return tokenizer.decode(model_config.hf_config.image_token_index)
133
+ if model_type in ("chameleon", "internvl_chat"):
134
+ return "<image>"
135
+
136
+ raise TypeError(f"Unknown model type: {model_type}")
137
+ elif modality == "audio":
138
+ raise TypeError("No audio models are supported yet.")
139
+ else:
140
+ raise TypeError(f"Unknown modality: {modality}")
141
+
142
+
143
+# TODO: Let user specify how to insert multimodal tokens into prompt
144
# (similar to chat template)
122
-def _get_full_image_text_prompt(image_token_str: str, text_prompt: str) -> str:
123
- """Combine image and text prompts for vision language model"""
145
+def _get_full_multimodal_text_prompt(placeholder_token_str: str,
146
+ text_prompt: str) -> str:
147
+ """Combine multimodal prompts for a multimodal language model"""
148
149
# NOTE: For now we assume all model architectures use the same
126
- # image + text prompt format. This may change in the future.
127
- return f"{image_token_str}\n{text_prompt}"
150
+ # placeholder + text prompt format. This may change in the future.
151
+ return f"{placeholder_token_str}\n{text_prompt}"
152
153
154
def _parse_chat_message_content_parts(
@@ -135,6 +159,7 @@ def _parse_chat_message_content_parts(
159
) -> ChatMessageParseResult:
160
texts: List[str] = []
161
mm_futures: List[Awaitable[MultiModalDataDict]] = []
162
+ modality: Literal["image", "audio"] = "image"
163
164
for part in parts:
165
part_type = part["type"]
@@ -142,9 +167,10 @@ def _parse_chat_message_content_parts(
167
text = cast(ChatCompletionContentPartTextParam, part)["text"]
168
texts.append(text)
169
elif part_type == "image_url":
170
+ modality = "image"
171
if len(mm_futures) > 0:
172
raise NotImplementedError(
147
- "Multiple 'image_url' input is currently not supported.")
173
+ "Multiple multimodal inputs is currently not supported.")
174
175
image_url = cast(ChatCompletionContentPartImageParam,
176
part)["image_url"]
@@ -156,21 +182,32 @@ def _parse_chat_message_content_parts(
182
183
image_future = async_get_and_parse_image(image_url["url"])
184
mm_futures.append(image_future)
185
+ elif part_type == "audio_url":
186
+ modality = "audio"
187
+ if len(mm_futures) > 0:
188
+ raise NotImplementedError(
189
+ "Multiple multimodal inputs is currently not supported.")
190
+
191
+ audio_url = cast(ChatCompletionContentPartAudioParam,
192
+ part)["audio_url"]
193
+ audio_future = async_get_and_parse_audio(audio_url["url"])
194
+ mm_futures.append(audio_future)
195
else:
196
raise NotImplementedError(f"Unknown part type: {part_type}")
197
198
text_prompt = "\n".join(texts)
199
200
if mm_futures:
165
- image_token_str = _image_token_str(model_config, tokenizer)
166
- if image_token_str is not None:
167
- if image_token_str in text_prompt:
201
+ placeholder_token_str = _mm_token_str(model_config, tokenizer,
202
+ modality)
203
+ if placeholder_token_str is not None:
204
+ if placeholder_token_str in text_prompt:
205
logger.warning(
169
- "Detected image token string in the text prompt. "
206
+ "Detected multi-modal token string in the text prompt. "
207
"Skipping prompt formatting.")
208
else:
172
- text_prompt = _get_full_image_text_prompt(
173
- image_token_str=image_token_str,
209
+ text_prompt = _get_full_multimodal_text_prompt(
210
+ placeholder_token_str=placeholder_token_str,
211
text_prompt=text_prompt,
212
)
213
vllm/envs.py
+6
@@ -44,6 +44,7 @@ if TYPE_CHECKING:
44
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
45
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
46
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
47
+ VLLM_AUDIO_FETCH_TIMEOUT: int = 5
48
VLLM_TARGET_DEVICE: str = "cuda"
49
MAX_JOBS: Optional[str] = None
50
NVCC_THREADS: Optional[str] = None
@@ -321,6 +322,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
322
"VLLM_IMAGE_FETCH_TIMEOUT":
323
lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
324
325
+ # Timeout for fetching audio when serving multimodal models
326
+ # Default is 5 seconds
327
+ "VLLM_AUDIO_FETCH_TIMEOUT":
328
+ lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "5")),
329
+
330
# Path to the XLA persistent cache directory.
331
# Only used for XLA devices such as TPUs.
332
"VLLM_XLA_CACHE_PATH":
vllm/model_executor/model_loader/loader.py
+2
-2
@@ -38,7 +38,7 @@ from vllm.model_executor.model_loader.weight_utils import (
38
safetensors_weights_iterator)
39
from vllm.model_executor.models.interfaces import (has_inner_state,
40
supports_lora,
41
- supports_vision)
41
+ supports_multimodal)
42
from vllm.model_executor.utils import set_weight_attrs
43
from vllm.platforms import current_platform
44
from vllm.utils import is_pin_memory_available
@@ -131,7 +131,7 @@ def _get_model_initialization_kwargs(
131
"be added in the future. If this is important to you, "
132
"please open an issue on github.")
133
134
- if supports_vision(model_class):
134
+ if supports_multimodal(model_class):
135
if multimodal_config is None:
136
raise ValueError("Provide vision related configurations "
137
"through LLM entrypoint or engine arguments.")
vllm/model_executor/models/blip2.py
+6
-6
@@ -20,8 +20,8 @@ from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
20
21
from .blip import (BlipVisionModel, dummy_image_for_blip,
22
get_max_blip_image_tokens)
23
-from .interfaces import SupportsVision
24
-from .utils import merge_vision_embeddings
23
+from .interfaces import SupportsMultiModal
24
+from .utils import merge_multimodal_embeddings
25
26
_KEYS_TO_MODIFY_MAPPING = {
27
"language_model.lm_head": "lm_head",
@@ -457,7 +457,7 @@ def input_processor_for_blip2(ctx: InputContext, llm_inputs: LLMInputs):
457
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
458
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2)
459
@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
460
-class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
460
+class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
461
462
def __init__(self,
463
config: Blip2Config,
@@ -621,9 +621,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
621
vision_embeddings = self._process_image_input(image_input)
622
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
623
624
- inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
625
- vision_embeddings,
626
- BLIP2_IMAGE_TOKEN_ID)
624
+ inputs_embeds = merge_multimodal_embeddings(
625
+ input_ids, inputs_embeds, vision_embeddings,
626
+ BLIP2_IMAGE_TOKEN_ID)
627
628
input_ids = None
629
else:
vllm/model_executor/models/chameleon.py
+2
-2
@@ -35,7 +35,7 @@ from vllm.multimodal.image import (cached_get_tokenizer,
35
from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
36
from vllm.utils import print_warning_once
37
38
-from .interfaces import SupportsVision
38
+from .interfaces import SupportsMultiModal
39
40
logger = init_logger(__name__)
41
@@ -886,7 +886,7 @@ class ChameleonModel(nn.Module):
886
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
887
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon)
888
@INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon)
889
-class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
889
+class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal):
890
891
def __init__(
892
self,
vllm/model_executor/models/fuyu.py
+6
-6
@@ -40,8 +40,8 @@ from vllm.multimodal.image import (cached_get_image_processor,
40
cached_get_tokenizer)
41
from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
42
43
-from .interfaces import SupportsVision
44
-from .utils import merge_vision_embeddings
43
+from .interfaces import SupportsMultiModal
44
+from .utils import merge_multimodal_embeddings
45
46
logger = init_logger(__name__)
47
@@ -209,7 +209,7 @@ def input_mapper_for_fuyu(ctx: InputContext, data: object):
209
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
210
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu)
211
@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
212
-class FuyuForCausalLM(nn.Module, SupportsVision):
212
+class FuyuForCausalLM(nn.Module, SupportsMultiModal):
213
214
def __init__(self,
215
config: FuyuConfig,
@@ -271,9 +271,9 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
271
if image_input is not None:
272
vision_embeddings = self._process_image_input(image_input)
273
inputs_embeds = self.language_model.model.embed_tokens(input_ids)
274
- inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
275
- vision_embeddings,
276
- self.image_token_id)
274
+ inputs_embeds = merge_multimodal_embeddings(
275
+ input_ids, inputs_embeds, vision_embeddings,
276
+ self.image_token_id)
277
278
else:
279
inputs_embeds = None
vllm/model_executor/models/interfaces.py
+16
-12
@@ -10,12 +10,15 @@ logger = init_logger(__name__)
10
11
12
@runtime_checkable
13
-class SupportsVision(Protocol):
14
- """The interface required for all vision language models (VLMs)."""
13
+class SupportsMultiModal(Protocol):
14
+ """
15
+ The interface required for all multimodal (vision or audio) language
16
+ models.
17
+ """
18
16
- supports_vision: ClassVar[Literal[True]] = True
19
+ supports_multimodal: ClassVar[Literal[True]] = True
20
"""
18
- A flag that indicates this model supports vision inputs.
21
+ A flag that indicates this model supports multimodal inputs.
22
23
Note:
24
There is no need to redefine this flag if this class is in the
@@ -29,30 +32,31 @@ class SupportsVision(Protocol):
32
# We can't use runtime_checkable with ClassVar for issubclass checks
33
# so we need to treat the class as an instance and use isinstance instead
34
@runtime_checkable
32
-class _SupportsVisionType(Protocol):
33
- supports_vision: Literal[True]
35
+class _SupportsMultiModalType(Protocol):
36
+ supports_multimodal: Literal[True]
37
38
def __call__(self, *, multimodal_config: MultiModalConfig) -> None:
39
...
40
41
42
@overload
40
-def supports_vision(model: Type[object]) -> TypeIs[Type[SupportsVision]]:
43
+def supports_multimodal(
44
+ model: Type[object]) -> TypeIs[Type[SupportsMultiModal]]:
45
...
46
47
48
@overload
45
-def supports_vision(model: object) -> TypeIs[SupportsVision]:
49
+def supports_multimodal(model: object) -> TypeIs[SupportsMultiModal]:
50
...
51
52
49
-def supports_vision(
53
+def supports_multimodal(
54
model: Union[Type[object], object],
51
-) -> Union[TypeIs[Type[SupportsVision]], TypeIs[SupportsVision]]:
55
+) -> Union[TypeIs[Type[SupportsMultiModal]], TypeIs[SupportsMultiModal]]:
56
if isinstance(model, type):
53
- return isinstance(model, _SupportsVisionType)
57
+ return isinstance(model, _SupportsMultiModalType)
58
55
- return isinstance(model, SupportsVision)
59
+ return isinstance(model, SupportsMultiModal)
60
61
62
@runtime_checkable
vllm/model_executor/models/internvl.py
+6
-6
@@ -27,9 +27,9 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
27
28
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
29
get_clip_num_patches)
30
-from .interfaces import SupportsVision
30
+from .interfaces import SupportsMultiModal
31
from .utils import (filter_weights, init_vllm_registered_model,
32
- merge_vision_embeddings)
32
+ merge_multimodal_embeddings)
33
34
IMG_START = '<img>'
35
IMG_END = '</img>'
@@ -292,7 +292,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
292
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
293
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_internvl)
294
@INPUT_REGISTRY.register_input_processor(input_processor_for_internvl)
295
-class InternVLChatModel(nn.Module, SupportsVision):
295
+class InternVLChatModel(nn.Module, SupportsMultiModal):
296
297
def __init__(self,
298
config: PretrainedConfig,
@@ -451,9 +451,9 @@ class InternVLChatModel(nn.Module, SupportsVision):
451
inputs_embeds = self.language_model.model.get_input_embeddings(
452
input_ids)
453
vision_embeddings = self._process_image_input(image_input)
454
- inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
455
- vision_embeddings,
456
- self.img_context_token_id)
454
+ inputs_embeds = merge_multimodal_embeddings(
455
+ input_ids, inputs_embeds, vision_embeddings,
456
+ self.img_context_token_id)
457
input_ids = None
458
else:
459
inputs_embeds = None
vllm/model_executor/models/llava.py
+4
-4
@@ -19,12 +19,12 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
19
from .clip import (CLIPVisionModel, dummy_image_for_clip,
20
dummy_seq_data_for_clip, get_max_clip_image_tokens,
21
input_processor_for_clip)
22
-from .interfaces import SupportsVision
22
+from .interfaces import SupportsMultiModal
23
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24
dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
25
input_processor_for_siglip)
26
from .utils import (filter_weights, init_vllm_registered_model,
27
- merge_vision_embeddings)
27
+ merge_multimodal_embeddings)
28
29
30
class LlavaImagePixelInputs(TypedDict):
@@ -181,7 +181,7 @@ def _init_vision_tower(hf_config: LlavaConfig):
181
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
182
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
183
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
184
-class LlavaForConditionalGeneration(nn.Module, SupportsVision):
184
+class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
185
186
def __init__(self,
187
config: LlavaConfig,
@@ -338,7 +338,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
338
inputs_embeds = self.language_model.model.get_input_embeddings(
339
input_ids)
340
341
- inputs_embeds = merge_vision_embeddings(
341
+ inputs_embeds = merge_multimodal_embeddings(
342
input_ids, inputs_embeds, vision_embeddings,
343
self.config.image_token_index)
344
vllm/model_executor/models/llava_next.py
+4
-4
@@ -23,13 +23,13 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
23
from .clip import (CLIPVisionModel, dummy_image_for_clip,
24
dummy_seq_data_for_clip, get_clip_image_feature_size,
25
get_clip_patch_grid_length, input_processor_for_clip)
26
-from .interfaces import SupportsVision
26
+from .interfaces import SupportsMultiModal
27
from .llava import LlavaMultiModalProjector
28
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
30
get_siglip_patch_grid_length, input_processor_for_siglip)
31
from .utils import (filter_weights, init_vllm_registered_model,
32
- merge_vision_embeddings)
32
+ merge_multimodal_embeddings)
33
34
logger = init_logger(__name__)
35
@@ -275,7 +275,7 @@ def _init_vision_tower(hf_config: LlavaNextConfig):
275
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
276
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
277
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next)
278
-class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
278
+class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
279
280
def __init__(self,
281
config: LlavaNextConfig,
@@ -571,7 +571,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
571
inputs_embeds = self.language_model.model.get_input_embeddings(
572
input_ids)
573
574
- inputs_embeds = merge_vision_embeddings(
574
+ inputs_embeds = merge_multimodal_embeddings(
575
input_ids, inputs_embeds, vision_embeddings,
576
self.config.image_token_index)
577
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -48,7 +48,7 @@ from vllm.model_executor.layers.sampler import Sampler
48
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
-from vllm.model_executor.models.interfaces import SupportsVision
51
+from vllm.model_executor.models.interfaces import SupportsMultiModal
52
from vllm.model_executor.models.llama import LlamaModel
53
from vllm.model_executor.models.minicpm import MiniCPMModel
54
from vllm.model_executor.models.qwen2 import Qwen2Model
@@ -479,7 +479,7 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
479
return llm_inputs
480
481
482
-class MiniCPMVBaseModel(nn.Module, SupportsVision):
482
+class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
483
"""
484
The abstract class of MiniCPMV can only be inherited, but cannot be
485
instantiated.
vllm/model_executor/models/paligemma.py
+4
-4
@@ -19,10 +19,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.image import cached_get_tokenizer
20
from vllm.sequence import IntermediateTensors, SamplerOutput
21
22
-from .interfaces import SupportsVision
22
+from .interfaces import SupportsMultiModal
23
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24
dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
25
-from .utils import merge_vision_embeddings
25
+from .utils import merge_multimodal_embeddings
26
27
logger = init_logger(__name__)
28
@@ -130,7 +130,7 @@ class PaliGemmaMultiModalProjector(nn.Module):
130
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens)
131
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)
132
@INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)
133
-class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
133
+class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
134
135
def __init__(self,
136
config: PaliGemmaConfig,
@@ -244,7 +244,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
244
245
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
246
247
- inputs_embeds = merge_vision_embeddings(
247
+ inputs_embeds = merge_multimodal_embeddings(
248
input_ids, inputs_embeds, vision_embeddings,
249
self.config.image_token_index)
250
vllm/model_executor/models/phi3v.py
+6
-6
@@ -42,8 +42,8 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
42
43
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
44
input_processor_for_clip)
45
-from .interfaces import SupportsVision
46
-from .utils import merge_vision_embeddings
45
+from .interfaces import SupportsMultiModal
46
+from .utils import merge_multimodal_embeddings
47
48
logger = init_logger(__name__)
49
@@ -453,7 +453,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
453
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
454
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
455
@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
456
-class Phi3VForCausalLM(nn.Module, SupportsVision):
456
+class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
457
458
def __init__(self,
459
config: PretrainedConfig,
@@ -568,9 +568,9 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
568
if image_input is not None:
569
vision_embeddings = self._process_image_input(image_input)
570
inputs_embeds = self.model.get_input_embeddings(input_ids)
571
- inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
572
- vision_embeddings,
573
- self.image_token_id)
571
+ inputs_embeds = merge_multimodal_embeddings(
572
+ input_ids, inputs_embeds, vision_embeddings,
573
+ self.image_token_id)
574
input_ids = None
575
else:
576
inputs_embeds = None
vllm/model_executor/models/utils.py
+15
-14
@@ -54,41 +54,42 @@ def init_vllm_registered_model(
54
)
55
56
57
-def merge_vision_embeddings(input_ids: torch.Tensor,
58
- inputs_embeds: torch.Tensor,
59
- vision_embeddings: BatchedTensors,
60
- image_token_id: int) -> torch.Tensor:
57
+def merge_multimodal_embeddings(input_ids: torch.Tensor,
58
+ inputs_embeds: torch.Tensor,
59
+ multimodal_embeddings: BatchedTensors,
60
+ placeholder_token_id: int) -> torch.Tensor:
61
"""
62
- Merge ``vision_embeddings`` into ``inputs_embeds`` by overwriting the
63
- positions in ``inputs_embeds`` corresponding to placeholder image tokens in
62
+ Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
63
+ positions in ``inputs_embeds`` corresponding to placeholder tokens in
64
``input_ids``.
65
66
Note:
67
This updates ``inputs_embeds`` in place.
68
"""
69
- mask = (input_ids == image_token_id)
69
+ mask = (input_ids == placeholder_token_id)
70
num_expected_tokens = mask.sum()
71
72
- if isinstance(vision_embeddings, torch.Tensor):
73
- batch_size, batch_tokens, *_, embed_dim = vision_embeddings.shape
72
+ if isinstance(multimodal_embeddings, torch.Tensor):
73
+ batch_size, batch_tokens, *_, embed_dim = multimodal_embeddings.shape
74
total_tokens = batch_size * batch_tokens
75
if num_expected_tokens != total_tokens:
76
expr = f"{batch_size} x {batch_tokens}"
77
raise ValueError(
78
f"Attempted to assign {expr} = {total_tokens} "
79
- f"image tokens to {num_expected_tokens} placeholders")
79
+ f"multimodal tokens to {num_expected_tokens} placeholders")
80
81
- inputs_embeds[mask] = vision_embeddings.view(total_tokens, embed_dim)
81
+ inputs_embeds[mask] = multimodal_embeddings.view(
82
+ total_tokens, embed_dim)
83
else:
83
- size_per_batch = [t.shape[0] for t in vision_embeddings]
84
+ size_per_batch = [t.shape[0] for t in multimodal_embeddings]
85
total_tokens = sum(size_per_batch)
86
if num_expected_tokens != total_tokens:
87
expr = ' + '.join(map(str, size_per_batch))
88
raise ValueError(
89
f"Attempted to assign {expr} = {total_tokens} "
89
- f"image tokens to {num_expected_tokens} placeholders")
90
+ f"multimodal tokens to {num_expected_tokens} placeholders")
91
91
- inputs_embeds[mask] = torch.cat(vision_embeddings)
92
+ inputs_embeds[mask] = torch.cat(multimodal_embeddings)
93
94
return inputs_embeds
95
vllm/multimodal/audio.py
new
+17
@@ -0,0 +1,17 @@
1
+from vllm.inputs.registry import InputContext
2
+from vllm.multimodal.base import MultiModalInputs, MultiModalPlugin
3
+
4
+
5
+class AudioPlugin(MultiModalPlugin):
6
+ """Plugin for audio data."""
7
+
8
+ def get_data_key(self) -> str:
9
+ return "audio"
10
+
11
+ def _default_input_mapper(self, ctx: InputContext,
12
+ data: object) -> MultiModalInputs:
13
+ raise NotImplementedError("There is no default audio input mapper")
14
+
15
+ def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
16
+ raise NotImplementedError(
17
+ "There is no default maximum multimodal tokens")
vllm/multimodal/base.py
+5
-1
@@ -3,8 +3,9 @@ from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
from typing import Any, Callable, Dict, List, Optional
5
from typing import Sequence as GenericSequence
6
-from typing import Type, TypedDict, TypeVar, Union, cast
6
+from typing import Tuple, Type, TypedDict, TypeVar, Union, cast
7
8
+import numpy as np
9
import torch
10
import torch.types
11
from PIL import Image
@@ -121,6 +122,9 @@ class MultiModalDataBuiltins(TypedDict, total=False):
122
image: Image.Image
123
"""The input image."""
124
125
+ audio: Tuple[np.ndarray, Union[int, float]]
126
+ """The input audio and its sampling rate."""
127
+
128
129
MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
130
"""
vllm/multimodal/registry.py
+2
-1
@@ -6,6 +6,7 @@ import torch
6
from vllm.config import ModelConfig
7
from vllm.logger import init_logger
8
9
+from .audio import AudioPlugin
10
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
11
MultiModalPlugin, MultiModalTokensCalc)
12
from .image import ImagePlugin
@@ -19,7 +20,7 @@ class MultiModalRegistry:
20
:class:`~vllm.multimodal.MultiModalPlugin` for each modality.
21
"""
22
22
- DEFAULT_PLUGINS = (ImagePlugin(), )
23
+ DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin())
24
25
def __init__(
26
self,
vllm/multimodal/utils.py
+56
-2
@@ -1,11 +1,14 @@
1
import base64
2
from io import BytesIO
3
-from typing import Union
3
+from typing import Tuple, Union
4
5
+import librosa
6
+import numpy as np
7
+import soundfile
8
from PIL import Image
9
10
from vllm.connections import global_http_connection
8
-from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
11
+from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
12
from vllm.multimodal.base import MultiModalDataDict
13
14
@@ -63,11 +66,62 @@ async def async_fetch_image(image_url: str,
66
return image.convert(image_mode)
67
68
69
+def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
70
+ """
71
+ Load audio from a URL.
72
+ """
73
+ if audio_url.startswith("http"):
74
+ audio_bytes = global_http_connection.get_bytes(
75
+ audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
76
+ elif audio_url.startswith("data:audio"):
77
+ _, audio_base64 = audio_url.split(",", 1)
78
+ audio_bytes = base64.b64decode(audio_base64)
79
+ else:
80
+ raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
81
+ "with either 'data:audio' or 'http'.")
82
+
83
+ return librosa.load(BytesIO(audio_bytes), sr=None)
84
+
85
+
86
+async def async_fetch_audio(
87
+ audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
88
+ """
89
+ Asynchronously fetch audio from a URL.
90
+ """
91
+ if audio_url.startswith("http"):
92
+ audio_bytes = await global_http_connection.async_get_bytes(
93
+ audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
94
+ elif audio_url.startswith("data:audio"):
95
+ _, audio_base64 = audio_url.split(",", 1)
96
+ audio_bytes = base64.b64decode(audio_base64)
97
+ else:
98
+ raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
99
+ "with either 'data:audio' or 'http'.")
100
+
101
+ return librosa.load(BytesIO(audio_bytes), sr=None)
102
+
103
+
104
+async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
105
+ audio, sr = await async_fetch_audio(audio_url)
106
+ return {"audio": (audio, sr)}
107
+
108
+
109
async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
110
image = await async_fetch_image(image_url)
111
return {"image": image}
112
113
114
+def encode_audio_base64(
115
+ audio: np.ndarray,
116
+ sampling_rate: int,
117
+) -> str:
118
+ """Encode audio as base64."""
119
+ buffered = BytesIO()
120
+ soundfile.write(buffered, audio, sampling_rate, format="WAV")
121
+
122
+ return base64.b64encode(buffered.getvalue()).decode('utf-8')
123
+
124
+
125
def encode_image_base64(
126
image: Image.Image,
127
*,
vllm/worker/model_runner.py
+4
-4
@@ -40,7 +40,7 @@ from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
40
from vllm.model_executor.model_loader import get_model
41
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
42
from vllm.model_executor.models.interfaces import (supports_lora,
43
- supports_vision)
43
+ supports_multimodal)
44
from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
45
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
46
MultiModalInputs)
@@ -900,9 +900,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
900
901
if self.lora_config:
902
assert supports_lora(self.model), "Model does not support LoRA"
903
- assert not supports_vision(
903
+ assert not supports_multimodal(
904
self.model
905
- ), "To be tested: vision language model with LoRA settings."
905
+ ), "To be tested: multimodal language model with LoRA settings."
906
907
self.lora_manager = LRUCacheWorkerLoRAManager(
908
self.scheduler_config.max_num_seqs,
@@ -1054,7 +1054,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1054
# of images processed.
1055
model_config = self.model_config
1056
1057
- if supports_vision(self.model):
1057
+ if supports_multimodal(self.model):
1058
max_mm_tokens = MULTIMODAL_REGISTRY \
1059
.get_max_multimodal_tokens(model_config)
1060
max_num_seqs_orig = max_num_seqs
vllm/worker/xpu_model_runner.py
+2
-2
@@ -12,7 +12,7 @@ from vllm.distributed import broadcast_tensor_dict
12
from vllm.inputs import INPUT_REGISTRY
13
from vllm.logger import init_logger
14
from vllm.model_executor.model_loader import get_model
15
-from vllm.model_executor.models.interfaces import supports_vision
15
+from vllm.model_executor.models.interfaces import supports_multimodal
16
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17
MultiModalInputs)
18
from vllm.sampling_params import SamplingParams
@@ -165,7 +165,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
165
# of images processed.
166
model_config = self.model_config
167
168
- if supports_vision(self.model):
168
+ if supports_multimodal(self.model):
169
max_mm_tokens = MULTIMODAL_REGISTRY \
170
.get_max_multimodal_tokens(model_config)
171
max_num_seqs_orig = max_num_seqs