A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Frontend] Add OpenAI API support for input_audio (#11027)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
kYLe committed
Dec 17, 2024 at 00:09 UTC
66d4b16724226e9f377551198cc7425c12ddafae
5 files changed
+301
-23
docs/source/serving/openai_compatible_server.md
+5
-5
@@ -34,11 +34,6 @@ We currently support the following OpenAI APIs:
34
- *Note: `suffix` parameter is not supported.*
35
- [Chat Completions API](#chat-api) (`/v1/chat/completions`)
36
- Only applicable to [text generation models](../models/generative_models.rst) (`--task generate`) with a [chat template](#chat-template).
37
- - [Vision](https://platform.openai.com/docs/guides/vision)-related parameters are supported; see [Multimodal Inputs](../usage/multimodal_inputs.rst).
38
- - *Note: `image_url.detail` parameter is not supported.*
39
- - We also support `audio_url` content type for audio files.
40
- - Refer to [vllm.entrypoints.chat_utils](https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/chat_utils.py) for the exact schema.
41
- - *TODO: Support `input_audio` content type as defined [here](https://github.com/openai/openai-python/blob/v1.52.2/src/openai/types/chat/chat_completion_content_part_input_audio_param.py).*
37
- *Note: `parallel_tool_calls` and `user` parameters are ignored.*
38
- [Embeddings API](#embeddings-api) (`/v1/embeddings`)
39
- Only applicable to [embedding models](../models/pooling_models.rst) (`--task embed`).
@@ -209,6 +204,11 @@ The following extra parameters are supported:
204
205
Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference/chat) for more details.
206
207
+We support both [Vision](https://platform.openai.com/docs/guides/vision)- and
208
+[Audio](https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in)-related parameters;
209
+see our [Multimodal Inputs](../usage/multimodal_inputs.rst) guide for more information.
210
+- *Note: `image_url.detail` parameter is not supported.*
211
+
212
#### Extra parameters
213
214
The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
docs/source/usage/multimodal_inputs.rst
+89
-1
@@ -315,7 +315,95 @@ You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/e
315
Audio
316
^^^^^
317
318
-Instead of :code:`image_url`, you can pass an audio file via :code:`audio_url`.
318
+Audio input is supported according to `OpenAI Audio API <https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in>`_.
319
+Here is a simple example using Ultravox-v0.3.
320
+
321
+First, launch the OpenAI-compatible server:
322
+
323
+.. code-block:: bash
324
+
325
+ vllm serve fixie-ai/ultravox-v0_3
326
+
327
+Then, you can use the OpenAI client as follows:
328
+
329
+.. code-block:: python
330
+
331
+ import base64
332
+ import requests
333
+ from openai import OpenAI
334
+ from vllm.assets.audio import AudioAsset
335
+
336
+ def encode_base64_content_from_url(content_url: str) -> str:
337
+ """Encode a content retrieved from a remote url to base64 format."""
338
+
339
+ with requests.get(content_url) as response:
340
+ response.raise_for_status()
341
+ result = base64.b64encode(response.content).decode('utf-8')
342
+
343
+ return result
344
+
345
+ openai_api_key = "EMPTY"
346
+ openai_api_base = "http://localhost:8000/v1"
347
+
348
+ client = OpenAI(
349
+ api_key=openai_api_key,
350
+ base_url=openai_api_base,
351
+ )
352
+
353
+ # Any format supported by librosa is supported
354
+ audio_url = AudioAsset("winning_call").url
355
+ audio_base64 = encode_base64_content_from_url(audio_url)
356
+
357
+ chat_completion_from_base64 = client.chat.completions.create(
358
+ messages=[{
359
+ "role": "user",
360
+ "content": [
361
+ {
362
+ "type": "text",
363
+ "text": "What's in this audio?"
364
+ },
365
+ {
366
+ "type": "input_audio",
367
+ "input_audio": {
368
+ "data": audio_base64,
369
+ "format": "wav"
370
+ },
371
+ },
372
+ ],
373
+ }],
374
+ model=model,
375
+ max_completion_tokens=64,
376
+ )
377
+
378
+ result = chat_completion_from_base64.choices[0].message.content
379
+ print("Chat completion output from input audio:", result)
380
+
381
+Alternatively, you can pass :code:`audio_url`, which is the audio counterpart of :code:`image_url` for image input:
382
+
383
+.. code-block:: python
384
+
385
+ chat_completion_from_url = client.chat.completions.create(
386
+ messages=[{
387
+ "role": "user",
388
+ "content": [
389
+ {
390
+ "type": "text",
391
+ "text": "What's in this audio?"
392
+ },
393
+ {
394
+ "type": "audio_url",
395
+ "audio_url": {
396
+ "url": audio_url
397
+ },
398
+ },
399
+ ],
400
+ }],
401
+ model=model,
402
+ max_completion_tokens=64,
403
+ )
404
+
405
+ result = chat_completion_from_url.choices[0].message.content
406
+ print("Chat completion output from audio url:", result)
407
408
A full code example can be found in `examples/openai_chat_completion_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py>`_.
409
examples/openai_chat_completion_client_for_multimodal.py
+31
-3
@@ -153,10 +153,37 @@ def run_multi_image() -> None:
153
154
# Audio input inference
155
def run_audio() -> None:
156
- # Any format supported by librosa is supported
156
audio_url = AudioAsset("winning_call").url
157
+ audio_base64 = encode_base64_content_from_url(audio_url)
158
+
159
+ # OpenAI-compatible schema (`input_audio`)
160
+ chat_completion_from_base64 = client.chat.completions.create(
161
+ messages=[{
162
+ "role":
163
+ "user",
164
+ "content": [
165
+ {
166
+ "type": "text",
167
+ "text": "What's in this audio?"
168
+ },
169
+ {
170
+ "type": "input_audio",
171
+ "input_audio": {
172
+ # Any format supported by librosa is supported
173
+ "data": audio_base64,
174
+ "format": "wav"
175
+ },
176
+ },
177
+ ],
178
+ }],
179
+ model=model,
180
+ max_completion_tokens=64,
181
+ )
182
+
183
+ result = chat_completion_from_base64.choices[0].message.content
184
+ print("Chat completion output from input audio:", result)
185
159
- # Use audio url in the payload
186
+ # HTTP URL
187
chat_completion_from_url = client.chat.completions.create(
188
messages=[{
189
"role":
@@ -169,6 +196,7 @@ def run_audio() -> None:
196
{
197
"type": "audio_url",
198
"audio_url": {
199
+ # Any format supported by librosa is supported
200
"url": audio_url
201
},
202
},
@@ -181,7 +209,7 @@ def run_audio() -> None:
209
result = chat_completion_from_url.choices[0].message.content
210
print("Chat completion output from audio url:", result)
211
184
- audio_base64 = encode_base64_content_from_url(audio_url)
212
+ # base64 URL
213
chat_completion_from_base64 = client.chat.completions.create(
214
messages=[{
215
"role":
tests/entrypoints/openai/test_audio.py
+121
-4
@@ -155,6 +155,61 @@ async def test_single_chat_session_audio_base64encoded(
155
assert message.content is not None and len(message.content) >= 0
156
157
158
+@pytest.mark.asyncio
159
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
160
+@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
161
+async def test_single_chat_session_input_audio(
162
+ client: openai.AsyncOpenAI, model_name: str, audio_url: str,
163
+ base64_encoded_audio: Dict[str, str]):
164
+ messages = [{
165
+ "role":
166
+ "user",
167
+ "content": [
168
+ {
169
+ "type": "input_audio",
170
+ "input_audio": {
171
+ "data": base64_encoded_audio[audio_url],
172
+ "format": "wav"
173
+ }
174
+ },
175
+ {
176
+ "type": "text",
177
+ "text": "What's happening in this audio?"
178
+ },
179
+ ],
180
+ }]
181
+
182
+ # test single completion
183
+ chat_completion = await client.chat.completions.create(
184
+ model=model_name,
185
+ messages=messages,
186
+ max_completion_tokens=10,
187
+ logprobs=True,
188
+ top_logprobs=5)
189
+ assert len(chat_completion.choices) == 1
190
+
191
+ choice = chat_completion.choices[0]
192
+ assert choice.finish_reason == "length"
193
+ assert chat_completion.usage == openai.types.CompletionUsage(
194
+ completion_tokens=10, prompt_tokens=202, total_tokens=212)
195
+
196
+ message = choice.message
197
+ message = chat_completion.choices[0].message
198
+ assert message.content is not None and len(message.content) >= 10
199
+ assert message.role == "assistant"
200
+ messages.append({"role": "assistant", "content": message.content})
201
+
202
+ # test multi-turn dialogue
203
+ messages.append({"role": "user", "content": "express your result in json"})
204
+ chat_completion = await client.chat.completions.create(
205
+ model=model_name,
206
+ messages=messages,
207
+ max_completion_tokens=10,
208
+ )
209
+ message = chat_completion.choices[0].message
210
+ assert message.content is not None and len(message.content) >= 0
211
+
212
+
213
@pytest.mark.asyncio
214
@pytest.mark.parametrize("model_name", [MODEL_NAME])
215
@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
@@ -212,11 +267,72 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
267
assert "".join(chunks) == output
268
269
270
+@pytest.mark.asyncio
271
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
272
+@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
273
+async def test_chat_streaming_input_audio(client: openai.AsyncOpenAI,
274
+ model_name: str, audio_url: str,
275
+ base64_encoded_audio: Dict[str,
276
+ str]):
277
+ messages = [{
278
+ "role":
279
+ "user",
280
+ "content": [
281
+ {
282
+ "type": "input_audio",
283
+ "input_audio": {
284
+ "data": base64_encoded_audio[audio_url],
285
+ "format": "wav"
286
+ }
287
+ },
288
+ {
289
+ "type": "text",
290
+ "text": "What's happening in this audio?"
291
+ },
292
+ ],
293
+ }]
294
+
295
+ # test single completion
296
+ chat_completion = await client.chat.completions.create(
297
+ model=model_name,
298
+ messages=messages,
299
+ max_completion_tokens=10,
300
+ temperature=0.0,
301
+ )
302
+ output = chat_completion.choices[0].message.content
303
+ stop_reason = chat_completion.choices[0].finish_reason
304
+
305
+ # test streaming
306
+ stream = await client.chat.completions.create(
307
+ model=model_name,
308
+ messages=messages,
309
+ max_completion_tokens=10,
310
+ temperature=0.0,
311
+ stream=True,
312
+ )
313
+ chunks: List[str] = []
314
+ finish_reason_count = 0
315
+ async for chunk in stream:
316
+ delta = chunk.choices[0].delta
317
+ if delta.role:
318
+ assert delta.role == "assistant"
319
+ if delta.content:
320
+ chunks.append(delta.content)
321
+ if chunk.choices[0].finish_reason is not None:
322
+ finish_reason_count += 1
323
+ # finish reason should only return in last block
324
+ assert finish_reason_count == 1
325
+ assert chunk.choices[0].finish_reason == stop_reason
326
+ assert delta.content
327
+ assert "".join(chunks) == output
328
+
329
+
330
@pytest.mark.asyncio
331
@pytest.mark.parametrize("model_name", [MODEL_NAME])
332
@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
333
async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
219
- audio_url: str):
334
+ audio_url: str,
335
+ base64_encoded_audio: Dict[str, str]):
336
337
messages = [{
338
"role":
@@ -229,9 +345,10 @@ async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
345
}
346
},
347
{
232
- "type": "audio_url",
233
- "audio_url": {
234
- "url": audio_url
348
+ "type": "input_audio",
349
+ "input_audio": {
350
+ "data": base64_encoded_audio[audio_url],
351
+ "format": "wav"
352
}
353
},
354
{
vllm/entrypoints/chat_utils.py
+55
-10
@@ -13,7 +13,8 @@ import transformers.utils.chat_template_utils as hf_chat_utils
13
# yapf conflicts with isort for this block
14
# yapf: disable
15
from openai.types.chat import (ChatCompletionAssistantMessageParam,
16
- ChatCompletionContentPartImageParam)
16
+ ChatCompletionContentPartImageParam,
17
+ ChatCompletionContentPartInputAudioParam)
18
from openai.types.chat import (
19
ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
20
from openai.types.chat import (ChatCompletionContentPartRefusalParam,
@@ -105,6 +106,7 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
106
107
ChatCompletionContentPartParam: TypeAlias = Union[
108
OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
109
+ ChatCompletionContentPartInputAudioParam,
110
ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,
111
CustomChatCompletionContentSimpleImageParam,
112
CustomChatCompletionContentSimpleAudioParam,
@@ -519,6 +521,10 @@ class BaseMultiModalContentParser(ABC):
521
def parse_audio(self, audio_url: str) -> None:
522
raise NotImplementedError
523
524
+ @abstractmethod
525
+ def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
526
+ raise NotImplementedError
527
+
528
@abstractmethod
529
def parse_video(self, video_url: str) -> None:
530
raise NotImplementedError
@@ -545,6 +551,15 @@ class MultiModalContentParser(BaseMultiModalContentParser):
551
placeholder = self._tracker.add("audio", audio)
552
self._add_placeholder(placeholder)
553
554
+ def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
555
+ input_audio_data = input_audio.get("data","")
556
+ input_audio_format = input_audio.get("format","")
557
+ audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
558
+ audio = get_and_parse_audio(audio_url)
559
+
560
+ placeholder = self._tracker.add("audio", audio)
561
+ self._add_placeholder(placeholder)
562
+
563
def parse_video(self, video_url: str) -> None:
564
video = get_and_parse_video(video_url)
565
@@ -574,6 +589,15 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
589
placeholder = self._tracker.add("audio", audio_coro)
590
self._add_placeholder(placeholder)
591
592
+ def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
593
+ input_audio_data = input_audio.get("data","")
594
+ input_audio_format = input_audio.get("format","")
595
+ audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
596
+ audio_coro = async_get_and_parse_audio(audio_url)
597
+
598
+ placeholder = self._tracker.add("audio", audio_coro)
599
+ self._add_placeholder(placeholder)
600
+
601
def parse_video(self, video_url: str) -> None:
602
video = async_get_and_parse_video(video_url)
603
@@ -667,17 +691,22 @@ def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
691
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
692
_ImageParser = partial(cast, ChatCompletionContentPartImageParam)
693
_AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
694
+_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
695
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
696
_VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
697
698
# Define a mapping from part types to their corresponding parsing functions.
674
-MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
699
+MM_PARSER_MAP: Dict[str,
700
+ Callable[[ChatCompletionContentPartParam],
701
+ Union[str, Dict[str,str]]]] = {
702
"text":
703
lambda part: _TextParser(part).get("text", ""),
704
"image_url":
705
lambda part: _ImageParser(part).get("image_url", {}).get("url", ""),
706
"audio_url":
707
lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
708
+ "input_audio":
709
+ lambda part: _InputAudioParser(part).get("input_audio", {}),
710
"refusal":
711
lambda part: _RefusalParser(part).get("refusal", ""),
712
"video_url":
@@ -686,7 +715,8 @@ MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
715
716
717
def _parse_chat_message_content_mm_part(
689
- part: ChatCompletionContentPartParam) -> Tuple[str, str]:
718
+ part: ChatCompletionContentPartParam) -> Tuple[str,
719
+ Union[str, Dict[str, str]]]:
720
"""
721
Parses a given multi-modal content part based on its type.
722
@@ -717,6 +747,7 @@ def _parse_chat_message_content_mm_part(
747
return part_type, content
748
749
# Handle missing 'type' but provided direct URL fields.
750
+ # 'type' is required field by pydantic
751
if part_type is None:
752
if part.get("image_url") is not None:
753
image_params = cast(CustomChatCompletionContentSimpleImageParam,
@@ -726,6 +757,9 @@ def _parse_chat_message_content_mm_part(
757
audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
758
part)
759
return "audio_url", audio_params.get("audio_url", "")
760
+ if part.get("input_audio") is not None:
761
+ input_audio_params = cast(Dict[str, str], part)
762
+ return "input_audio", input_audio_params
763
if part.get("video_url") is not None:
764
video_params = cast(CustomChatCompletionContentSimpleVideoParam,
765
part)
@@ -739,7 +773,7 @@ def _parse_chat_message_content_mm_part(
773
774
775
VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
742
- "audio_url", "video_url")
776
+ "audio_url", "input_audio", "video_url")
777
778
779
def _parse_chat_message_content_parts(
@@ -795,7 +829,7 @@ def _parse_chat_message_content_part(
829
# Handle structured dictionary parts
830
part_type, content = _parse_chat_message_content_mm_part(part)
831
798
- # if part_type is text/refusal/image_url/audio_url/video_url but
832
+ # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
833
# content is empty, log a warning and skip
834
if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
835
logger.warning(
@@ -804,18 +838,30 @@ def _parse_chat_message_content_part(
838
return None
839
840
if part_type in ("text", "refusal"):
807
- return {'type': 'text', 'text': content} if wrap_dicts else content
841
+ str_content = cast(str, content)
842
+ if wrap_dicts:
843
+ return {'type': 'text', 'text': str_content}
844
+ else:
845
+ return str_content
846
847
if part_type == "image_url":
810
- mm_parser.parse_image(content)
848
+ str_content = cast(str, content)
849
+ mm_parser.parse_image(str_content)
850
return {'type': 'image'} if wrap_dicts else None
851
852
if part_type == "audio_url":
814
- mm_parser.parse_audio(content)
853
+ str_content = cast(str, content)
854
+ mm_parser.parse_audio(str_content)
855
+ return {'type': 'audio'} if wrap_dicts else None
856
+
857
+ if part_type == "input_audio":
858
+ dict_content = cast(Dict[str, str], content)
859
+ mm_parser.parse_input_audio(dict_content)
860
return {'type': 'audio'} if wrap_dicts else None
861
862
if part_type == "video_url":
818
- mm_parser.parse_video(content)
863
+ str_content = cast(str, content)
864
+ mm_parser.parse_video(str_content)
865
return {'type': 'video'} if wrap_dicts else None
866
867
raise NotImplementedError(f"Unknown part type: {part_type}")
@@ -840,7 +886,6 @@ def _parse_chat_message_content(
886
content = [
887
ChatCompletionContentPartTextParam(type="text", text=content)
888
]
843
-
889
result = _parse_chat_message_content_parts(
890
role,
891
content, # type: ignore