@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66d4b16724
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Frontend] Add OpenAI API support for input_audio (#11027)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

kYLe committed Dec 17, 2024 at 00:09 UTC 66d4b16724226e9f377551198cc7425c12ddafae
5 files changed +301 -23
docs/source/serving/openai_compatible_server.md
+5 -5
@@ -34,11 +34,6 @@ We currently support the following OpenAI APIs:
34 - *Note: `suffix` parameter is not supported.*
35 - [Chat Completions API](#chat-api) (`/v1/chat/completions`)
36 - Only applicable to [text generation models](../models/generative_models.rst) (`--task generate`) with a [chat template](#chat-template).
37 - - [Vision](https://platform.openai.com/docs/guides/vision)-related parameters are supported; see [Multimodal Inputs](../usage/multimodal_inputs.rst).
38 - - *Note: `image_url.detail` parameter is not supported.*
39 - - We also support `audio_url` content type for audio files.
40 - - Refer to [vllm.entrypoints.chat_utils](https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/chat_utils.py) for the exact schema.
41 - - *TODO: Support `input_audio` content type as defined [here](https://github.com/openai/openai-python/blob/v1.52.2/src/openai/types/chat/chat_completion_content_part_input_audio_param.py).*
37 - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
38 - [Embeddings API](#embeddings-api) (`/v1/embeddings`)
39 - Only applicable to [embedding models](../models/pooling_models.rst) (`--task embed`).
@@ -209,6 +204,11 @@ The following extra parameters are supported:
204
205 Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference/chat) for more details.
206
207 +We support both [Vision](https://platform.openai.com/docs/guides/vision)- and
208 +[Audio](https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in)-related parameters;
209 +see our [Multimodal Inputs](../usage/multimodal_inputs.rst) guide for more information.
210 +- *Note: `image_url.detail` parameter is not supported.*
211 +
212 #### Extra parameters
213
214 The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
docs/source/usage/multimodal_inputs.rst
+89 -1
@@ -315,7 +315,95 @@ You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/e
315 Audio
316 ^^^^^
317
318 -Instead of :code:`image_url`, you can pass an audio file via :code:`audio_url`.
318 +Audio input is supported according to `OpenAI Audio API <https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in>`_.
319 +Here is a simple example using Ultravox-v0.3.
320 +
321 +First, launch the OpenAI-compatible server:
322 +
323 +.. code-block:: bash
324 +
325 + vllm serve fixie-ai/ultravox-v0_3
326 +
327 +Then, you can use the OpenAI client as follows:
328 +
329 +.. code-block:: python
330 +
331 + import base64
332 + import requests
333 + from openai import OpenAI
334 + from vllm.assets.audio import AudioAsset
335 +
336 + def encode_base64_content_from_url(content_url: str) -> str:
337 + """Encode a content retrieved from a remote url to base64 format."""
338 +
339 + with requests.get(content_url) as response:
340 + response.raise_for_status()
341 + result = base64.b64encode(response.content).decode('utf-8')
342 +
343 + return result
344 +
345 + openai_api_key = "EMPTY"
346 + openai_api_base = "http://localhost:8000/v1"
347 +
348 + client = OpenAI(
349 + api_key=openai_api_key,
350 + base_url=openai_api_base,
351 + )
352 +
353 + # Any format supported by librosa is supported
354 + audio_url = AudioAsset("winning_call").url
355 + audio_base64 = encode_base64_content_from_url(audio_url)
356 +
357 + chat_completion_from_base64 = client.chat.completions.create(
358 + messages=[{
359 + "role": "user",
360 + "content": [
361 + {
362 + "type": "text",
363 + "text": "What's in this audio?"
364 + },
365 + {
366 + "type": "input_audio",
367 + "input_audio": {
368 + "data": audio_base64,
369 + "format": "wav"
370 + },
371 + },
372 + ],
373 + }],
374 + model=model,
375 + max_completion_tokens=64,
376 + )
377 +
378 + result = chat_completion_from_base64.choices[0].message.content
379 + print("Chat completion output from input audio:", result)
380 +
381 +Alternatively, you can pass :code:`audio_url`, which is the audio counterpart of :code:`image_url` for image input:
382 +
383 +.. code-block:: python
384 +
385 + chat_completion_from_url = client.chat.completions.create(
386 + messages=[{
387 + "role": "user",
388 + "content": [
389 + {
390 + "type": "text",
391 + "text": "What's in this audio?"
392 + },
393 + {
394 + "type": "audio_url",
395 + "audio_url": {
396 + "url": audio_url
397 + },
398 + },
399 + ],
400 + }],
401 + model=model,
402 + max_completion_tokens=64,
403 + )
404 +
405 + result = chat_completion_from_url.choices[0].message.content
406 + print("Chat completion output from audio url:", result)
407
408 A full code example can be found in `examples/openai_chat_completion_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py>`_.
409
examples/openai_chat_completion_client_for_multimodal.py
+31 -3
@@ -153,10 +153,37 @@ def run_multi_image() -> None:
153
154 # Audio input inference
155 def run_audio() -> None:
156 - # Any format supported by librosa is supported
156 audio_url = AudioAsset("winning_call").url
157 + audio_base64 = encode_base64_content_from_url(audio_url)
158 +
159 + # OpenAI-compatible schema (`input_audio`)
160 + chat_completion_from_base64 = client.chat.completions.create(
161 + messages=[{
162 + "role":
163 + "user",
164 + "content": [
165 + {
166 + "type": "text",
167 + "text": "What's in this audio?"
168 + },
169 + {
170 + "type": "input_audio",
171 + "input_audio": {
172 + # Any format supported by librosa is supported
173 + "data": audio_base64,
174 + "format": "wav"
175 + },
176 + },
177 + ],
178 + }],
179 + model=model,
180 + max_completion_tokens=64,
181 + )
182 +
183 + result = chat_completion_from_base64.choices[0].message.content
184 + print("Chat completion output from input audio:", result)
185
159 - # Use audio url in the payload
186 + # HTTP URL
187 chat_completion_from_url = client.chat.completions.create(
188 messages=[{
189 "role":
@@ -169,6 +196,7 @@ def run_audio() -> None:
196 {
197 "type": "audio_url",
198 "audio_url": {
199 + # Any format supported by librosa is supported
200 "url": audio_url
201 },
202 },
@@ -181,7 +209,7 @@ def run_audio() -> None:
209 result = chat_completion_from_url.choices[0].message.content
210 print("Chat completion output from audio url:", result)
211
184 - audio_base64 = encode_base64_content_from_url(audio_url)
212 + # base64 URL
213 chat_completion_from_base64 = client.chat.completions.create(
214 messages=[{
215 "role":
tests/entrypoints/openai/test_audio.py
+121 -4
@@ -155,6 +155,61 @@ async def test_single_chat_session_audio_base64encoded(
155 assert message.content is not None and len(message.content) >= 0
156
157
158 +@pytest.mark.asyncio
159 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
160 +@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
161 +async def test_single_chat_session_input_audio(
162 + client: openai.AsyncOpenAI, model_name: str, audio_url: str,
163 + base64_encoded_audio: Dict[str, str]):
164 + messages = [{
165 + "role":
166 + "user",
167 + "content": [
168 + {
169 + "type": "input_audio",
170 + "input_audio": {
171 + "data": base64_encoded_audio[audio_url],
172 + "format": "wav"
173 + }
174 + },
175 + {
176 + "type": "text",
177 + "text": "What's happening in this audio?"
178 + },
179 + ],
180 + }]
181 +
182 + # test single completion
183 + chat_completion = await client.chat.completions.create(
184 + model=model_name,
185 + messages=messages,
186 + max_completion_tokens=10,
187 + logprobs=True,
188 + top_logprobs=5)
189 + assert len(chat_completion.choices) == 1
190 +
191 + choice = chat_completion.choices[0]
192 + assert choice.finish_reason == "length"
193 + assert chat_completion.usage == openai.types.CompletionUsage(
194 + completion_tokens=10, prompt_tokens=202, total_tokens=212)
195 +
196 + message = choice.message
197 + message = chat_completion.choices[0].message
198 + assert message.content is not None and len(message.content) >= 10
199 + assert message.role == "assistant"
200 + messages.append({"role": "assistant", "content": message.content})
201 +
202 + # test multi-turn dialogue
203 + messages.append({"role": "user", "content": "express your result in json"})
204 + chat_completion = await client.chat.completions.create(
205 + model=model_name,
206 + messages=messages,
207 + max_completion_tokens=10,
208 + )
209 + message = chat_completion.choices[0].message
210 + assert message.content is not None and len(message.content) >= 0
211 +
212 +
213 @pytest.mark.asyncio
214 @pytest.mark.parametrize("model_name", [MODEL_NAME])
215 @pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
@@ -212,11 +267,72 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
267 assert "".join(chunks) == output
268
269
270 +@pytest.mark.asyncio
271 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
272 +@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
273 +async def test_chat_streaming_input_audio(client: openai.AsyncOpenAI,
274 + model_name: str, audio_url: str,
275 + base64_encoded_audio: Dict[str,
276 + str]):
277 + messages = [{
278 + "role":
279 + "user",
280 + "content": [
281 + {
282 + "type": "input_audio",
283 + "input_audio": {
284 + "data": base64_encoded_audio[audio_url],
285 + "format": "wav"
286 + }
287 + },
288 + {
289 + "type": "text",
290 + "text": "What's happening in this audio?"
291 + },
292 + ],
293 + }]
294 +
295 + # test single completion
296 + chat_completion = await client.chat.completions.create(
297 + model=model_name,
298 + messages=messages,
299 + max_completion_tokens=10,
300 + temperature=0.0,
301 + )
302 + output = chat_completion.choices[0].message.content
303 + stop_reason = chat_completion.choices[0].finish_reason
304 +
305 + # test streaming
306 + stream = await client.chat.completions.create(
307 + model=model_name,
308 + messages=messages,
309 + max_completion_tokens=10,
310 + temperature=0.0,
311 + stream=True,
312 + )
313 + chunks: List[str] = []
314 + finish_reason_count = 0
315 + async for chunk in stream:
316 + delta = chunk.choices[0].delta
317 + if delta.role:
318 + assert delta.role == "assistant"
319 + if delta.content:
320 + chunks.append(delta.content)
321 + if chunk.choices[0].finish_reason is not None:
322 + finish_reason_count += 1
323 + # finish reason should only return in last block
324 + assert finish_reason_count == 1
325 + assert chunk.choices[0].finish_reason == stop_reason
326 + assert delta.content
327 + assert "".join(chunks) == output
328 +
329 +
330 @pytest.mark.asyncio
331 @pytest.mark.parametrize("model_name", [MODEL_NAME])
332 @pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
333 async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
219 - audio_url: str):
334 + audio_url: str,
335 + base64_encoded_audio: Dict[str, str]):
336
337 messages = [{
338 "role":
@@ -229,9 +345,10 @@ async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
345 }
346 },
347 {
232 - "type": "audio_url",
233 - "audio_url": {
234 - "url": audio_url
348 + "type": "input_audio",
349 + "input_audio": {
350 + "data": base64_encoded_audio[audio_url],
351 + "format": "wav"
352 }
353 },
354 {
vllm/entrypoints/chat_utils.py
+55 -10
@@ -13,7 +13,8 @@ import transformers.utils.chat_template_utils as hf_chat_utils
13 # yapf conflicts with isort for this block
14 # yapf: disable
15 from openai.types.chat import (ChatCompletionAssistantMessageParam,
16 - ChatCompletionContentPartImageParam)
16 + ChatCompletionContentPartImageParam,
17 + ChatCompletionContentPartInputAudioParam)
18 from openai.types.chat import (
19 ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
20 from openai.types.chat import (ChatCompletionContentPartRefusalParam,
@@ -105,6 +106,7 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
106
107 ChatCompletionContentPartParam: TypeAlias = Union[
108 OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
109 + ChatCompletionContentPartInputAudioParam,
110 ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,
111 CustomChatCompletionContentSimpleImageParam,
112 CustomChatCompletionContentSimpleAudioParam,
@@ -519,6 +521,10 @@ class BaseMultiModalContentParser(ABC):
521 def parse_audio(self, audio_url: str) -> None:
522 raise NotImplementedError
523
524 + @abstractmethod
525 + def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
526 + raise NotImplementedError
527 +
528 @abstractmethod
529 def parse_video(self, video_url: str) -> None:
530 raise NotImplementedError
@@ -545,6 +551,15 @@ class MultiModalContentParser(BaseMultiModalContentParser):
551 placeholder = self._tracker.add("audio", audio)
552 self._add_placeholder(placeholder)
553
554 + def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
555 + input_audio_data = input_audio.get("data","")
556 + input_audio_format = input_audio.get("format","")
557 + audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
558 + audio = get_and_parse_audio(audio_url)
559 +
560 + placeholder = self._tracker.add("audio", audio)
561 + self._add_placeholder(placeholder)
562 +
563 def parse_video(self, video_url: str) -> None:
564 video = get_and_parse_video(video_url)
565
@@ -574,6 +589,15 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
589 placeholder = self._tracker.add("audio", audio_coro)
590 self._add_placeholder(placeholder)
591
592 + def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
593 + input_audio_data = input_audio.get("data","")
594 + input_audio_format = input_audio.get("format","")
595 + audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
596 + audio_coro = async_get_and_parse_audio(audio_url)
597 +
598 + placeholder = self._tracker.add("audio", audio_coro)
599 + self._add_placeholder(placeholder)
600 +
601 def parse_video(self, video_url: str) -> None:
602 video = async_get_and_parse_video(video_url)
603
@@ -667,17 +691,22 @@ def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
691 _TextParser = partial(cast, ChatCompletionContentPartTextParam)
692 _ImageParser = partial(cast, ChatCompletionContentPartImageParam)
693 _AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
694 +_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
695 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
696 _VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
697
698 # Define a mapping from part types to their corresponding parsing functions.
674 -MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
699 +MM_PARSER_MAP: Dict[str,
700 + Callable[[ChatCompletionContentPartParam],
701 + Union[str, Dict[str,str]]]] = {
702 "text":
703 lambda part: _TextParser(part).get("text", ""),
704 "image_url":
705 lambda part: _ImageParser(part).get("image_url", {}).get("url", ""),
706 "audio_url":
707 lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
708 + "input_audio":
709 + lambda part: _InputAudioParser(part).get("input_audio", {}),
710 "refusal":
711 lambda part: _RefusalParser(part).get("refusal", ""),
712 "video_url":
@@ -686,7 +715,8 @@ MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
715
716
717 def _parse_chat_message_content_mm_part(
689 - part: ChatCompletionContentPartParam) -> Tuple[str, str]:
718 + part: ChatCompletionContentPartParam) -> Tuple[str,
719 + Union[str, Dict[str, str]]]:
720 """
721 Parses a given multi-modal content part based on its type.
722
@@ -717,6 +747,7 @@ def _parse_chat_message_content_mm_part(
747 return part_type, content
748
749 # Handle missing 'type' but provided direct URL fields.
750 + # 'type' is required field by pydantic
751 if part_type is None:
752 if part.get("image_url") is not None:
753 image_params = cast(CustomChatCompletionContentSimpleImageParam,
@@ -726,6 +757,9 @@ def _parse_chat_message_content_mm_part(
757 audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
758 part)
759 return "audio_url", audio_params.get("audio_url", "")
760 + if part.get("input_audio") is not None:
761 + input_audio_params = cast(Dict[str, str], part)
762 + return "input_audio", input_audio_params
763 if part.get("video_url") is not None:
764 video_params = cast(CustomChatCompletionContentSimpleVideoParam,
765 part)
@@ -739,7 +773,7 @@ def _parse_chat_message_content_mm_part(
773
774
775 VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
742 - "audio_url", "video_url")
776 + "audio_url", "input_audio", "video_url")
777
778
779 def _parse_chat_message_content_parts(
@@ -795,7 +829,7 @@ def _parse_chat_message_content_part(
829 # Handle structured dictionary parts
830 part_type, content = _parse_chat_message_content_mm_part(part)
831
798 - # if part_type is text/refusal/image_url/audio_url/video_url but
832 + # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
833 # content is empty, log a warning and skip
834 if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
835 logger.warning(
@@ -804,18 +838,30 @@ def _parse_chat_message_content_part(
838 return None
839
840 if part_type in ("text", "refusal"):
807 - return {'type': 'text', 'text': content} if wrap_dicts else content
841 + str_content = cast(str, content)
842 + if wrap_dicts:
843 + return {'type': 'text', 'text': str_content}
844 + else:
845 + return str_content
846
847 if part_type == "image_url":
810 - mm_parser.parse_image(content)
848 + str_content = cast(str, content)
849 + mm_parser.parse_image(str_content)
850 return {'type': 'image'} if wrap_dicts else None
851
852 if part_type == "audio_url":
814 - mm_parser.parse_audio(content)
853 + str_content = cast(str, content)
854 + mm_parser.parse_audio(str_content)
855 + return {'type': 'audio'} if wrap_dicts else None
856 +
857 + if part_type == "input_audio":
858 + dict_content = cast(Dict[str, str], content)
859 + mm_parser.parse_input_audio(dict_content)
860 return {'type': 'audio'} if wrap_dicts else None
861
862 if part_type == "video_url":
818 - mm_parser.parse_video(content)
863 + str_content = cast(str, content)
864 + mm_parser.parse_video(str_content)
865 return {'type': 'video'} if wrap_dicts else None
866
867 raise NotImplementedError(f"Unknown part type: {part_type}")
@@ -840,7 +886,6 @@ def _parse_chat_message_content(
886 content = [
887 ChatCompletionContentPartTextParam(type="text", text=content)
888 ]
843 -
889 result = _parse_chat_message_content_parts(
890 role,
891 content, # type: ignore