A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model] Add UltravoxModel and UltravoxConfig (#7615)
Peter Salas committed
Aug 21, 2024 at 15:49 UTC
1ca0d4f86bd6db76bd601df16647fed53e495a0a
33 files changed
+1090
-264
docs/source/models/supported_models.rst
+6
-1
@@ -186,7 +186,7 @@ Multimodal Language Models
186
187
* - Architecture
188
- Models
189
- - Supported Modality(ies)
189
+ - Supported Modalities
190
- Example HuggingFace Models
191
- :ref:`LoRA <lora>`
192
* - :code:`Blip2ForConditionalGeneration`
@@ -234,6 +234,11 @@ Multimodal Language Models
234
- Image
235
- :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
236
-
237
+ * - :code: `UltravoxModel`
238
+ - Ultravox
239
+ - Audio
240
+ - :code: `fixie-ai/ultravox-v0_3`
241
+ -
242
243
.. note::
244
For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
examples/offline_inference_audio_language.py
new
+97
@@ -0,0 +1,97 @@
1
+"""
2
+This example shows how to use vLLM for running offline inference
3
+with the correct prompt format on vision language models.
4
+
5
+For most models, the prompt format should follow corresponding examples
6
+on HuggingFace model repository.
7
+"""
8
+from transformers import AutoTokenizer
9
+
10
+from vllm import LLM, SamplingParams
11
+from vllm.assets.audio import AudioAsset
12
+from vllm.utils import FlexibleArgumentParser
13
+
14
+# Input audio and question
15
+audio_and_sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
16
+question = "What is recited in the audio?"
17
+
18
+
19
+# Ultravox 0.3
20
+def run_ultravox(question):
21
+ model_name = "fixie-ai/ultravox-v0_3"
22
+
23
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
24
+ messages = [{
25
+ 'role': 'user',
26
+ 'content': f"<|reserved_special_token_0|>\n{question}"
27
+ }]
28
+ prompt = tokenizer.apply_chat_template(messages,
29
+ tokenize=False,
30
+ add_generation_prompt=True)
31
+
32
+ llm = LLM(model=model_name)
33
+ stop_token_ids = None
34
+ return llm, prompt, stop_token_ids
35
+
36
+
37
+model_example_map = {
38
+ "ultravox": run_ultravox,
39
+}
40
+
41
+
42
+def main(args):
43
+ model = args.model_type
44
+ if model not in model_example_map:
45
+ raise ValueError(f"Model type {model} is not supported.")
46
+
47
+ llm, prompt, stop_token_ids = model_example_map[model](question)
48
+
49
+ # We set temperature to 0.2 so that outputs can be different
50
+ # even when all prompts are identical when running batch inference.
51
+ sampling_params = SamplingParams(temperature=0.2,
52
+ max_tokens=64,
53
+ stop_token_ids=stop_token_ids)
54
+
55
+ assert args.num_prompts > 0
56
+ if args.num_prompts == 1:
57
+ # Single inference
58
+ inputs = {
59
+ "prompt": prompt,
60
+ "multi_modal_data": {
61
+ "audio": audio_and_sample_rate
62
+ },
63
+ }
64
+
65
+ else:
66
+ # Batch inference
67
+ inputs = [{
68
+ "prompt": prompt,
69
+ "multi_modal_data": {
70
+ "audio": audio_and_sample_rate
71
+ },
72
+ } for _ in range(args.num_prompts)]
73
+
74
+ outputs = llm.generate(inputs, sampling_params=sampling_params)
75
+
76
+ for o in outputs:
77
+ generated_text = o.outputs[0].text
78
+ print(generated_text)
79
+
80
+
81
+if __name__ == "__main__":
82
+ parser = FlexibleArgumentParser(
83
+ description='Demo on using vLLM for offline inference with '
84
+ 'audio language models')
85
+ parser.add_argument('--model-type',
86
+ '-m',
87
+ type=str,
88
+ default="ultravox",
89
+ choices=model_example_map.keys(),
90
+ help='Huggingface "model_type".')
91
+ parser.add_argument('--num-prompts',
92
+ type=int,
93
+ default=1,
94
+ help='Number of prompts to run.')
95
+
96
+ args = parser.parse_args()
97
+ main(args)
examples/openai_audio_api_client.py
new
+90
@@ -0,0 +1,90 @@
1
+"""An example showing how to use vLLM to serve VLMs.
2
+
3
+Launch the vLLM server with the following command:
4
+vllm serve fixie-ai/ultravox-v0_3
5
+"""
6
+import base64
7
+
8
+import requests
9
+from openai import OpenAI
10
+
11
+from vllm.assets.audio import AudioAsset
12
+
13
+# Modify OpenAI's API key and API base to use vLLM's API server.
14
+openai_api_key = "EMPTY"
15
+openai_api_base = "http://localhost:8000/v1"
16
+
17
+client = OpenAI(
18
+ # defaults to os.environ.get("OPENAI_API_KEY")
19
+ api_key=openai_api_key,
20
+ base_url=openai_api_base,
21
+)
22
+
23
+models = client.models.list()
24
+model = models.data[0].id
25
+
26
+# Any format supported by librosa is supported
27
+audio_url = AudioAsset("winning_call").url
28
+
29
+# Use audio url in the payload
30
+chat_completion_from_url = client.chat.completions.create(
31
+ messages=[{
32
+ "role":
33
+ "user",
34
+ "content": [
35
+ {
36
+ "type": "text",
37
+ "text": "What's in this audio?"
38
+ },
39
+ {
40
+ "type": "audio_url",
41
+ "audio_url": {
42
+ "url": audio_url
43
+ },
44
+ },
45
+ ],
46
+ }],
47
+ model=model,
48
+ max_tokens=64,
49
+)
50
+
51
+result = chat_completion_from_url.choices[0].message.content
52
+print(f"Chat completion output:{result}")
53
+
54
+
55
+# Use base64 encoded audio in the payload
56
+def encode_audio_base64_from_url(audio_url: str) -> str:
57
+ """Encode an audio retrieved from a remote url to base64 format."""
58
+
59
+ with requests.get(audio_url) as response:
60
+ response.raise_for_status()
61
+ result = base64.b64encode(response.content).decode('utf-8')
62
+
63
+ return result
64
+
65
+
66
+audio_base64 = encode_audio_base64_from_url(audio_url=audio_url)
67
+chat_completion_from_base64 = client.chat.completions.create(
68
+ messages=[{
69
+ "role":
70
+ "user",
71
+ "content": [
72
+ {
73
+ "type": "text",
74
+ "text": "What's in this audio?"
75
+ },
76
+ {
77
+ "type": "audio_url",
78
+ "audio_url": {
79
+ # Any format supported by librosa is supported
80
+ "url": f"data:audio/ogg;base64,{audio_base64}"
81
+ },
82
+ },
83
+ ],
84
+ }],
85
+ model=model,
86
+ max_tokens=64,
87
+)
88
+
89
+result = chat_completion_from_base64.choices[0].message.content
90
+print(f"Chat completion output:{result}")
tests/conftest.py
+19
-12
@@ -9,14 +9,14 @@ from enum import Enum
9
from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
10
TypeVar, Union)
11
12
+import numpy as np
13
import pytest
14
import torch
15
import torch.nn as nn
16
import torch.nn.functional as F
17
from huggingface_hub import snapshot_download
18
from PIL import Image
18
-from transformers import (AutoModelForCausalLM, AutoModelForSeq2SeqLM,
19
- AutoModelForVision2Seq, AutoTokenizer, BatchEncoding,
19
+from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
20
BatchFeature)
21
22
from vllm import LLM, SamplingParams
@@ -216,8 +216,7 @@ class HfRunner:
216
*,
217
model_kwargs: Optional[Dict[str, Any]] = None,
218
is_embedding_model: bool = False,
219
- is_vision_model: bool = False,
220
- is_encoder_decoder_model: bool = False,
219
+ auto_cls=AutoModelForCausalLM,
220
postprocess_inputs: Callable[[BatchEncoding],
221
BatchEncoding] = identity,
222
) -> None:
@@ -234,13 +233,6 @@ class HfRunner:
233
device="cpu",
234
).to(dtype=torch_dtype))
235
else:
237
- if is_vision_model:
238
- auto_cls = AutoModelForVision2Seq
239
- elif is_encoder_decoder_model:
240
- auto_cls = AutoModelForSeq2SeqLM
241
- else:
242
- auto_cls = AutoModelForCausalLM
243
-
236
model_kwargs = model_kwargs if model_kwargs is not None else {}
237
self.model = self.wrap_device(
238
auto_cls.from_pretrained(
@@ -432,6 +424,7 @@ class HfRunner:
424
max_tokens: int,
425
num_logprobs: int,
426
images: Optional[List[Image.Image]] = None,
427
+ audios: Optional[List[Tuple[np.ndarray, int]]] = None,
428
**kwargs: Any,
429
) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
430
all_logprobs: List[List[Dict[int, float]]] = []
@@ -446,6 +439,11 @@ class HfRunner:
439
if images is not None and images[i] is not None:
440
processor_kwargs["images"] = images[i]
441
442
+ if audios is not None:
443
+ audio, sr = audios[i]
444
+ processor_kwargs["audio"] = audio
445
+ processor_kwargs["sampling_rate"] = sr
446
+
447
inputs = self.processor(**processor_kwargs)
448
inputs = self.postprocess_inputs(inputs)
449
@@ -627,6 +625,8 @@ class VllmRunner:
625
sampling_params: SamplingParams,
626
images: Optional[Union[List[Image.Image],
627
List[List[Image.Image]]]] = None,
628
+ audios: Optional[Union[List[Tuple[np.ndarray, int]],
629
+ List[List[Tuple[np.ndarray, int]]]]] = None
630
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
631
assert sampling_params.logprobs is not None
632
@@ -638,6 +638,10 @@ class VllmRunner:
638
for i, image in enumerate(images):
639
inputs[i]["multi_modal_data"] = {"image": image}
640
641
+ if audios is not None:
642
+ for i, audio in enumerate(audios):
643
+ inputs[i]["multi_modal_data"] = {"audio": audio}
644
+
645
req_outputs = self.model.generate(inputs,
646
sampling_params=sampling_params)
647
return self._final_steps_generate_w_logprobs(req_outputs)
@@ -674,6 +678,8 @@ class VllmRunner:
678
num_logprobs: int,
679
images: Optional[Union[List[Image.Image],
680
List[List[Image.Image]]]] = None,
681
+ audios: Optional[Union[List[Tuple[np.ndarray, int]],
682
+ List[List[Tuple[np.ndarray, int]]]]] = None,
683
stop_token_ids: Optional[List[int]] = None,
684
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
685
greedy_logprobs_params = SamplingParams(temperature=0.0,
@@ -682,7 +688,8 @@ class VllmRunner:
688
stop_token_ids=stop_token_ids)
689
outputs = self.generate_w_logprobs(prompts,
690
greedy_logprobs_params,
685
- images=images)
691
+ images=images,
692
+ audios=audios)
693
694
return [(output_ids, output_str, output_logprobs)
695
for output_ids, output_str, output_logprobs in outputs]
tests/distributed/test_basic_distributed_correctness_enc_dec.py
+2
-1
@@ -10,6 +10,7 @@ pytest distributed/test_basic_distributed_correctness_enc_dec.py
10
"""
11
12
import pytest
13
+from transformers import AutoModelForSeq2SeqLM
14
15
from vllm.utils import cuda_device_count_stateless
16
@@ -85,7 +86,7 @@ def test_models(
86
}
87
88
with hf_runner(model, dtype=dtype,
88
- is_encoder_decoder_model=True) as hf_model:
89
+ auto_cls=AutoModelForSeq2SeqLM) as hf_model:
90
hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
91
test_prompts,
92
max_tokens,
tests/entrypoints/openai/test_audio.py
+23
-125
@@ -1,138 +1,36 @@
1
-import math
2
-import sys
3
-import time
4
-from typing import Dict, List, Optional, Tuple, Union, cast
5
-from unittest.mock import patch
6
-
7
-import librosa
8
-import numpy as np
1
+from typing import Dict, List
2
+
3
import openai
4
import pytest
11
-import requests
12
-import torch
13
-
14
-from vllm import ModelRegistry
15
-from vllm.config import MultiModalConfig
16
-from vllm.inputs import INPUT_REGISTRY
17
-from vllm.inputs.data import LLMInputs
18
-from vllm.inputs.registry import InputContext
19
-from vllm.model_executor.models.interfaces import SupportsMultiModal
20
-from vllm.model_executor.models.opt import OPTForCausalLM
21
-from vllm.multimodal import MULTIMODAL_REGISTRY
22
-from vllm.multimodal.base import MultiModalInputs
23
-from vllm.multimodal.image import (cached_get_tokenizer,
24
- repeat_and_pad_image_tokens)
25
-from vllm.multimodal.utils import encode_audio_base64, fetch_audio
26
-from vllm.utils import get_open_port
5
28
-from ...utils import VLLM_PATH
6
+from vllm.assets.audio import AudioAsset
7
+from vllm.multimodal.utils import encode_audio_base64, fetch_audio
8
30
-chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
31
-assert chatml_jinja_path.exists()
9
+from ...utils import RemoteOpenAIServer
10
33
-MODEL_NAME = "facebook/opt-125m"
11
+MODEL_NAME = "fixie-ai/ultravox-v0_3"
12
TEST_AUDIO_URLS = [
35
- "https://upload.wikimedia.org/wikipedia/en/b/bf/Dave_Niehaus_Winning_Call_1995_AL_Division_Series.ogg",
13
+ AudioAsset("winning_call").url,
14
]
15
16
39
-def server_function(port):
40
-
41
- def fake_input_mapper(ctx: InputContext, data: object):
42
- assert isinstance(data, tuple)
43
- (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
44
-
45
- # Resample it to 1 sample per second
46
- audio = librosa.resample(audio, orig_sr=sr, target_sr=1)
47
- return MultiModalInputs({"processed_audio": torch.from_numpy(audio)})
48
-
49
- def fake_input_processor(ctx: InputContext, llm_inputs: LLMInputs):
50
- multi_modal_data = llm_inputs.get("multi_modal_data")
51
- if multi_modal_data is None or "audio" not in multi_modal_data:
52
- return llm_inputs
53
-
54
- audio, sr = multi_modal_data.get("audio")
55
- audio_duration = math.ceil(len(audio) / sr)
56
-
57
- new_prompt, new_token_ids = repeat_and_pad_image_tokens(
58
- cached_get_tokenizer(ctx.model_config.tokenizer),
59
- llm_inputs.get("prompt"),
60
- llm_inputs["prompt_token_ids"],
61
- image_token_id=62, # "_"
62
- repeat_count=audio_duration)
63
-
64
- return LLMInputs(prompt_token_ids=new_token_ids,
65
- prompt=new_prompt,
66
- multi_modal_data=multi_modal_data)
67
-
68
- @MULTIMODAL_REGISTRY.register_input_mapper("audio", fake_input_mapper)
69
- @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
70
- "audio", lambda *_, **__: 100)
71
- @INPUT_REGISTRY.register_input_processor(fake_input_processor)
72
- class FakeAudioModel(OPTForCausalLM, SupportsMultiModal):
73
-
74
- def __init__(self, *args, multimodal_config: MultiModalConfig,
75
- **kwargs):
76
- assert multimodal_config is not None
77
- super().__init__(*args, **kwargs)
78
-
79
- def forward(
80
- self,
81
- *args,
82
- processed_audio: Optional[torch.Tensor] = None,
83
- **kwargs,
84
- ) -> torch.Tensor:
85
- return super().forward(*args, **kwargs)
86
-
87
- ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
88
-
89
- with patch(
90
- "vllm.entrypoints.chat_utils._mm_token_str",
91
- lambda *_, **__: "_"), patch(
92
- "vllm.model_executor.models.ModelRegistry.is_multimodal_model"
93
- ) as mock:
94
- mock.return_value = True
95
- sys.argv = ["placeholder.py"] + \
96
- (f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
97
- "--dtype bfloat16 --enforce-eager --api-key token-abc123 "
98
- f"--port {port} --chat-template {chatml_jinja_path} "
99
- "--disable-frontend-multiprocessing").split()
100
- import runpy
101
- runpy.run_module('vllm.entrypoints.openai.api_server',
102
- run_name='__main__')
17
+@pytest.fixture(scope="module")
18
+def server():
19
+ args = [
20
+ "--dtype",
21
+ "bfloat16",
22
+ "--max-model-len",
23
+ "4096",
24
+ "--enforce-eager",
25
+ ]
26
+
27
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
28
+ yield remote_server
29
30
31
@pytest.fixture(scope="module")
106
-def client():
107
- port = get_open_port()
108
- ctx = torch.multiprocessing.get_context("spawn")
109
- server = ctx.Process(target=server_function, args=(port, ))
110
- server.start()
111
- MAX_SERVER_START_WAIT_S = 60
112
- client = openai.AsyncOpenAI(
113
- base_url=f"http://localhost:{port}/v1",
114
- api_key="token-abc123",
115
- )
116
- # run health check
117
- health_url = f"http://localhost:{port}/health"
118
- start = time.time()
119
- while True:
120
- try:
121
- if requests.get(health_url).status_code == 200:
122
- break
123
- except Exception as err:
124
- result = server.exitcode
125
- if result is not None:
126
- raise RuntimeError("Server exited unexpectedly.") from err
127
-
128
- time.sleep(0.5)
129
- if time.time() - start > MAX_SERVER_START_WAIT_S:
130
- raise RuntimeError("Server failed to start in time.") from err
131
-
132
- try:
133
- yield client
134
- finally:
135
- server.kill()
32
+def client(server):
33
+ return server.get_async_client()
34
35
36
@pytest.fixture(scope="session")
@@ -176,7 +74,7 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
74
choice = chat_completion.choices[0]
75
assert choice.finish_reason == "length"
76
assert chat_completion.usage == openai.types.CompletionUsage(
179
- completion_tokens=10, prompt_tokens=36, total_tokens=46)
77
+ completion_tokens=10, prompt_tokens=202, total_tokens=212)
78
79
message = choice.message
80
message = chat_completion.choices[0].message
@@ -231,7 +129,7 @@ async def test_single_chat_session_audio_base64encoded(
129
choice = chat_completion.choices[0]
130
assert choice.finish_reason == "length"
131
assert chat_completion.usage == openai.types.CompletionUsage(
234
- completion_tokens=10, prompt_tokens=36, total_tokens=46)
132
+ completion_tokens=10, prompt_tokens=202, total_tokens=212)
133
134
message = choice.message
135
message = chat_completion.choices[0].message
tests/models/test_bart.py
+2
-1
@@ -12,6 +12,7 @@ if not is_cpu():
12
# (xFormers, etc.)
13
14
import pytest
15
+ from transformers import AutoModelForSeq2SeqLM
16
17
from vllm.sequence import SampleLogprobs
18
@@ -131,7 +132,7 @@ if not is_cpu():
132
}
133
134
with hf_runner(model, dtype=dtype,
134
- is_encoder_decoder_model=True) as hf_model:
135
+ auto_cls=AutoModelForSeq2SeqLM) as hf_model:
136
hf_outputs = (
137
hf_model.generate_encoder_decoder_greedy_logprobs_limit(
138
test_case_prompts,
tests/models/test_blip2.py
+3
-2
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Tuple
2
3
import pytest
4
-from transformers import AutoTokenizer
4
+from transformers import AutoModelForVision2Seq, AutoTokenizer
5
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
@@ -80,7 +80,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
80
for prompts, images in inputs_per_image
81
]
82
83
- with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
83
+ with hf_runner(model, dtype=dtype,
84
+ auto_cls=AutoModelForVision2Seq) as hf_model:
85
hf_outputs_per_image = [
86
hf_model.generate_greedy_logprobs_limit(prompts,
87
max_tokens,
tests/models/test_chameleon.py
+2
-2
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Type
2
3
import pytest
4
-from transformers import BatchEncoding
4
+from transformers import AutoModelForVision2Seq, BatchEncoding
5
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
@@ -74,7 +74,7 @@ def run_test(
74
with hf_runner(model,
75
dtype=dtype,
76
postprocess_inputs=process,
77
- is_vision_model=True) as hf_model:
77
+ auto_cls=AutoModelForVision2Seq) as hf_model:
78
hf_outputs_per_image = [
79
hf_model.generate_greedy_logprobs_limit(prompts,
80
max_tokens,
tests/models/test_llava.py
+3
-2
@@ -1,7 +1,8 @@
1
from typing import List, Optional, Tuple, Type
2
3
import pytest
4
-from transformers import AutoConfig, AutoTokenizer, BatchEncoding
4
+from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5
+ BatchEncoding)
6
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
@@ -124,7 +125,7 @@ def run_test(
125
with hf_runner(model,
126
dtype=dtype,
127
postprocess_inputs=process,
127
- is_vision_model=True) as hf_model:
128
+ auto_cls=AutoModelForVision2Seq) as hf_model:
129
hf_outputs_per_image = [
130
hf_model.generate_greedy_logprobs_limit(prompts,
131
max_tokens,
tests/models/test_llava_image_embeds.py
+3
-2
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Tuple, Type
2
3
import pytest
4
-from transformers import AutoConfig, AutoTokenizer
4
+from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6
from vllm.sequence import SampleLogprobs
7
@@ -105,7 +105,8 @@ def run_test(
105
for prompts, images in vllm_inputs_per_image
106
]
107
108
- with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
108
+ with hf_runner(model, dtype=dtype,
109
+ auto_cls=AutoModelForVision2Seq) as hf_model:
110
hf_outputs_per_image = [
111
hf_model.generate_greedy_logprobs_limit(prompts,
112
max_tokens,
tests/models/test_llava_next.py
+3
-2
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Tuple, Type, overload
2
3
import pytest
4
-from transformers import AutoConfig, AutoTokenizer
4
+from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
@@ -129,7 +129,8 @@ def run_test(
129
for prompts, images in inputs_per_image
130
]
131
132
- with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
132
+ with hf_runner(model, dtype=dtype,
133
+ auto_cls=AutoModelForVision2Seq) as hf_model:
134
hf_outputs_per_image = [
135
hf_model.generate_greedy_logprobs_limit(prompts,
136
max_tokens,
tests/models/test_paligemma.py
+3
-2
@@ -2,7 +2,7 @@ import os
2
from typing import List, Optional, Tuple, Type
3
4
import pytest
5
-from transformers import AutoConfig, AutoTokenizer
5
+from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
@@ -102,7 +102,8 @@ def run_test(
102
for prompts, images in inputs_per_image
103
]
104
105
- with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
105
+ with hf_runner(model, dtype=dtype,
106
+ auto_cls=AutoModelForVision2Seq) as hf_model:
107
hf_outputs_per_image = [
108
hf_model.generate_greedy_logprobs_limit(prompts,
109
max_tokens,
tests/models/test_qwen.py
+1
-1
@@ -26,7 +26,7 @@ def test_text_only_qwen_model(
26
# for qwen-vl is still unsupported in VLLM. In the near-future, the
27
# implementation and this test will be extended to consider
28
# visual inputs as well.
29
- with hf_runner(model, dtype=dtype, is_vision_model=False) as hf_model:
29
+ with hf_runner(model, dtype=dtype) as hf_model:
30
hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
example_prompts,
32
max_tokens,
tests/models/test_ultravox.py
new
+151
@@ -0,0 +1,151 @@
1
+from typing import List, Optional, Tuple, Type
2
+
3
+import librosa
4
+import numpy as np
5
+import pytest
6
+from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
+
8
+from vllm.assets.audio import AudioAsset
9
+from vllm.sequence import SampleLogprobs
10
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
+
12
+from ..conftest import HfRunner, VllmRunner
13
+from .utils import check_logprobs_close
14
+
15
+pytestmark = pytest.mark.vlm
16
+
17
+MODEL_NAME = "fixie-ai/ultravox-v0_3"
18
+
19
+AudioTuple = Tuple[np.ndarray, int]
20
+
21
+
22
+@pytest.fixture(scope="session")
23
+def audio_and_sample_rate():
24
+ return AudioAsset("mary_had_lamb").audio_and_sample_rate
25
+
26
+
27
+@pytest.fixture
28
+def prompts_and_audios(audio_and_sample_rate):
29
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
30
+
31
+ vllm_placeholder = "<|reserved_special_token_0|>"
32
+ hf_placeholder = "<|audio|>"
33
+
34
+ question = "What's in the audio?"
35
+ vllm_prompt = tokenizer.apply_chat_template(
36
+ [{
37
+ 'role': 'user',
38
+ 'content': f"{vllm_placeholder}\n{question}"
39
+ }],
40
+ tokenize=False,
41
+ add_generation_prompt=True)
42
+ hf_prompt = tokenizer.apply_chat_template(
43
+ [{
44
+ 'role': 'user',
45
+ 'content': f"{hf_placeholder}\n{question}"
46
+ }],
47
+ tokenize=False,
48
+ add_generation_prompt=True)
49
+
50
+ return [(vllm_prompt, hf_prompt, audio_and_sample_rate)]
51
+
52
+
53
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
54
+ Optional[SampleLogprobs]],
55
+ model: str):
56
+ """Sanitize vllm output to be comparable with hf output."""
57
+ output_ids, output_str, out_logprobs = vllm_output
58
+
59
+ tokenizer = AutoTokenizer.from_pretrained(model)
60
+ eos_token_id = tokenizer.eos_token_id
61
+
62
+ hf_output_ids = output_ids[:]
63
+ hf_output_str = output_str
64
+ if hf_output_ids[-1] == eos_token_id:
65
+ hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
66
+
67
+ return hf_output_ids, hf_output_str, out_logprobs
68
+
69
+
70
+def run_test(
71
+ hf_runner: Type[HfRunner],
72
+ vllm_runner: Type[VllmRunner],
73
+ prompts_and_audios: List[Tuple[str, str, AudioTuple]],
74
+ model: str,
75
+ *,
76
+ dtype: str,
77
+ max_tokens: int,
78
+ num_logprobs: int,
79
+ tensor_parallel_size: int,
80
+ distributed_executor_backend: Optional[str] = None,
81
+):
82
+ """Inference result should be the same between hf and vllm."""
83
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
84
+
85
+ # NOTE: take care of the order. run vLLM first, and then run HF.
86
+ # vLLM needs a fresh new process without cuda initialization.
87
+ # if we run HF first, the cuda initialization will be done and it
88
+ # will hurt multiprocessing backend with fork method (the default method).
89
+
90
+ with vllm_runner(model,
91
+ dtype=dtype,
92
+ tensor_parallel_size=tensor_parallel_size,
93
+ distributed_executor_backend=distributed_executor_backend,
94
+ enforce_eager=True) as vllm_model:
95
+ vllm_outputs_per_audio = [
96
+ vllm_model.generate_greedy_logprobs([vllm_prompt],
97
+ max_tokens,
98
+ num_logprobs=num_logprobs,
99
+ audios=[audio])
100
+ for vllm_prompt, _, audio in prompts_and_audios
101
+ ]
102
+
103
+ def process(hf_inputs: BatchEncoding):
104
+ hf_inputs["audio_values"] = hf_inputs["audio_values"] \
105
+ .to(torch_dtype) # type: ignore
106
+ return hf_inputs
107
+
108
+ with hf_runner(model,
109
+ dtype=dtype,
110
+ postprocess_inputs=process,
111
+ auto_cls=AutoModel) as hf_model:
112
+
113
+ hf_outputs_per_audio = [
114
+ hf_model.generate_greedy_logprobs_limit(
115
+ [hf_prompt],
116
+ max_tokens,
117
+ num_logprobs=num_logprobs,
118
+ audios=[(librosa.resample(audio[0],
119
+ orig_sr=audio[1],
120
+ target_sr=16000), 16000)])
121
+ for _, hf_prompt, audio in prompts_and_audios
122
+ ]
123
+
124
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_audio,
125
+ vllm_outputs_per_audio):
126
+ check_logprobs_close(
127
+ outputs_0_lst=hf_outputs,
128
+ outputs_1_lst=[
129
+ vllm_to_hf_output(vllm_output, model)
130
+ for vllm_output in vllm_outputs
131
+ ],
132
+ name_0="hf",
133
+ name_1="vllm",
134
+ )
135
+
136
+
137
+@pytest.mark.parametrize("dtype", ["half"])
138
+@pytest.mark.parametrize("max_tokens", [128])
139
+@pytest.mark.parametrize("num_logprobs", [5])
140
+def test_models(hf_runner, vllm_runner, prompts_and_audios, dtype: str,
141
+ max_tokens: int, num_logprobs: int) -> None:
142
+ run_test(
143
+ hf_runner,
144
+ vllm_runner,
145
+ prompts_and_audios,
146
+ MODEL_NAME,
147
+ dtype=dtype,
148
+ max_tokens=max_tokens,
149
+ num_logprobs=num_logprobs,
150
+ tensor_parallel_size=1,
151
+ )
vllm/assets/audio.py
new
+26
@@ -0,0 +1,26 @@
1
+from dataclasses import dataclass
2
+from typing import Literal, Tuple
3
+from urllib.parse import urljoin
4
+
5
+import librosa
6
+import numpy as np
7
+
8
+from vllm.assets.base import get_vllm_public_assets, vLLM_S3_BUCKET_URL
9
+
10
+ASSET_DIR = "multimodal_asset"
11
+
12
+
13
+@dataclass(frozen=True)
14
+class AudioAsset:
15
+ name: Literal["winning_call", "mary_had_lamb"]
16
+
17
+ @property
18
+ def audio_and_sample_rate(self) -> Tuple[np.ndarray, int]:
19
+
20
+ audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
21
+ s3_prefix=ASSET_DIR)
22
+ return librosa.load(audio_path, sr=None)
23
+
24
+ @property
25
+ def url(self) -> str:
26
+ return urljoin(vLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
vllm/entrypoints/chat_utils.py
+4
-2
@@ -117,8 +117,8 @@ def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
117
modality: Literal["image", "audio"]) -> Optional[str]:
118
# TODO: Let user specify how to insert image tokens into prompt
119
# (similar to chat template)
120
+ model_type = model_config.hf_config.model_type
121
if modality == "image":
121
- model_type = model_config.hf_config.model_type
122
if model_type == "phi3_v":
123
# Workaround since this token is not defined in the tokenizer
124
return "<|image_1|>"
@@ -134,7 +134,9 @@ def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
134
135
raise TypeError(f"Unknown model type: {model_type}")
136
elif modality == "audio":
137
- raise TypeError("No audio models are supported yet.")
137
+ if model_type == "ultravox":
138
+ return "<|reserved_special_token_0|>"
139
+ raise TypeError(f"Unknown model type: {model_type}")
140
else:
141
raise TypeError(f"Unknown modality: {modality}")
142
vllm/model_executor/models/__init__.py
+2
-1
@@ -61,7 +61,7 @@ _GENERATION_MODELS = {
61
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
62
"MedusaModel": ("medusa", "Medusa"),
63
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
64
- "JambaForCausalLM": ("jamba", "JambaForCausalLM")
64
+ "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
65
}
66
67
_EMBEDDING_MODELS = {
@@ -83,6 +83,7 @@ _MULTIMODAL_MODELS = {
83
"PaliGemmaForConditionalGeneration": ("paligemma",
84
"PaliGemmaForConditionalGeneration"),
85
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
86
+ "UltravoxModel": ("ultravox", "UltravoxModel"),
87
}
88
_CONDITIONAL_GENERATION_MODELS = {
89
"BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/blip.py
+4
-4
@@ -15,8 +15,8 @@ from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
RowParallelLinear)
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
-from vllm.multimodal.image import (cached_get_tokenizer,
19
- repeat_and_pad_image_tokens)
18
+from vllm.multimodal.utils import (cached_get_tokenizer,
19
+ repeat_and_pad_placeholder_tokens)
20
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
21
22
@@ -97,11 +97,11 @@ def input_processor_for_blip(
97
else:
98
image_feature_size = image_feature_size_override
99
100
- new_prompt, new_token_ids = repeat_and_pad_image_tokens(
100
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
101
tokenizer,
102
llm_inputs.get("prompt"),
103
llm_inputs["prompt_token_ids"],
104
- image_token_id=image_token_id,
104
+ placeholder_token_id=image_token_id,
105
repeat_count=image_feature_size,
106
)
107
vllm/model_executor/models/chameleon.py
+4
-4
@@ -30,8 +30,8 @@ from vllm.model_executor.model_loader.weight_utils import (
30
from vllm.model_executor.sampling_metadata import SamplingMetadata
31
from vllm.model_executor.utils import set_weight_attrs
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
-from vllm.multimodal.image import (cached_get_tokenizer,
34
- repeat_and_pad_image_tokens)
33
+from vllm.multimodal.utils import (cached_get_tokenizer,
34
+ repeat_and_pad_placeholder_tokens)
35
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36
SamplerOutput, SequenceData)
37
from vllm.utils import print_warning_once
@@ -124,11 +124,11 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
124
125
model_config = ctx.model_config
126
tokenizer = cached_get_tokenizer(model_config.tokenizer)
127
- new_prompt, new_token_ids = repeat_and_pad_image_tokens(
127
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
128
tokenizer,
129
llm_inputs.get("prompt"),
130
llm_inputs["prompt_token_ids"],
131
- image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
131
+ placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID,
132
repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
133
pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
134
pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID,
vllm/model_executor/models/clip.py
+4
-4
@@ -16,8 +16,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
RowParallelLinear)
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19
-from vllm.multimodal.image import (cached_get_tokenizer,
20
- repeat_and_pad_image_tokens)
19
+from vllm.multimodal.utils import (cached_get_tokenizer,
20
+ repeat_and_pad_placeholder_tokens)
21
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
22
23
@@ -103,11 +103,11 @@ def input_processor_for_clip(
103
else:
104
image_feature_size = image_feature_size_override
105
106
- new_prompt, new_token_ids = repeat_and_pad_image_tokens(
106
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
107
tokenizer,
108
llm_inputs.get("prompt"),
109
llm_inputs["prompt_token_ids"],
110
- image_token_id=image_token_id,
110
+ placeholder_token_id=image_token_id,
111
repeat_count=image_feature_size,
112
)
113
vllm/model_executor/models/fuyu.py
+2
-2
@@ -36,8 +36,8 @@ from vllm.model_executor.models.persimmon import PersimmonForCausalLM
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
38
from vllm.multimodal.base import MultiModalInputs
39
-from vllm.multimodal.image import (cached_get_image_processor,
40
- cached_get_tokenizer)
39
+from vllm.multimodal.image import cached_get_image_processor
40
+from vllm.multimodal.utils import cached_get_tokenizer
41
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
SamplerOutput, SequenceData)
43
vllm/model_executor/models/internvl.py
+1
-1
@@ -23,7 +23,7 @@ from vllm.model_executor.models.intern_vit import InternVisionModel
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.base import MultiModalInputs
26
-from vllm.multimodal.image import cached_get_tokenizer
26
+from vllm.multimodal.utils import cached_get_tokenizer
27
from vllm.sequence import IntermediateTensors, SamplerOutput
28
29
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -54,8 +54,8 @@ from vllm.model_executor.models.minicpm import MiniCPMModel
54
from vllm.model_executor.models.qwen2 import Qwen2Model
55
from vllm.model_executor.sampling_metadata import SamplingMetadata
56
from vllm.multimodal import MULTIMODAL_REGISTRY
57
-from vllm.multimodal.image import (cached_get_image_processor,
58
- cached_get_tokenizer)
57
+from vllm.multimodal.image import cached_get_image_processor
58
+from vllm.multimodal.utils import cached_get_tokenizer
59
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60
SamplerOutput, SequenceData)
61
vllm/model_executor/models/paligemma.py
+1
-1
@@ -16,7 +16,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.models.gemma import GemmaModel
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
-from vllm.multimodal.image import cached_get_tokenizer
19
+from vllm.multimodal.utils import cached_get_tokenizer
20
from vllm.sequence import IntermediateTensors, SamplerOutput
21
22
from .interfaces import SupportsMultiModal
vllm/model_executor/models/phi3v.py
+1
-1
@@ -37,7 +37,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
37
from vllm.model_executor.models.llama import LlamaModel
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.multimodal import MULTIMODAL_REGISTRY
40
-from vllm.multimodal.image import cached_get_tokenizer
40
+from vllm.multimodal.utils import cached_get_tokenizer
41
from vllm.sequence import IntermediateTensors, SamplerOutput
42
43
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
vllm/model_executor/models/siglip.py
+4
-4
@@ -24,8 +24,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
VocabParallelEmbedding)
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
-from vllm.multimodal.image import (cached_get_tokenizer,
28
- repeat_and_pad_image_tokens)
27
+from vllm.multimodal.utils import (cached_get_tokenizer,
28
+ repeat_and_pad_placeholder_tokens)
29
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
30
31
@@ -112,11 +112,11 @@ def input_processor_for_siglip(
112
else:
113
image_feature_size = image_feature_size_override
114
115
- new_prompt, new_token_ids = repeat_and_pad_image_tokens(
115
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
116
tokenizer,
117
llm_inputs.get("prompt"),
118
llm_inputs["prompt_token_ids"],
119
- image_token_id=image_token_id,
119
+ placeholder_token_id=image_token_id,
120
repeat_count=image_feature_size,
121
)
122
vllm/model_executor/models/ultravox.py
new
+435
@@ -0,0 +1,435 @@
1
+# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
2
+"""PyTorch Ultravox model."""
3
+
4
+import itertools
5
+import math
6
+from array import array
7
+from functools import lru_cache
8
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9
+ TypedDict, Union, cast)
10
+
11
+import librosa
12
+import numpy as np
13
+import torch
14
+import torch.utils.checkpoint
15
+from torch import nn
16
+from torch.nn import functional as F
17
+from transformers.models.whisper import WhisperFeatureExtractor
18
+from transformers.models.whisper.modeling_whisper import WhisperEncoder
19
+
20
+from vllm.attention import AttentionMetadata
21
+from vllm.config import CacheConfig, MultiModalConfig
22
+from vllm.inputs import INPUT_REGISTRY
23
+from vllm.inputs.data import LLMInputs
24
+from vllm.inputs.registry import InputContext
25
+from vllm.logger import init_logger
26
+from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
27
+from vllm.model_executor.layers.layernorm import RMSNorm
28
+from vllm.model_executor.layers.quantization.base_config import (
29
+ QuantizationConfig)
30
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31
+from vllm.model_executor.models.interfaces import SupportsMultiModal
32
+from vllm.model_executor.models.utils import (filter_weights,
33
+ init_vllm_registered_model,
34
+ merge_multimodal_embeddings)
35
+from vllm.model_executor.sampling_metadata import SamplingMetadata
36
+from vllm.multimodal import MULTIMODAL_REGISTRY
37
+from vllm.multimodal.base import MultiModalInputs
38
+from vllm.multimodal.utils import (cached_get_tokenizer,
39
+ repeat_and_pad_placeholder_tokens)
40
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SamplerOutput, SequenceData
41
+from vllm.transformers_utils.configs.ultravox import UltravoxConfig
42
+
43
+_AUDIO_PLACEHOLDER_TOKEN = 128002
44
+_AUDIO_TOKENS_PER_SECOND = 6.25
45
+
46
+logger = init_logger(__name__)
47
+
48
+
49
+class UltravoxAudioFeatureInputs(TypedDict):
50
+ type: Literal["audio_features"]
51
+ data: Union[torch.Tensor, List[torch.Tensor]]
52
+ """Shape: `(batch_size, 80, M)"""
53
+
54
+
55
+class UltravoxAudioEmbeddingInputs(TypedDict):
56
+ type: Literal["audio_embeds"]
57
+ data: torch.Tensor
58
+
59
+
60
+UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
61
+ UltravoxAudioEmbeddingInputs]
62
+
63
+
64
+@lru_cache
65
+def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor:
66
+ return WhisperFeatureExtractor.from_pretrained(model_id)
67
+
68
+
69
+def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor:
70
+ return cached_feature_extractor(
71
+ ctx.get_hf_config(UltravoxConfig).audio_model_id)
72
+
73
+
74
+def get_ultravox_max_audio_tokens(ctx: InputContext):
75
+ feature_extractor = whisper_feature_extractor(ctx)
76
+ return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND)
77
+
78
+
79
+def dummy_data_for_ultravox(
80
+ ctx: InputContext,
81
+ seq_len: int,
82
+ mm_counts: Mapping[str, int],
83
+):
84
+ feature_extractor = whisper_feature_extractor(ctx)
85
+
86
+ audio_count = mm_counts["audio"]
87
+
88
+ audio_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [
89
+ _AUDIO_PLACEHOLDER_TOKEN
90
+ ]) * get_ultravox_max_audio_tokens(ctx) * audio_count
91
+ other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
92
+ [0]) * (seq_len - len(audio_token_ids))
93
+
94
+ audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1)
95
+ mm_dict = {
96
+ "audio":
97
+ audio_and_sr if audio_count == 1 else [audio_and_sr] * audio_count
98
+ }
99
+
100
+ return (SequenceData(audio_token_ids + other_token_ids), mm_dict)
101
+
102
+
103
+def input_mapper_for_ultravox(ctx: InputContext, data: object):
104
+ if isinstance(data, tuple):
105
+ (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
106
+ feature_extractor = whisper_feature_extractor(ctx)
107
+
108
+ if sr != feature_extractor.sampling_rate:
109
+ audio = librosa.resample(audio,
110
+ orig_sr=sr,
111
+ target_sr=feature_extractor.sampling_rate)
112
+ sr = feature_extractor.sampling_rate
113
+
114
+ minimum_audio_length = feature_extractor.n_fft // 2 + 1
115
+ if len(audio) < minimum_audio_length:
116
+ # Not enough audio; pad it.
117
+ audio = np.pad(audio, (0, minimum_audio_length - len(audio)))
118
+
119
+ return MultiModalInputs({
120
+ "audio_features":
121
+ feature_extractor(audio,
122
+ sampling_rate=sr,
123
+ padding="longest",
124
+ return_tensors="pt")["input_features"]
125
+ })
126
+
127
+ raise NotImplementedError(f"Unsupported data type: {type(data)}")
128
+
129
+
130
+def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
131
+ multi_modal_data = llm_inputs.get("multi_modal_data")
132
+ if multi_modal_data is None or "audio" not in multi_modal_data:
133
+ return llm_inputs
134
+
135
+ feature_extractor = whisper_feature_extractor(ctx)
136
+ audio_data, sample_rate = multi_modal_data["audio"]
137
+
138
+ audio_length = audio_data.shape[0]
139
+ if sample_rate != feature_extractor.sampling_rate:
140
+ # Account for resampling.
141
+ adjustment = feature_extractor.sampling_rate / sample_rate
142
+ audio_length = math.ceil(adjustment * audio_length)
143
+
144
+ feature_extractor_output_length = math.ceil(
145
+ (audio_length -
146
+ (feature_extractor.hop_length - 1)) / feature_extractor.hop_length)
147
+
148
+ uv_config = ctx.get_hf_config(UltravoxConfig)
149
+ audio_num_tokens = min(
150
+ max(
151
+ 1,
152
+ math.ceil(feature_extractor_output_length /
153
+ (uv_config.stack_factor * 2))),
154
+ get_ultravox_max_audio_tokens(ctx))
155
+ tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
156
+
157
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
158
+ tokenizer,
159
+ llm_inputs.get("prompt"),
160
+ llm_inputs["prompt_token_ids"],
161
+ placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN,
162
+ repeat_count=audio_num_tokens,
163
+ )
164
+
165
+ # NOTE: Create a defensive copy of the original inputs
166
+ return LLMInputs(prompt_token_ids=new_token_ids,
167
+ prompt=new_prompt,
168
+ multi_modal_data=multi_modal_data)
169
+
170
+
171
+class StackAudioFrames(nn.Module):
172
+ """
173
+ Stack the audio embedding frames to reduce the sequence length by a factor
174
+ of `stack_factor`.
175
+ """
176
+
177
+ def __init__(self, stack_factor: int = 8):
178
+ super().__init__()
179
+ self.stack_factor = stack_factor
180
+
181
+ def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
182
+ B, T, C = audio_embeds.shape
183
+ T_pad = (T + self.stack_factor -
184
+ 1) // self.stack_factor * self.stack_factor
185
+ audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T))
186
+ B, T, C = audio_embeds.shape
187
+ audio_embeds = audio_embeds.view(B, T // self.stack_factor,
188
+ C * self.stack_factor)
189
+ return audio_embeds
190
+
191
+
192
+class FlippedSiluAndMul(SiluAndMul):
193
+ """Ultravox is trained with SwiGLU with flipped halves."""
194
+
195
+ def forward(self, x: torch.Tensor):
196
+ a, b = x.chunk(2, dim=-1)
197
+ flipped = torch.cat((b, a), dim=-1)
198
+ return super().forward(flipped)
199
+
200
+
201
+class UltravoxProjector(nn.Module):
202
+
203
+ def __init__(self, config: UltravoxConfig):
204
+ super().__init__()
205
+ self.hidden_dim = config.hidden_size
206
+ self._pad_and_stack = StackAudioFrames(config.stack_factor)
207
+ dim = config.audio_config.hidden_size * config.stack_factor
208
+ self.ln_pre = RMSNorm(dim)
209
+ self.linear_1 = nn.Linear(dim, self.hidden_dim, bias=False)
210
+ dim = self.hidden_dim
211
+
212
+ if config.projector_act == "swiglu":
213
+ self.act = FlippedSiluAndMul()
214
+ dim = dim // 2
215
+ else:
216
+ self.act = get_act_fn(config.projector_act)
217
+
218
+ self.linear_2 = nn.Linear(dim,
219
+ config.text_config.hidden_size,
220
+ bias=False)
221
+ self.ln_post = RMSNorm(config.text_config.hidden_size)
222
+
223
+ def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
224
+ audio_features = self._pad_and_stack(audio_features)
225
+ audio_features = self.ln_pre(audio_features)
226
+ hidden_states = self.linear_1(audio_features)
227
+ hidden_states = self.act(hidden_states)
228
+ hidden_states = self.linear_2(hidden_states)
229
+ hidden_states = self.ln_post(hidden_states)
230
+ return hidden_states
231
+
232
+
233
+class ModifiedWhisperEncoder(WhisperEncoder):
234
+ """
235
+ Encoder portion of OpenAI's Whisper model.
236
+
237
+ This implementation is a slightly modified version of HF Transformers'
238
+ Whisper Encoder, with only a few fixes:
239
+ 1. base_model_prefix updated to allow for doing `.from_pretrained`
240
+ directly on the encoder
241
+ 2. allow less than 30 second of audio padding to be passed in:
242
+ - relaxed ValueError check for `input_features` length to be less
243
+ than or equal to `expected_seq_length` instead of strictly equal
244
+ - embed_pos is now sliced to match the length of `inputs_embeds`
245
+
246
+ Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
247
+ See commentary: https://github.com/huggingface/transformers/issues/25744
248
+ """
249
+
250
+ base_model_prefix = "model.encoder"
251
+
252
+ def forward(
253
+ self,
254
+ input_features,
255
+ ):
256
+ expected_seq_length = (self.config.max_source_positions *
257
+ self.conv1.stride[0] * self.conv2.stride[0])
258
+ if input_features.shape[-1] > expected_seq_length:
259
+ raise ValueError(
260
+ f"Whisper expects the mel input features to be of length "
261
+ f"{expected_seq_length} or less, but found "
262
+ f"{input_features.shape[-1]}. Make sure to pad the input mel "
263
+ f"features to {expected_seq_length}.")
264
+
265
+ inputs_embeds = nn.functional.gelu(self.conv1(input_features))
266
+ inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
267
+
268
+ inputs_embeds = inputs_embeds.permute(0, 2, 1)
269
+ embed_pos = self.embed_positions.weight[:inputs_embeds.size(-2)]
270
+
271
+ hidden_states = inputs_embeds + embed_pos
272
+ hidden_states = nn.functional.dropout(hidden_states,
273
+ p=self.dropout,
274
+ training=self.training)
275
+
276
+ for encoder_layer in self.layers:
277
+ layer_outputs = encoder_layer(
278
+ hidden_states,
279
+ None,
280
+ layer_head_mask=None,
281
+ )
282
+
283
+ hidden_states = layer_outputs[0]
284
+
285
+ hidden_states = self.layer_norm(hidden_states)
286
+ return hidden_states
287
+
288
+
289
+@MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_ultravox)
290
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
291
+ "audio", get_ultravox_max_audio_tokens)
292
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_ultravox)
293
+@INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox)
294
+class UltravoxModel(nn.Module, SupportsMultiModal):
295
+
296
+ def __init__(self,
297
+ config: UltravoxConfig,
298
+ multimodal_config: MultiModalConfig,
299
+ cache_config: Optional[CacheConfig] = None,
300
+ quant_config: Optional["QuantizationConfig"] = None):
301
+ super().__init__()
302
+ self.config = config
303
+ self.multi_modal_config = multimodal_config
304
+ assert self.multi_modal_config
305
+
306
+ if config.audio_model_id is not None:
307
+ self.audio_tower = ModifiedWhisperEncoder.from_pretrained(
308
+ config.audio_model_id)
309
+ else:
310
+ self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
311
+ self.multi_modal_projector = UltravoxProjector(config)
312
+ self.language_model = init_vllm_registered_model(
313
+ config.text_config, cache_config, quant_config)
314
+
315
+ def _audio_features_to_embeddings(
316
+ self, input_features: torch.Tensor) -> torch.Tensor:
317
+ audio_input = input_features.to(self.audio_tower.dtype)
318
+ audio_features = self.audio_tower(audio_input)
319
+ audio_features = audio_features.to(self.audio_tower.dtype)
320
+ audio_embeddings = self.multi_modal_projector(audio_features)
321
+ return audio_embeddings
322
+
323
+ def _parse_and_validate_audio_input(
324
+ self, **kwargs: object) -> Optional[UltravoxAudioInputs]:
325
+ audio_features = kwargs.pop("audio_features", None)
326
+ audio_embeds = kwargs.pop("audio_embeds", None)
327
+
328
+ if audio_features is None and audio_embeds is None:
329
+ return None
330
+
331
+ if audio_features is not None:
332
+ if not isinstance(audio_features, (torch.Tensor, list)):
333
+ raise ValueError("Incorrect type of audio features. "
334
+ f"Got type: {type(audio_features)}")
335
+
336
+ return UltravoxAudioFeatureInputs(type="audio_features",
337
+ data=audio_features)
338
+
339
+ if audio_embeds is not None:
340
+ if not isinstance(audio_embeds, torch.Tensor):
341
+ raise ValueError("Incorrect type of audio embeds. "
342
+ f"Got type: {type(audio_embeds)}")
343
+
344
+ return UltravoxAudioEmbeddingInputs(type="audio_embeds",
345
+ data=audio_embeds)
346
+
347
+ raise AssertionError("This line should be unreachable.")
348
+
349
+ def _process_audio_input(
350
+ self, audio_input: UltravoxAudioInputs
351
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
352
+ if audio_input["type"] == "audio_embeds":
353
+ return audio_input["data"]
354
+
355
+ audio_features = audio_input["data"]
356
+ if isinstance(audio_features, list):
357
+ # TODO: Batch these through the encoder/projector instead of
358
+ # serializing them.
359
+ return [
360
+ self._audio_features_to_embeddings(
361
+ features.unsqueeze(0)).squeeze(0)
362
+ for features in audio_features
363
+ ]
364
+ else:
365
+ return self._audio_features_to_embeddings(audio_features)
366
+
367
+ def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
368
+ kv_caches: List[torch.Tensor],
369
+ attn_metadata: AttentionMetadata,
370
+ intermediate_tensors: Optional[torch.Tensor],
371
+ **kwargs) -> SamplerOutput:
372
+ """Run forward pass for Ultravox
373
+
374
+ One key thing to understand is the `input_ids` already accounts for the
375
+ positions of the to-be-inserted audio embeddings. The to-be-inserted
376
+ audio has a size that is essentially 6.25 tokens per second of audio.
377
+
378
+ This way, the `positions` and `attn_metadata` are consistent
379
+ with the `input_ids`.
380
+
381
+ Args:
382
+ input_features: A batch of audio inputs, [1, 80, M].
383
+ """
384
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
385
+ if audio_input is not None:
386
+ audio_embeddings = self._process_audio_input(audio_input)
387
+ inputs_embeds = self.language_model.model.get_input_embeddings(
388
+ input_ids)
389
+
390
+ inputs_embeds = merge_multimodal_embeddings(
391
+ input_ids, inputs_embeds, audio_embeddings,
392
+ _AUDIO_PLACEHOLDER_TOKEN)
393
+ input_ids = None
394
+ else:
395
+ inputs_embeds = None
396
+
397
+ hidden_states = self.language_model.model(
398
+ input_ids=input_ids,
399
+ positions=positions,
400
+ kv_caches=kv_caches,
401
+ attn_metadata=attn_metadata,
402
+ intermediate_tensors=intermediate_tensors,
403
+ inputs_embeds=inputs_embeds)
404
+ return hidden_states
405
+
406
+ def compute_logits(self, hidden_states: torch.Tensor,
407
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
408
+ return self.language_model.compute_logits(hidden_states,
409
+ sampling_metadata)
410
+
411
+ def sample(
412
+ self,
413
+ logits: torch.Tensor,
414
+ sampling_metadata: SamplingMetadata,
415
+ ) -> Optional[SamplerOutput]:
416
+ return self.language_model.sample(logits, sampling_metadata)
417
+
418
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
419
+ # prepare weight iterators for components
420
+ projector_weights, llm_weights = itertools.tee(weights, 2)
421
+
422
+ # load projector weights
423
+ projector_weights = filter_weights(projector_weights,
424
+ "multi_modal_projector")
425
+ projector_params_dict = dict(
426
+ self.multi_modal_projector.named_parameters())
427
+ for name, loaded_weight in projector_weights:
428
+ param = projector_params_dict[name]
429
+ weight_loader = getattr(param, "weight_loader",
430
+ default_weight_loader)
431
+ weight_loader(param, loaded_weight)
432
+
433
+ # load llm backbone
434
+ llm_weights = filter_weights(llm_weights, "language_model")
435
+ self.language_model.load_weights(llm_weights)
vllm/multimodal/image.py
-83
@@ -1,5 +1,4 @@
1
from functools import lru_cache
2
-from typing import List, Optional, Tuple, TypeVar
2
3
import torch
4
from PIL import Image
@@ -8,7 +7,6 @@ from vllm.config import ModelConfig
7
from vllm.inputs.registry import InputContext
8
from vllm.logger import init_logger
9
from vllm.transformers_utils.image_processor import get_image_processor
11
-from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
10
from vllm.utils import is_list_of
11
12
from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
@@ -16,87 +14,6 @@ from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
14
logger = init_logger(__name__)
15
16
cached_get_image_processor = lru_cache(get_image_processor)
19
-cached_get_tokenizer = lru_cache(get_tokenizer)
20
-
21
-# Utilities for image input processors
22
-_T = TypeVar("_T", str, int)
23
-
24
-
25
-def repeat_and_pad_token(
26
- token: _T,
27
- *,
28
- repeat_count: int = 1,
29
- pad_token_left: Optional[_T] = None,
30
- pad_token_right: Optional[_T] = None,
31
-) -> List[_T]:
32
- replacement = [token] * repeat_count
33
- if pad_token_left is not None:
34
- replacement = [pad_token_left] + replacement
35
- if pad_token_right is not None:
36
- replacement = replacement + [pad_token_right]
37
-
38
- return replacement
39
-
40
-
41
-def repeat_and_pad_image_tokens(
42
- tokenizer: AnyTokenizer,
43
- prompt: Optional[str],
44
- prompt_token_ids: List[int],
45
- *,
46
- image_token_id: int,
47
- repeat_count: int = 1,
48
- pad_token_left: Optional[int] = None,
49
- pad_token_right: Optional[int] = None,
50
-) -> Tuple[Optional[str], List[int]]:
51
- if prompt is None:
52
- new_prompt = None
53
- else:
54
- image_token_str = tokenizer.decode(image_token_id)
55
- pad_token_str_left = (None if pad_token_left is None else
56
- tokenizer.decode(pad_token_left))
57
- pad_token_str_right = (None if pad_token_right is None else
58
- tokenizer.decode(pad_token_right))
59
- replacement_str = "".join(
60
- repeat_and_pad_token(
61
- image_token_str,
62
- repeat_count=repeat_count,
63
- pad_token_left=pad_token_str_left,
64
- pad_token_right=pad_token_str_right,
65
- ))
66
-
67
- image_token_count = prompt.count(image_token_str)
68
- # This is an arbitrary number to distinguish between the two cases
69
- if image_token_count > 16:
70
- logger.warning(
71
- "Please follow the prompt format that is "
72
- "documented on HuggingFace which does not involve "
73
- "repeating %s tokens.", image_token_str)
74
- elif image_token_count > 1:
75
- logger.warning("Multiple image input is not supported yet, "
76
- "so any extra image tokens will be treated "
77
- "as plain text.")
78
-
79
- # The image tokens are removed to be consistent with HuggingFace
80
- new_prompt = prompt.replace(image_token_str, replacement_str, 1)
81
-
82
- new_token_ids: List[int] = []
83
- for i, token in enumerate(prompt_token_ids):
84
- if token == image_token_id:
85
- replacement_ids = repeat_and_pad_token(
86
- image_token_id,
87
- repeat_count=repeat_count,
88
- pad_token_left=pad_token_left,
89
- pad_token_right=pad_token_right,
90
- )
91
- new_token_ids.extend(replacement_ids)
92
-
93
- # No need to further scan the list since we only replace once
94
- new_token_ids.extend(prompt_token_ids[i + 1:])
95
- break
96
- else:
97
- new_token_ids.append(token)
98
-
99
- return new_prompt, new_token_ids
17
18
19
class ImagePlugin(MultiModalPlugin):
vllm/multimodal/utils.py
+89
-1
@@ -1,6 +1,7 @@
1
import base64
2
+from functools import lru_cache
3
from io import BytesIO
3
-from typing import Tuple, Union
4
+from typing import List, Optional, Tuple, TypeVar, Union
5
6
import librosa
7
import numpy as np
@@ -9,7 +10,13 @@ from PIL import Image
10
11
from vllm.connections import global_http_connection
12
from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
13
+from vllm.logger import init_logger
14
from vllm.multimodal.base import MultiModalDataDict
15
+from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16
+
17
+logger = init_logger(__name__)
18
+
19
+cached_get_tokenizer = lru_cache(get_tokenizer)
20
21
22
def _load_image_from_bytes(b: bytes):
@@ -154,3 +161,84 @@ def rescale_image_size(image: Image.Image,
161
if transpose >= 0:
162
image = image.transpose(Image.Transpose(transpose))
163
return image
164
+
165
+
166
+# Utilities for input processors
167
+_T = TypeVar("_T", str, int)
168
+
169
+
170
+def repeat_and_pad_token(
171
+ token: _T,
172
+ *,
173
+ repeat_count: int = 1,
174
+ pad_token_left: Optional[_T] = None,
175
+ pad_token_right: Optional[_T] = None,
176
+) -> List[_T]:
177
+ replacement = [token] * repeat_count
178
+ if pad_token_left is not None:
179
+ replacement = [pad_token_left] + replacement
180
+ if pad_token_right is not None:
181
+ replacement = replacement + [pad_token_right]
182
+
183
+ return replacement
184
+
185
+
186
+def repeat_and_pad_placeholder_tokens(
187
+ tokenizer: AnyTokenizer,
188
+ prompt: Optional[str],
189
+ prompt_token_ids: List[int],
190
+ *,
191
+ placeholder_token_id: int,
192
+ repeat_count: int = 1,
193
+ pad_token_left: Optional[int] = None,
194
+ pad_token_right: Optional[int] = None,
195
+) -> Tuple[Optional[str], List[int]]:
196
+ if prompt is None:
197
+ new_prompt = None
198
+ else:
199
+ placeholder_token_str = tokenizer.decode(placeholder_token_id)
200
+ pad_token_str_left = (None if pad_token_left is None else
201
+ tokenizer.decode(pad_token_left))
202
+ pad_token_str_right = (None if pad_token_right is None else
203
+ tokenizer.decode(pad_token_right))
204
+ replacement_str = "".join(
205
+ repeat_and_pad_token(
206
+ placeholder_token_str,
207
+ repeat_count=repeat_count,
208
+ pad_token_left=pad_token_str_left,
209
+ pad_token_right=pad_token_str_right,
210
+ ))
211
+
212
+ placeholder_token_count = prompt.count(placeholder_token_str)
213
+ # This is an arbitrary number to distinguish between the two cases
214
+ if placeholder_token_count > 16:
215
+ logger.warning(
216
+ "Please follow the prompt format that is "
217
+ "documented on HuggingFace which does not involve "
218
+ "repeating %s tokens.", placeholder_token_str)
219
+ elif placeholder_token_count > 1:
220
+ logger.warning("Multiple multi-modal input is not supported yet, "
221
+ "so any extra placeholder tokens will be treated "
222
+ "as plain text.")
223
+
224
+ # The image tokens are removed to be consistent with HuggingFace
225
+ new_prompt = prompt.replace(placeholder_token_str, replacement_str, 1)
226
+
227
+ new_token_ids: List[int] = []
228
+ for i, token in enumerate(prompt_token_ids):
229
+ if token == placeholder_token_id:
230
+ replacement_ids = repeat_and_pad_token(
231
+ placeholder_token_id,
232
+ repeat_count=repeat_count,
233
+ pad_token_left=pad_token_left,
234
+ pad_token_right=pad_token_right,
235
+ )
236
+ new_token_ids.extend(replacement_ids)
237
+
238
+ # No need to further scan the list since we only replace once
239
+ new_token_ids.extend(prompt_token_ids[i + 1:])
240
+ break
241
+ else:
242
+ new_token_ids.append(token)
243
+
244
+ return new_prompt, new_token_ids
vllm/transformers_utils/config.py
+2
-1
@@ -12,7 +12,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
12
InternVLChatConfig, JAISConfig,
13
MedusaConfig, MLPSpeculatorConfig,
14
MPTConfig, NemotronConfig,
15
- RWConfig)
15
+ RWConfig, UltravoxConfig)
16
17
if VLLM_USE_MODELSCOPE:
18
from modelscope import AutoConfig
@@ -32,6 +32,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
32
"medusa": MedusaConfig,
33
"internvl_chat": InternVLChatConfig,
34
"nemotron": NemotronConfig,
35
+ "ultravox": UltravoxConfig,
36
}
37
38
for name, cls in _CONFIG_REGISTRY.items():
vllm/transformers_utils/configs/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.transformers_utils.configs.medusa import MedusaConfig
10
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
11
from vllm.transformers_utils.configs.mpt import MPTConfig
12
from vllm.transformers_utils.configs.nemotron import NemotronConfig
13
+from vllm.transformers_utils.configs.ultravox import UltravoxConfig
14
15
__all__ = [
16
"ChatGLMConfig",
@@ -21,4 +22,5 @@ __all__ = [
22
"MedusaConfig",
23
"MLPSpeculatorConfig",
24
"NemotronConfig",
25
+ "UltravoxConfig",
26
]
vllm/transformers_utils/configs/ultravox.py
new
+99
@@ -0,0 +1,99 @@
1
+# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py
2
+from typing import Any, Dict, Optional
3
+
4
+import transformers
5
+
6
+
7
+class UltravoxConfig(transformers.PretrainedConfig):
8
+ r"""
9
+ This is the configuration class to store the configuration of a
10
+ [`UltravoxForConditionalGeneration`]. It is used to instantiate an
11
+ Ultravox model according to the specified arguments, defining the model
12
+ architecture.
13
+
14
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to
15
+ control the model outputs. Read the documentation from [`PretrainedConfig`]
16
+ for more information.
17
+
18
+ Args:
19
+ audio_config (`Union[AutoConfig, dict]`, *optional*):
20
+ Custom audio config or dict
21
+ text_config (`Union[AutoConfig, dict]`, *optional*):
22
+ The config object of the text backbone. Can be any of `LlamaConfig`
23
+ or `MistralConfig`.
24
+ ignore_index (`int`, *optional*, defaults to -100):
25
+ The ignore index for the loss function.
26
+ audio_token_index (`int`, *optional*, defaults to 32000):
27
+ The audio token index to encode the audio prompt.
28
+ stack_factor (`int`, *optional*, defaults to 8):
29
+ Audio downsampling factor for the multimodal projector.
30
+ norm_init (`float`, *optional*, defaults to 0.4):
31
+ The initialization value for the layer normalization.
32
+ projector_act (`str`, *optional*, defaults to `"swiglu"`):
33
+ The activation function used by the multimodal projector.
34
+ text_model_lora_config (`LoraConfigSimplified`, *optional*):
35
+ The LoRA configuration for finetuning the text model.
36
+ audio_model_lora_config (`LoraConfigSimplified`, *optional*):
37
+ The LoRA configuration for finetuning the audio model.
38
+ """
39
+
40
+ model_type = "ultravox"
41
+ is_composition = False
42
+
43
+ def __init__(
44
+ self,
45
+ audio_config: Optional[Dict[str, Any]] = None,
46
+ text_config: Optional[Dict[str, Any]] = None,
47
+ audio_model_id: Optional[str] = None,
48
+ text_model_id: Optional[str] = None,
49
+ ignore_index: int = -100,
50
+ audio_token_index: int = 32000,
51
+ hidden_size: int = 4096,
52
+ stack_factor: int = 8,
53
+ norm_init: float = 0.4,
54
+ projector_act: str = "swiglu",
55
+ text_model_lora_config: Optional[Dict[str, Any]] = None,
56
+ audio_model_lora_config: Optional[Dict[str, Any]] = None,
57
+ **kwargs,
58
+ ):
59
+ self.ignore_index = ignore_index
60
+
61
+ self.audio_model_id = audio_model_id
62
+ self.text_model_id = text_model_id
63
+ self.audio_token_index = audio_token_index
64
+
65
+ self.hidden_size = hidden_size
66
+ self.stack_factor = stack_factor
67
+ self.norm_init = norm_init
68
+ self.projector_act = projector_act
69
+
70
+ if text_model_id is not None:
71
+ # Avoid circular import
72
+ from vllm.transformers_utils.config import get_config
73
+
74
+ self.text_config = get_config(text_model_id,
75
+ trust_remote_code=False)
76
+ else:
77
+ text_config = text_config or {}
78
+ self.text_config = transformers.CONFIG_MAPPING[text_config.get(
79
+ "model_type", "llama")](**text_config)
80
+
81
+ if audio_model_id is not None:
82
+ # Avoid circular import
83
+ from vllm.transformers_utils.config import get_config
84
+
85
+ self.audio_config = get_config(audio_model_id,
86
+ trust_remote_code=False)
87
+ else:
88
+ audio_config = audio_config or {}
89
+ self.audio_config = transformers.CONFIG_MAPPING[audio_config.get(
90
+ "model_type", "whisper")](**audio_config)
91
+
92
+ self.text_model_lora_config = text_model_lora_config or {}
93
+ self.audio_model_lora_config = audio_model_lora_config or {}
94
+
95
+ self.vocab_size = self.text_config.vocab_size
96
+
97
+ self.initializer_range = self.text_config.initializer_range
98
+
99
+ super().__init__(**kwargs)