A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Model] Ultravox Model: Support v0.5 Release (#12912)
Signed-off-by: Farzad Abdolhosseini <farzad@fixie.ai>
Farzad Abdolhosseini committed
Feb 10, 2025 at 14:02 UTC
08b2d845d6261309bfdb46933f872eebe4e2bb31
12 files changed
+36
-22
docs/source/models/supported_models.md
+1
-1
@@ -856,7 +856,7 @@ See [this page](#generative-models) for more information on how to use generativ
856
- * `UltravoxModel`
857
* Ultravox
858
* T + A<sup>E+</sup>
859
- * `fixie-ai/ultravox-v0_3`
859
+ * `fixie-ai/ultravox-v0_5-llama-3_2-1b`
860
* ✅︎
861
* ✅︎
862
* ✅︎
docs/source/serving/multimodal_inputs.md
+2
-2
@@ -359,12 +359,12 @@ export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>
359
### Audio
360
361
Audio input is supported according to [OpenAI Audio API](https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in).
362
-Here is a simple example using Ultravox-v0.3.
362
+Here is a simple example using Ultravox-v0.5-1B.
363
364
First, launch the OpenAI-compatible server:
365
366
```bash
367
-vllm serve fixie-ai/ultravox-v0_3
367
+vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b
368
```
369
370
Then, you can use the OpenAI client as follows:
examples/offline_inference/audio_language.py
+2
-2
@@ -24,9 +24,9 @@ question_per_audio_count = {
24
# Unless specified, these settings have been tested to work on a single L4.
25
26
27
-# Ultravox 0.3
27
+# Ultravox 0.5-1B
28
def run_ultravox(question: str, audio_count: int):
29
- model_name = "fixie-ai/ultravox-v0_3"
29
+ model_name = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
30
31
tokenizer = AutoTokenizer.from_pretrained(model_name)
32
messages = [{
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+1
-1
@@ -12,7 +12,7 @@ vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
12
--trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
13
14
(audio inference with Ultravox)
15
-vllm serve fixie-ai/ultravox-v0_3 --max-model-len 4096
15
+vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b --max-model-len 4096
16
"""
17
import base64
18
tests/distributed/test_pipeline_parallel.py
+2
-2
@@ -215,7 +215,7 @@ MULTIMODAL_MODELS = {
215
"Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
216
"Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
217
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
218
- "fixie-ai/ultravox-v0_3": PPTestSettings.fast(trust_remote_code=True),
218
+ "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
219
# [Encoder-decoder]
220
# TODO: Implement PP
221
# "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
@@ -234,7 +234,7 @@ TEST_MODELS = [
234
# [MULTIMODAL GENERATION]
235
"OpenGVLab/InternVL2-1B",
236
"microsoft/Phi-3-vision-128k-instruct",
237
- "fixie-ai/ultravox-v0_3",
237
+ "fixie-ai/ultravox-v0_5-llama-3_2-1b",
238
# [LANGUAGE GENERATION - HYBRID ARCH]
239
"ai21labs/Jamba-tiny-dev",
240
]
tests/entrypoints/openai/test_audio.py
+1
-1
@@ -11,7 +11,7 @@ from vllm.multimodal.utils import encode_audio_base64, fetch_audio
11
12
from ...utils import RemoteOpenAIServer
13
14
-MODEL_NAME = "fixie-ai/ultravox-v0_3"
14
+MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
15
TEST_AUDIO_URLS = [
16
AudioAsset("winning_call").url,
17
]
tests/entrypoints/test_chat_utils.py
+1
-1
@@ -21,7 +21,7 @@ from ..utils import VLLM_PATH
21
EXAMPLES_DIR = VLLM_PATH / "examples"
22
23
PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
24
-ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_3"
24
+ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
25
QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
26
MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
27
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
tests/models/decoder_only/audio_language/test_ultravox.py
+1
-1
@@ -15,7 +15,7 @@ from ....conftest import HfRunner, VllmRunner
15
from ....utils import RemoteOpenAIServer
16
from ...utils import check_logprobs_close
17
18
-MODEL_NAME = "fixie-ai/ultravox-v0_3"
18
+MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
19
20
AudioTuple = Tuple[np.ndarray, int]
21
tests/models/multimodal/processing/test_common.py
+1
-1
@@ -164,7 +164,7 @@ def _test_processing_correctness(
164
"Qwen/Qwen2-VL-2B-Instruct",
165
"Qwen/Qwen2.5-VL-3B-Instruct",
166
"Qwen/Qwen2-Audio-7B-Instruct",
167
- "fixie-ai/ultravox-v0_3",
167
+ "fixie-ai/ultravox-v0_5-llama-3_2-1b",
168
])
169
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
170
@pytest.mark.parametrize("num_batches", [32])
tests/models/registry.py
+1
-1
@@ -267,7 +267,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
267
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
268
"Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
269
min_transformers_version="4.49"), # noqa: E501
270
- "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3",
270
+ "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b",
271
trust_remote_code=True),
272
# [Encoder-decoder]
273
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
vllm/model_executor/models/ultravox.py
+17
-9
@@ -258,27 +258,35 @@ class UltravoxProjector(nn.Module):
258
super().__init__()
259
self.hidden_dim = config.hidden_size
260
self._pad_and_stack = StackAudioFrames(config.stack_factor)
261
- dim = config.audio_config.hidden_size * config.stack_factor
262
- self.ln_pre = RMSNorm(dim)
263
- self.linear_1 = nn.Linear(dim, self.hidden_dim, bias=False)
264
- dim = self.hidden_dim
261
+ dim_in = config.audio_config.hidden_size * config.stack_factor
262
+ self.ln_pre = RMSNorm(dim_in)
263
+ self.linear_1 = nn.Linear(dim_in, self.hidden_dim, bias=False)
264
+ dim_mid = self.hidden_dim
265
266
if config.projector_act == "swiglu":
267
self.act = MulAndSilu()
268
- dim = dim // 2
268
+ dim_mid = dim_mid // 2
269
else:
270
self.act = get_act_fn(config.projector_act)
271
272
- self.linear_2 = nn.Linear(dim,
273
- config.text_config.hidden_size,
274
- bias=False)
275
- self.ln_post = RMSNorm(config.text_config.hidden_size)
272
+ dim_out = config.text_config.hidden_size
273
+ self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
274
+
275
+ # Ultravox v0.4.1 and below use layer_norm after the second linear layer
276
+ # while v0.5.0 and above uses layer_norm after the first linear layer.
277
+ if config.projector_ln_mid:
278
+ self.ln_mid: nn.Module = RMSNorm(dim_mid)
279
+ self.ln_post = nn.Identity()
280
+ else:
281
+ self.ln_mid = nn.Identity()
282
+ self.ln_post = RMSNorm(dim_out)
283
284
def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
285
audio_features = self._pad_and_stack(audio_features)
286
audio_features = self.ln_pre(audio_features)
287
hidden_states = self.linear_1(audio_features)
288
hidden_states = self.act(hidden_states)
289
+ hidden_states = self.ln_mid(hidden_states)
290
hidden_states = self.linear_2(hidden_states)
291
hidden_states = self.ln_post(hidden_states)
292
return hidden_states
vllm/transformers_utils/configs/ultravox.py
+6
@@ -37,6 +37,10 @@ class UltravoxConfig(transformers.PretrainedConfig):
37
The LoRA configuration for finetuning the text model.
38
audio_model_lora_config (`LoraConfigSimplified`, *optional*):
39
The LoRA configuration for finetuning the audio model.
40
+ projector_ln_mid (`bool`, *optional*, defaults to `False`):
41
+ Whether to apply layer normalization at the middle of the
42
+ projector or at the end. Versions v0.4.1 and below
43
+ use `False`, but v0.5 and above use `True`.
44
"""
45
46
model_type = "ultravox"
@@ -56,6 +60,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
60
projector_act: str = "swiglu",
61
text_model_lora_config: Optional[Dict[str, Any]] = None,
62
audio_model_lora_config: Optional[Dict[str, Any]] = None,
63
+ projector_ln_mid: bool = False,
64
**kwargs,
65
):
66
self.ignore_index = ignore_index
@@ -68,6 +73,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
73
self.stack_factor = stack_factor
74
self.norm_init = norm_init
75
self.projector_act = projector_act
76
+ self.projector_ln_mid = projector_ln_mid
77
78
if text_model_id is not None:
79
# Avoid circular import