@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ca0d4f86b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model] Add UltravoxModel and UltravoxConfig (#7615)

Peter Salas committed Aug 21, 2024 at 15:49 UTC 1ca0d4f86bd6db76bd601df16647fed53e495a0a
33 files changed +1090 -264
docs/source/models/supported_models.rst
+6 -1
@@ -186,7 +186,7 @@ Multimodal Language Models
186
187 * - Architecture
188 - Models
189 - - Supported Modality(ies)
189 + - Supported Modalities
190 - Example HuggingFace Models
191 - :ref:`LoRA <lora>`
192 * - :code:`Blip2ForConditionalGeneration`
@@ -234,6 +234,11 @@ Multimodal Language Models
234 - Image
235 - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
236 -
237 + * - :code: `UltravoxModel`
238 + - Ultravox
239 + - Audio
240 + - :code: `fixie-ai/ultravox-v0_3`
241 + -
242
243 .. note::
244 For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
examples/offline_inference_audio_language.py new
+97
@@ -0,0 +1,97 @@
1 +"""
2 +This example shows how to use vLLM for running offline inference
3 +with the correct prompt format on vision language models.
4 +
5 +For most models, the prompt format should follow corresponding examples
6 +on HuggingFace model repository.
7 +"""
8 +from transformers import AutoTokenizer
9 +
10 +from vllm import LLM, SamplingParams
11 +from vllm.assets.audio import AudioAsset
12 +from vllm.utils import FlexibleArgumentParser
13 +
14 +# Input audio and question
15 +audio_and_sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
16 +question = "What is recited in the audio?"
17 +
18 +
19 +# Ultravox 0.3
20 +def run_ultravox(question):
21 + model_name = "fixie-ai/ultravox-v0_3"
22 +
23 + tokenizer = AutoTokenizer.from_pretrained(model_name)
24 + messages = [{
25 + 'role': 'user',
26 + 'content': f"<|reserved_special_token_0|>\n{question}"
27 + }]
28 + prompt = tokenizer.apply_chat_template(messages,
29 + tokenize=False,
30 + add_generation_prompt=True)
31 +
32 + llm = LLM(model=model_name)
33 + stop_token_ids = None
34 + return llm, prompt, stop_token_ids
35 +
36 +
37 +model_example_map = {
38 + "ultravox": run_ultravox,
39 +}
40 +
41 +
42 +def main(args):
43 + model = args.model_type
44 + if model not in model_example_map:
45 + raise ValueError(f"Model type {model} is not supported.")
46 +
47 + llm, prompt, stop_token_ids = model_example_map[model](question)
48 +
49 + # We set temperature to 0.2 so that outputs can be different
50 + # even when all prompts are identical when running batch inference.
51 + sampling_params = SamplingParams(temperature=0.2,
52 + max_tokens=64,
53 + stop_token_ids=stop_token_ids)
54 +
55 + assert args.num_prompts > 0
56 + if args.num_prompts == 1:
57 + # Single inference
58 + inputs = {
59 + "prompt": prompt,
60 + "multi_modal_data": {
61 + "audio": audio_and_sample_rate
62 + },
63 + }
64 +
65 + else:
66 + # Batch inference
67 + inputs = [{
68 + "prompt": prompt,
69 + "multi_modal_data": {
70 + "audio": audio_and_sample_rate
71 + },
72 + } for _ in range(args.num_prompts)]
73 +
74 + outputs = llm.generate(inputs, sampling_params=sampling_params)
75 +
76 + for o in outputs:
77 + generated_text = o.outputs[0].text
78 + print(generated_text)
79 +
80 +
81 +if __name__ == "__main__":
82 + parser = FlexibleArgumentParser(
83 + description='Demo on using vLLM for offline inference with '
84 + 'audio language models')
85 + parser.add_argument('--model-type',
86 + '-m',
87 + type=str,
88 + default="ultravox",
89 + choices=model_example_map.keys(),
90 + help='Huggingface "model_type".')
91 + parser.add_argument('--num-prompts',
92 + type=int,
93 + default=1,
94 + help='Number of prompts to run.')
95 +
96 + args = parser.parse_args()
97 + main(args)
examples/openai_audio_api_client.py new
+90
@@ -0,0 +1,90 @@
1 +"""An example showing how to use vLLM to serve VLMs.
2 +
3 +Launch the vLLM server with the following command:
4 +vllm serve fixie-ai/ultravox-v0_3
5 +"""
6 +import base64
7 +
8 +import requests
9 +from openai import OpenAI
10 +
11 +from vllm.assets.audio import AudioAsset
12 +
13 +# Modify OpenAI's API key and API base to use vLLM's API server.
14 +openai_api_key = "EMPTY"
15 +openai_api_base = "http://localhost:8000/v1"
16 +
17 +client = OpenAI(
18 + # defaults to os.environ.get("OPENAI_API_KEY")
19 + api_key=openai_api_key,
20 + base_url=openai_api_base,
21 +)
22 +
23 +models = client.models.list()
24 +model = models.data[0].id
25 +
26 +# Any format supported by librosa is supported
27 +audio_url = AudioAsset("winning_call").url
28 +
29 +# Use audio url in the payload
30 +chat_completion_from_url = client.chat.completions.create(
31 + messages=[{
32 + "role":
33 + "user",
34 + "content": [
35 + {
36 + "type": "text",
37 + "text": "What's in this audio?"
38 + },
39 + {
40 + "type": "audio_url",
41 + "audio_url": {
42 + "url": audio_url
43 + },
44 + },
45 + ],
46 + }],
47 + model=model,
48 + max_tokens=64,
49 +)
50 +
51 +result = chat_completion_from_url.choices[0].message.content
52 +print(f"Chat completion output:{result}")
53 +
54 +
55 +# Use base64 encoded audio in the payload
56 +def encode_audio_base64_from_url(audio_url: str) -> str:
57 + """Encode an audio retrieved from a remote url to base64 format."""
58 +
59 + with requests.get(audio_url) as response:
60 + response.raise_for_status()
61 + result = base64.b64encode(response.content).decode('utf-8')
62 +
63 + return result
64 +
65 +
66 +audio_base64 = encode_audio_base64_from_url(audio_url=audio_url)
67 +chat_completion_from_base64 = client.chat.completions.create(
68 + messages=[{
69 + "role":
70 + "user",
71 + "content": [
72 + {
73 + "type": "text",
74 + "text": "What's in this audio?"
75 + },
76 + {
77 + "type": "audio_url",
78 + "audio_url": {
79 + # Any format supported by librosa is supported
80 + "url": f"data:audio/ogg;base64,{audio_base64}"
81 + },
82 + },
83 + ],
84 + }],
85 + model=model,
86 + max_tokens=64,
87 +)
88 +
89 +result = chat_completion_from_base64.choices[0].message.content
90 +print(f"Chat completion output:{result}")
tests/conftest.py
+19 -12
@@ -9,14 +9,14 @@ from enum import Enum
9 from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
10 TypeVar, Union)
11
12 +import numpy as np
13 import pytest
14 import torch
15 import torch.nn as nn
16 import torch.nn.functional as F
17 from huggingface_hub import snapshot_download
18 from PIL import Image
18 -from transformers import (AutoModelForCausalLM, AutoModelForSeq2SeqLM,
19 - AutoModelForVision2Seq, AutoTokenizer, BatchEncoding,
19 +from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
20 BatchFeature)
21
22 from vllm import LLM, SamplingParams
@@ -216,8 +216,7 @@ class HfRunner:
216 *,
217 model_kwargs: Optional[Dict[str, Any]] = None,
218 is_embedding_model: bool = False,
219 - is_vision_model: bool = False,
220 - is_encoder_decoder_model: bool = False,
219 + auto_cls=AutoModelForCausalLM,
220 postprocess_inputs: Callable[[BatchEncoding],
221 BatchEncoding] = identity,
222 ) -> None:
@@ -234,13 +233,6 @@ class HfRunner:
233 device="cpu",
234 ).to(dtype=torch_dtype))
235 else:
237 - if is_vision_model:
238 - auto_cls = AutoModelForVision2Seq
239 - elif is_encoder_decoder_model:
240 - auto_cls = AutoModelForSeq2SeqLM
241 - else:
242 - auto_cls = AutoModelForCausalLM
243 -
236 model_kwargs = model_kwargs if model_kwargs is not None else {}
237 self.model = self.wrap_device(
238 auto_cls.from_pretrained(
@@ -432,6 +424,7 @@ class HfRunner:
424 max_tokens: int,
425 num_logprobs: int,
426 images: Optional[List[Image.Image]] = None,
427 + audios: Optional[List[Tuple[np.ndarray, int]]] = None,
428 **kwargs: Any,
429 ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
430 all_logprobs: List[List[Dict[int, float]]] = []
@@ -446,6 +439,11 @@ class HfRunner:
439 if images is not None and images[i] is not None:
440 processor_kwargs["images"] = images[i]
441
442 + if audios is not None:
443 + audio, sr = audios[i]
444 + processor_kwargs["audio"] = audio
445 + processor_kwargs["sampling_rate"] = sr
446 +
447 inputs = self.processor(**processor_kwargs)
448 inputs = self.postprocess_inputs(inputs)
449
@@ -627,6 +625,8 @@ class VllmRunner:
625 sampling_params: SamplingParams,
626 images: Optional[Union[List[Image.Image],
627 List[List[Image.Image]]]] = None,
628 + audios: Optional[Union[List[Tuple[np.ndarray, int]],
629 + List[List[Tuple[np.ndarray, int]]]]] = None
630 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
631 assert sampling_params.logprobs is not None
632
@@ -638,6 +638,10 @@ class VllmRunner:
638 for i, image in enumerate(images):
639 inputs[i]["multi_modal_data"] = {"image": image}
640
641 + if audios is not None:
642 + for i, audio in enumerate(audios):
643 + inputs[i]["multi_modal_data"] = {"audio": audio}
644 +
645 req_outputs = self.model.generate(inputs,
646 sampling_params=sampling_params)
647 return self._final_steps_generate_w_logprobs(req_outputs)
@@ -674,6 +678,8 @@ class VllmRunner:
678 num_logprobs: int,
679 images: Optional[Union[List[Image.Image],
680 List[List[Image.Image]]]] = None,
681 + audios: Optional[Union[List[Tuple[np.ndarray, int]],
682 + List[List[Tuple[np.ndarray, int]]]]] = None,
683 stop_token_ids: Optional[List[int]] = None,
684 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
685 greedy_logprobs_params = SamplingParams(temperature=0.0,
@@ -682,7 +688,8 @@ class VllmRunner:
688 stop_token_ids=stop_token_ids)
689 outputs = self.generate_w_logprobs(prompts,
690 greedy_logprobs_params,
685 - images=images)
691 + images=images,
692 + audios=audios)
693
694 return [(output_ids, output_str, output_logprobs)
695 for output_ids, output_str, output_logprobs in outputs]
tests/distributed/test_basic_distributed_correctness_enc_dec.py
+2 -1
@@ -10,6 +10,7 @@ pytest distributed/test_basic_distributed_correctness_enc_dec.py
10 """
11
12 import pytest
13 +from transformers import AutoModelForSeq2SeqLM
14
15 from vllm.utils import cuda_device_count_stateless
16
@@ -85,7 +86,7 @@ def test_models(
86 }
87
88 with hf_runner(model, dtype=dtype,
88 - is_encoder_decoder_model=True) as hf_model:
89 + auto_cls=AutoModelForSeq2SeqLM) as hf_model:
90 hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
91 test_prompts,
92 max_tokens,
tests/entrypoints/openai/test_audio.py
+23 -125
@@ -1,138 +1,36 @@
1 -import math
2 -import sys
3 -import time
4 -from typing import Dict, List, Optional, Tuple, Union, cast
5 -from unittest.mock import patch
6 -
7 -import librosa
8 -import numpy as np
1 +from typing import Dict, List
2 +
3 import openai
4 import pytest
11 -import requests
12 -import torch
13 -
14 -from vllm import ModelRegistry
15 -from vllm.config import MultiModalConfig
16 -from vllm.inputs import INPUT_REGISTRY
17 -from vllm.inputs.data import LLMInputs
18 -from vllm.inputs.registry import InputContext
19 -from vllm.model_executor.models.interfaces import SupportsMultiModal
20 -from vllm.model_executor.models.opt import OPTForCausalLM
21 -from vllm.multimodal import MULTIMODAL_REGISTRY
22 -from vllm.multimodal.base import MultiModalInputs
23 -from vllm.multimodal.image import (cached_get_tokenizer,
24 - repeat_and_pad_image_tokens)
25 -from vllm.multimodal.utils import encode_audio_base64, fetch_audio
26 -from vllm.utils import get_open_port
5
28 -from ...utils import VLLM_PATH
6 +from vllm.assets.audio import AudioAsset
7 +from vllm.multimodal.utils import encode_audio_base64, fetch_audio
8
30 -chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
31 -assert chatml_jinja_path.exists()
9 +from ...utils import RemoteOpenAIServer
10
33 -MODEL_NAME = "facebook/opt-125m"
11 +MODEL_NAME = "fixie-ai/ultravox-v0_3"
12 TEST_AUDIO_URLS = [
35 - "https://upload.wikimedia.org/wikipedia/en/b/bf/Dave_Niehaus_Winning_Call_1995_AL_Division_Series.ogg",
13 + AudioAsset("winning_call").url,
14 ]
15
16
39 -def server_function(port):
40 -
41 - def fake_input_mapper(ctx: InputContext, data: object):
42 - assert isinstance(data, tuple)
43 - (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
44 -
45 - # Resample it to 1 sample per second
46 - audio = librosa.resample(audio, orig_sr=sr, target_sr=1)
47 - return MultiModalInputs({"processed_audio": torch.from_numpy(audio)})
48 -
49 - def fake_input_processor(ctx: InputContext, llm_inputs: LLMInputs):
50 - multi_modal_data = llm_inputs.get("multi_modal_data")
51 - if multi_modal_data is None or "audio" not in multi_modal_data:
52 - return llm_inputs
53 -
54 - audio, sr = multi_modal_data.get("audio")
55 - audio_duration = math.ceil(len(audio) / sr)
56 -
57 - new_prompt, new_token_ids = repeat_and_pad_image_tokens(
58 - cached_get_tokenizer(ctx.model_config.tokenizer),
59 - llm_inputs.get("prompt"),
60 - llm_inputs["prompt_token_ids"],
61 - image_token_id=62, # "_"
62 - repeat_count=audio_duration)
63 -
64 - return LLMInputs(prompt_token_ids=new_token_ids,
65 - prompt=new_prompt,
66 - multi_modal_data=multi_modal_data)
67 -
68 - @MULTIMODAL_REGISTRY.register_input_mapper("audio", fake_input_mapper)
69 - @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
70 - "audio", lambda *_, **__: 100)
71 - @INPUT_REGISTRY.register_input_processor(fake_input_processor)
72 - class FakeAudioModel(OPTForCausalLM, SupportsMultiModal):
73 -
74 - def __init__(self, *args, multimodal_config: MultiModalConfig,
75 - **kwargs):
76 - assert multimodal_config is not None
77 - super().__init__(*args, **kwargs)
78 -
79 - def forward(
80 - self,
81 - *args,
82 - processed_audio: Optional[torch.Tensor] = None,
83 - **kwargs,
84 - ) -> torch.Tensor:
85 - return super().forward(*args, **kwargs)
86 -
87 - ModelRegistry.register_model("OPTForCausalLM", FakeAudioModel)
88 -
89 - with patch(
90 - "vllm.entrypoints.chat_utils._mm_token_str",
91 - lambda *_, **__: "_"), patch(
92 - "vllm.model_executor.models.ModelRegistry.is_multimodal_model"
93 - ) as mock:
94 - mock.return_value = True
95 - sys.argv = ["placeholder.py"] + \
96 - (f"--model {MODEL_NAME} --gpu-memory-utilization 0.10 "
97 - "--dtype bfloat16 --enforce-eager --api-key token-abc123 "
98 - f"--port {port} --chat-template {chatml_jinja_path} "
99 - "--disable-frontend-multiprocessing").split()
100 - import runpy
101 - runpy.run_module('vllm.entrypoints.openai.api_server',
102 - run_name='__main__')
17 +@pytest.fixture(scope="module")
18 +def server():
19 + args = [
20 + "--dtype",
21 + "bfloat16",
22 + "--max-model-len",
23 + "4096",
24 + "--enforce-eager",
25 + ]
26 +
27 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
28 + yield remote_server
29
30
31 @pytest.fixture(scope="module")
106 -def client():
107 - port = get_open_port()
108 - ctx = torch.multiprocessing.get_context("spawn")
109 - server = ctx.Process(target=server_function, args=(port, ))
110 - server.start()
111 - MAX_SERVER_START_WAIT_S = 60
112 - client = openai.AsyncOpenAI(
113 - base_url=f"http://localhost:{port}/v1",
114 - api_key="token-abc123",
115 - )
116 - # run health check
117 - health_url = f"http://localhost:{port}/health"
118 - start = time.time()
119 - while True:
120 - try:
121 - if requests.get(health_url).status_code == 200:
122 - break
123 - except Exception as err:
124 - result = server.exitcode
125 - if result is not None:
126 - raise RuntimeError("Server exited unexpectedly.") from err
127 -
128 - time.sleep(0.5)
129 - if time.time() - start > MAX_SERVER_START_WAIT_S:
130 - raise RuntimeError("Server failed to start in time.") from err
131 -
132 - try:
133 - yield client
134 - finally:
135 - server.kill()
32 +def client(server):
33 + return server.get_async_client()
34
35
36 @pytest.fixture(scope="session")
@@ -176,7 +74,7 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
74 choice = chat_completion.choices[0]
75 assert choice.finish_reason == "length"
76 assert chat_completion.usage == openai.types.CompletionUsage(
179 - completion_tokens=10, prompt_tokens=36, total_tokens=46)
77 + completion_tokens=10, prompt_tokens=202, total_tokens=212)
78
79 message = choice.message
80 message = chat_completion.choices[0].message
@@ -231,7 +129,7 @@ async def test_single_chat_session_audio_base64encoded(
129 choice = chat_completion.choices[0]
130 assert choice.finish_reason == "length"
131 assert chat_completion.usage == openai.types.CompletionUsage(
234 - completion_tokens=10, prompt_tokens=36, total_tokens=46)
132 + completion_tokens=10, prompt_tokens=202, total_tokens=212)
133
134 message = choice.message
135 message = chat_completion.choices[0].message
tests/models/test_bart.py
+2 -1
@@ -12,6 +12,7 @@ if not is_cpu():
12 # (xFormers, etc.)
13
14 import pytest
15 + from transformers import AutoModelForSeq2SeqLM
16
17 from vllm.sequence import SampleLogprobs
18
@@ -131,7 +132,7 @@ if not is_cpu():
132 }
133
134 with hf_runner(model, dtype=dtype,
134 - is_encoder_decoder_model=True) as hf_model:
135 + auto_cls=AutoModelForSeq2SeqLM) as hf_model:
136 hf_outputs = (
137 hf_model.generate_encoder_decoder_greedy_logprobs_limit(
138 test_case_prompts,
tests/models/test_blip2.py
+3 -2
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Tuple
2
3 import pytest
4 -from transformers import AutoTokenizer
4 +from transformers import AutoModelForVision2Seq, AutoTokenizer
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
@@ -80,7 +80,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
80 for prompts, images in inputs_per_image
81 ]
82
83 - with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
83 + with hf_runner(model, dtype=dtype,
84 + auto_cls=AutoModelForVision2Seq) as hf_model:
85 hf_outputs_per_image = [
86 hf_model.generate_greedy_logprobs_limit(prompts,
87 max_tokens,
tests/models/test_chameleon.py
+2 -2
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Type
2
3 import pytest
4 -from transformers import BatchEncoding
4 +from transformers import AutoModelForVision2Seq, BatchEncoding
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
@@ -74,7 +74,7 @@ def run_test(
74 with hf_runner(model,
75 dtype=dtype,
76 postprocess_inputs=process,
77 - is_vision_model=True) as hf_model:
77 + auto_cls=AutoModelForVision2Seq) as hf_model:
78 hf_outputs_per_image = [
79 hf_model.generate_greedy_logprobs_limit(prompts,
80 max_tokens,
tests/models/test_llava.py
+3 -2
@@ -1,7 +1,8 @@
1 from typing import List, Optional, Tuple, Type
2
3 import pytest
4 -from transformers import AutoConfig, AutoTokenizer, BatchEncoding
4 +from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5 + BatchEncoding)
6
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
@@ -124,7 +125,7 @@ def run_test(
125 with hf_runner(model,
126 dtype=dtype,
127 postprocess_inputs=process,
127 - is_vision_model=True) as hf_model:
128 + auto_cls=AutoModelForVision2Seq) as hf_model:
129 hf_outputs_per_image = [
130 hf_model.generate_greedy_logprobs_limit(prompts,
131 max_tokens,
tests/models/test_llava_image_embeds.py
+3 -2
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Tuple, Type
2
3 import pytest
4 -from transformers import AutoConfig, AutoTokenizer
4 +from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6 from vllm.sequence import SampleLogprobs
7
@@ -105,7 +105,8 @@ def run_test(
105 for prompts, images in vllm_inputs_per_image
106 ]
107
108 - with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
108 + with hf_runner(model, dtype=dtype,
109 + auto_cls=AutoModelForVision2Seq) as hf_model:
110 hf_outputs_per_image = [
111 hf_model.generate_greedy_logprobs_limit(prompts,
112 max_tokens,
tests/models/test_llava_next.py
+3 -2
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 -from transformers import AutoConfig, AutoTokenizer
4 +from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
@@ -129,7 +129,8 @@ def run_test(
129 for prompts, images in inputs_per_image
130 ]
131
132 - with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
132 + with hf_runner(model, dtype=dtype,
133 + auto_cls=AutoModelForVision2Seq) as hf_model:
134 hf_outputs_per_image = [
135 hf_model.generate_greedy_logprobs_limit(prompts,
136 max_tokens,
tests/models/test_paligemma.py
+3 -2
@@ -2,7 +2,7 @@ import os
2 from typing import List, Optional, Tuple, Type
3
4 import pytest
5 -from transformers import AutoConfig, AutoTokenizer
5 +from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
@@ -102,7 +102,8 @@ def run_test(
102 for prompts, images in inputs_per_image
103 ]
104
105 - with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
105 + with hf_runner(model, dtype=dtype,
106 + auto_cls=AutoModelForVision2Seq) as hf_model:
107 hf_outputs_per_image = [
108 hf_model.generate_greedy_logprobs_limit(prompts,
109 max_tokens,
tests/models/test_qwen.py
+1 -1
@@ -26,7 +26,7 @@ def test_text_only_qwen_model(
26 # for qwen-vl is still unsupported in VLLM. In the near-future, the
27 # implementation and this test will be extended to consider
28 # visual inputs as well.
29 - with hf_runner(model, dtype=dtype, is_vision_model=False) as hf_model:
29 + with hf_runner(model, dtype=dtype) as hf_model:
30 hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 example_prompts,
32 max_tokens,
tests/models/test_ultravox.py new
+151
@@ -0,0 +1,151 @@
1 +from typing import List, Optional, Tuple, Type
2 +
3 +import librosa
4 +import numpy as np
5 +import pytest
6 +from transformers import AutoModel, AutoTokenizer, BatchEncoding
7 +
8 +from vllm.assets.audio import AudioAsset
9 +from vllm.sequence import SampleLogprobs
10 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11 +
12 +from ..conftest import HfRunner, VllmRunner
13 +from .utils import check_logprobs_close
14 +
15 +pytestmark = pytest.mark.vlm
16 +
17 +MODEL_NAME = "fixie-ai/ultravox-v0_3"
18 +
19 +AudioTuple = Tuple[np.ndarray, int]
20 +
21 +
22 +@pytest.fixture(scope="session")
23 +def audio_and_sample_rate():
24 + return AudioAsset("mary_had_lamb").audio_and_sample_rate
25 +
26 +
27 +@pytest.fixture
28 +def prompts_and_audios(audio_and_sample_rate):
29 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
30 +
31 + vllm_placeholder = "<|reserved_special_token_0|>"
32 + hf_placeholder = "<|audio|>"
33 +
34 + question = "What's in the audio?"
35 + vllm_prompt = tokenizer.apply_chat_template(
36 + [{
37 + 'role': 'user',
38 + 'content': f"{vllm_placeholder}\n{question}"
39 + }],
40 + tokenize=False,
41 + add_generation_prompt=True)
42 + hf_prompt = tokenizer.apply_chat_template(
43 + [{
44 + 'role': 'user',
45 + 'content': f"{hf_placeholder}\n{question}"
46 + }],
47 + tokenize=False,
48 + add_generation_prompt=True)
49 +
50 + return [(vllm_prompt, hf_prompt, audio_and_sample_rate)]
51 +
52 +
53 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
54 + Optional[SampleLogprobs]],
55 + model: str):
56 + """Sanitize vllm output to be comparable with hf output."""
57 + output_ids, output_str, out_logprobs = vllm_output
58 +
59 + tokenizer = AutoTokenizer.from_pretrained(model)
60 + eos_token_id = tokenizer.eos_token_id
61 +
62 + hf_output_ids = output_ids[:]
63 + hf_output_str = output_str
64 + if hf_output_ids[-1] == eos_token_id:
65 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
66 +
67 + return hf_output_ids, hf_output_str, out_logprobs
68 +
69 +
70 +def run_test(
71 + hf_runner: Type[HfRunner],
72 + vllm_runner: Type[VllmRunner],
73 + prompts_and_audios: List[Tuple[str, str, AudioTuple]],
74 + model: str,
75 + *,
76 + dtype: str,
77 + max_tokens: int,
78 + num_logprobs: int,
79 + tensor_parallel_size: int,
80 + distributed_executor_backend: Optional[str] = None,
81 +):
82 + """Inference result should be the same between hf and vllm."""
83 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
84 +
85 + # NOTE: take care of the order. run vLLM first, and then run HF.
86 + # vLLM needs a fresh new process without cuda initialization.
87 + # if we run HF first, the cuda initialization will be done and it
88 + # will hurt multiprocessing backend with fork method (the default method).
89 +
90 + with vllm_runner(model,
91 + dtype=dtype,
92 + tensor_parallel_size=tensor_parallel_size,
93 + distributed_executor_backend=distributed_executor_backend,
94 + enforce_eager=True) as vllm_model:
95 + vllm_outputs_per_audio = [
96 + vllm_model.generate_greedy_logprobs([vllm_prompt],
97 + max_tokens,
98 + num_logprobs=num_logprobs,
99 + audios=[audio])
100 + for vllm_prompt, _, audio in prompts_and_audios
101 + ]
102 +
103 + def process(hf_inputs: BatchEncoding):
104 + hf_inputs["audio_values"] = hf_inputs["audio_values"] \
105 + .to(torch_dtype) # type: ignore
106 + return hf_inputs
107 +
108 + with hf_runner(model,
109 + dtype=dtype,
110 + postprocess_inputs=process,
111 + auto_cls=AutoModel) as hf_model:
112 +
113 + hf_outputs_per_audio = [
114 + hf_model.generate_greedy_logprobs_limit(
115 + [hf_prompt],
116 + max_tokens,
117 + num_logprobs=num_logprobs,
118 + audios=[(librosa.resample(audio[0],
119 + orig_sr=audio[1],
120 + target_sr=16000), 16000)])
121 + for _, hf_prompt, audio in prompts_and_audios
122 + ]
123 +
124 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_audio,
125 + vllm_outputs_per_audio):
126 + check_logprobs_close(
127 + outputs_0_lst=hf_outputs,
128 + outputs_1_lst=[
129 + vllm_to_hf_output(vllm_output, model)
130 + for vllm_output in vllm_outputs
131 + ],
132 + name_0="hf",
133 + name_1="vllm",
134 + )
135 +
136 +
137 +@pytest.mark.parametrize("dtype", ["half"])
138 +@pytest.mark.parametrize("max_tokens", [128])
139 +@pytest.mark.parametrize("num_logprobs", [5])
140 +def test_models(hf_runner, vllm_runner, prompts_and_audios, dtype: str,
141 + max_tokens: int, num_logprobs: int) -> None:
142 + run_test(
143 + hf_runner,
144 + vllm_runner,
145 + prompts_and_audios,
146 + MODEL_NAME,
147 + dtype=dtype,
148 + max_tokens=max_tokens,
149 + num_logprobs=num_logprobs,
150 + tensor_parallel_size=1,
151 + )
vllm/assets/audio.py new
+26
@@ -0,0 +1,26 @@
1 +from dataclasses import dataclass
2 +from typing import Literal, Tuple
3 +from urllib.parse import urljoin
4 +
5 +import librosa
6 +import numpy as np
7 +
8 +from vllm.assets.base import get_vllm_public_assets, vLLM_S3_BUCKET_URL
9 +
10 +ASSET_DIR = "multimodal_asset"
11 +
12 +
13 +@dataclass(frozen=True)
14 +class AudioAsset:
15 + name: Literal["winning_call", "mary_had_lamb"]
16 +
17 + @property
18 + def audio_and_sample_rate(self) -> Tuple[np.ndarray, int]:
19 +
20 + audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
21 + s3_prefix=ASSET_DIR)
22 + return librosa.load(audio_path, sr=None)
23 +
24 + @property
25 + def url(self) -> str:
26 + return urljoin(vLLM_S3_BUCKET_URL, f"{ASSET_DIR}/{self.name}.ogg")
vllm/entrypoints/chat_utils.py
+4 -2
@@ -117,8 +117,8 @@ def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
117 modality: Literal["image", "audio"]) -> Optional[str]:
118 # TODO: Let user specify how to insert image tokens into prompt
119 # (similar to chat template)
120 + model_type = model_config.hf_config.model_type
121 if modality == "image":
121 - model_type = model_config.hf_config.model_type
122 if model_type == "phi3_v":
123 # Workaround since this token is not defined in the tokenizer
124 return "<|image_1|>"
@@ -134,7 +134,9 @@ def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
134
135 raise TypeError(f"Unknown model type: {model_type}")
136 elif modality == "audio":
137 - raise TypeError("No audio models are supported yet.")
137 + if model_type == "ultravox":
138 + return "<|reserved_special_token_0|>"
139 + raise TypeError(f"Unknown model type: {model_type}")
140 else:
141 raise TypeError(f"Unknown modality: {modality}")
142
vllm/model_executor/models/__init__.py
+2 -1
@@ -61,7 +61,7 @@ _GENERATION_MODELS = {
61 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
62 "MedusaModel": ("medusa", "Medusa"),
63 "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
64 - "JambaForCausalLM": ("jamba", "JambaForCausalLM")
64 + "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
65 }
66
67 _EMBEDDING_MODELS = {
@@ -83,6 +83,7 @@ _MULTIMODAL_MODELS = {
83 "PaliGemmaForConditionalGeneration": ("paligemma",
84 "PaliGemmaForConditionalGeneration"),
85 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
86 + "UltravoxModel": ("ultravox", "UltravoxModel"),
87 }
88 _CONDITIONAL_GENERATION_MODELS = {
89 "BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/blip.py
+4 -4
@@ -15,8 +15,8 @@ from vllm.model_executor.layers.activation import get_act_fn
15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 RowParallelLinear)
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 -from vllm.multimodal.image import (cached_get_tokenizer,
19 - repeat_and_pad_image_tokens)
18 +from vllm.multimodal.utils import (cached_get_tokenizer,
19 + repeat_and_pad_placeholder_tokens)
20 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
21
22
@@ -97,11 +97,11 @@ def input_processor_for_blip(
97 else:
98 image_feature_size = image_feature_size_override
99
100 - new_prompt, new_token_ids = repeat_and_pad_image_tokens(
100 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
101 tokenizer,
102 llm_inputs.get("prompt"),
103 llm_inputs["prompt_token_ids"],
104 - image_token_id=image_token_id,
104 + placeholder_token_id=image_token_id,
105 repeat_count=image_feature_size,
106 )
107
vllm/model_executor/models/chameleon.py
+4 -4
@@ -30,8 +30,8 @@ from vllm.model_executor.model_loader.weight_utils import (
30 from vllm.model_executor.sampling_metadata import SamplingMetadata
31 from vllm.model_executor.utils import set_weight_attrs
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 -from vllm.multimodal.image import (cached_get_tokenizer,
34 - repeat_and_pad_image_tokens)
33 +from vllm.multimodal.utils import (cached_get_tokenizer,
34 + repeat_and_pad_placeholder_tokens)
35 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36 SamplerOutput, SequenceData)
37 from vllm.utils import print_warning_once
@@ -124,11 +124,11 @@ def input_processor_for_chameleon(ctx: InputContext, llm_inputs: LLMInputs):
124
125 model_config = ctx.model_config
126 tokenizer = cached_get_tokenizer(model_config.tokenizer)
127 - new_prompt, new_token_ids = repeat_and_pad_image_tokens(
127 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
128 tokenizer,
129 llm_inputs.get("prompt"),
130 llm_inputs["prompt_token_ids"],
131 - image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
131 + placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID,
132 repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
133 pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
134 pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID,
vllm/model_executor/models/clip.py
+4 -4
@@ -16,8 +16,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 RowParallelLinear)
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19 -from vllm.multimodal.image import (cached_get_tokenizer,
20 - repeat_and_pad_image_tokens)
19 +from vllm.multimodal.utils import (cached_get_tokenizer,
20 + repeat_and_pad_placeholder_tokens)
21 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
22
23
@@ -103,11 +103,11 @@ def input_processor_for_clip(
103 else:
104 image_feature_size = image_feature_size_override
105
106 - new_prompt, new_token_ids = repeat_and_pad_image_tokens(
106 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
107 tokenizer,
108 llm_inputs.get("prompt"),
109 llm_inputs["prompt_token_ids"],
110 - image_token_id=image_token_id,
110 + placeholder_token_id=image_token_id,
111 repeat_count=image_feature_size,
112 )
113
vllm/model_executor/models/fuyu.py
+2 -2
@@ -36,8 +36,8 @@ from vllm.model_executor.models.persimmon import PersimmonForCausalLM
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.multimodal import MULTIMODAL_REGISTRY
38 from vllm.multimodal.base import MultiModalInputs
39 -from vllm.multimodal.image import (cached_get_image_processor,
40 - cached_get_tokenizer)
39 +from vllm.multimodal.image import cached_get_image_processor
40 +from vllm.multimodal.utils import cached_get_tokenizer
41 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 SamplerOutput, SequenceData)
43
vllm/model_executor/models/internvl.py
+1 -1
@@ -23,7 +23,7 @@ from vllm.model_executor.models.intern_vit import InternVisionModel
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.multimodal.base import MultiModalInputs
26 -from vllm.multimodal.image import cached_get_tokenizer
26 +from vllm.multimodal.utils import cached_get_tokenizer
27 from vllm.sequence import IntermediateTensors, SamplerOutput
28
29 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
vllm/model_executor/models/minicpmv.py
+2 -2
@@ -54,8 +54,8 @@ from vllm.model_executor.models.minicpm import MiniCPMModel
54 from vllm.model_executor.models.qwen2 import Qwen2Model
55 from vllm.model_executor.sampling_metadata import SamplingMetadata
56 from vllm.multimodal import MULTIMODAL_REGISTRY
57 -from vllm.multimodal.image import (cached_get_image_processor,
58 - cached_get_tokenizer)
57 +from vllm.multimodal.image import cached_get_image_processor
58 +from vllm.multimodal.utils import cached_get_tokenizer
59 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60 SamplerOutput, SequenceData)
61
vllm/model_executor/models/paligemma.py
+1 -1
@@ -16,7 +16,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.models.gemma import GemmaModel
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 -from vllm.multimodal.image import cached_get_tokenizer
19 +from vllm.multimodal.utils import cached_get_tokenizer
20 from vllm.sequence import IntermediateTensors, SamplerOutput
21
22 from .interfaces import SupportsMultiModal
vllm/model_executor/models/phi3v.py
+1 -1
@@ -37,7 +37,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
37 from vllm.model_executor.models.llama import LlamaModel
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.multimodal import MULTIMODAL_REGISTRY
40 -from vllm.multimodal.image import cached_get_tokenizer
40 +from vllm.multimodal.utils import cached_get_tokenizer
41 from vllm.sequence import IntermediateTensors, SamplerOutput
42
43 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
vllm/model_executor/models/siglip.py
+4 -4
@@ -24,8 +24,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 VocabParallelEmbedding)
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 -from vllm.multimodal.image import (cached_get_tokenizer,
28 - repeat_and_pad_image_tokens)
27 +from vllm.multimodal.utils import (cached_get_tokenizer,
28 + repeat_and_pad_placeholder_tokens)
29 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
30
31
@@ -112,11 +112,11 @@ def input_processor_for_siglip(
112 else:
113 image_feature_size = image_feature_size_override
114
115 - new_prompt, new_token_ids = repeat_and_pad_image_tokens(
115 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
116 tokenizer,
117 llm_inputs.get("prompt"),
118 llm_inputs["prompt_token_ids"],
119 - image_token_id=image_token_id,
119 + placeholder_token_id=image_token_id,
120 repeat_count=image_feature_size,
121 )
122
vllm/model_executor/models/ultravox.py new
+435
@@ -0,0 +1,435 @@
1 +# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
2 +"""PyTorch Ultravox model."""
3 +
4 +import itertools
5 +import math
6 +from array import array
7 +from functools import lru_cache
8 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9 + TypedDict, Union, cast)
10 +
11 +import librosa
12 +import numpy as np
13 +import torch
14 +import torch.utils.checkpoint
15 +from torch import nn
16 +from torch.nn import functional as F
17 +from transformers.models.whisper import WhisperFeatureExtractor
18 +from transformers.models.whisper.modeling_whisper import WhisperEncoder
19 +
20 +from vllm.attention import AttentionMetadata
21 +from vllm.config import CacheConfig, MultiModalConfig
22 +from vllm.inputs import INPUT_REGISTRY
23 +from vllm.inputs.data import LLMInputs
24 +from vllm.inputs.registry import InputContext
25 +from vllm.logger import init_logger
26 +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
27 +from vllm.model_executor.layers.layernorm import RMSNorm
28 +from vllm.model_executor.layers.quantization.base_config import (
29 + QuantizationConfig)
30 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31 +from vllm.model_executor.models.interfaces import SupportsMultiModal
32 +from vllm.model_executor.models.utils import (filter_weights,
33 + init_vllm_registered_model,
34 + merge_multimodal_embeddings)
35 +from vllm.model_executor.sampling_metadata import SamplingMetadata
36 +from vllm.multimodal import MULTIMODAL_REGISTRY
37 +from vllm.multimodal.base import MultiModalInputs
38 +from vllm.multimodal.utils import (cached_get_tokenizer,
39 + repeat_and_pad_placeholder_tokens)
40 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SamplerOutput, SequenceData
41 +from vllm.transformers_utils.configs.ultravox import UltravoxConfig
42 +
43 +_AUDIO_PLACEHOLDER_TOKEN = 128002
44 +_AUDIO_TOKENS_PER_SECOND = 6.25
45 +
46 +logger = init_logger(__name__)
47 +
48 +
49 +class UltravoxAudioFeatureInputs(TypedDict):
50 + type: Literal["audio_features"]
51 + data: Union[torch.Tensor, List[torch.Tensor]]
52 + """Shape: `(batch_size, 80, M)"""
53 +
54 +
55 +class UltravoxAudioEmbeddingInputs(TypedDict):
56 + type: Literal["audio_embeds"]
57 + data: torch.Tensor
58 +
59 +
60 +UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
61 + UltravoxAudioEmbeddingInputs]
62 +
63 +
64 +@lru_cache
65 +def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor:
66 + return WhisperFeatureExtractor.from_pretrained(model_id)
67 +
68 +
69 +def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor:
70 + return cached_feature_extractor(
71 + ctx.get_hf_config(UltravoxConfig).audio_model_id)
72 +
73 +
74 +def get_ultravox_max_audio_tokens(ctx: InputContext):
75 + feature_extractor = whisper_feature_extractor(ctx)
76 + return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND)
77 +
78 +
79 +def dummy_data_for_ultravox(
80 + ctx: InputContext,
81 + seq_len: int,
82 + mm_counts: Mapping[str, int],
83 +):
84 + feature_extractor = whisper_feature_extractor(ctx)
85 +
86 + audio_count = mm_counts["audio"]
87 +
88 + audio_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [
89 + _AUDIO_PLACEHOLDER_TOKEN
90 + ]) * get_ultravox_max_audio_tokens(ctx) * audio_count
91 + other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
92 + [0]) * (seq_len - len(audio_token_ids))
93 +
94 + audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1)
95 + mm_dict = {
96 + "audio":
97 + audio_and_sr if audio_count == 1 else [audio_and_sr] * audio_count
98 + }
99 +
100 + return (SequenceData(audio_token_ids + other_token_ids), mm_dict)
101 +
102 +
103 +def input_mapper_for_ultravox(ctx: InputContext, data: object):
104 + if isinstance(data, tuple):
105 + (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
106 + feature_extractor = whisper_feature_extractor(ctx)
107 +
108 + if sr != feature_extractor.sampling_rate:
109 + audio = librosa.resample(audio,
110 + orig_sr=sr,
111 + target_sr=feature_extractor.sampling_rate)
112 + sr = feature_extractor.sampling_rate
113 +
114 + minimum_audio_length = feature_extractor.n_fft // 2 + 1
115 + if len(audio) < minimum_audio_length:
116 + # Not enough audio; pad it.
117 + audio = np.pad(audio, (0, minimum_audio_length - len(audio)))
118 +
119 + return MultiModalInputs({
120 + "audio_features":
121 + feature_extractor(audio,
122 + sampling_rate=sr,
123 + padding="longest",
124 + return_tensors="pt")["input_features"]
125 + })
126 +
127 + raise NotImplementedError(f"Unsupported data type: {type(data)}")
128 +
129 +
130 +def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
131 + multi_modal_data = llm_inputs.get("multi_modal_data")
132 + if multi_modal_data is None or "audio" not in multi_modal_data:
133 + return llm_inputs
134 +
135 + feature_extractor = whisper_feature_extractor(ctx)
136 + audio_data, sample_rate = multi_modal_data["audio"]
137 +
138 + audio_length = audio_data.shape[0]
139 + if sample_rate != feature_extractor.sampling_rate:
140 + # Account for resampling.
141 + adjustment = feature_extractor.sampling_rate / sample_rate
142 + audio_length = math.ceil(adjustment * audio_length)
143 +
144 + feature_extractor_output_length = math.ceil(
145 + (audio_length -
146 + (feature_extractor.hop_length - 1)) / feature_extractor.hop_length)
147 +
148 + uv_config = ctx.get_hf_config(UltravoxConfig)
149 + audio_num_tokens = min(
150 + max(
151 + 1,
152 + math.ceil(feature_extractor_output_length /
153 + (uv_config.stack_factor * 2))),
154 + get_ultravox_max_audio_tokens(ctx))
155 + tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
156 +
157 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
158 + tokenizer,
159 + llm_inputs.get("prompt"),
160 + llm_inputs["prompt_token_ids"],
161 + placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN,
162 + repeat_count=audio_num_tokens,
163 + )
164 +
165 + # NOTE: Create a defensive copy of the original inputs
166 + return LLMInputs(prompt_token_ids=new_token_ids,
167 + prompt=new_prompt,
168 + multi_modal_data=multi_modal_data)
169 +
170 +
171 +class StackAudioFrames(nn.Module):
172 + """
173 + Stack the audio embedding frames to reduce the sequence length by a factor
174 + of `stack_factor`.
175 + """
176 +
177 + def __init__(self, stack_factor: int = 8):
178 + super().__init__()
179 + self.stack_factor = stack_factor
180 +
181 + def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
182 + B, T, C = audio_embeds.shape
183 + T_pad = (T + self.stack_factor -
184 + 1) // self.stack_factor * self.stack_factor
185 + audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T))
186 + B, T, C = audio_embeds.shape
187 + audio_embeds = audio_embeds.view(B, T // self.stack_factor,
188 + C * self.stack_factor)
189 + return audio_embeds
190 +
191 +
192 +class FlippedSiluAndMul(SiluAndMul):
193 + """Ultravox is trained with SwiGLU with flipped halves."""
194 +
195 + def forward(self, x: torch.Tensor):
196 + a, b = x.chunk(2, dim=-1)
197 + flipped = torch.cat((b, a), dim=-1)
198 + return super().forward(flipped)
199 +
200 +
201 +class UltravoxProjector(nn.Module):
202 +
203 + def __init__(self, config: UltravoxConfig):
204 + super().__init__()
205 + self.hidden_dim = config.hidden_size
206 + self._pad_and_stack = StackAudioFrames(config.stack_factor)
207 + dim = config.audio_config.hidden_size * config.stack_factor
208 + self.ln_pre = RMSNorm(dim)
209 + self.linear_1 = nn.Linear(dim, self.hidden_dim, bias=False)
210 + dim = self.hidden_dim
211 +
212 + if config.projector_act == "swiglu":
213 + self.act = FlippedSiluAndMul()
214 + dim = dim // 2
215 + else:
216 + self.act = get_act_fn(config.projector_act)
217 +
218 + self.linear_2 = nn.Linear(dim,
219 + config.text_config.hidden_size,
220 + bias=False)
221 + self.ln_post = RMSNorm(config.text_config.hidden_size)
222 +
223 + def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
224 + audio_features = self._pad_and_stack(audio_features)
225 + audio_features = self.ln_pre(audio_features)
226 + hidden_states = self.linear_1(audio_features)
227 + hidden_states = self.act(hidden_states)
228 + hidden_states = self.linear_2(hidden_states)
229 + hidden_states = self.ln_post(hidden_states)
230 + return hidden_states
231 +
232 +
233 +class ModifiedWhisperEncoder(WhisperEncoder):
234 + """
235 + Encoder portion of OpenAI's Whisper model.
236 +
237 + This implementation is a slightly modified version of HF Transformers'
238 + Whisper Encoder, with only a few fixes:
239 + 1. base_model_prefix updated to allow for doing `.from_pretrained`
240 + directly on the encoder
241 + 2. allow less than 30 second of audio padding to be passed in:
242 + - relaxed ValueError check for `input_features` length to be less
243 + than or equal to `expected_seq_length` instead of strictly equal
244 + - embed_pos is now sliced to match the length of `inputs_embeds`
245 +
246 + Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
247 + See commentary: https://github.com/huggingface/transformers/issues/25744
248 + """
249 +
250 + base_model_prefix = "model.encoder"
251 +
252 + def forward(
253 + self,
254 + input_features,
255 + ):
256 + expected_seq_length = (self.config.max_source_positions *
257 + self.conv1.stride[0] * self.conv2.stride[0])
258 + if input_features.shape[-1] > expected_seq_length:
259 + raise ValueError(
260 + f"Whisper expects the mel input features to be of length "
261 + f"{expected_seq_length} or less, but found "
262 + f"{input_features.shape[-1]}. Make sure to pad the input mel "
263 + f"features to {expected_seq_length}.")
264 +
265 + inputs_embeds = nn.functional.gelu(self.conv1(input_features))
266 + inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
267 +
268 + inputs_embeds = inputs_embeds.permute(0, 2, 1)
269 + embed_pos = self.embed_positions.weight[:inputs_embeds.size(-2)]
270 +
271 + hidden_states = inputs_embeds + embed_pos
272 + hidden_states = nn.functional.dropout(hidden_states,
273 + p=self.dropout,
274 + training=self.training)
275 +
276 + for encoder_layer in self.layers:
277 + layer_outputs = encoder_layer(
278 + hidden_states,
279 + None,
280 + layer_head_mask=None,
281 + )
282 +
283 + hidden_states = layer_outputs[0]
284 +
285 + hidden_states = self.layer_norm(hidden_states)
286 + return hidden_states
287 +
288 +
289 +@MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_ultravox)
290 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
291 + "audio", get_ultravox_max_audio_tokens)
292 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_ultravox)
293 +@INPUT_REGISTRY.register_input_processor(input_processor_for_ultravox)
294 +class UltravoxModel(nn.Module, SupportsMultiModal):
295 +
296 + def __init__(self,
297 + config: UltravoxConfig,
298 + multimodal_config: MultiModalConfig,
299 + cache_config: Optional[CacheConfig] = None,
300 + quant_config: Optional["QuantizationConfig"] = None):
301 + super().__init__()
302 + self.config = config
303 + self.multi_modal_config = multimodal_config
304 + assert self.multi_modal_config
305 +
306 + if config.audio_model_id is not None:
307 + self.audio_tower = ModifiedWhisperEncoder.from_pretrained(
308 + config.audio_model_id)
309 + else:
310 + self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
311 + self.multi_modal_projector = UltravoxProjector(config)
312 + self.language_model = init_vllm_registered_model(
313 + config.text_config, cache_config, quant_config)
314 +
315 + def _audio_features_to_embeddings(
316 + self, input_features: torch.Tensor) -> torch.Tensor:
317 + audio_input = input_features.to(self.audio_tower.dtype)
318 + audio_features = self.audio_tower(audio_input)
319 + audio_features = audio_features.to(self.audio_tower.dtype)
320 + audio_embeddings = self.multi_modal_projector(audio_features)
321 + return audio_embeddings
322 +
323 + def _parse_and_validate_audio_input(
324 + self, **kwargs: object) -> Optional[UltravoxAudioInputs]:
325 + audio_features = kwargs.pop("audio_features", None)
326 + audio_embeds = kwargs.pop("audio_embeds", None)
327 +
328 + if audio_features is None and audio_embeds is None:
329 + return None
330 +
331 + if audio_features is not None:
332 + if not isinstance(audio_features, (torch.Tensor, list)):
333 + raise ValueError("Incorrect type of audio features. "
334 + f"Got type: {type(audio_features)}")
335 +
336 + return UltravoxAudioFeatureInputs(type="audio_features",
337 + data=audio_features)
338 +
339 + if audio_embeds is not None:
340 + if not isinstance(audio_embeds, torch.Tensor):
341 + raise ValueError("Incorrect type of audio embeds. "
342 + f"Got type: {type(audio_embeds)}")
343 +
344 + return UltravoxAudioEmbeddingInputs(type="audio_embeds",
345 + data=audio_embeds)
346 +
347 + raise AssertionError("This line should be unreachable.")
348 +
349 + def _process_audio_input(
350 + self, audio_input: UltravoxAudioInputs
351 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
352 + if audio_input["type"] == "audio_embeds":
353 + return audio_input["data"]
354 +
355 + audio_features = audio_input["data"]
356 + if isinstance(audio_features, list):
357 + # TODO: Batch these through the encoder/projector instead of
358 + # serializing them.
359 + return [
360 + self._audio_features_to_embeddings(
361 + features.unsqueeze(0)).squeeze(0)
362 + for features in audio_features
363 + ]
364 + else:
365 + return self._audio_features_to_embeddings(audio_features)
366 +
367 + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
368 + kv_caches: List[torch.Tensor],
369 + attn_metadata: AttentionMetadata,
370 + intermediate_tensors: Optional[torch.Tensor],
371 + **kwargs) -> SamplerOutput:
372 + """Run forward pass for Ultravox
373 +
374 + One key thing to understand is the `input_ids` already accounts for the
375 + positions of the to-be-inserted audio embeddings. The to-be-inserted
376 + audio has a size that is essentially 6.25 tokens per second of audio.
377 +
378 + This way, the `positions` and `attn_metadata` are consistent
379 + with the `input_ids`.
380 +
381 + Args:
382 + input_features: A batch of audio inputs, [1, 80, M].
383 + """
384 + audio_input = self._parse_and_validate_audio_input(**kwargs)
385 + if audio_input is not None:
386 + audio_embeddings = self._process_audio_input(audio_input)
387 + inputs_embeds = self.language_model.model.get_input_embeddings(
388 + input_ids)
389 +
390 + inputs_embeds = merge_multimodal_embeddings(
391 + input_ids, inputs_embeds, audio_embeddings,
392 + _AUDIO_PLACEHOLDER_TOKEN)
393 + input_ids = None
394 + else:
395 + inputs_embeds = None
396 +
397 + hidden_states = self.language_model.model(
398 + input_ids=input_ids,
399 + positions=positions,
400 + kv_caches=kv_caches,
401 + attn_metadata=attn_metadata,
402 + intermediate_tensors=intermediate_tensors,
403 + inputs_embeds=inputs_embeds)
404 + return hidden_states
405 +
406 + def compute_logits(self, hidden_states: torch.Tensor,
407 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
408 + return self.language_model.compute_logits(hidden_states,
409 + sampling_metadata)
410 +
411 + def sample(
412 + self,
413 + logits: torch.Tensor,
414 + sampling_metadata: SamplingMetadata,
415 + ) -> Optional[SamplerOutput]:
416 + return self.language_model.sample(logits, sampling_metadata)
417 +
418 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
419 + # prepare weight iterators for components
420 + projector_weights, llm_weights = itertools.tee(weights, 2)
421 +
422 + # load projector weights
423 + projector_weights = filter_weights(projector_weights,
424 + "multi_modal_projector")
425 + projector_params_dict = dict(
426 + self.multi_modal_projector.named_parameters())
427 + for name, loaded_weight in projector_weights:
428 + param = projector_params_dict[name]
429 + weight_loader = getattr(param, "weight_loader",
430 + default_weight_loader)
431 + weight_loader(param, loaded_weight)
432 +
433 + # load llm backbone
434 + llm_weights = filter_weights(llm_weights, "language_model")
435 + self.language_model.load_weights(llm_weights)
vllm/multimodal/image.py
-83
@@ -1,5 +1,4 @@
1 from functools import lru_cache
2 -from typing import List, Optional, Tuple, TypeVar
2
3 import torch
4 from PIL import Image
@@ -8,7 +7,6 @@ from vllm.config import ModelConfig
7 from vllm.inputs.registry import InputContext
8 from vllm.logger import init_logger
9 from vllm.transformers_utils.image_processor import get_image_processor
11 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
10 from vllm.utils import is_list_of
11
12 from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
@@ -16,87 +14,6 @@ from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
14 logger = init_logger(__name__)
15
16 cached_get_image_processor = lru_cache(get_image_processor)
19 -cached_get_tokenizer = lru_cache(get_tokenizer)
20 -
21 -# Utilities for image input processors
22 -_T = TypeVar("_T", str, int)
23 -
24 -
25 -def repeat_and_pad_token(
26 - token: _T,
27 - *,
28 - repeat_count: int = 1,
29 - pad_token_left: Optional[_T] = None,
30 - pad_token_right: Optional[_T] = None,
31 -) -> List[_T]:
32 - replacement = [token] * repeat_count
33 - if pad_token_left is not None:
34 - replacement = [pad_token_left] + replacement
35 - if pad_token_right is not None:
36 - replacement = replacement + [pad_token_right]
37 -
38 - return replacement
39 -
40 -
41 -def repeat_and_pad_image_tokens(
42 - tokenizer: AnyTokenizer,
43 - prompt: Optional[str],
44 - prompt_token_ids: List[int],
45 - *,
46 - image_token_id: int,
47 - repeat_count: int = 1,
48 - pad_token_left: Optional[int] = None,
49 - pad_token_right: Optional[int] = None,
50 -) -> Tuple[Optional[str], List[int]]:
51 - if prompt is None:
52 - new_prompt = None
53 - else:
54 - image_token_str = tokenizer.decode(image_token_id)
55 - pad_token_str_left = (None if pad_token_left is None else
56 - tokenizer.decode(pad_token_left))
57 - pad_token_str_right = (None if pad_token_right is None else
58 - tokenizer.decode(pad_token_right))
59 - replacement_str = "".join(
60 - repeat_and_pad_token(
61 - image_token_str,
62 - repeat_count=repeat_count,
63 - pad_token_left=pad_token_str_left,
64 - pad_token_right=pad_token_str_right,
65 - ))
66 -
67 - image_token_count = prompt.count(image_token_str)
68 - # This is an arbitrary number to distinguish between the two cases
69 - if image_token_count > 16:
70 - logger.warning(
71 - "Please follow the prompt format that is "
72 - "documented on HuggingFace which does not involve "
73 - "repeating %s tokens.", image_token_str)
74 - elif image_token_count > 1:
75 - logger.warning("Multiple image input is not supported yet, "
76 - "so any extra image tokens will be treated "
77 - "as plain text.")
78 -
79 - # The image tokens are removed to be consistent with HuggingFace
80 - new_prompt = prompt.replace(image_token_str, replacement_str, 1)
81 -
82 - new_token_ids: List[int] = []
83 - for i, token in enumerate(prompt_token_ids):
84 - if token == image_token_id:
85 - replacement_ids = repeat_and_pad_token(
86 - image_token_id,
87 - repeat_count=repeat_count,
88 - pad_token_left=pad_token_left,
89 - pad_token_right=pad_token_right,
90 - )
91 - new_token_ids.extend(replacement_ids)
92 -
93 - # No need to further scan the list since we only replace once
94 - new_token_ids.extend(prompt_token_ids[i + 1:])
95 - break
96 - else:
97 - new_token_ids.append(token)
98 -
99 - return new_prompt, new_token_ids
17
18
19 class ImagePlugin(MultiModalPlugin):
vllm/multimodal/utils.py
+89 -1
@@ -1,6 +1,7 @@
1 import base64
2 +from functools import lru_cache
3 from io import BytesIO
3 -from typing import Tuple, Union
4 +from typing import List, Optional, Tuple, TypeVar, Union
5
6 import librosa
7 import numpy as np
@@ -9,7 +10,13 @@ from PIL import Image
10
11 from vllm.connections import global_http_connection
12 from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
13 +from vllm.logger import init_logger
14 from vllm.multimodal.base import MultiModalDataDict
15 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16 +
17 +logger = init_logger(__name__)
18 +
19 +cached_get_tokenizer = lru_cache(get_tokenizer)
20
21
22 def _load_image_from_bytes(b: bytes):
@@ -154,3 +161,84 @@ def rescale_image_size(image: Image.Image,
161 if transpose >= 0:
162 image = image.transpose(Image.Transpose(transpose))
163 return image
164 +
165 +
166 +# Utilities for input processors
167 +_T = TypeVar("_T", str, int)
168 +
169 +
170 +def repeat_and_pad_token(
171 + token: _T,
172 + *,
173 + repeat_count: int = 1,
174 + pad_token_left: Optional[_T] = None,
175 + pad_token_right: Optional[_T] = None,
176 +) -> List[_T]:
177 + replacement = [token] * repeat_count
178 + if pad_token_left is not None:
179 + replacement = [pad_token_left] + replacement
180 + if pad_token_right is not None:
181 + replacement = replacement + [pad_token_right]
182 +
183 + return replacement
184 +
185 +
186 +def repeat_and_pad_placeholder_tokens(
187 + tokenizer: AnyTokenizer,
188 + prompt: Optional[str],
189 + prompt_token_ids: List[int],
190 + *,
191 + placeholder_token_id: int,
192 + repeat_count: int = 1,
193 + pad_token_left: Optional[int] = None,
194 + pad_token_right: Optional[int] = None,
195 +) -> Tuple[Optional[str], List[int]]:
196 + if prompt is None:
197 + new_prompt = None
198 + else:
199 + placeholder_token_str = tokenizer.decode(placeholder_token_id)
200 + pad_token_str_left = (None if pad_token_left is None else
201 + tokenizer.decode(pad_token_left))
202 + pad_token_str_right = (None if pad_token_right is None else
203 + tokenizer.decode(pad_token_right))
204 + replacement_str = "".join(
205 + repeat_and_pad_token(
206 + placeholder_token_str,
207 + repeat_count=repeat_count,
208 + pad_token_left=pad_token_str_left,
209 + pad_token_right=pad_token_str_right,
210 + ))
211 +
212 + placeholder_token_count = prompt.count(placeholder_token_str)
213 + # This is an arbitrary number to distinguish between the two cases
214 + if placeholder_token_count > 16:
215 + logger.warning(
216 + "Please follow the prompt format that is "
217 + "documented on HuggingFace which does not involve "
218 + "repeating %s tokens.", placeholder_token_str)
219 + elif placeholder_token_count > 1:
220 + logger.warning("Multiple multi-modal input is not supported yet, "
221 + "so any extra placeholder tokens will be treated "
222 + "as plain text.")
223 +
224 + # The image tokens are removed to be consistent with HuggingFace
225 + new_prompt = prompt.replace(placeholder_token_str, replacement_str, 1)
226 +
227 + new_token_ids: List[int] = []
228 + for i, token in enumerate(prompt_token_ids):
229 + if token == placeholder_token_id:
230 + replacement_ids = repeat_and_pad_token(
231 + placeholder_token_id,
232 + repeat_count=repeat_count,
233 + pad_token_left=pad_token_left,
234 + pad_token_right=pad_token_right,
235 + )
236 + new_token_ids.extend(replacement_ids)
237 +
238 + # No need to further scan the list since we only replace once
239 + new_token_ids.extend(prompt_token_ids[i + 1:])
240 + break
241 + else:
242 + new_token_ids.append(token)
243 +
244 + return new_prompt, new_token_ids
vllm/transformers_utils/config.py
+2 -1
@@ -12,7 +12,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
12 InternVLChatConfig, JAISConfig,
13 MedusaConfig, MLPSpeculatorConfig,
14 MPTConfig, NemotronConfig,
15 - RWConfig)
15 + RWConfig, UltravoxConfig)
16
17 if VLLM_USE_MODELSCOPE:
18 from modelscope import AutoConfig
@@ -32,6 +32,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
32 "medusa": MedusaConfig,
33 "internvl_chat": InternVLChatConfig,
34 "nemotron": NemotronConfig,
35 + "ultravox": UltravoxConfig,
36 }
37
38 for name, cls in _CONFIG_REGISTRY.items():
vllm/transformers_utils/configs/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.transformers_utils.configs.medusa import MedusaConfig
10 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
11 from vllm.transformers_utils.configs.mpt import MPTConfig
12 from vllm.transformers_utils.configs.nemotron import NemotronConfig
13 +from vllm.transformers_utils.configs.ultravox import UltravoxConfig
14
15 __all__ = [
16 "ChatGLMConfig",
@@ -21,4 +22,5 @@ __all__ = [
22 "MedusaConfig",
23 "MLPSpeculatorConfig",
24 "NemotronConfig",
25 + "UltravoxConfig",
26 ]
vllm/transformers_utils/configs/ultravox.py new
+99
@@ -0,0 +1,99 @@
1 +# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py
2 +from typing import Any, Dict, Optional
3 +
4 +import transformers
5 +
6 +
7 +class UltravoxConfig(transformers.PretrainedConfig):
8 + r"""
9 + This is the configuration class to store the configuration of a
10 + [`UltravoxForConditionalGeneration`]. It is used to instantiate an
11 + Ultravox model according to the specified arguments, defining the model
12 + architecture.
13 +
14 + Configuration objects inherit from [`PretrainedConfig`] and can be used to
15 + control the model outputs. Read the documentation from [`PretrainedConfig`]
16 + for more information.
17 +
18 + Args:
19 + audio_config (`Union[AutoConfig, dict]`, *optional*):
20 + Custom audio config or dict
21 + text_config (`Union[AutoConfig, dict]`, *optional*):
22 + The config object of the text backbone. Can be any of `LlamaConfig`
23 + or `MistralConfig`.
24 + ignore_index (`int`, *optional*, defaults to -100):
25 + The ignore index for the loss function.
26 + audio_token_index (`int`, *optional*, defaults to 32000):
27 + The audio token index to encode the audio prompt.
28 + stack_factor (`int`, *optional*, defaults to 8):
29 + Audio downsampling factor for the multimodal projector.
30 + norm_init (`float`, *optional*, defaults to 0.4):
31 + The initialization value for the layer normalization.
32 + projector_act (`str`, *optional*, defaults to `"swiglu"`):
33 + The activation function used by the multimodal projector.
34 + text_model_lora_config (`LoraConfigSimplified`, *optional*):
35 + The LoRA configuration for finetuning the text model.
36 + audio_model_lora_config (`LoraConfigSimplified`, *optional*):
37 + The LoRA configuration for finetuning the audio model.
38 + """
39 +
40 + model_type = "ultravox"
41 + is_composition = False
42 +
43 + def __init__(
44 + self,
45 + audio_config: Optional[Dict[str, Any]] = None,
46 + text_config: Optional[Dict[str, Any]] = None,
47 + audio_model_id: Optional[str] = None,
48 + text_model_id: Optional[str] = None,
49 + ignore_index: int = -100,
50 + audio_token_index: int = 32000,
51 + hidden_size: int = 4096,
52 + stack_factor: int = 8,
53 + norm_init: float = 0.4,
54 + projector_act: str = "swiglu",
55 + text_model_lora_config: Optional[Dict[str, Any]] = None,
56 + audio_model_lora_config: Optional[Dict[str, Any]] = None,
57 + **kwargs,
58 + ):
59 + self.ignore_index = ignore_index
60 +
61 + self.audio_model_id = audio_model_id
62 + self.text_model_id = text_model_id
63 + self.audio_token_index = audio_token_index
64 +
65 + self.hidden_size = hidden_size
66 + self.stack_factor = stack_factor
67 + self.norm_init = norm_init
68 + self.projector_act = projector_act
69 +
70 + if text_model_id is not None:
71 + # Avoid circular import
72 + from vllm.transformers_utils.config import get_config
73 +
74 + self.text_config = get_config(text_model_id,
75 + trust_remote_code=False)
76 + else:
77 + text_config = text_config or {}
78 + self.text_config = transformers.CONFIG_MAPPING[text_config.get(
79 + "model_type", "llama")](**text_config)
80 +
81 + if audio_model_id is not None:
82 + # Avoid circular import
83 + from vllm.transformers_utils.config import get_config
84 +
85 + self.audio_config = get_config(audio_model_id,
86 + trust_remote_code=False)
87 + else:
88 + audio_config = audio_config or {}
89 + self.audio_config = transformers.CONFIG_MAPPING[audio_config.get(
90 + "model_type", "whisper")](**audio_config)
91 +
92 + self.text_model_lora_config = text_model_lora_config or {}
93 + self.audio_model_lora_config = audio_model_lora_config or {}
94 +
95 + self.vocab_size = self.text_config.vocab_size
96 +
97 + self.initializer_range = self.text_config.initializer_range
98 +
99 + super().__init__(**kwargs)