A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] Add Ultravox support for multiple audio chunks (#7963)
Peter Salas committed
Sep 3, 2024 at 21:38 UTC
2be8ec6e71473573a9732460fcde9392cf52be45
3 files changed
+198
-115
examples/offline_inference_audio_language.py
+34
-24
@@ -11,25 +11,33 @@ from vllm import LLM, SamplingParams
11
from vllm.assets.audio import AudioAsset
12
from vllm.utils import FlexibleArgumentParser
13
14
-# Input audio and question
15
-audio_and_sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
16
-question = "What is recited in the audio?"
14
+audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
15
+question_per_audio_count = [
16
+ "What is recited in the audio?",
17
+ "What sport and what nursery rhyme are referenced?"
18
+]
19
20
21
# Ultravox 0.3
20
-def run_ultravox(question):
22
+def run_ultravox(question, audio_count):
23
model_name = "fixie-ai/ultravox-v0_3"
24
25
tokenizer = AutoTokenizer.from_pretrained(model_name)
26
messages = [{
25
- 'role': 'user',
26
- 'content': f"<|reserved_special_token_0|>\n{question}"
27
+ 'role':
28
+ 'user',
29
+ 'content':
30
+ "<|reserved_special_token_0|>\n" * audio_count + question
31
}]
32
prompt = tokenizer.apply_chat_template(messages,
33
tokenize=False,
34
add_generation_prompt=True)
35
32
- llm = LLM(model=model_name)
36
+ llm = LLM(model=model_name,
37
+ enforce_eager=True,
38
+ enable_chunked_prefill=False,
39
+ max_model_len=8192,
40
+ limit_mm_per_prompt={"audio": audio_count})
41
stop_token_ids = None
42
return llm, prompt, stop_token_ids
43
@@ -44,7 +52,9 @@ def main(args):
52
if model not in model_example_map:
53
raise ValueError(f"Model type {model} is not supported.")
54
47
- llm, prompt, stop_token_ids = model_example_map[model](question)
55
+ audio_count = args.num_audios
56
+ llm, prompt, stop_token_ids = model_example_map[model](
57
+ question_per_audio_count[audio_count - 1], audio_count)
58
59
# We set temperature to 0.2 so that outputs can be different
60
# even when all prompts are identical when running batch inference.
@@ -53,23 +63,18 @@ def main(args):
63
stop_token_ids=stop_token_ids)
64
65
assert args.num_prompts > 0
56
- if args.num_prompts == 1:
57
- # Single inference
58
- inputs = {
59
- "prompt": prompt,
60
- "multi_modal_data": {
61
- "audio": audio_and_sample_rate
62
- },
63
- }
64
-
65
- else:
66
+ inputs = {
67
+ "prompt": prompt,
68
+ "multi_modal_data": {
69
+ "audio": [
70
+ asset.audio_and_sample_rate
71
+ for asset in audio_assets[:audio_count]
72
+ ]
73
+ },
74
+ }
75
+ if args.num_prompts > 1:
76
# Batch inference
67
- inputs = [{
68
- "prompt": prompt,
69
- "multi_modal_data": {
70
- "audio": audio_and_sample_rate
71
- },
72
- } for _ in range(args.num_prompts)]
77
+ inputs = [inputs] * args.num_prompts
78
79
outputs = llm.generate(inputs, sampling_params=sampling_params)
80
@@ -92,6 +97,11 @@ if __name__ == "__main__":
97
type=int,
98
default=1,
99
help='Number of prompts to run.')
100
+ parser.add_argument("--num-audios",
101
+ type=int,
102
+ default=1,
103
+ choices=[1, 2],
104
+ help="Number of audio items per prompt.")
105
106
args = parser.parse_args()
107
main(args)
tests/models/test_ultravox.py
+77
-26
@@ -16,37 +16,32 @@ MODEL_NAME = "fixie-ai/ultravox-v0_3"
16
17
AudioTuple = Tuple[np.ndarray, int]
18
19
+VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
20
+HF_PLACEHOLDER = "<|audio|>"
21
+
22
23
@pytest.fixture(scope="session")
21
-def audio_and_sample_rate():
24
+def audio_assets():
25
from vllm.assets.audio import AudioAsset
23
- return AudioAsset("mary_had_lamb").audio_and_sample_rate
26
+ return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
27
28
26
-@pytest.fixture
27
-def prompts_and_audios(audio_and_sample_rate):
28
- tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
29
+@pytest.fixture(scope="module", params=("mary_had_lamb", "winning_call"))
30
+def audio(request):
31
+ from vllm.assets.audio import AudioAsset
32
+ return AudioAsset(request.param)
33
30
- vllm_placeholder = "<|reserved_special_token_0|>"
31
- hf_placeholder = "<|audio|>"
34
33
- question = "What's in the audio?"
34
- vllm_prompt = tokenizer.apply_chat_template(
35
- [{
36
- 'role': 'user',
37
- 'content': f"{vllm_placeholder}\n{question}"
38
- }],
39
- tokenize=False,
40
- add_generation_prompt=True)
41
- hf_prompt = tokenizer.apply_chat_template(
42
- [{
43
- 'role': 'user',
44
- 'content': f"{hf_placeholder}\n{question}"
45
- }],
46
- tokenize=False,
47
- add_generation_prompt=True)
35
+def _get_prompt(audio_count, question, placeholder):
36
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
37
+ placeholder = f"{placeholder}\n" * audio_count
38
49
- return [(vllm_prompt, hf_prompt, audio_and_sample_rate)]
39
+ return tokenizer.apply_chat_template([{
40
+ 'role': 'user',
41
+ 'content': f"{placeholder}{question}"
42
+ }],
43
+ tokenize=False,
44
+ add_generation_prompt=True)
45
46
47
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -134,15 +129,71 @@ def run_test(
129
)
130
131
132
+def run_multi_audio_test(
133
+ vllm_runner: Type[VllmRunner],
134
+ prompts_and_audios: List[Tuple[str, List[AudioTuple]]],
135
+ model: str,
136
+ *,
137
+ dtype: str,
138
+ max_tokens: int,
139
+ num_logprobs: int,
140
+ tensor_parallel_size: int,
141
+ distributed_executor_backend: Optional[str] = None,
142
+):
143
+ with vllm_runner(model,
144
+ dtype=dtype,
145
+ tensor_parallel_size=tensor_parallel_size,
146
+ distributed_executor_backend=distributed_executor_backend,
147
+ enforce_eager=True,
148
+ limit_mm_per_prompt={
149
+ "audio":
150
+ max((len(audio) for _, audio in prompts_and_audios))
151
+ }) as vllm_model:
152
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
153
+ [prompt for prompt, _ in prompts_and_audios],
154
+ max_tokens,
155
+ num_logprobs=num_logprobs,
156
+ audios=[audios for _, audios in prompts_and_audios])
157
+
158
+ # The HuggingFace model doesn't support multiple audios yet, so
159
+ # just assert that some tokens were generated.
160
+ assert all(tokens for tokens, *_ in vllm_outputs)
161
+
162
+
163
@pytest.mark.parametrize("dtype", ["half"])
164
@pytest.mark.parametrize("max_tokens", [128])
165
@pytest.mark.parametrize("num_logprobs", [5])
140
-def test_models(hf_runner, vllm_runner, prompts_and_audios, dtype: str,
141
- max_tokens: int, num_logprobs: int) -> None:
166
+def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
167
+ num_logprobs: int) -> None:
168
+
169
+ vllm_prompt = _get_prompt(1, "Describe the audio above.", VLLM_PLACEHOLDER)
170
+ hf_prompt = _get_prompt(1, "Describe the audio above.", HF_PLACEHOLDER)
171
run_test(
172
hf_runner,
173
vllm_runner,
145
- prompts_and_audios,
174
+ [(vllm_prompt, hf_prompt, audio.audio_and_sample_rate)],
175
+ MODEL_NAME,
176
+ dtype=dtype,
177
+ max_tokens=max_tokens,
178
+ num_logprobs=num_logprobs,
179
+ tensor_parallel_size=1,
180
+ )
181
+
182
+
183
+@pytest.mark.parametrize("dtype", ["half"])
184
+@pytest.mark.parametrize("max_tokens", [128])
185
+@pytest.mark.parametrize("num_logprobs", [5])
186
+def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
187
+ max_tokens: int,
188
+ num_logprobs: int) -> None:
189
+
190
+ vllm_prompt = _get_prompt(len(audio_assets),
191
+ "Describe each of the audios above.",
192
+ VLLM_PLACEHOLDER)
193
+ run_multi_audio_test(
194
+ vllm_runner,
195
+ [(vllm_prompt, [audio.audio_and_sample_rate
196
+ for audio in audio_assets])],
197
MODEL_NAME,
198
dtype=dtype,
199
max_tokens=max_tokens,
vllm/model_executor/models/ultravox.py
+87
-65
@@ -29,12 +29,12 @@ from vllm.model_executor.layers.quantization.base_config import (
29
from vllm.model_executor.layers.sampler import SamplerOutput
30
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31
from vllm.model_executor.models.interfaces import SupportsMultiModal
32
-from vllm.model_executor.models.utils import (filter_weights,
32
+from vllm.model_executor.models.utils import (filter_weights, flatten_bn,
33
init_vllm_registered_model,
34
merge_multimodal_embeddings)
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
from vllm.multimodal import MULTIMODAL_REGISTRY
37
-from vllm.multimodal.base import MultiModalInputs
37
+from vllm.multimodal.base import MultiModalInputs, NestedTensors
38
from vllm.multimodal.utils import (cached_get_tokenizer,
39
repeat_and_pad_placeholder_tokens)
40
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
@@ -48,13 +48,14 @@ logger = init_logger(__name__)
48
49
class UltravoxAudioFeatureInputs(TypedDict):
50
type: Literal["audio_features"]
51
- data: Union[torch.Tensor, List[torch.Tensor]]
52
- """Shape: `(batch_size * num_audios, 80, M)"""
51
+ data: NestedTensors
52
+ """Shape: `(batch_size, num_audios, 80, M)"""
53
54
55
class UltravoxAudioEmbeddingInputs(TypedDict):
56
type: Literal["audio_embeds"]
57
- data: torch.Tensor
57
+ data: NestedTensors
58
+ """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
59
60
61
UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
@@ -85,24 +86,33 @@ def dummy_data_for_ultravox(
86
87
audio_count = mm_counts["audio"]
88
88
- audio_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [
89
- _AUDIO_PLACEHOLDER_TOKEN
90
- ]) * get_ultravox_max_audio_tokens(ctx) * audio_count
89
+ audio_placeholder = array(
90
+ VLLM_TOKEN_ID_ARRAY_TYPE,
91
+ [_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx)
92
+
93
+ # Add a separator between each chunk.
94
+ audio_token_ids = (audio_placeholder +
95
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [0])) * audio_count
96
other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
97
[0]) * (seq_len - len(audio_token_ids))
98
99
audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1)
95
- mm_dict = {
96
- "audio":
97
- audio_and_sr if audio_count == 1 else [audio_and_sr] * audio_count
98
- }
100
+ mm_dict = {"audio": [audio_and_sr] * audio_count}
101
102
return (SequenceData(audio_token_ids + other_token_ids), mm_dict)
103
104
105
def input_mapper_for_ultravox(ctx: InputContext, data: object):
104
- if isinstance(data, tuple):
105
- (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
106
+ if not isinstance(data, list):
107
+ data = [data]
108
+
109
+ audio_features = []
110
+ for audio_input in data:
111
+ if not isinstance(audio_input, tuple):
112
+ raise NotImplementedError(
113
+ f"Unsupported data type: {type(audio_input)}")
114
+
115
+ (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], audio_input)
116
feature_extractor = whisper_feature_extractor(ctx)
117
118
if sr != feature_extractor.sampling_rate:
@@ -121,15 +131,14 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
131
# Not enough audio; pad it.
132
audio = np.pad(audio, (0, minimum_audio_length - len(audio)))
133
124
- return MultiModalInputs({
125
- "audio_features":
126
- feature_extractor(audio,
127
- sampling_rate=sr,
128
- padding="longest",
129
- return_tensors="pt")["input_features"]
130
- })
134
+ single_audio_features = feature_extractor(
135
+ audio, sampling_rate=sr, padding="longest",
136
+ return_tensors="pt")["input_features"]
137
132
- raise NotImplementedError(f"Unsupported data type: {type(data)}")
138
+ # Remove the batch dimension because we're wrapping it in a list.
139
+ audio_features.append(single_audio_features.squeeze(0))
140
+
141
+ return MultiModalInputs({"audio_features": audio_features})
142
143
144
def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
@@ -138,25 +147,31 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
147
return llm_inputs
148
149
feature_extractor = whisper_feature_extractor(ctx)
141
- audio_data, sample_rate = multi_modal_data["audio"]
142
-
143
- audio_length = audio_data.shape[0]
144
- if sample_rate != feature_extractor.sampling_rate:
145
- # Account for resampling.
146
- adjustment = feature_extractor.sampling_rate / sample_rate
147
- audio_length = math.ceil(adjustment * audio_length)
148
-
149
- feature_extractor_output_length = math.ceil(
150
- (audio_length -
151
- (feature_extractor.hop_length - 1)) / feature_extractor.hop_length)
152
-
153
- uv_config = ctx.get_hf_config(UltravoxConfig)
154
- audio_num_tokens = min(
155
- max(
156
- 1,
157
- math.ceil(feature_extractor_output_length /
158
- (uv_config.stack_factor * 2))),
159
- get_ultravox_max_audio_tokens(ctx))
150
+ audios = multi_modal_data["audio"]
151
+ if not isinstance(audios, list):
152
+ audios = [audios]
153
+
154
+ audio_token_counts = []
155
+ for audio_data, sample_rate in audios:
156
+ audio_length = audio_data.shape[0]
157
+ if sample_rate != feature_extractor.sampling_rate:
158
+ # Account for resampling.
159
+ adjustment = feature_extractor.sampling_rate / sample_rate
160
+ audio_length = math.ceil(adjustment * audio_length)
161
+
162
+ feature_extractor_output_length = math.ceil(
163
+ (audio_length - (feature_extractor.hop_length - 1)) /
164
+ feature_extractor.hop_length)
165
+
166
+ uv_config = ctx.get_hf_config(UltravoxConfig)
167
+ audio_num_tokens = min(
168
+ max(
169
+ 1,
170
+ math.ceil(feature_extractor_output_length /
171
+ (uv_config.stack_factor * 2))),
172
+ get_ultravox_max_audio_tokens(ctx))
173
+ audio_token_counts.append(audio_num_tokens)
174
+
175
tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
176
177
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
@@ -164,7 +179,7 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
179
llm_inputs.get("prompt"),
180
llm_inputs["prompt_token_ids"],
181
placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN,
167
- repeat_count=audio_num_tokens,
182
+ repeat_count=audio_token_counts,
183
)
184
185
# NOTE: Create a defensive copy of the original inputs
@@ -338,45 +353,52 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
353
raise ValueError("Incorrect type of audio features. "
354
f"Got type: {type(audio_features)}")
355
341
- # Remove the N dimension until multiple audios are supported.
342
- if isinstance(audio_features, torch.Tensor):
343
- audio_features = audio_features.squeeze(1)
344
- else:
345
- audio_features = [t.squeeze(0) for t in audio_features]
346
-
356
return UltravoxAudioFeatureInputs(type="audio_features",
357
data=audio_features)
358
359
if audio_embeds is not None:
351
- if not isinstance(audio_embeds, torch.Tensor):
360
+ if not isinstance(audio_embeds, (torch.Tensor, list)):
361
raise ValueError("Incorrect type of audio embeds. "
362
f"Got type: {type(audio_embeds)}")
363
355
- # Remove the N dimension until multiple audios are supported.
356
- audio_embeds = audio_embeds.squeeze(1)
357
-
364
return UltravoxAudioEmbeddingInputs(type="audio_embeds",
365
data=audio_embeds)
366
367
raise AssertionError("This line should be unreachable.")
368
369
def _process_audio_input(
364
- self, audio_input: UltravoxAudioInputs
365
- ) -> Union[torch.Tensor, List[torch.Tensor]]:
370
+ self, audio_input: UltravoxAudioInputs) -> NestedTensors:
371
if audio_input["type"] == "audio_embeds":
372
return audio_input["data"]
373
374
audio_features = audio_input["data"]
370
- if isinstance(audio_features, list):
371
- # TODO: Batch these through the encoder/projector instead of
372
- # serializing them.
373
- return [
374
- self._audio_features_to_embeddings(
375
- features.unsqueeze(0)).squeeze(0)
376
- for features in audio_features
377
- ]
378
- else:
379
- return self._audio_features_to_embeddings(audio_features)
375
+ if isinstance(audio_features, torch.Tensor):
376
+ # Combine the B and N dimensions for the encoder/projector
377
+ flattened = flatten_bn(audio_features)
378
+ flattened_embeddings = self._audio_features_to_embeddings(
379
+ flattened)
380
+
381
+ # Restore the original dimensions
382
+ embeddings = flattened_embeddings.unflatten(
383
+ 0, audio_features.shape[:2])
384
+ return embeddings
385
+
386
+ result = []
387
+ # TODO: Batch heterogeneous tensors through the encoder/projector
388
+ for audio_features_item in audio_features:
389
+ if isinstance(audio_features_item, torch.Tensor):
390
+ result.append(
391
+ self._audio_features_to_embeddings(audio_features_item))
392
+ else:
393
+ embeddings = [
394
+ # Add a batch dimension to embed it, then remove it.
395
+ self._audio_features_to_embeddings(tensor.unsqueeze(0)
396
+ ).squeeze(0)
397
+ for tensor in audio_features_item
398
+ ]
399
+ result.append(embeddings)
400
+
401
+ return result
402
403
def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
404
kv_caches: List[torch.Tensor],
@@ -393,7 +415,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
415
with the `input_ids`.
416
417
Args:
396
- input_features: A batch of audio inputs, [1, 80, M].
418
+ audio_features: A batch of audio inputs [B, N, 80, M].
419
"""
420
audio_input = self._parse_and_validate_audio_input(**kwargs)
421
if audio_input is not None: