@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2be8ec6e71
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model] Add Ultravox support for multiple audio chunks (#7963)

Peter Salas committed Sep 3, 2024 at 21:38 UTC 2be8ec6e71473573a9732460fcde9392cf52be45
3 files changed +198 -115
examples/offline_inference_audio_language.py
+34 -24
@@ -11,25 +11,33 @@ from vllm import LLM, SamplingParams
11 from vllm.assets.audio import AudioAsset
12 from vllm.utils import FlexibleArgumentParser
13
14 -# Input audio and question
15 -audio_and_sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
16 -question = "What is recited in the audio?"
14 +audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
15 +question_per_audio_count = [
16 + "What is recited in the audio?",
17 + "What sport and what nursery rhyme are referenced?"
18 +]
19
20
21 # Ultravox 0.3
20 -def run_ultravox(question):
22 +def run_ultravox(question, audio_count):
23 model_name = "fixie-ai/ultravox-v0_3"
24
25 tokenizer = AutoTokenizer.from_pretrained(model_name)
26 messages = [{
25 - 'role': 'user',
26 - 'content': f"<|reserved_special_token_0|>\n{question}"
27 + 'role':
28 + 'user',
29 + 'content':
30 + "<|reserved_special_token_0|>\n" * audio_count + question
31 }]
32 prompt = tokenizer.apply_chat_template(messages,
33 tokenize=False,
34 add_generation_prompt=True)
35
32 - llm = LLM(model=model_name)
36 + llm = LLM(model=model_name,
37 + enforce_eager=True,
38 + enable_chunked_prefill=False,
39 + max_model_len=8192,
40 + limit_mm_per_prompt={"audio": audio_count})
41 stop_token_ids = None
42 return llm, prompt, stop_token_ids
43
@@ -44,7 +52,9 @@ def main(args):
52 if model not in model_example_map:
53 raise ValueError(f"Model type {model} is not supported.")
54
47 - llm, prompt, stop_token_ids = model_example_map[model](question)
55 + audio_count = args.num_audios
56 + llm, prompt, stop_token_ids = model_example_map[model](
57 + question_per_audio_count[audio_count - 1], audio_count)
58
59 # We set temperature to 0.2 so that outputs can be different
60 # even when all prompts are identical when running batch inference.
@@ -53,23 +63,18 @@ def main(args):
63 stop_token_ids=stop_token_ids)
64
65 assert args.num_prompts > 0
56 - if args.num_prompts == 1:
57 - # Single inference
58 - inputs = {
59 - "prompt": prompt,
60 - "multi_modal_data": {
61 - "audio": audio_and_sample_rate
62 - },
63 - }
64 -
65 - else:
66 + inputs = {
67 + "prompt": prompt,
68 + "multi_modal_data": {
69 + "audio": [
70 + asset.audio_and_sample_rate
71 + for asset in audio_assets[:audio_count]
72 + ]
73 + },
74 + }
75 + if args.num_prompts > 1:
76 # Batch inference
67 - inputs = [{
68 - "prompt": prompt,
69 - "multi_modal_data": {
70 - "audio": audio_and_sample_rate
71 - },
72 - } for _ in range(args.num_prompts)]
77 + inputs = [inputs] * args.num_prompts
78
79 outputs = llm.generate(inputs, sampling_params=sampling_params)
80
@@ -92,6 +97,11 @@ if __name__ == "__main__":
97 type=int,
98 default=1,
99 help='Number of prompts to run.')
100 + parser.add_argument("--num-audios",
101 + type=int,
102 + default=1,
103 + choices=[1, 2],
104 + help="Number of audio items per prompt.")
105
106 args = parser.parse_args()
107 main(args)
tests/models/test_ultravox.py
+77 -26
@@ -16,37 +16,32 @@ MODEL_NAME = "fixie-ai/ultravox-v0_3"
16
17 AudioTuple = Tuple[np.ndarray, int]
18
19 +VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
20 +HF_PLACEHOLDER = "<|audio|>"
21 +
22
23 @pytest.fixture(scope="session")
21 -def audio_and_sample_rate():
24 +def audio_assets():
25 from vllm.assets.audio import AudioAsset
23 - return AudioAsset("mary_had_lamb").audio_and_sample_rate
26 + return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
27
28
26 -@pytest.fixture
27 -def prompts_and_audios(audio_and_sample_rate):
28 - tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
29 +@pytest.fixture(scope="module", params=("mary_had_lamb", "winning_call"))
30 +def audio(request):
31 + from vllm.assets.audio import AudioAsset
32 + return AudioAsset(request.param)
33
30 - vllm_placeholder = "<|reserved_special_token_0|>"
31 - hf_placeholder = "<|audio|>"
34
33 - question = "What's in the audio?"
34 - vllm_prompt = tokenizer.apply_chat_template(
35 - [{
36 - 'role': 'user',
37 - 'content': f"{vllm_placeholder}\n{question}"
38 - }],
39 - tokenize=False,
40 - add_generation_prompt=True)
41 - hf_prompt = tokenizer.apply_chat_template(
42 - [{
43 - 'role': 'user',
44 - 'content': f"{hf_placeholder}\n{question}"
45 - }],
46 - tokenize=False,
47 - add_generation_prompt=True)
35 +def _get_prompt(audio_count, question, placeholder):
36 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
37 + placeholder = f"{placeholder}\n" * audio_count
38
49 - return [(vllm_prompt, hf_prompt, audio_and_sample_rate)]
39 + return tokenizer.apply_chat_template([{
40 + 'role': 'user',
41 + 'content': f"{placeholder}{question}"
42 + }],
43 + tokenize=False,
44 + add_generation_prompt=True)
45
46
47 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -134,15 +129,71 @@ def run_test(
129 )
130
131
132 +def run_multi_audio_test(
133 + vllm_runner: Type[VllmRunner],
134 + prompts_and_audios: List[Tuple[str, List[AudioTuple]]],
135 + model: str,
136 + *,
137 + dtype: str,
138 + max_tokens: int,
139 + num_logprobs: int,
140 + tensor_parallel_size: int,
141 + distributed_executor_backend: Optional[str] = None,
142 +):
143 + with vllm_runner(model,
144 + dtype=dtype,
145 + tensor_parallel_size=tensor_parallel_size,
146 + distributed_executor_backend=distributed_executor_backend,
147 + enforce_eager=True,
148 + limit_mm_per_prompt={
149 + "audio":
150 + max((len(audio) for _, audio in prompts_and_audios))
151 + }) as vllm_model:
152 + vllm_outputs = vllm_model.generate_greedy_logprobs(
153 + [prompt for prompt, _ in prompts_and_audios],
154 + max_tokens,
155 + num_logprobs=num_logprobs,
156 + audios=[audios for _, audios in prompts_and_audios])
157 +
158 + # The HuggingFace model doesn't support multiple audios yet, so
159 + # just assert that some tokens were generated.
160 + assert all(tokens for tokens, *_ in vllm_outputs)
161 +
162 +
163 @pytest.mark.parametrize("dtype", ["half"])
164 @pytest.mark.parametrize("max_tokens", [128])
165 @pytest.mark.parametrize("num_logprobs", [5])
140 -def test_models(hf_runner, vllm_runner, prompts_and_audios, dtype: str,
141 - max_tokens: int, num_logprobs: int) -> None:
166 +def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
167 + num_logprobs: int) -> None:
168 +
169 + vllm_prompt = _get_prompt(1, "Describe the audio above.", VLLM_PLACEHOLDER)
170 + hf_prompt = _get_prompt(1, "Describe the audio above.", HF_PLACEHOLDER)
171 run_test(
172 hf_runner,
173 vllm_runner,
145 - prompts_and_audios,
174 + [(vllm_prompt, hf_prompt, audio.audio_and_sample_rate)],
175 + MODEL_NAME,
176 + dtype=dtype,
177 + max_tokens=max_tokens,
178 + num_logprobs=num_logprobs,
179 + tensor_parallel_size=1,
180 + )
181 +
182 +
183 +@pytest.mark.parametrize("dtype", ["half"])
184 +@pytest.mark.parametrize("max_tokens", [128])
185 +@pytest.mark.parametrize("num_logprobs", [5])
186 +def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
187 + max_tokens: int,
188 + num_logprobs: int) -> None:
189 +
190 + vllm_prompt = _get_prompt(len(audio_assets),
191 + "Describe each of the audios above.",
192 + VLLM_PLACEHOLDER)
193 + run_multi_audio_test(
194 + vllm_runner,
195 + [(vllm_prompt, [audio.audio_and_sample_rate
196 + for audio in audio_assets])],
197 MODEL_NAME,
198 dtype=dtype,
199 max_tokens=max_tokens,
vllm/model_executor/models/ultravox.py
+87 -65
@@ -29,12 +29,12 @@ from vllm.model_executor.layers.quantization.base_config import (
29 from vllm.model_executor.layers.sampler import SamplerOutput
30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31 from vllm.model_executor.models.interfaces import SupportsMultiModal
32 -from vllm.model_executor.models.utils import (filter_weights,
32 +from vllm.model_executor.models.utils import (filter_weights, flatten_bn,
33 init_vllm_registered_model,
34 merge_multimodal_embeddings)
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 from vllm.multimodal import MULTIMODAL_REGISTRY
37 -from vllm.multimodal.base import MultiModalInputs
37 +from vllm.multimodal.base import MultiModalInputs, NestedTensors
38 from vllm.multimodal.utils import (cached_get_tokenizer,
39 repeat_and_pad_placeholder_tokens)
40 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
@@ -48,13 +48,14 @@ logger = init_logger(__name__)
48
49 class UltravoxAudioFeatureInputs(TypedDict):
50 type: Literal["audio_features"]
51 - data: Union[torch.Tensor, List[torch.Tensor]]
52 - """Shape: `(batch_size * num_audios, 80, M)"""
51 + data: NestedTensors
52 + """Shape: `(batch_size, num_audios, 80, M)"""
53
54
55 class UltravoxAudioEmbeddingInputs(TypedDict):
56 type: Literal["audio_embeds"]
57 - data: torch.Tensor
57 + data: NestedTensors
58 + """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
59
60
61 UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
@@ -85,24 +86,33 @@ def dummy_data_for_ultravox(
86
87 audio_count = mm_counts["audio"]
88
88 - audio_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [
89 - _AUDIO_PLACEHOLDER_TOKEN
90 - ]) * get_ultravox_max_audio_tokens(ctx) * audio_count
89 + audio_placeholder = array(
90 + VLLM_TOKEN_ID_ARRAY_TYPE,
91 + [_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx)
92 +
93 + # Add a separator between each chunk.
94 + audio_token_ids = (audio_placeholder +
95 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [0])) * audio_count
96 other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
97 [0]) * (seq_len - len(audio_token_ids))
98
99 audio_and_sr = (np.array([0.0] * feature_extractor.chunk_length), 1)
95 - mm_dict = {
96 - "audio":
97 - audio_and_sr if audio_count == 1 else [audio_and_sr] * audio_count
98 - }
100 + mm_dict = {"audio": [audio_and_sr] * audio_count}
101
102 return (SequenceData(audio_token_ids + other_token_ids), mm_dict)
103
104
105 def input_mapper_for_ultravox(ctx: InputContext, data: object):
104 - if isinstance(data, tuple):
105 - (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], data)
106 + if not isinstance(data, list):
107 + data = [data]
108 +
109 + audio_features = []
110 + for audio_input in data:
111 + if not isinstance(audio_input, tuple):
112 + raise NotImplementedError(
113 + f"Unsupported data type: {type(audio_input)}")
114 +
115 + (audio, sr) = cast(Tuple[np.ndarray, Union[float, int]], audio_input)
116 feature_extractor = whisper_feature_extractor(ctx)
117
118 if sr != feature_extractor.sampling_rate:
@@ -121,15 +131,14 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
131 # Not enough audio; pad it.
132 audio = np.pad(audio, (0, minimum_audio_length - len(audio)))
133
124 - return MultiModalInputs({
125 - "audio_features":
126 - feature_extractor(audio,
127 - sampling_rate=sr,
128 - padding="longest",
129 - return_tensors="pt")["input_features"]
130 - })
134 + single_audio_features = feature_extractor(
135 + audio, sampling_rate=sr, padding="longest",
136 + return_tensors="pt")["input_features"]
137
132 - raise NotImplementedError(f"Unsupported data type: {type(data)}")
138 + # Remove the batch dimension because we're wrapping it in a list.
139 + audio_features.append(single_audio_features.squeeze(0))
140 +
141 + return MultiModalInputs({"audio_features": audio_features})
142
143
144 def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
@@ -138,25 +147,31 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
147 return llm_inputs
148
149 feature_extractor = whisper_feature_extractor(ctx)
141 - audio_data, sample_rate = multi_modal_data["audio"]
142 -
143 - audio_length = audio_data.shape[0]
144 - if sample_rate != feature_extractor.sampling_rate:
145 - # Account for resampling.
146 - adjustment = feature_extractor.sampling_rate / sample_rate
147 - audio_length = math.ceil(adjustment * audio_length)
148 -
149 - feature_extractor_output_length = math.ceil(
150 - (audio_length -
151 - (feature_extractor.hop_length - 1)) / feature_extractor.hop_length)
152 -
153 - uv_config = ctx.get_hf_config(UltravoxConfig)
154 - audio_num_tokens = min(
155 - max(
156 - 1,
157 - math.ceil(feature_extractor_output_length /
158 - (uv_config.stack_factor * 2))),
159 - get_ultravox_max_audio_tokens(ctx))
150 + audios = multi_modal_data["audio"]
151 + if not isinstance(audios, list):
152 + audios = [audios]
153 +
154 + audio_token_counts = []
155 + for audio_data, sample_rate in audios:
156 + audio_length = audio_data.shape[0]
157 + if sample_rate != feature_extractor.sampling_rate:
158 + # Account for resampling.
159 + adjustment = feature_extractor.sampling_rate / sample_rate
160 + audio_length = math.ceil(adjustment * audio_length)
161 +
162 + feature_extractor_output_length = math.ceil(
163 + (audio_length - (feature_extractor.hop_length - 1)) /
164 + feature_extractor.hop_length)
165 +
166 + uv_config = ctx.get_hf_config(UltravoxConfig)
167 + audio_num_tokens = min(
168 + max(
169 + 1,
170 + math.ceil(feature_extractor_output_length /
171 + (uv_config.stack_factor * 2))),
172 + get_ultravox_max_audio_tokens(ctx))
173 + audio_token_counts.append(audio_num_tokens)
174 +
175 tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
176
177 new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
@@ -164,7 +179,7 @@ def input_processor_for_ultravox(ctx: InputContext, llm_inputs: LLMInputs):
179 llm_inputs.get("prompt"),
180 llm_inputs["prompt_token_ids"],
181 placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN,
167 - repeat_count=audio_num_tokens,
182 + repeat_count=audio_token_counts,
183 )
184
185 # NOTE: Create a defensive copy of the original inputs
@@ -338,45 +353,52 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
353 raise ValueError("Incorrect type of audio features. "
354 f"Got type: {type(audio_features)}")
355
341 - # Remove the N dimension until multiple audios are supported.
342 - if isinstance(audio_features, torch.Tensor):
343 - audio_features = audio_features.squeeze(1)
344 - else:
345 - audio_features = [t.squeeze(0) for t in audio_features]
346 -
356 return UltravoxAudioFeatureInputs(type="audio_features",
357 data=audio_features)
358
359 if audio_embeds is not None:
351 - if not isinstance(audio_embeds, torch.Tensor):
360 + if not isinstance(audio_embeds, (torch.Tensor, list)):
361 raise ValueError("Incorrect type of audio embeds. "
362 f"Got type: {type(audio_embeds)}")
363
355 - # Remove the N dimension until multiple audios are supported.
356 - audio_embeds = audio_embeds.squeeze(1)
357 -
364 return UltravoxAudioEmbeddingInputs(type="audio_embeds",
365 data=audio_embeds)
366
367 raise AssertionError("This line should be unreachable.")
368
369 def _process_audio_input(
364 - self, audio_input: UltravoxAudioInputs
365 - ) -> Union[torch.Tensor, List[torch.Tensor]]:
370 + self, audio_input: UltravoxAudioInputs) -> NestedTensors:
371 if audio_input["type"] == "audio_embeds":
372 return audio_input["data"]
373
374 audio_features = audio_input["data"]
370 - if isinstance(audio_features, list):
371 - # TODO: Batch these through the encoder/projector instead of
372 - # serializing them.
373 - return [
374 - self._audio_features_to_embeddings(
375 - features.unsqueeze(0)).squeeze(0)
376 - for features in audio_features
377 - ]
378 - else:
379 - return self._audio_features_to_embeddings(audio_features)
375 + if isinstance(audio_features, torch.Tensor):
376 + # Combine the B and N dimensions for the encoder/projector
377 + flattened = flatten_bn(audio_features)
378 + flattened_embeddings = self._audio_features_to_embeddings(
379 + flattened)
380 +
381 + # Restore the original dimensions
382 + embeddings = flattened_embeddings.unflatten(
383 + 0, audio_features.shape[:2])
384 + return embeddings
385 +
386 + result = []
387 + # TODO: Batch heterogeneous tensors through the encoder/projector
388 + for audio_features_item in audio_features:
389 + if isinstance(audio_features_item, torch.Tensor):
390 + result.append(
391 + self._audio_features_to_embeddings(audio_features_item))
392 + else:
393 + embeddings = [
394 + # Add a batch dimension to embed it, then remove it.
395 + self._audio_features_to_embeddings(tensor.unsqueeze(0)
396 + ).squeeze(0)
397 + for tensor in audio_features_item
398 + ]
399 + result.append(embeddings)
400 +
401 + return result
402
403 def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
404 kv_caches: List[torch.Tensor],
@@ -393,7 +415,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
415 with the `input_ids`.
416
417 Args:
396 - input_features: A batch of audio inputs, [1, 80, M].
418 + audio_features: A batch of audio inputs [B, N, 80, M].
419 """
420 audio_input = self._parse_and_validate_audio_input(**kwargs)
421 if audio_input is not None: