A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Model] LoRA Support for Ultravox model (#11253)
Sumit Vij committed
Feb 5, 2025 at 19:54 UTC
d88506dda45f69cf1f56c4325282c2a881eaaaf7
4 files changed
+160
-7
docs/source/models/supported_models.md
+1
-1
@@ -857,7 +857,7 @@ See [this page](#generative-models) for more information on how to use generativ
857
* Ultravox
858
* T + A<sup>E+</sup>
859
* `fixie-ai/ultravox-v0_3`
860
- *
860
+ * ✅︎
861
* ✅︎
862
* ✅︎
863
:::
tests/conftest.py
+12
-4
@@ -737,6 +737,7 @@ class VllmRunner:
737
images: Optional[PromptImageInput] = None,
738
videos: Optional[PromptVideoInput] = None,
739
audios: Optional[PromptAudioInput] = None,
740
+ **kwargs: Any,
741
) -> List[Tuple[List[List[int]], List[str]]]:
742
inputs = self.get_inputs(prompts,
743
images=images,
@@ -744,7 +745,8 @@ class VllmRunner:
745
audios=audios)
746
747
req_outputs = self.model.generate(inputs,
747
- sampling_params=sampling_params)
748
+ sampling_params=sampling_params,
749
+ **kwargs)
750
751
outputs: List[Tuple[List[List[int]], List[str]]] = []
752
for req_output in req_outputs:
@@ -782,6 +784,7 @@ class VllmRunner:
784
images: Optional[PromptImageInput] = None,
785
audios: Optional[PromptAudioInput] = None,
786
videos: Optional[PromptVideoInput] = None,
787
+ **kwargs: Any,
788
) -> Union[List[TokensTextLogprobs],
789
List[TokensTextLogprobsPromptLogprobs]]:
790
inputs = self.get_inputs(prompts,
@@ -790,7 +793,8 @@ class VllmRunner:
793
audios=audios)
794
795
req_outputs = self.model.generate(inputs,
793
- sampling_params=sampling_params)
796
+ sampling_params=sampling_params,
797
+ **kwargs)
798
799
toks_str_logsprobs_prompt_logprobs = (
800
self._final_steps_generate_w_logprobs(req_outputs))
@@ -826,13 +830,15 @@ class VllmRunner:
830
images: Optional[PromptImageInput] = None,
831
videos: Optional[PromptVideoInput] = None,
832
audios: Optional[PromptAudioInput] = None,
833
+ **kwargs: Any,
834
) -> List[Tuple[List[int], str]]:
835
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
836
outputs = self.generate(prompts,
837
greedy_params,
838
images=images,
839
videos=videos,
835
- audios=audios)
840
+ audios=audios,
841
+ **kwargs)
842
return [(output_ids[0], output_str[0])
843
for output_ids, output_str in outputs]
844
@@ -847,6 +853,7 @@ class VllmRunner:
853
videos: Optional[PromptVideoInput] = None,
854
stop_token_ids: Optional[List[int]] = None,
855
stop: Optional[List[str]] = None,
856
+ **kwargs: Any,
857
) -> Union[List[TokensTextLogprobs],
858
List[TokensTextLogprobsPromptLogprobs]]:
859
greedy_logprobs_params = SamplingParams(
@@ -861,7 +868,8 @@ class VllmRunner:
868
greedy_logprobs_params,
869
images=images,
870
audios=audios,
864
- videos=videos)
871
+ videos=videos,
872
+ **kwargs)
873
874
def generate_encoder_decoder_greedy_logprobs(
875
self,
tests/lora/test_ultravox.py
new
+121
@@ -0,0 +1,121 @@
1
+import shutil
2
+from os import path
3
+from tempfile import TemporaryDirectory
4
+from typing import List, Tuple
5
+
6
+import torch
7
+from huggingface_hub import snapshot_download
8
+from safetensors.torch import load_file, save_file
9
+from transformers import AutoTokenizer
10
+
11
+from vllm.lora.request import LoRARequest
12
+
13
+from ..models.utils import check_outputs_equal
14
+
15
+ULTRAVOX_MODEL_NAME = "fixie-ai/ultravox-v0_3"
16
+LLMA_MODEL_NAME = "meta-llama/Llama-3.1-8B-Instruct"
17
+
18
+VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
19
+
20
+PROMPT = "Tell me about a Fool's mate move in 20 words. Provide the moves!"
21
+
22
+
23
+def llama3_1_8b_chess_lora_path():
24
+ return snapshot_download(
25
+ repo_id="mkopecki/chess-lora-adapter-llama-3.1-8b")
26
+
27
+
28
+# can't use llama lora adapter without module name transformation
29
+# because ultravox nest language model
30
+def transform_module_names_for_ultravox(state_dict):
31
+ transformed_state_dict = {}
32
+ for key, value in state_dict.items():
33
+ new_key = key.replace("base_model.model",
34
+ "base_model.model.language_model")
35
+ transformed_state_dict[new_key] = value
36
+ return transformed_state_dict
37
+
38
+
39
+def mk_llama3_1_8b_ultravox_chess_lora(source_repo, target_path):
40
+ tensor_file = "adapter_model.safetensors"
41
+ state_dict = load_file(path.join(source_repo, tensor_file))
42
+ transformed_state_dict = transform_module_names_for_ultravox(state_dict)
43
+
44
+ save_file(transformed_state_dict, path.join(target_path, tensor_file))
45
+
46
+ config_file = "adapter_config.json"
47
+ shutil.copyfile(path.join(source_repo, config_file),
48
+ path.join(target_path, config_file))
49
+ return target_path
50
+
51
+
52
+def _get_prompt(audio_count, question, placeholder, model_name) -> str:
53
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
54
+ placeholder = f"{placeholder}\n" * audio_count
55
+
56
+ return tokenizer.apply_chat_template([{
57
+ 'role': 'user',
58
+ 'content': f"{placeholder}{question}"
59
+ }],
60
+ tokenize=False,
61
+ add_generation_prompt=True)
62
+
63
+
64
+def test_ultravox_lora(vllm_runner):
65
+ """
66
+ TODO: Train an Ultravox LoRA instead of using a Llama LoRA.
67
+ """
68
+ # Workaround to prevent device mismatch in Whisper.
69
+ # Can be removed when it is fixed upstream in transformer
70
+ # https://github.com/huggingface/transformers/pull/35866
71
+ torch.set_default_device("cpu")
72
+
73
+ llama3_1_8b_chess_lora = llama3_1_8b_chess_lora_path()
74
+ with TemporaryDirectory() as temp_ultravox_lora_dir:
75
+ llama3_1_8b_ultravox_chess_lora = mk_llama3_1_8b_ultravox_chess_lora(
76
+ llama3_1_8b_chess_lora, temp_ultravox_lora_dir)
77
+ with vllm_runner(
78
+ ULTRAVOX_MODEL_NAME,
79
+ enforce_eager=True,
80
+ max_num_seqs=2,
81
+ enable_lora=True,
82
+ max_loras=1,
83
+ max_lora_rank=128,
84
+ dtype="bfloat16",
85
+ max_model_len=1024,
86
+ ) as vllm_model:
87
+ ultravox_outputs: List[Tuple[
88
+ List[int], str]] = vllm_model.generate_greedy(
89
+ [
90
+ _get_prompt(0, PROMPT, VLLM_PLACEHOLDER,
91
+ ULTRAVOX_MODEL_NAME)
92
+ ],
93
+ 256,
94
+ lora_request=LoRARequest(str(1), 1,
95
+ llama3_1_8b_ultravox_chess_lora),
96
+ )
97
+
98
+ # run llama with and without lora to compare outputs with above
99
+ with vllm_runner(
100
+ LLMA_MODEL_NAME,
101
+ enforce_eager=True,
102
+ max_num_seqs=2,
103
+ enable_lora=True,
104
+ max_loras=1,
105
+ max_lora_rank=128,
106
+ dtype="bfloat16",
107
+ max_model_len=1024,
108
+ ) as vllm_model:
109
+ llama_outputs: List[Tuple[List[int], str]] = (
110
+ vllm_model.generate_greedy(
111
+ [_get_prompt(0, PROMPT, VLLM_PLACEHOLDER, LLMA_MODEL_NAME)],
112
+ 256,
113
+ lora_request=LoRARequest(str(1), 1, llama3_1_8b_chess_lora),
114
+ ))
115
+
116
+ check_outputs_equal(
117
+ outputs_0_lst=ultravox_outputs,
118
+ outputs_1_lst=llama_outputs,
119
+ name_0="ultravox",
120
+ name_1="llama",
121
+ )
vllm/model_executor/models/ultravox.py
+26
-2
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
22
from vllm.model_executor.layers.layernorm import RMSNorm
23
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24
from vllm.model_executor.model_loader.loader import DefaultModelLoader
25
+from vllm.model_executor.models.module_mapping import MultiModelKeys
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.multimodal import MULTIMODAL_REGISTRY
28
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
@@ -33,7 +34,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
34
from vllm.sequence import IntermediateTensors
35
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
36
36
-from .interfaces import SupportsMultiModal, SupportsPP
37
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
38
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39
init_vllm_registered_model, maybe_prefix,
40
merge_multimodal_embeddings,
@@ -343,7 +344,20 @@ class ModifiedWhisperEncoder(WhisperEncoder):
344
UltravoxMultiModalProcessor,
345
info=UltravoxProcessingInfo,
346
dummy_inputs=UltravoxDummyInputsBuilder)
346
-class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
347
+class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
348
+
349
+ packed_modules_mapping = {
350
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
351
+ "gate_up_proj": ["gate_proj", "up_proj"]
352
+ }
353
+
354
+ # LoRA specific attributes
355
+ # TODO : Add LoRA to the audio tower and projector.
356
+ supported_lora_modules = [
357
+ "qkv_proj", "o_proj", "gate_up_proj", "down_proj"
358
+ ]
359
+ embedding_modules = {}
360
+ embedding_padding_modules = []
361
362
hf_to_vllm_mapper = WeightsMapper(
363
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
@@ -391,6 +405,16 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
405
406
return get_sampler()
407
408
+ def get_mm_mapping(self) -> MultiModelKeys:
409
+ """
410
+ Get the module prefix in multimodal models
411
+ """
412
+ return MultiModelKeys.from_string_field(
413
+ language_model="language_model.",
414
+ connector="multi_modal_projector.",
415
+ tower_model="audio_tower.",
416
+ )
417
+
418
def _audio_features_to_embeddings(
419
self, input_features: torch.Tensor) -> torch.Tensor:
420
audio_input = input_features.to(self.audio_tower.dtype)