@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d88506dda4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Model] LoRA Support for Ultravox model (#11253)

Sumit Vij committed Feb 5, 2025 at 19:54 UTC d88506dda45f69cf1f56c4325282c2a881eaaaf7
4 files changed +160 -7
docs/source/models/supported_models.md
+1 -1
@@ -857,7 +857,7 @@ See [this page](#generative-models) for more information on how to use generativ
857 * Ultravox
858 * T + A<sup>E+</sup>
859 * `fixie-ai/ultravox-v0_3`
860 - *
860 + * ✅︎
861 * ✅︎
862 * ✅︎
863 :::
tests/conftest.py
+12 -4
@@ -737,6 +737,7 @@ class VllmRunner:
737 images: Optional[PromptImageInput] = None,
738 videos: Optional[PromptVideoInput] = None,
739 audios: Optional[PromptAudioInput] = None,
740 + **kwargs: Any,
741 ) -> List[Tuple[List[List[int]], List[str]]]:
742 inputs = self.get_inputs(prompts,
743 images=images,
@@ -744,7 +745,8 @@ class VllmRunner:
745 audios=audios)
746
747 req_outputs = self.model.generate(inputs,
747 - sampling_params=sampling_params)
748 + sampling_params=sampling_params,
749 + **kwargs)
750
751 outputs: List[Tuple[List[List[int]], List[str]]] = []
752 for req_output in req_outputs:
@@ -782,6 +784,7 @@ class VllmRunner:
784 images: Optional[PromptImageInput] = None,
785 audios: Optional[PromptAudioInput] = None,
786 videos: Optional[PromptVideoInput] = None,
787 + **kwargs: Any,
788 ) -> Union[List[TokensTextLogprobs],
789 List[TokensTextLogprobsPromptLogprobs]]:
790 inputs = self.get_inputs(prompts,
@@ -790,7 +793,8 @@ class VllmRunner:
793 audios=audios)
794
795 req_outputs = self.model.generate(inputs,
793 - sampling_params=sampling_params)
796 + sampling_params=sampling_params,
797 + **kwargs)
798
799 toks_str_logsprobs_prompt_logprobs = (
800 self._final_steps_generate_w_logprobs(req_outputs))
@@ -826,13 +830,15 @@ class VllmRunner:
830 images: Optional[PromptImageInput] = None,
831 videos: Optional[PromptVideoInput] = None,
832 audios: Optional[PromptAudioInput] = None,
833 + **kwargs: Any,
834 ) -> List[Tuple[List[int], str]]:
835 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
836 outputs = self.generate(prompts,
837 greedy_params,
838 images=images,
839 videos=videos,
835 - audios=audios)
840 + audios=audios,
841 + **kwargs)
842 return [(output_ids[0], output_str[0])
843 for output_ids, output_str in outputs]
844
@@ -847,6 +853,7 @@ class VllmRunner:
853 videos: Optional[PromptVideoInput] = None,
854 stop_token_ids: Optional[List[int]] = None,
855 stop: Optional[List[str]] = None,
856 + **kwargs: Any,
857 ) -> Union[List[TokensTextLogprobs],
858 List[TokensTextLogprobsPromptLogprobs]]:
859 greedy_logprobs_params = SamplingParams(
@@ -861,7 +868,8 @@ class VllmRunner:
868 greedy_logprobs_params,
869 images=images,
870 audios=audios,
864 - videos=videos)
871 + videos=videos,
872 + **kwargs)
873
874 def generate_encoder_decoder_greedy_logprobs(
875 self,
tests/lora/test_ultravox.py new
+121
@@ -0,0 +1,121 @@
1 +import shutil
2 +from os import path
3 +from tempfile import TemporaryDirectory
4 +from typing import List, Tuple
5 +
6 +import torch
7 +from huggingface_hub import snapshot_download
8 +from safetensors.torch import load_file, save_file
9 +from transformers import AutoTokenizer
10 +
11 +from vllm.lora.request import LoRARequest
12 +
13 +from ..models.utils import check_outputs_equal
14 +
15 +ULTRAVOX_MODEL_NAME = "fixie-ai/ultravox-v0_3"
16 +LLMA_MODEL_NAME = "meta-llama/Llama-3.1-8B-Instruct"
17 +
18 +VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
19 +
20 +PROMPT = "Tell me about a Fool's mate move in 20 words. Provide the moves!"
21 +
22 +
23 +def llama3_1_8b_chess_lora_path():
24 + return snapshot_download(
25 + repo_id="mkopecki/chess-lora-adapter-llama-3.1-8b")
26 +
27 +
28 +# can't use llama lora adapter without module name transformation
29 +# because ultravox nest language model
30 +def transform_module_names_for_ultravox(state_dict):
31 + transformed_state_dict = {}
32 + for key, value in state_dict.items():
33 + new_key = key.replace("base_model.model",
34 + "base_model.model.language_model")
35 + transformed_state_dict[new_key] = value
36 + return transformed_state_dict
37 +
38 +
39 +def mk_llama3_1_8b_ultravox_chess_lora(source_repo, target_path):
40 + tensor_file = "adapter_model.safetensors"
41 + state_dict = load_file(path.join(source_repo, tensor_file))
42 + transformed_state_dict = transform_module_names_for_ultravox(state_dict)
43 +
44 + save_file(transformed_state_dict, path.join(target_path, tensor_file))
45 +
46 + config_file = "adapter_config.json"
47 + shutil.copyfile(path.join(source_repo, config_file),
48 + path.join(target_path, config_file))
49 + return target_path
50 +
51 +
52 +def _get_prompt(audio_count, question, placeholder, model_name) -> str:
53 + tokenizer = AutoTokenizer.from_pretrained(model_name)
54 + placeholder = f"{placeholder}\n" * audio_count
55 +
56 + return tokenizer.apply_chat_template([{
57 + 'role': 'user',
58 + 'content': f"{placeholder}{question}"
59 + }],
60 + tokenize=False,
61 + add_generation_prompt=True)
62 +
63 +
64 +def test_ultravox_lora(vllm_runner):
65 + """
66 + TODO: Train an Ultravox LoRA instead of using a Llama LoRA.
67 + """
68 + # Workaround to prevent device mismatch in Whisper.
69 + # Can be removed when it is fixed upstream in transformer
70 + # https://github.com/huggingface/transformers/pull/35866
71 + torch.set_default_device("cpu")
72 +
73 + llama3_1_8b_chess_lora = llama3_1_8b_chess_lora_path()
74 + with TemporaryDirectory() as temp_ultravox_lora_dir:
75 + llama3_1_8b_ultravox_chess_lora = mk_llama3_1_8b_ultravox_chess_lora(
76 + llama3_1_8b_chess_lora, temp_ultravox_lora_dir)
77 + with vllm_runner(
78 + ULTRAVOX_MODEL_NAME,
79 + enforce_eager=True,
80 + max_num_seqs=2,
81 + enable_lora=True,
82 + max_loras=1,
83 + max_lora_rank=128,
84 + dtype="bfloat16",
85 + max_model_len=1024,
86 + ) as vllm_model:
87 + ultravox_outputs: List[Tuple[
88 + List[int], str]] = vllm_model.generate_greedy(
89 + [
90 + _get_prompt(0, PROMPT, VLLM_PLACEHOLDER,
91 + ULTRAVOX_MODEL_NAME)
92 + ],
93 + 256,
94 + lora_request=LoRARequest(str(1), 1,
95 + llama3_1_8b_ultravox_chess_lora),
96 + )
97 +
98 + # run llama with and without lora to compare outputs with above
99 + with vllm_runner(
100 + LLMA_MODEL_NAME,
101 + enforce_eager=True,
102 + max_num_seqs=2,
103 + enable_lora=True,
104 + max_loras=1,
105 + max_lora_rank=128,
106 + dtype="bfloat16",
107 + max_model_len=1024,
108 + ) as vllm_model:
109 + llama_outputs: List[Tuple[List[int], str]] = (
110 + vllm_model.generate_greedy(
111 + [_get_prompt(0, PROMPT, VLLM_PLACEHOLDER, LLMA_MODEL_NAME)],
112 + 256,
113 + lora_request=LoRARequest(str(1), 1, llama3_1_8b_chess_lora),
114 + ))
115 +
116 + check_outputs_equal(
117 + outputs_0_lst=ultravox_outputs,
118 + outputs_1_lst=llama_outputs,
119 + name_0="ultravox",
120 + name_1="llama",
121 + )
vllm/model_executor/models/ultravox.py
+26 -2
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
22 from vllm.model_executor.layers.layernorm import RMSNorm
23 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.model_loader.loader import DefaultModelLoader
25 +from vllm.model_executor.models.module_mapping import MultiModelKeys
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.multimodal import MULTIMODAL_REGISTRY
28 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
@@ -33,7 +34,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
34 from vllm.sequence import IntermediateTensors
35 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
36
36 -from .interfaces import SupportsMultiModal, SupportsPP
37 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
38 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39 init_vllm_registered_model, maybe_prefix,
40 merge_multimodal_embeddings,
@@ -343,7 +344,20 @@ class ModifiedWhisperEncoder(WhisperEncoder):
344 UltravoxMultiModalProcessor,
345 info=UltravoxProcessingInfo,
346 dummy_inputs=UltravoxDummyInputsBuilder)
346 -class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
347 +class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
348 +
349 + packed_modules_mapping = {
350 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
351 + "gate_up_proj": ["gate_proj", "up_proj"]
352 + }
353 +
354 + # LoRA specific attributes
355 + # TODO : Add LoRA to the audio tower and projector.
356 + supported_lora_modules = [
357 + "qkv_proj", "o_proj", "gate_up_proj", "down_proj"
358 + ]
359 + embedding_modules = {}
360 + embedding_padding_modules = []
361
362 hf_to_vllm_mapper = WeightsMapper(
363 orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
@@ -391,6 +405,16 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
405
406 return get_sampler()
407
408 + def get_mm_mapping(self) -> MultiModelKeys:
409 + """
410 + Get the module prefix in multimodal models
411 + """
412 + return MultiModelKeys.from_string_field(
413 + language_model="language_model.",
414 + connector="multi_modal_projector.",
415 + tower_model="audio_tower.",
416 + )
417 +
418 def _audio_features_to_embeddings(
419 self, input_features: torch.Tensor) -> torch.Tensor:
420 audio_input = input_features.to(self.audio_tower.dtype)