A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[CI/Build] Delete ultravox LoRA test (#14730)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 13, 2025 at 15:57 UTC
bd44b812cb328ba0b7291ab714f9390d63c5eec5
1 file changed
-131
tests/lora/test_ultravox.py
deleted
-131
@@ -1,131 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-import shutil
4
-from os import path
5
-from tempfile import TemporaryDirectory
6
-
7
-import pytest
8
-import torch
9
-from huggingface_hub import snapshot_download
10
-from safetensors.torch import load_file, save_file
11
-from transformers import AutoTokenizer
12
-
13
-from vllm.lora.request import LoRARequest
14
-
15
-from ..models.utils import check_outputs_equal
16
-
17
-ULTRAVOX_MODEL_NAME = "fixie-ai/ultravox-v0_3"
18
-LLMA_MODEL_NAME = "meta-llama/Llama-3.1-8B-Instruct"
19
-
20
-VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
21
-
22
-PROMPT = "Tell me about a Fool's mate move in 20 words. Provide the moves!"
23
-
24
-
25
-@pytest.fixture(autouse=True)
26
-def v1(run_with_both_engines_lora):
27
- # Simple autouse wrapper to run both engines for each test
28
- # This can be promoted up to conftest.py to run for every
29
- # test in a package
30
- pass
31
-
32
-
33
-def llama3_1_8b_chess_lora_path():
34
- return snapshot_download(
35
- repo_id="mkopecki/chess-lora-adapter-llama-3.1-8b")
36
-
37
-
38
-# can't use llama lora adapter without module name transformation
39
-# because ultravox nest language model
40
-def transform_module_names_for_ultravox(state_dict):
41
- transformed_state_dict = {}
42
- for key, value in state_dict.items():
43
- new_key = key.replace("base_model.model",
44
- "base_model.model.language_model")
45
- transformed_state_dict[new_key] = value
46
- return transformed_state_dict
47
-
48
-
49
-def mk_llama3_1_8b_ultravox_chess_lora(source_repo, target_path):
50
- tensor_file = "adapter_model.safetensors"
51
- state_dict = load_file(path.join(source_repo, tensor_file))
52
- transformed_state_dict = transform_module_names_for_ultravox(state_dict)
53
-
54
- save_file(transformed_state_dict, path.join(target_path, tensor_file))
55
-
56
- config_file = "adapter_config.json"
57
- shutil.copyfile(path.join(source_repo, config_file),
58
- path.join(target_path, config_file))
59
- return target_path
60
-
61
-
62
-def _get_prompt(audio_count, question, placeholder, model_name) -> str:
63
- tokenizer = AutoTokenizer.from_pretrained(model_name)
64
- placeholder = f"{placeholder}\n" * audio_count
65
-
66
- return tokenizer.apply_chat_template([{
67
- 'role': 'user',
68
- 'content': f"{placeholder}{question}"
69
- }],
70
- tokenize=False,
71
- add_generation_prompt=True)
72
-
73
-
74
-def test_ultravox_lora(vllm_runner):
75
- """
76
- TODO: Train an Ultravox LoRA instead of using a Llama LoRA.
77
- """
78
- # Workaround to prevent device mismatch in Whisper.
79
- # Can be removed when it is fixed upstream in transformer
80
- # https://github.com/huggingface/transformers/pull/35866
81
- torch.set_default_device("cpu")
82
-
83
- llama3_1_8b_chess_lora = llama3_1_8b_chess_lora_path()
84
- with TemporaryDirectory() as temp_ultravox_lora_dir:
85
- llama3_1_8b_ultravox_chess_lora = mk_llama3_1_8b_ultravox_chess_lora(
86
- llama3_1_8b_chess_lora, temp_ultravox_lora_dir)
87
- with vllm_runner(
88
- ULTRAVOX_MODEL_NAME,
89
- enforce_eager=True,
90
- max_num_seqs=2,
91
- enable_lora=True,
92
- max_loras=1,
93
- max_lora_rank=128,
94
- dtype="bfloat16",
95
- max_model_len=1024,
96
- ) as vllm_model:
97
- ultravox_outputs: list[tuple[
98
- list[int], str]] = vllm_model.generate_greedy(
99
- [
100
- _get_prompt(0, PROMPT, VLLM_PLACEHOLDER,
101
- ULTRAVOX_MODEL_NAME)
102
- ],
103
- 256,
104
- lora_request=LoRARequest(str(1), 1,
105
- llama3_1_8b_ultravox_chess_lora),
106
- )
107
-
108
- # run llama with and without lora to compare outputs with above
109
- with vllm_runner(
110
- LLMA_MODEL_NAME,
111
- enforce_eager=True,
112
- max_num_seqs=2,
113
- enable_lora=True,
114
- max_loras=1,
115
- max_lora_rank=128,
116
- dtype="bfloat16",
117
- max_model_len=1024,
118
- ) as vllm_model:
119
- llama_outputs: list[tuple[list[int], str]] = (
120
- vllm_model.generate_greedy(
121
- [_get_prompt(0, PROMPT, VLLM_PLACEHOLDER, LLMA_MODEL_NAME)],
122
- 256,
123
- lora_request=LoRARequest(str(1), 1, llama3_1_8b_chess_lora),
124
- ))
125
-
126
- check_outputs_equal(
127
- outputs_0_lst=ultravox_outputs,
128
- outputs_1_lst=llama_outputs,
129
- name_0="ultravox",
130
- name_1="llama",
131
- )