@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 180 lines 6.39 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Tests for applying default registered multimodal loras."""
4
5 import os
6 import unittest.mock as mock
7
8 import pytest
9
10 from vllm.lora.request import LoRARequest
11 from vllm.platforms import current_platform
12 from vllm.transformers_utils.repo_utils import hf_api
13
14 from ..conftest import AudioTestAssets, VllmRunner
15 from ..utils import create_new_process_for_each_test
16
17 MODEL_PATH = hf_api().snapshot_download("microsoft/Phi-4-multimodal-instruct")
18 AUDIO_LORA_PATH = os.path.join(MODEL_PATH, "speech-lora")
19 IMAGE_LORA_PATH = os.path.join(MODEL_PATH, "vision-lora")
20
21 AUDIO_PROMPT = "<|user|><|audio_1|>Can you transcribe this audio?<|end|><|assistant|>" # noqa: E501
22
23 # Responses are greedy decoded; we just check the end of
24 # the generated text. If the lora is inactive, this model
25 # generates commentary on the transcription.
26 RESPONSE_SUFFIX_WITH_LORA = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
27 RESPONSE_SUFFIX_WITHOUT_LORA = "Certainly! Here is the transcription of the audio you provided:\n\nThe first words I spoke in the original phonograph record: A little piece of practical poetry. Mary had a little lamb; its fleece was white as snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
28
29 VLLM_RUNNER_BASE_KWARGS = {
30 "model_name": MODEL_PATH,
31 "dtype": "half",
32 "enable_lora": "True",
33 "max_num_seqs": 2,
34 "max_lora_rank": 320,
35 # Keep these LoRA tests on short-RoPE for determinism post-LongRoPE change.
36 "max_model_len": 4096,
37 "gpu_memory_utilization": 0.8,
38 "limit_mm_per_prompt": {"audio": 1},
39 "enforce_eager": True,
40 }
41
42
43 def run_test(
44 vllm_runner, audio_assets, monkeypatch, lora_request, expected_suffix, **kwargs
45 ):
46 monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
47
48 inputs = [([AUDIO_PROMPT], [audio_assets[0].audio_and_sample_rate[0]])]
49
50 # Apply any additional kwargs as overrides to the base kwargs
51 vllm_runner_kwargs = {**VLLM_RUNNER_BASE_KWARGS, **kwargs}
52
53 with vllm_runner(**vllm_runner_kwargs) as vllm_model:
54 vllm_outputs_with_default_lora = [
55 vllm_model.generate_greedy(
56 prompts,
57 max_tokens=128,
58 audios=audios,
59 lora_request=lora_request,
60 )
61 for prompts, audios in inputs
62 ]
63
64 assert vllm_outputs_with_default_lora[-1][-1][-1].endswith(expected_suffix)
65
66
67 @create_new_process_for_each_test()
68 def test_active_default_mm_lora(
69 vllm_runner: type[VllmRunner],
70 audio_assets: AudioTestAssets,
71 monkeypatch: pytest.MonkeyPatch,
72 ):
73 """Ensure that we can use the default audio lora."""
74 run_test(
75 vllm_runner,
76 audio_assets,
77 monkeypatch,
78 lora_request=None,
79 default_mm_loras={"audio": AUDIO_LORA_PATH},
80 expected_suffix=RESPONSE_SUFFIX_WITH_LORA,
81 )
82
83
84 @pytest.mark.skipif(
85 current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
86 )
87 @create_new_process_for_each_test()
88 def test_inactive_default_mm_lora(
89 vllm_runner: type[VllmRunner],
90 audio_assets: AudioTestAssets,
91 monkeypatch: pytest.MonkeyPatch,
92 ):
93 """Ensure that modalities are filtered properly."""
94 # Default image lora won't be active since we only pass audio
95 run_test(
96 vllm_runner,
97 audio_assets,
98 monkeypatch,
99 lora_request=None,
100 default_mm_loras={"image": IMAGE_LORA_PATH},
101 expected_suffix=RESPONSE_SUFFIX_WITHOUT_LORA,
102 )
103
104
105 @pytest.mark.skipif(
106 current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
107 )
108 @create_new_process_for_each_test()
109 def test_default_mm_lora_succeeds_with_redundant_lora_request(
110 vllm_runner: type[VllmRunner],
111 audio_assets: AudioTestAssets,
112 monkeypatch: pytest.MonkeyPatch,
113 ):
114 """Ensure that redundantly providing the lora works."""
115 run_test(
116 vllm_runner,
117 audio_assets,
118 monkeypatch,
119 lora_request=LoRARequest("audio", 1, AUDIO_LORA_PATH),
120 default_mm_loras={"audio": AUDIO_LORA_PATH},
121 expected_suffix=RESPONSE_SUFFIX_WITH_LORA,
122 )
123
124
125 @pytest.mark.skipif(
126 current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
127 )
128 @create_new_process_for_each_test()
129 def test_default_mm_lora_fails_with_overridden_lora_request(
130 vllm_runner: type[VllmRunner],
131 audio_assets: AudioTestAssets,
132 monkeypatch: pytest.MonkeyPatch,
133 ):
134 """Ensure that if the lora_request conflicts with default_mm_loras,
135 we use the lora_request."""
136 run_test(
137 vllm_runner,
138 audio_assets,
139 monkeypatch,
140 lora_request=LoRARequest("speech", 2, AUDIO_LORA_PATH),
141 default_mm_loras={"audio": IMAGE_LORA_PATH},
142 expected_suffix=RESPONSE_SUFFIX_WITH_LORA,
143 )
144
145
146 @create_new_process_for_each_test()
147 def test_default_mm_lora_does_not_expand_string_reqs(vllm_runner, monkeypatch):
148 # See run_test: force spawn to avoid the forked-child CUDA re-init crash.
149 monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
150
151 class MockEngineException(Exception):
152 pass
153
154 # Regression test for ensuring default multimodal lora resolution
155 # does not expand the lora req if the prompt type is a string.
156 vllm_runner_kwargs = {
157 **VLLM_RUNNER_BASE_KWARGS,
158 **{"default_mm_loras": {"audio": AUDIO_LORA_PATH}},
159 }
160
161 # Avoid the full generation call since these tests are expensive;
162 # just check what lora request is actually submitted to the engine
163 mock_err = "Engine is mocked for this test"
164
165 with (
166 mock.patch(
167 "vllm.v1.engine.llm_engine.LLMEngine.add_request",
168 side_effect=MockEngineException(mock_err),
169 ) as mock_add_request,
170 vllm_runner(**vllm_runner_kwargs) as vllm_model,
171 ):
172 # Die once we actually submit the request to the engine
173 with pytest.raises(MockEngineException):
174 vllm_model.llm.generate(prompts=AUDIO_PROMPT)
175
176 # Then check to make sure the submitted lora request
177 # and text prompt were zipped together correctly
178 engine_args, engine_kwargs = mock_add_request.call_args
179 assert engine_args[1]["prompt"] == AUDIO_PROMPT
180 assert engine_kwargs["lora_request"] is None