A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 02:30 UTC
[Bugfix] Fix lora tests (#34834)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Cyrus Leung committed
Feb 19, 2026 at 05:22 UTC
61cf0876805c1ac04b9b811c27e3645eedfc13c8
3 files changed
+28
-44
tests/lora/test_default_mm_loras.py
+1
-1
@@ -153,5 +153,5 @@ def test_default_mm_lora_does_not_expand_string_reqs(vllm_runner):
153
# Then check to make sure the submitted lora request
154
# and text prompt were zipped together correctly
155
engine_args, engine_kwargs = mock_add_request.call_args
156
+ assert engine_args[1]["prompt"] == AUDIO_PROMPT
157
assert engine_kwargs["lora_request"] is None
157
- assert engine_kwargs["prompt_text"] == AUDIO_PROMPT
tests/lora/test_qwenvl.py
+10
-9
@@ -88,9 +88,8 @@ class Qwen2VLTester:
88
# Validate outputs
89
for generated, expected in zip(generated_texts, expected_outputs):
90
assert expected.startswith(generated), (
91
- f"Generated text {generated} doesn't "
91
+ f"Generated text {generated} doesn't match expected pattern {expected}"
92
)
93
- f"match expected pattern {expected}"
93
94
def run_beam_search_test(
95
self,
@@ -118,11 +117,14 @@ class Qwen2VLTester:
117
inputs, beam_search_params, lora_request=lora_request
118
)
119
121
- for output_obj, expected_outs in zip(outputs, expected_outputs):
120
+ for output_obj, expected_texts in zip(outputs, expected_outputs):
121
output_texts = [seq.text for seq in output_obj.sequences]
123
- assert output_texts == expected_outs, (
124
- f"Generated texts {output_texts} do not match expected {expected_outs}"
125
- ) # noqa: E501
122
+
123
+ for output_text, expected_text in zip(output_texts, expected_texts):
124
+ # NOTE beam search .text contains the whole text including inputs
125
+ assert output_text.endswith(expected_text), (
126
+ f"Generated {output_text} does not match expected {expected_text}"
127
+ )
128
129
130
TEST_IMAGES = [
@@ -151,11 +153,10 @@ EXPECTED_OUTPUTS_VISION_NO_CONNECTOR = [
153
"A closeup shot of the Tokyo Skytree with pink flowers in the foreground.",
154
]
155
154
-# NOTE - beam search .text contains the whole text
156
EXPECTED_BEAM_SEARCH_OUTPUTS = [
157
[
157
- "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>What is in the image?<|im_end|>\n<|im_start|>assistant\nA majestic skyscraper stands", # noqa: E501
158
- "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>What is in the image?<|im_end|>\n<|im_start|>assistant\nA majestic tower stands tall", # noqa: E501
158
+ "A majestic skyscraper stands",
159
+ "A majestic tower stands tall",
160
],
161
]
162
vllm/entrypoints/llm.py
+17
-34
@@ -542,51 +542,31 @@ class LLM:
542
outputs = self._run_engine(use_tqdm=use_tqdm)
543
return self.engine_class.validate_outputs(outputs, RequestOutput)
544
545
- def _resolve_lora_reqs(
546
- self,
547
- prompts: Sequence[ProcessorInputs],
548
- lora_request: Sequence[LoRARequest | None] | LoRARequest | None,
549
- ):
550
- lora_config = self.llm_engine.vllm_config.lora_config
551
- seq_lora_requests = self._lora_request_to_seq(lora_request, len(prompts))
552
-
553
- if (
554
- lora_config is None
555
- or not self.model_config.is_multimodal_model
556
- or (lora_config and lora_config.default_mm_loras is None)
557
- ):
558
- return seq_lora_requests
559
-
560
- return [
561
- self._resolve_single_prompt_mm_lora(
562
- prompt,
563
- lora_req,
564
- lora_config.default_mm_loras,
565
- )
566
- for prompt, lora_req in zip(prompts, seq_lora_requests)
567
- ]
568
-
569
- def _resolve_single_prompt_mm_lora(
545
+ def _resolve_mm_lora(
546
self,
547
prompt: ProcessorInputs,
548
lora_request: LoRARequest | None,
573
- default_mm_loras: dict[str, str] | None,
574
- ):
575
- if not default_mm_loras or prompt["type"] != "multimodal":
549
+ ) -> LoRARequest | None:
550
+ if prompt["type"] != "multimodal":
551
+ return lora_request
552
+
553
+ lora_config = self.llm_engine.vllm_config.lora_config
554
+ default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
555
+ if not default_mm_loras:
556
return lora_request
557
558
prompt_modalities = prompt["mm_placeholders"].keys()
559
intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
560
if not intersection:
561
return lora_request
562
+
563
if len(intersection) > 1:
564
# TODO: Would be nice to be able to have multiple loras per prompt
565
logger.warning(
585
- "Multiple modality specific loras were registered and would be"
586
- " used by a single prompt consuming several modalities; "
587
- " currently we only support one lora per request; as such,"
588
- " lora(s) registered with modalities: %s"
589
- " will be skipped",
566
+ "Multiple modality specific loras were registered and would be "
567
+ "used by a single prompt consuming several modalities; "
568
+ "currently we only support one lora per request; as such, "
569
+ "lora(s) registered with modalities: %s will be skipped",
570
intersection,
571
)
572
return lora_request
@@ -1915,7 +1895,10 @@ class LLM:
1895
request_id = self._add_request(
1896
prompt,
1897
params[i],
1918
- lora_request=None if lora_requests is None else lora_requests[i],
1898
+ lora_request=self._resolve_mm_lora(
1899
+ prompt,
1900
+ None if lora_requests is None else lora_requests[i],
1901
+ ),
1902
priority=0 if priorities is None else priorities[i],
1903
)
1904
added_request_ids.append(request_id)