A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc] Optimize Qwen2-VL LoRA test (#11663)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 1, 2025 at 14:42 UTC
11d8a091c6c775575a53d37408c94faa0b07730f
2 files changed
+21
-4
tests/lora/test_qwen2vl.py
+2
-3
@@ -7,7 +7,7 @@ from vllm.assets.image import ImageAsset
7
from vllm.lora.request import LoRARequest
8
from vllm.platforms import current_platform
9
10
-MODEL_PATH = "Qwen/Qwen2-VL-7B-Instruct"
10
+MODEL_PATH = "Qwen/Qwen2-VL-2B-Instruct"
11
12
PROMPT_TEMPLATE = (
13
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
@@ -49,10 +49,9 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
49
# Print the outputs.
50
generated_texts: List[str] = []
51
for output in outputs:
52
- prompt = output.prompt
52
generated_text = output.outputs[0].text.strip()
53
generated_texts.append(generated_text)
55
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
54
+ print(f"Generated text: {generated_text!r}")
55
return generated_texts
56
57
vllm/model_executor/models/qwen2_vl.py
+19
-1
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
52
GPTQMarlinConfig)
53
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
54
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
55
+from vllm.model_executor.models.module_mapping import MultiModelKeys
56
from vllm.multimodal import MULTIMODAL_REGISTRY
57
from vllm.multimodal.inputs import (ImageItem, ModalityData,
58
MultiModalFieldConfig, MultiModalKwargs,
@@ -926,15 +927,23 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
927
}
928
929
# LoRA specific attributes
929
- # TODO Support LoRA for the visual encoder in the future.
930
supported_lora_modules = [
931
"qkv_proj",
932
"o_proj",
933
"gate_up_proj",
934
"down_proj",
935
+ # vision tower
936
+ "qkv",
937
+ "attn.proj", # Distinguish patch_embed.proj
938
+ "fc1",
939
+ "fc2",
940
+ # projector
941
+ "mlp.0",
942
+ "mlp.2"
943
]
944
embedding_modules = {}
945
embedding_padding_modules = []
946
+
947
# To ensure correct weight loading and mapping.
948
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
949
"lm_head.": "language_model.lm_head.",
@@ -1231,3 +1240,12 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1240
1241
loader = AutoWeightsLoader(self)
1242
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1243
+
1244
+ def get_mm_mapping(self) -> MultiModelKeys:
1245
+ """
1246
+ Get the module prefix in multimodal models
1247
+ """
1248
+ return MultiModelKeys.from_string_field(
1249
+ language_model="language_model",
1250
+ connector="visual.",
1251
+ tower_model="visual.merger.")