@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 11d8a091c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Misc] Optimize Qwen2-VL LoRA test (#11663)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 1, 2025 at 14:42 UTC 11d8a091c6c775575a53d37408c94faa0b07730f
2 files changed +21 -4
tests/lora/test_qwen2vl.py
+2 -3
@@ -7,7 +7,7 @@ from vllm.assets.image import ImageAsset
7 from vllm.lora.request import LoRARequest
8 from vllm.platforms import current_platform
9
10 -MODEL_PATH = "Qwen/Qwen2-VL-7B-Instruct"
10 +MODEL_PATH = "Qwen/Qwen2-VL-2B-Instruct"
11
12 PROMPT_TEMPLATE = (
13 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
@@ -49,10 +49,9 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
49 # Print the outputs.
50 generated_texts: List[str] = []
51 for output in outputs:
52 - prompt = output.prompt
52 generated_text = output.outputs[0].text.strip()
53 generated_texts.append(generated_text)
55 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
54 + print(f"Generated text: {generated_text!r}")
55 return generated_texts
56
57
vllm/model_executor/models/qwen2_vl.py
+19 -1
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
52 GPTQMarlinConfig)
53 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
54 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
55 +from vllm.model_executor.models.module_mapping import MultiModelKeys
56 from vllm.multimodal import MULTIMODAL_REGISTRY
57 from vllm.multimodal.inputs import (ImageItem, ModalityData,
58 MultiModalFieldConfig, MultiModalKwargs,
@@ -926,15 +927,23 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
927 }
928
929 # LoRA specific attributes
929 - # TODO Support LoRA for the visual encoder in the future.
930 supported_lora_modules = [
931 "qkv_proj",
932 "o_proj",
933 "gate_up_proj",
934 "down_proj",
935 + # vision tower
936 + "qkv",
937 + "attn.proj", # Distinguish patch_embed.proj
938 + "fc1",
939 + "fc2",
940 + # projector
941 + "mlp.0",
942 + "mlp.2"
943 ]
944 embedding_modules = {}
945 embedding_padding_modules = []
946 +
947 # To ensure correct weight loading and mapping.
948 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
949 "lm_head.": "language_model.lm_head.",
@@ -1231,3 +1240,12 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1240
1241 loader = AutoWeightsLoader(self)
1242 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1243 +
1244 + def get_mm_mapping(self) -> MultiModelKeys:
1245 + """
1246 + Get the module prefix in multimodal models
1247 + """
1248 + return MultiModelKeys.from_string_field(
1249 + language_model="language_model",
1250 + connector="visual.",
1251 + tower_model="visual.merger.")