@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9c62db07ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Support Qwen-VL and Qwen-VL-Chat models with text-only inputs (#5710)

Co-authored-by: Roger Wang <ywang@roblox.com>

Jie Fu (傅杰) committed Jun 22, 2024 at 10:07 UTC 9c62db07ed8ee28d9f1a0e6ac215446d49532008
1 file changed +10
vllm/model_executor/models/qwen.py
+10
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
28 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 from vllm.model_executor.sampling_metadata import SamplingMetadata
30 from vllm.sequence import SamplerOutput
31 +from vllm.utils import print_warning_once
32
33
34 class QWenMLP(nn.Module):
@@ -288,6 +289,15 @@ class QWenLMHeadModel(nn.Module):
289 # Skip loading extra bias for GPTQ models.
290 if name.endswith(".bias") and name not in params_dict:
291 continue
292 + # Skip loading visual weights to support Qwen-VL models
293 + # in cases with text-only inputs
294 + # TODO: add support for Qwen-VL
295 + if (name not in params_dict
296 + and name.startswith("transformer.visual.")):
297 + print_warning_once(
298 + "Only text inputs are allowed. Images won't be handled "
299 + "until Qwen-VL models are fully supported.")
300 + continue
301 param = params_dict[name]
302 weight_loader = getattr(param, "weight_loader",
303 default_weight_loader)