A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Support Qwen-VL and Qwen-VL-Chat models with text-only inputs (#5710)
Co-authored-by: Roger Wang <ywang@roblox.com>
Jie Fu (傅杰) committed
Jun 22, 2024 at 10:07 UTC
9c62db07ed8ee28d9f1a0e6ac215446d49532008
1 file changed
+10
vllm/model_executor/models/qwen.py
+10
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
28
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
from vllm.sequence import SamplerOutput
31
+from vllm.utils import print_warning_once
32
33
34
class QWenMLP(nn.Module):
@@ -288,6 +289,15 @@ class QWenLMHeadModel(nn.Module):
289
# Skip loading extra bias for GPTQ models.
290
if name.endswith(".bias") and name not in params_dict:
291
continue
292
+ # Skip loading visual weights to support Qwen-VL models
293
+ # in cases with text-only inputs
294
+ # TODO: add support for Qwen-VL
295
+ if (name not in params_dict
296
+ and name.startswith("transformer.visual.")):
297
+ print_warning_once(
298
+ "Only text inputs are allowed. Images won't be handled "
299
+ "until Qwen-VL models are fully supported.")
300
+ continue
301
param = params_dict[name]
302
weight_loader = getattr(param, "weight_loader",
303
default_weight_loader)