A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix `_init_vision_model` in NVLM_D model (#9611)
Co-authored-by: Isotr0py <2037008807@qq.com>
Cyrus Leung committed
Oct 23, 2024 at 22:09 UTC
e7116c017c86cb547f4d1888edaf13a9be2a4562
1 file changed
+28
-9
vllm/model_executor/models/nvlm_d.py
+28
-9
@@ -58,12 +58,31 @@ class NVLM_D_Model(InternVLChatModel):
58
nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False),
59
)
60
61
- def _init_vision_model(self, config: PretrainedConfig,
62
- quant_config: Optional[QuantizationConfig],
63
- num_hidden_layers: int):
64
- # We added additional dummy heads to the original num of heads to make
65
- # the number of heads divisible by 8.
66
- return InternVisionModel(config.vision_config,
67
- quant_config=quant_config,
68
- num_hidden_layers_override=num_hidden_layers,
69
- num_dummy_heads=7)
61
+ def _init_vision_model(
62
+ self,
63
+ config: PretrainedConfig,
64
+ quant_config: Optional[QuantizationConfig],
65
+ *,
66
+ is_mono: bool,
67
+ prefix: str,
68
+ ):
69
+ if not is_mono:
70
+ vision_feature_layer = config.select_layer
71
+ if vision_feature_layer < 0:
72
+ num_hidden_layers = config.vision_config.num_hidden_layers \
73
+ + vision_feature_layer + 1
74
+ else:
75
+ num_hidden_layers = vision_feature_layer + 1
76
+
77
+ # We added additional dummy heads to the original num of heads to
78
+ # make the number of heads divisible by 8.
79
+ return InternVisionModel(
80
+ config.vision_config,
81
+ quant_config=quant_config,
82
+ num_hidden_layers_override=num_hidden_layers,
83
+ num_dummy_heads=7,
84
+ prefix=prefix,
85
+ )
86
+ else:
87
+ msg = "Monolith mode is not applicable to NVLM_D"
88
+ raise NotImplementedError(msg)