A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix]Disable the post_norm layer of the vision encoder for LLaVA models (#9653)
litianjian committed
Oct 24, 2024 at 22:52 UTC
f58454968fe1c5ddf84199b341a6ed5c99f0c0cc
4 files changed
+8
-4
vllm/model_executor/models/llava.py
+2
-1
@@ -273,7 +273,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
273
config.projector_hidden_act = "gelu"
274
275
# TODO: Optionally initializes this for supporting embeddings.
276
- self.vision_tower = init_vision_tower_for_llava(config, quant_config)
276
+ self.vision_tower = init_vision_tower_for_llava(
277
+ config, quant_config, require_post_norm=False)
278
self.multi_modal_projector = LlavaMultiModalProjector(
279
vision_hidden_size=config.vision_config.hidden_size,
280
text_hidden_size=config.text_config.hidden_size,
vllm/model_executor/models/llava_next.py
+2
-1
@@ -277,7 +277,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
277
self.multimodal_config = multimodal_config
278
279
# TODO: Optionally initializes this for supporting embeddings.
280
- self.vision_tower = init_vision_tower_for_llava(config, quant_config)
280
+ self.vision_tower = init_vision_tower_for_llava(
281
+ config, quant_config, require_post_norm=False)
282
self.image_newline = nn.Parameter(
283
torch.empty(config.text_config.hidden_size))
284
self.multi_modal_projector = LlavaMultiModalProjector(
vllm/model_executor/models/llava_next_video.py
+2
-1
@@ -256,7 +256,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
256
self.multimodal_config = multimodal_config
257
258
# Initialize the vision tower only up to the required feature layer
259
- self.vision_tower = init_vision_tower_for_llava(config, quant_config)
259
+ self.vision_tower = init_vision_tower_for_llava(
260
+ config, quant_config, require_post_norm=False)
261
self.vision_resampler = LlavaNextVideoPooler(config)
262
self.multi_modal_projector = LlavaNextMultiModalProjector(
263
vision_hidden_size=config.vision_config.hidden_size,
vllm/model_executor/models/llava_onevision.py
+2
-1
@@ -400,7 +400,8 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
400
self.multimodal_config = multimodal_config
401
402
# Initialize the vision tower only up to the required feature layer
403
- self.vision_tower = init_vision_tower_for_llava(config, quant_config)
403
+ self.vision_tower = init_vision_tower_for_llava(
404
+ config, quant_config, require_post_norm=False)
405
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
406
self.language_model = init_vllm_registered_model(
407
config.text_config, cache_config, quant_config)