@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f58454968f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix]Disable the post_norm layer of the vision encoder for LLaVA models (#9653)

litianjian committed Oct 24, 2024 at 22:52 UTC f58454968fe1c5ddf84199b341a6ed5c99f0c0cc
4 files changed +8 -4
vllm/model_executor/models/llava.py
+2 -1
@@ -273,7 +273,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
273 config.projector_hidden_act = "gelu"
274
275 # TODO: Optionally initializes this for supporting embeddings.
276 - self.vision_tower = init_vision_tower_for_llava(config, quant_config)
276 + self.vision_tower = init_vision_tower_for_llava(
277 + config, quant_config, require_post_norm=False)
278 self.multi_modal_projector = LlavaMultiModalProjector(
279 vision_hidden_size=config.vision_config.hidden_size,
280 text_hidden_size=config.text_config.hidden_size,
vllm/model_executor/models/llava_next.py
+2 -1
@@ -277,7 +277,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
277 self.multimodal_config = multimodal_config
278
279 # TODO: Optionally initializes this for supporting embeddings.
280 - self.vision_tower = init_vision_tower_for_llava(config, quant_config)
280 + self.vision_tower = init_vision_tower_for_llava(
281 + config, quant_config, require_post_norm=False)
282 self.image_newline = nn.Parameter(
283 torch.empty(config.text_config.hidden_size))
284 self.multi_modal_projector = LlavaMultiModalProjector(
vllm/model_executor/models/llava_next_video.py
+2 -1
@@ -256,7 +256,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
256 self.multimodal_config = multimodal_config
257
258 # Initialize the vision tower only up to the required feature layer
259 - self.vision_tower = init_vision_tower_for_llava(config, quant_config)
259 + self.vision_tower = init_vision_tower_for_llava(
260 + config, quant_config, require_post_norm=False)
261 self.vision_resampler = LlavaNextVideoPooler(config)
262 self.multi_modal_projector = LlavaNextMultiModalProjector(
263 vision_hidden_size=config.vision_config.hidden_size,
vllm/model_executor/models/llava_onevision.py
+2 -1
@@ -400,7 +400,8 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
400 self.multimodal_config = multimodal_config
401
402 # Initialize the vision tower only up to the required feature layer
403 - self.vision_tower = init_vision_tower_for_llava(config, quant_config)
403 + self.vision_tower = init_vision_tower_for_llava(
404 + config, quant_config, require_post_norm=False)
405 self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
406 self.language_model = init_vllm_registered_model(
407 config.text_config, cache_config, quant_config)