@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da1a844e61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix missing `post_layernorm` in CLIP (#8155)

Cyrus Leung committed Sep 10, 2024 at 16:22 UTC da1a844e61366b473cef6b3f7437ea5dc41876a1
2 files changed +42 -19
vllm/model_executor/models/clip.py
+25 -4
@@ -355,6 +355,19 @@ class CLIPVisionTransformer(nn.Module):
355 quant_config=quant_config,
356 num_hidden_layers_override=num_hidden_layers_override)
357
358 + if len(self.encoder.layers) > config.num_hidden_layers:
359 + raise ValueError(
360 + f"The original encoder only has {config.num_hidden_layers} "
361 + f"layers, but you requested {len(self.encoder.layers)} layers."
362 + )
363 + elif len(self.encoder.layers) == config.num_hidden_layers:
364 + self.post_layernorm = nn.LayerNorm(embed_dim,
365 + eps=config.layer_norm_eps)
366 + else:
367 + # post_layernorm is unused when we extract intermediate features
368 + # In this case, we can skip it to conserve memory
369 + self.post_layernorm = None
370 +
371 def forward(
372 self,
373 pixel_values: torch.Tensor,
@@ -364,7 +377,10 @@ class CLIPVisionTransformer(nn.Module):
377 hidden_states = self.pre_layrnorm(hidden_states)
378 hidden_states = self.encoder(inputs_embeds=hidden_states)
379
367 - return hidden_states
380 + if self.post_layernorm is None:
381 + return hidden_states
382 +
383 + return self.post_layernorm(hidden_states)
384
385
386 class CLIPVisionModel(nn.Module):
@@ -386,9 +402,12 @@ class CLIPVisionModel(nn.Module):
402 quant_config=quant_config,
403 num_hidden_layers_override=num_hidden_layers_override)
404
389 - def forward(self, pixel_values: Optional[torch.Tensor] = None):
405 + @property
406 + def _require_post_layernorm(self) -> bool:
407 + return self.vision_model.post_layernorm is not None
408
391 - return self.vision_model(pixel_values=pixel_values)
409 + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
410 + return self.vision_model(pixel_values)
411
412 @property
413 def device(self):
@@ -408,8 +427,10 @@ class CLIPVisionModel(nn.Module):
427
428 for name, loaded_weight in weights:
429 # post_layernorm is not needed in CLIPVisionModel
411 - if "vision_model.post_layernorm" in name:
430 + if ("vision_model.post_layernorm" in name
431 + and not self._require_post_layernorm):
432 continue
433 +
434 # omit layers when num_hidden_layers_override is set
435 if "vision_model.encoder.layers." in name:
436 layer_idx = int(name.split(".")[3])
vllm/model_executor/models/siglip.py
+17 -15
@@ -443,27 +443,26 @@ class SiglipVisionTransformer(nn.Module):
443 self.config = config
444 embed_dim = config.hidden_size
445
446 - if (num_hidden_layers_override is None
447 - or num_hidden_layers_override == config.num_hidden_layers):
448 - self.need_post_layernorm = True
449 - elif num_hidden_layers_override > config.num_hidden_layers:
450 - raise ValueError(
451 - "num_hidden_layers_override cannot be greater than "
452 - "num_hidden_layers")
453 - else:
454 - self.need_post_layernorm = False
455 -
446 self.embeddings = SiglipVisionEmbeddings(config)
447 self.encoder = SiglipEncoder(
448 config,
449 quant_config=quant_config,
450 num_hidden_layers_override=num_hidden_layers_override,
451 )
462 - if self.need_post_layernorm:
452 +
453 + if len(self.encoder.layers) > config.num_hidden_layers:
454 + raise ValueError(
455 + f"The original encoder only has {config.num_hidden_layers} "
456 + f"layers, but you requested {len(self.encoder.layers)} layers."
457 + )
458 + elif len(self.encoder.layers) == config.num_hidden_layers:
459 self.post_layernorm = nn.LayerNorm(embed_dim,
460 eps=config.layer_norm_eps)
461 else:
466 - self.post_layernorm = nn.Identity()
462 + # post_layernorm is unused when we extract intermediate features
463 + # In this case, we can skip it to conserve memory
464 + self.post_layernorm = None
465 +
466 self.use_head = (True if not hasattr(config, "vision_use_head") else
467 config.vision_use_head)
468 if self.use_head:
@@ -482,6 +481,9 @@ class SiglipVisionTransformer(nn.Module):
481
482 encoder_outputs = self.encoder(inputs_embeds=hidden_states)
483
484 + if self.post_layernorm is None:
485 + return encoder_outputs
486 +
487 last_hidden_state = self.post_layernorm(encoder_outputs)
488 # TODO: add this back when pooled_output is used in inference
489 # if self.use_head:
@@ -512,8 +514,8 @@ class SiglipVisionModel(nn.Module):
514 )
515
516 @property
515 - def need_post_layernorm(self):
516 - return self.vision_model.need_post_layernorm
517 + def _require_post_layernorm(self) -> bool:
518 + return self.vision_model.post_layernorm is not None
519
520 def get_input_embeddings(self) -> nn.Module:
521 return self.vision_model.embeddings.patch_embedding
@@ -541,7 +543,7 @@ class SiglipVisionModel(nn.Module):
543 for name, loaded_weight in weights:
544 # post_layernorm is optional in SiglipVisionModel
545 if ("vision_model.post_layernorm" in name
544 - and not self.need_post_layernorm):
546 + and not self._require_post_layernorm):
547 continue
548
549 # omit layers when num_hidden_layers_override is set