@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3311562fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] remove post_layernorm in siglip (#8106)

wnma committed Sep 4, 2024 at 18:55 UTC d3311562fbe740a883e7f03f0b59620587cabb29
1 file changed +24 -3
vllm/model_executor/models/siglip.py
+24 -3
@@ -443,14 +443,27 @@ class SiglipVisionTransformer(nn.Module):
443 self.config = config
444 embed_dim = config.hidden_size
445
446 + if (num_hidden_layers_override is None
447 + or num_hidden_layers_override == config.num_hidden_layers):
448 + self.need_post_layernorm = True
449 + elif num_hidden_layers_override > config.num_hidden_layers:
450 + raise ValueError(
451 + "num_hidden_layers_override cannot be greater than "
452 + "num_hidden_layers")
453 + else:
454 + self.need_post_layernorm = False
455 +
456 self.embeddings = SiglipVisionEmbeddings(config)
457 self.encoder = SiglipEncoder(
458 config,
459 quant_config=quant_config,
460 num_hidden_layers_override=num_hidden_layers_override,
461 )
452 - self.post_layernorm = nn.LayerNorm(embed_dim,
453 - eps=config.layer_norm_eps)
462 + if self.need_post_layernorm:
463 + self.post_layernorm = nn.LayerNorm(embed_dim,
464 + eps=config.layer_norm_eps)
465 + else:
466 + self.post_layernorm = nn.Identity()
467 self.use_head = (True if not hasattr(config, "vision_use_head") else
468 config.vision_use_head)
469 if self.use_head:
@@ -470,7 +483,6 @@ class SiglipVisionTransformer(nn.Module):
483 encoder_outputs = self.encoder(inputs_embeds=hidden_states)
484
485 last_hidden_state = self.post_layernorm(encoder_outputs)
473 -
486 # TODO: add this back when pooled_output is used in inference
487 # if self.use_head:
488 # pooled_output = self.head(last_hidden_state)
@@ -499,6 +511,10 @@ class SiglipVisionModel(nn.Module):
511 num_hidden_layers_override=num_hidden_layers_override,
512 )
513
514 + @property
515 + def need_post_layernorm(self):
516 + return self.vision_model.need_post_layernorm
517 +
518 def get_input_embeddings(self) -> nn.Module:
519 return self.vision_model.embeddings.patch_embedding
520
@@ -517,6 +533,11 @@ class SiglipVisionModel(nn.Module):
533 layer_count = len(self.vision_model.encoder.layers)
534
535 for name, loaded_weight in weights:
536 + # post_layernorm is optional in SiglipVisionModel
537 + if ("vision_model.post_layernorm" in name
538 + and not self.need_post_layernorm):
539 + continue
540 +
541 # omit layers when num_hidden_layers_override is set
542 if "vision_model.encoder.layers." in name:
543 layer_idx = int(name.split(".")[3])