A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] remove post_layernorm in siglip (#8106)
wnma committed
Sep 4, 2024 at 18:55 UTC
d3311562fbe740a883e7f03f0b59620587cabb29
1 file changed
+24
-3
vllm/model_executor/models/siglip.py
+24
-3
@@ -443,14 +443,27 @@ class SiglipVisionTransformer(nn.Module):
443
self.config = config
444
embed_dim = config.hidden_size
445
446
+ if (num_hidden_layers_override is None
447
+ or num_hidden_layers_override == config.num_hidden_layers):
448
+ self.need_post_layernorm = True
449
+ elif num_hidden_layers_override > config.num_hidden_layers:
450
+ raise ValueError(
451
+ "num_hidden_layers_override cannot be greater than "
452
+ "num_hidden_layers")
453
+ else:
454
+ self.need_post_layernorm = False
455
+
456
self.embeddings = SiglipVisionEmbeddings(config)
457
self.encoder = SiglipEncoder(
458
config,
459
quant_config=quant_config,
460
num_hidden_layers_override=num_hidden_layers_override,
461
)
452
- self.post_layernorm = nn.LayerNorm(embed_dim,
453
- eps=config.layer_norm_eps)
462
+ if self.need_post_layernorm:
463
+ self.post_layernorm = nn.LayerNorm(embed_dim,
464
+ eps=config.layer_norm_eps)
465
+ else:
466
+ self.post_layernorm = nn.Identity()
467
self.use_head = (True if not hasattr(config, "vision_use_head") else
468
config.vision_use_head)
469
if self.use_head:
@@ -470,7 +483,6 @@ class SiglipVisionTransformer(nn.Module):
483
encoder_outputs = self.encoder(inputs_embeds=hidden_states)
484
485
last_hidden_state = self.post_layernorm(encoder_outputs)
473
-
486
# TODO: add this back when pooled_output is used in inference
487
# if self.use_head:
488
# pooled_output = self.head(last_hidden_state)
@@ -499,6 +511,10 @@ class SiglipVisionModel(nn.Module):
511
num_hidden_layers_override=num_hidden_layers_override,
512
)
513
514
+ @property
515
+ def need_post_layernorm(self):
516
+ return self.vision_model.need_post_layernorm
517
+
518
def get_input_embeddings(self) -> nn.Module:
519
return self.vision_model.embeddings.patch_embedding
520
@@ -517,6 +533,11 @@ class SiglipVisionModel(nn.Module):
533
layer_count = len(self.vision_model.encoder.layers)
534
535
for name, loaded_weight in weights:
536
+ # post_layernorm is optional in SiglipVisionModel
537
+ if ("vision_model.post_layernorm" in name
538
+ and not self.need_post_layernorm):
539
+ continue
540
+
541
# omit layers when num_hidden_layers_override is set
542
if "vision_model.encoder.layers." in name:
543
layer_idx = int(name.split(".")[3])