A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix missing `post_layernorm` in CLIP (#8155)
Cyrus Leung committed
Sep 10, 2024 at 16:22 UTC
da1a844e61366b473cef6b3f7437ea5dc41876a1
2 files changed
+42
-19
vllm/model_executor/models/clip.py
+25
-4
@@ -355,6 +355,19 @@ class CLIPVisionTransformer(nn.Module):
355
quant_config=quant_config,
356
num_hidden_layers_override=num_hidden_layers_override)
357
358
+ if len(self.encoder.layers) > config.num_hidden_layers:
359
+ raise ValueError(
360
+ f"The original encoder only has {config.num_hidden_layers} "
361
+ f"layers, but you requested {len(self.encoder.layers)} layers."
362
+ )
363
+ elif len(self.encoder.layers) == config.num_hidden_layers:
364
+ self.post_layernorm = nn.LayerNorm(embed_dim,
365
+ eps=config.layer_norm_eps)
366
+ else:
367
+ # post_layernorm is unused when we extract intermediate features
368
+ # In this case, we can skip it to conserve memory
369
+ self.post_layernorm = None
370
+
371
def forward(
372
self,
373
pixel_values: torch.Tensor,
@@ -364,7 +377,10 @@ class CLIPVisionTransformer(nn.Module):
377
hidden_states = self.pre_layrnorm(hidden_states)
378
hidden_states = self.encoder(inputs_embeds=hidden_states)
379
367
- return hidden_states
380
+ if self.post_layernorm is None:
381
+ return hidden_states
382
+
383
+ return self.post_layernorm(hidden_states)
384
385
386
class CLIPVisionModel(nn.Module):
@@ -386,9 +402,12 @@ class CLIPVisionModel(nn.Module):
402
quant_config=quant_config,
403
num_hidden_layers_override=num_hidden_layers_override)
404
389
- def forward(self, pixel_values: Optional[torch.Tensor] = None):
405
+ @property
406
+ def _require_post_layernorm(self) -> bool:
407
+ return self.vision_model.post_layernorm is not None
408
391
- return self.vision_model(pixel_values=pixel_values)
409
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
410
+ return self.vision_model(pixel_values)
411
412
@property
413
def device(self):
@@ -408,8 +427,10 @@ class CLIPVisionModel(nn.Module):
427
428
for name, loaded_weight in weights:
429
# post_layernorm is not needed in CLIPVisionModel
411
- if "vision_model.post_layernorm" in name:
430
+ if ("vision_model.post_layernorm" in name
431
+ and not self._require_post_layernorm):
432
continue
433
+
434
# omit layers when num_hidden_layers_override is set
435
if "vision_model.encoder.layers." in name:
436
layer_idx = int(name.split(".")[3])
vllm/model_executor/models/siglip.py
+17
-15
@@ -443,27 +443,26 @@ class SiglipVisionTransformer(nn.Module):
443
self.config = config
444
embed_dim = config.hidden_size
445
446
- if (num_hidden_layers_override is None
447
- or num_hidden_layers_override == config.num_hidden_layers):
448
- self.need_post_layernorm = True
449
- elif num_hidden_layers_override > config.num_hidden_layers:
450
- raise ValueError(
451
- "num_hidden_layers_override cannot be greater than "
452
- "num_hidden_layers")
453
- else:
454
- self.need_post_layernorm = False
455
-
446
self.embeddings = SiglipVisionEmbeddings(config)
447
self.encoder = SiglipEncoder(
448
config,
449
quant_config=quant_config,
450
num_hidden_layers_override=num_hidden_layers_override,
451
)
462
- if self.need_post_layernorm:
452
+
453
+ if len(self.encoder.layers) > config.num_hidden_layers:
454
+ raise ValueError(
455
+ f"The original encoder only has {config.num_hidden_layers} "
456
+ f"layers, but you requested {len(self.encoder.layers)} layers."
457
+ )
458
+ elif len(self.encoder.layers) == config.num_hidden_layers:
459
self.post_layernorm = nn.LayerNorm(embed_dim,
460
eps=config.layer_norm_eps)
461
else:
466
- self.post_layernorm = nn.Identity()
462
+ # post_layernorm is unused when we extract intermediate features
463
+ # In this case, we can skip it to conserve memory
464
+ self.post_layernorm = None
465
+
466
self.use_head = (True if not hasattr(config, "vision_use_head") else
467
config.vision_use_head)
468
if self.use_head:
@@ -482,6 +481,9 @@ class SiglipVisionTransformer(nn.Module):
481
482
encoder_outputs = self.encoder(inputs_embeds=hidden_states)
483
484
+ if self.post_layernorm is None:
485
+ return encoder_outputs
486
+
487
last_hidden_state = self.post_layernorm(encoder_outputs)
488
# TODO: add this back when pooled_output is used in inference
489
# if self.use_head:
@@ -512,8 +514,8 @@ class SiglipVisionModel(nn.Module):
514
)
515
516
@property
515
- def need_post_layernorm(self):
516
- return self.vision_model.need_post_layernorm
517
+ def _require_post_layernorm(self) -> bool:
518
+ return self.vision_model.post_layernorm is not None
519
520
def get_input_embeddings(self) -> nn.Module:
521
return self.vision_model.embeddings.patch_embedding
@@ -541,7 +543,7 @@ class SiglipVisionModel(nn.Module):
543
for name, loaded_weight in weights:
544
# post_layernorm is optional in SiglipVisionModel
545
if ("vision_model.post_layernorm" in name
544
- and not self.need_post_layernorm):
546
+ and not self._require_post_layernorm):
547
continue
548
549
# omit layers when num_hidden_layers_override is set