A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Model] Use in-place adds in SigLIP (#18922)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
May 30, 2025 at 10:12 UTC
c3bb9f23315f8cdc5007349717fd58d16a833ff6
1 file changed
+6
-6
vllm/model_executor/models/siglip.py
+6
-6
@@ -130,11 +130,10 @@ class SiglipVisionEmbeddings(nn.Module):
130
embeddings = patch_embeds.flatten(2).transpose(1, 2)
131
132
if interpolate_pos_encoding:
133
- embeddings = embeddings + self.interpolate_pos_encoding(
133
+ embeddings += self.interpolate_pos_encoding(
134
embeddings, height, width)
135
else:
136
- embeddings = embeddings + self.position_embedding(
137
- self.position_ids)
136
+ embeddings += self.position_embedding(self.position_ids)
137
return embeddings
138
139
@@ -271,12 +270,12 @@ class SiglipEncoderLayer(nn.Module):
270
271
hidden_states = self.layer_norm1(hidden_states)
272
hidden_states, _ = self.self_attn(hidden_states=hidden_states)
274
- hidden_states = residual + hidden_states
273
+ hidden_states += residual
274
275
residual = hidden_states
276
hidden_states = self.layer_norm2(hidden_states)
277
hidden_states = self.mlp(hidden_states)
279
- hidden_states = residual + hidden_states
278
+ hidden_states += residual
279
280
return hidden_states, None
281
@@ -354,7 +353,8 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
353
354
residual = hidden_state
355
hidden_state = self.layernorm(hidden_state)
357
- hidden_state = residual + self.mlp(hidden_state)
356
+ hidden_state = self.mlp(hidden_state)
357
+ hidden_state += residual
358
359
return hidden_state[:, 0]
360