A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix RMSNorm forward in InternViT attention qk_layernorm (#6992)
Isotr0py committed
Aug 2, 2024 at 03:00 UTC
2dd34371a6054966d30971dae89b0c431d7f0f08
1 file changed
+4
-4
vllm/model_executor/models/intern_vit.py
+4
-4
@@ -113,10 +113,10 @@ class InternAttention(nn.Module):
113
114
if self.qk_normalization:
115
B_, H_, N_, D_ = q.shape
116
- q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(
117
- B_, N_, H_, D_).transpose(1, 2)
118
- k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(
119
- B_, N_, H_, D_).transpose(1, 2)
116
+ q = self.q_norm.forward_native(q.transpose(1, 2).flatten(
117
+ -2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
118
+ k = self.k_norm.forward_native(k.transpose(1, 2).flatten(
119
+ -2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
120
121
x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
122
x = x.transpose(1, 2).reshape(B, N, C)