@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b2e0ad3b59
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Perf] Reduce peak memory usage of llama (#10339)

Signed-off-by: andoorve <37849411+andoorve@users.noreply.github.com>

Murali Andoorveedu committed Nov 14, 2024 at 16:38 UTC b2e0ad3b598ed0e022cdbd678a20821d411873c2
1 file changed +2 -2
vllm/model_executor/models/llama.py
+2 -2
@@ -90,8 +90,8 @@ class LlamaMLP(nn.Module):
90 self.act_fn = SiluAndMul()
91
92 def forward(self, x):
93 - gate_up, _ = self.gate_up_proj(x)
94 - x = self.act_fn(gate_up)
93 + x, _ = self.gate_up_proj(x)
94 + x = self.act_fn(x)
95 x, _ = self.down_proj(x)
96 return x
97