A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Kernel] Revert the API change of Attention.forward (#12038)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Jan 14, 2025 at 20:59 UTC
1f18adb2451e9b45048d17023169bd8cbb39747e
1 file changed
+2
-2
vllm/attention/layer.py
+2
-2
@@ -134,8 +134,8 @@ class Attention(nn.Module):
134
query: torch.Tensor,
135
key: torch.Tensor,
136
value: torch.Tensor,
137
- _kv_cache: torch.Tensor,
138
- _attn_metadata: AttentionMetadata,
137
+ kv_cache: torch.Tensor,
138
+ attn_metadata: AttentionMetadata,
139
) -> torch.Tensor:
140
if self.use_output:
141
output = torch.empty_like(query)