A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] Remove unused code in attention (#2384)
Woosuk Kwon committed
Jan 8, 2024 at 13:13 UTC
28c3f121040dd80d3540fb7d36e0b7a4817da28c
1 file changed
+9
-14
vllm/model_executor/layers/attention.py
+9
-14
@@ -156,20 +156,15 @@ class PagedAttention(nn.Module):
156
output = out.view_as(query)
157
else:
158
# Decoding run.
159
- if key_cache is not None and value_cache is not None:
160
- output = _paged_attention(
161
- query,
162
- key_cache,
163
- value_cache,
164
- input_metadata,
165
- self.num_kv_heads,
166
- self.scale,
167
- self.alibi_slopes,
168
- )
169
- else:
170
- # This happens during the initial memory profiling run for
171
- # CUDA graphs.
172
- output = torch.zeros_like(query)
159
+ output = _paged_attention(
160
+ query,
161
+ key_cache,
162
+ value_cache,
163
+ input_metadata,
164
+ self.num_kv_heads,
165
+ self.scale,
166
+ self.alibi_slopes,
167
+ )
168
169
# Reshape the output tensor.
170
return output.view(batch_size, seq_len, hidden_size)