@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28c3f12104
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Remove unused code in attention (#2384)

Woosuk Kwon committed Jan 8, 2024 at 13:13 UTC 28c3f121040dd80d3540fb7d36e0b7a4817da28c
1 file changed +9 -14
vllm/model_executor/layers/attention.py
+9 -14
@@ -156,20 +156,15 @@ class PagedAttention(nn.Module):
156 output = out.view_as(query)
157 else:
158 # Decoding run.
159 - if key_cache is not None and value_cache is not None:
160 - output = _paged_attention(
161 - query,
162 - key_cache,
163 - value_cache,
164 - input_metadata,
165 - self.num_kv_heads,
166 - self.scale,
167 - self.alibi_slopes,
168 - )
169 - else:
170 - # This happens during the initial memory profiling run for
171 - # CUDA graphs.
172 - output = torch.zeros_like(query)
159 + output = _paged_attention(
160 + query,
161 + key_cache,
162 + value_cache,
163 + input_metadata,
164 + self.num_kv_heads,
165 + self.scale,
166 + self.alibi_slopes,
167 + )
168
169 # Reshape the output tensor.
170 return output.view(batch_size, seq_len, hidden_size)