A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix][Hardware][Gaudi] Bump vllm_hpu_extension version (#11028)
Signed-off-by: Konrad Zawora <kzawora@habana.ai>
Konrad Zawora committed
Dec 9, 2024 at 22:21 UTC
cbcbdb1ceb9c219d13b2386e101992c399410551
2 files changed
+12
-1
requirements-hpu.txt
+1
-1
@@ -8,4 +8,4 @@ pandas
8
tabulate
9
setuptools>=61
10
setuptools-scm>=8
11
-vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@fd7f2e6
11
+vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@e096d6f
vllm/attention/backends/hpu_attn.py
+11
@@ -111,8 +111,16 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
111
self.matmul_qk = Matmul()
112
self.softmax = Softmax()
113
self.matmul_av = Matmul()
114
+ self.batch2block_matmul = Matmul()
115
+ self.block2batch_matmul = Matmul()
116
+ # NOTE(kzawora): Contiguous PA is off until model runner supports it
117
self.k_cache = VLLMKVCache()
118
+ self.k_cache.use_contiguous_pa = False
119
self.v_cache = VLLMKVCache()
120
+ self.v_cache.use_contiguous_pa = False
121
+ # NOTE(kzawora): Pipelined PA is off until model runner supports it
122
+ ops.pa_impl = ops.pa
123
+
124
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
125
self.sliding_window = sliding_window
126
self.alibi_slopes = alibi_slopes
@@ -228,9 +236,12 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
236
block_mapping=attn_metadata.block_mapping,
237
block_bias=attn_metadata.attn_bias,
238
block_scales=attn_metadata.block_scales,
239
+ block_groups=None,
240
scale=self.scale,
241
matmul_qk_op=self.matmul_qk,
242
matmul_av_op=self.matmul_av,
243
+ batch2block_matmul_op=self.batch2block_matmul,
244
+ block2batch_matmul_op=self.block2batch_matmul,
245
keys_fetch_func=self.k_cache.fetch_from_cache,
246
values_fetch_func=self.v_cache.fetch_from_cache)
247
# Reshape the output tensor.