@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cbcbdb1ceb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix][Hardware][Gaudi] Bump vllm_hpu_extension version (#11028)

Signed-off-by: Konrad Zawora <kzawora@habana.ai>

Konrad Zawora committed Dec 9, 2024 at 22:21 UTC cbcbdb1ceb9c219d13b2386e101992c399410551
2 files changed +12 -1
requirements-hpu.txt
+1 -1
@@ -8,4 +8,4 @@ pandas
8 tabulate
9 setuptools>=61
10 setuptools-scm>=8
11 -vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@fd7f2e6
11 +vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@e096d6f
vllm/attention/backends/hpu_attn.py
+11
@@ -111,8 +111,16 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
111 self.matmul_qk = Matmul()
112 self.softmax = Softmax()
113 self.matmul_av = Matmul()
114 + self.batch2block_matmul = Matmul()
115 + self.block2batch_matmul = Matmul()
116 + # NOTE(kzawora): Contiguous PA is off until model runner supports it
117 self.k_cache = VLLMKVCache()
118 + self.k_cache.use_contiguous_pa = False
119 self.v_cache = VLLMKVCache()
120 + self.v_cache.use_contiguous_pa = False
121 + # NOTE(kzawora): Pipelined PA is off until model runner supports it
122 + ops.pa_impl = ops.pa
123 +
124 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
125 self.sliding_window = sliding_window
126 self.alibi_slopes = alibi_slopes
@@ -228,9 +236,12 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
236 block_mapping=attn_metadata.block_mapping,
237 block_bias=attn_metadata.attn_bias,
238 block_scales=attn_metadata.block_scales,
239 + block_groups=None,
240 scale=self.scale,
241 matmul_qk_op=self.matmul_qk,
242 matmul_av_op=self.matmul_av,
243 + batch2block_matmul_op=self.batch2block_matmul,
244 + block2batch_matmul_op=self.block2batch_matmul,
245 keys_fetch_func=self.k_cache.fetch_from_cache,
246 values_fetch_func=self.v_cache.fetch_from_cache)
247 # Reshape the output tensor.