A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[TPU] Update pallas.py to support trillium (#8871)
Brittany committed
Sep 27, 2024 at 01:16 UTC
8df2dc3c8812c0abb97ce3e2913411d88524e59f
1 file changed
+1
-1
vllm/attention/backends/pallas.py
+1
-1
@@ -130,7 +130,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
130
assert tpu_type is not None
131
tpu_type = tpu_type.lower()
132
133
- if "lite" not in tpu_type:
133
+ if (("lite" not in tpu_type) and ("v6" not in tpu_type)):
134
if self.num_kv_heads % 2 == 0:
135
self.megacore_mode = "kv_head"
136
else: