@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8df2dc3c88
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[TPU] Update pallas.py to support trillium (#8871)

Brittany committed Sep 27, 2024 at 01:16 UTC 8df2dc3c8812c0abb97ce3e2913411d88524e59f
1 file changed +1 -1
vllm/attention/backends/pallas.py
+1 -1
@@ -130,7 +130,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
130 assert tpu_type is not None
131 tpu_type = tpu_type.lower()
132
133 - if "lite" not in tpu_type:
133 + if (("lite" not in tpu_type) and ("v6" not in tpu_type)):
134 if self.num_kv_heads % 2 == 0:
135 self.megacore_mode = "kv_head"
136 else: