A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix typo in Pallas backend (#5558)
Woosuk Kwon committed
Jun 14, 2024 at 14:40 UTC
28c145eb5755902505c066dc3b1e5315572cc6e7
1 file changed
+1
-1
vllm/attention/backends/pallas.py
+1
-1
@@ -110,7 +110,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
110
raise NotImplementedError("TPU version must be 4 or higher.")
111
112
self.megacore_mode = None
113
- tpu_type = torch_xla.tpu.get_tp_groupu_env()["TYPE"].lower()
113
+ tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
114
if not tpu_type.endswith("lite"):
115
if self.num_kv_heads % 2 == 0:
116
self.megacore_mode = "kv_head"