@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28c145eb57
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix typo in Pallas backend (#5558)

Woosuk Kwon committed Jun 14, 2024 at 14:40 UTC 28c145eb5755902505c066dc3b1e5315572cc6e7
1 file changed +1 -1
vllm/attention/backends/pallas.py
+1 -1
@@ -110,7 +110,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
110 raise NotImplementedError("TPU version must be 4 or higher.")
111
112 self.megacore_mode = None
113 - tpu_type = torch_xla.tpu.get_tp_groupu_env()["TYPE"].lower()
113 + tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
114 if not tpu_type.endswith("lite"):
115 if self.num_kv_heads % 2 == 0:
116 self.megacore_mode = "kv_head"