@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2684efc467
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[TPU][Bugfix] Fix tpu type api (#8035)

Woosuk Kwon committed Aug 30, 2024 at 09:01 UTC 2684efc4678eb46d1dc7fe4311365a99215e2dc6
1 file changed +4 -1
vllm/attention/backends/pallas.py
+4 -1
@@ -124,7 +124,10 @@ class PallasAttentionBackendImpl(AttentionImpl):
124
125 self.megacore_mode = None
126 tpu_env = torch_xla.tpu.get_tpu_env()
127 - tpu_type = tpu_env.get("TYPE") or tpu_env.get("ACCELERATOR_TYPE")
127 + tpu_type = (tpu_env.get("ACCELERATOR_TYPE", None)
128 + or tpu_env.get("TYPE", None)
129 + or tpu_env.get("TPU_ACCELERATOR_TYPE", None))
130 + assert tpu_type is not None
131 tpu_type = tpu_type.lower()
132
133 if "lite" not in tpu_type: