A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[TPU][Bugfix] Fix tpu type api (#8035)
Woosuk Kwon committed
Aug 30, 2024 at 09:01 UTC
2684efc4678eb46d1dc7fe4311365a99215e2dc6
1 file changed
+4
-1
vllm/attention/backends/pallas.py
+4
-1
@@ -124,7 +124,10 @@ class PallasAttentionBackendImpl(AttentionImpl):
124
125
self.megacore_mode = None
126
tpu_env = torch_xla.tpu.get_tpu_env()
127
- tpu_type = tpu_env.get("TYPE") or tpu_env.get("ACCELERATOR_TYPE")
127
+ tpu_type = (tpu_env.get("ACCELERATOR_TYPE", None)
128
+ or tpu_env.get("TYPE", None)
129
+ or tpu_env.get("TPU_ACCELERATOR_TYPE", None))
130
+ assert tpu_type is not None
131
tpu_type = tpu_type.lower()
132
133
if "lite" not in tpu_type: