A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[TPU] Avoid initializing TPU runtime in is_tpu (#7763)
Woosuk Kwon committed
Aug 21, 2024 at 21:31 UTC
eeee1c3b1ae30a9714dffe7a58bdbed10b1e2e38
1 file changed
+4
-2
vllm/platforms/__init__.py
+4
-2
@@ -8,8 +8,10 @@ current_platform: Platform
8
9
is_tpu = False
10
try:
11
- import torch_xla.core.xla_model as xm
12
- xm.xla_device(devkind="TPU")
11
+ # While it's technically possible to install libtpu on a non-TPU machine,
12
+ # this is a very uncommon scenario. Therefore, we assume that libtpu is
13
+ # installed if and only if the machine has TPUs.
14
+ import libtpu # noqa: F401
15
is_tpu = True
16
except Exception:
17
pass