A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[doc] fix doc build error caused by msgspec (#7659)
youkaichao committed
Aug 19, 2024 at 17:50 UTC
e54ebc2f8f9d78f3113fb2b531058977e6031609
2 files changed
+40
-8
docs/requirements-docs.txt
+1
@@ -3,6 +3,7 @@ sphinx-book-theme==1.0.1
3
sphinx-copybutton==0.5.2
4
myst-parser==2.0.0
5
sphinx-argparse==0.4.0
6
+msgspec
7
8
# packages to install to build the documentation
9
pydantic
vllm/platforms/__init__.py
+39
-8
@@ -1,23 +1,54 @@
1
-import torch
2
-
1
from .interface import Platform, PlatformEnum, UnspecifiedPlatform
2
3
current_platform: Platform
4
5
+# NOTE: we don't use `torch.version.cuda` / `torch.version.hip` because
6
+# they only indicate the build configuration, not the runtime environment.
7
+# For example, people can install a cuda build of pytorch but run on tpu.
8
+
9
+is_tpu = False
10
+try:
11
+ import torch_xla.core.xla_model as xm
12
+ xm.xla_device(devkind="TPU")
13
+ is_tpu = True
14
+except Exception:
15
+ pass
16
+
17
+is_cuda = False
18
+
19
+try:
20
+ import pynvml
21
+ pynvml.nvmlInit()
22
+ try:
23
+ if pynvml.nvmlDeviceGetCount() > 0:
24
+ is_cuda = True
25
+ finally:
26
+ pynvml.nvmlShutdown()
27
+except Exception:
28
+ pass
29
+
30
+is_rocm = False
31
+
32
try:
8
- import libtpu
9
-except ImportError:
10
- libtpu = None
33
+ import amdsmi
34
+ amdsmi.amdsmi_init()
35
+ try:
36
+ if len(amdsmi.amdsmi_get_processor_handles()) > 0:
37
+ is_rocm = True
38
+ finally:
39
+ amdsmi.amdsmi_shut_down()
40
+except Exception:
41
+ pass
42
12
-if libtpu is not None:
43
+if is_tpu:
44
# people might install pytorch built with cuda but run on tpu
45
# so we need to check tpu first
46
from .tpu import TpuPlatform
47
current_platform = TpuPlatform()
17
-elif torch.version.cuda is not None:
48
+elif is_cuda:
49
from .cuda import CudaPlatform
50
current_platform = CudaPlatform()
20
-elif torch.version.hip is not None:
51
+elif is_rocm:
52
from .rocm import RocmPlatform
53
current_platform = RocmPlatform()
54
else: