@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e54ebc2f8f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[doc] fix doc build error caused by msgspec (#7659)

youkaichao committed Aug 19, 2024 at 17:50 UTC e54ebc2f8f9d78f3113fb2b531058977e6031609
2 files changed +40 -8
docs/requirements-docs.txt
+1
@@ -3,6 +3,7 @@ sphinx-book-theme==1.0.1
3 sphinx-copybutton==0.5.2
4 myst-parser==2.0.0
5 sphinx-argparse==0.4.0
6 +msgspec
7
8 # packages to install to build the documentation
9 pydantic
vllm/platforms/__init__.py
+39 -8
@@ -1,23 +1,54 @@
1 -import torch
2 -
1 from .interface import Platform, PlatformEnum, UnspecifiedPlatform
2
3 current_platform: Platform
4
5 +# NOTE: we don't use `torch.version.cuda` / `torch.version.hip` because
6 +# they only indicate the build configuration, not the runtime environment.
7 +# For example, people can install a cuda build of pytorch but run on tpu.
8 +
9 +is_tpu = False
10 +try:
11 + import torch_xla.core.xla_model as xm
12 + xm.xla_device(devkind="TPU")
13 + is_tpu = True
14 +except Exception:
15 + pass
16 +
17 +is_cuda = False
18 +
19 +try:
20 + import pynvml
21 + pynvml.nvmlInit()
22 + try:
23 + if pynvml.nvmlDeviceGetCount() > 0:
24 + is_cuda = True
25 + finally:
26 + pynvml.nvmlShutdown()
27 +except Exception:
28 + pass
29 +
30 +is_rocm = False
31 +
32 try:
8 - import libtpu
9 -except ImportError:
10 - libtpu = None
33 + import amdsmi
34 + amdsmi.amdsmi_init()
35 + try:
36 + if len(amdsmi.amdsmi_get_processor_handles()) > 0:
37 + is_rocm = True
38 + finally:
39 + amdsmi.amdsmi_shut_down()
40 +except Exception:
41 + pass
42
12 -if libtpu is not None:
43 +if is_tpu:
44 # people might install pytorch built with cuda but run on tpu
45 # so we need to check tpu first
46 from .tpu import TpuPlatform
47 current_platform = TpuPlatform()
17 -elif torch.version.cuda is not None:
48 +elif is_cuda:
49 from .cuda import CudaPlatform
50 current_platform = CudaPlatform()
20 -elif torch.version.hip is not None:
51 +elif is_rocm:
52 from .rocm import RocmPlatform
53 current_platform = RocmPlatform()
54 else: