A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Refactor]A simple device-related refactor (#11163)
Signed-off-by: noemotiovon <noemotiovon@gmail.com> Co-authored-by: noemotiovon <noemotiovon@gmail.com>
Chenguang Li committed
Dec 13, 2024 at 21:39 UTC
d1fa714cb1c9a708d7da0de27c99f7eee07fe663
7 files changed
+51
-31
vllm/platforms/cpu.py
+5
@@ -98,3 +98,8 @@ class CpuPlatform(Platform):
98
"vllm.worker.cpu_worker.CPUWorker"
99
else:
100
parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
101
+
102
+ @classmethod
103
+ def is_pin_memory_available(cls) -> bool:
104
+ logger.warning("Pin memory is not supported on CPU.")
105
+ return False
vllm/platforms/hpu.py
+9
@@ -2,6 +2,8 @@ from typing import TYPE_CHECKING, Optional
2
3
import torch
4
5
+from vllm.logger import init_logger
6
+
7
from .interface import Platform, PlatformEnum, _Backend
8
9
if TYPE_CHECKING:
@@ -9,6 +11,8 @@ if TYPE_CHECKING:
11
else:
12
VllmConfig = None
13
14
+logger = init_logger(__name__)
15
+
16
17
class HpuPlatform(Platform):
18
_enum = PlatformEnum.HPU
@@ -43,3 +47,8 @@ class HpuPlatform(Platform):
47
parallel_config = vllm_config.parallel_config
48
if parallel_config.worker_cls == "auto":
49
parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"
50
+
51
+ @classmethod
52
+ def is_pin_memory_available(cls):
53
+ logger.warning("Pin memory is not supported on HPU.")
54
+ return False
vllm/platforms/interface.py
+17
@@ -1,6 +1,7 @@
1
import enum
2
import platform
3
import random
4
+from platform import uname
5
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
6
7
import numpy as np
@@ -16,6 +17,11 @@ else:
17
logger = init_logger(__name__)
18
19
20
+def in_wsl() -> bool:
21
+ # Reference: https://github.com/microsoft/WSL/issues/4071
22
+ return "microsoft" in " ".join(uname()).lower()
23
+
24
+
25
class _Backend(enum.Enum):
26
FLASH_ATTN = enum.auto()
27
FLASH_ATTN_VLLM_V1 = enum.auto()
@@ -221,6 +227,17 @@ class Platform:
227
228
return CpuArchEnum.OTHER if machine else CpuArchEnum.UNKNOWN
229
230
+ @classmethod
231
+ def is_pin_memory_available(cls) -> bool:
232
+ """Checks whether pin memory is available on the current platform."""
233
+ if in_wsl():
234
+ # Pinning memory in WSL is not supported.
235
+ # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
236
+ logger.warning("Using 'pin_memory=False' as WSL is detected. "
237
+ "This may slow down the performance.")
238
+ return False
239
+ return True
240
+
241
242
class UnspecifiedPlatform(Platform):
243
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/neuron.py
+9
@@ -1,5 +1,7 @@
1
from typing import TYPE_CHECKING, Optional
2
3
+from vllm.logger import init_logger
4
+
5
from .interface import Platform, PlatformEnum
6
7
if TYPE_CHECKING:
@@ -7,6 +9,8 @@ if TYPE_CHECKING:
9
else:
10
VllmConfig = None
11
12
+logger = init_logger(__name__)
13
+
14
15
class NeuronPlatform(Platform):
16
_enum = PlatformEnum.NEURON
@@ -28,3 +32,8 @@ class NeuronPlatform(Platform):
32
if parallel_config.worker_cls == "auto":
33
parallel_config.worker_cls = \
34
"vllm.worker.neuron_worker.NeuronWorker"
35
+
36
+ @classmethod
37
+ def is_pin_memory_available(cls) -> bool:
38
+ logger.warning("Pin memory is not supported on Neuron.")
39
+ return False
vllm/platforms/openvino.py
+5
-5
@@ -34,7 +34,7 @@ class OpenVinoPlatform(Platform):
34
return _Backend.OPENVINO
35
36
@classmethod
37
- def get_device_name(self, device_id: int = 0) -> str:
37
+ def get_device_name(cls, device_id: int = 0) -> str:
38
return "openvino"
39
40
@classmethod
@@ -42,19 +42,19 @@ class OpenVinoPlatform(Platform):
42
return False
43
44
@classmethod
45
- def inference_mode(self):
45
+ def inference_mode(cls):
46
return torch.inference_mode(mode=True)
47
48
@classmethod
49
- def is_openvino_cpu(self) -> bool:
49
+ def is_openvino_cpu(cls) -> bool:
50
return "CPU" in envs.VLLM_OPENVINO_DEVICE
51
52
@classmethod
53
- def is_openvino_gpu(self) -> bool:
53
+ def is_openvino_gpu(cls) -> bool:
54
return "GPU" in envs.VLLM_OPENVINO_DEVICE
55
56
@classmethod
57
- def is_pin_memory_available(self) -> bool:
57
+ def is_pin_memory_available(cls) -> bool:
58
logger.warning("Pin memory is not supported on OpenViNO.")
59
return False
60
vllm/platforms/xpu.py
+5
@@ -78,3 +78,8 @@ class XPUPlatform(Platform):
78
parallel_config.distributed_executor_backend = "ray"
79
if parallel_config.worker_cls == "auto":
80
parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
81
+
82
+ @classmethod
83
+ def is_pin_memory_available(cls):
84
+ logger.warning("Pin memory is not supported on XPU.")
85
+ return False
vllm/utils.py
+1
-26
@@ -24,7 +24,6 @@ from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
24
from collections import UserDict, defaultdict
25
from collections.abc import Iterable, Mapping
26
from functools import lru_cache, partial, wraps
27
-from platform import uname
27
from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
28
Dict, Generic, Hashable, List, Literal, Optional,
29
OrderedDict, Set, Tuple, Type, TypeVar, Union, overload)
@@ -344,12 +343,6 @@ def random_uuid() -> str:
343
return str(uuid.uuid4().hex)
344
345
347
-@lru_cache(maxsize=None)
348
-def in_wsl() -> bool:
349
- # Reference: https://github.com/microsoft/WSL/issues/4071
350
- return "microsoft" in " ".join(uname()).lower()
351
-
352
-
346
def make_async(
347
func: Callable[P, T],
348
executor: Optional[concurrent.futures.Executor] = None
@@ -729,25 +722,7 @@ def print_warning_once(msg: str) -> None:
722
723
@lru_cache(maxsize=None)
724
def is_pin_memory_available() -> bool:
732
-
733
- if in_wsl():
734
- # Pinning memory in WSL is not supported.
735
- # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
736
- print_warning_once("Using 'pin_memory=False' as WSL is detected. "
737
- "This may slow down the performance.")
738
- return False
739
- elif current_platform.is_xpu():
740
- print_warning_once("Pin memory is not supported on XPU.")
741
- return False
742
- elif current_platform.is_neuron():
743
- print_warning_once("Pin memory is not supported on Neuron.")
744
- return False
745
- elif current_platform.is_hpu():
746
- print_warning_once("Pin memory is not supported on HPU.")
747
- return False
748
- elif current_platform.is_cpu() or current_platform.is_openvino():
749
- return False
750
- return True
725
+ return current_platform.is_pin_memory_available()
726
727
728
class DeviceMemoryProfiler: