@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1fa714cb1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Refactor]A simple device-related refactor (#11163)

Signed-off-by: noemotiovon <noemotiovon@gmail.com> Co-authored-by: noemotiovon <noemotiovon@gmail.com>

Chenguang Li committed Dec 13, 2024 at 21:39 UTC d1fa714cb1c9a708d7da0de27c99f7eee07fe663
7 files changed +51 -31
vllm/platforms/cpu.py
+5
@@ -98,3 +98,8 @@ class CpuPlatform(Platform):
98 "vllm.worker.cpu_worker.CPUWorker"
99 else:
100 parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
101 +
102 + @classmethod
103 + def is_pin_memory_available(cls) -> bool:
104 + logger.warning("Pin memory is not supported on CPU.")
105 + return False
vllm/platforms/hpu.py
+9
@@ -2,6 +2,8 @@ from typing import TYPE_CHECKING, Optional
2
3 import torch
4
5 +from vllm.logger import init_logger
6 +
7 from .interface import Platform, PlatformEnum, _Backend
8
9 if TYPE_CHECKING:
@@ -9,6 +11,8 @@ if TYPE_CHECKING:
11 else:
12 VllmConfig = None
13
14 +logger = init_logger(__name__)
15 +
16
17 class HpuPlatform(Platform):
18 _enum = PlatformEnum.HPU
@@ -43,3 +47,8 @@ class HpuPlatform(Platform):
47 parallel_config = vllm_config.parallel_config
48 if parallel_config.worker_cls == "auto":
49 parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"
50 +
51 + @classmethod
52 + def is_pin_memory_available(cls):
53 + logger.warning("Pin memory is not supported on HPU.")
54 + return False
vllm/platforms/interface.py
+17
@@ -1,6 +1,7 @@
1 import enum
2 import platform
3 import random
4 +from platform import uname
5 from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
6
7 import numpy as np
@@ -16,6 +17,11 @@ else:
17 logger = init_logger(__name__)
18
19
20 +def in_wsl() -> bool:
21 + # Reference: https://github.com/microsoft/WSL/issues/4071
22 + return "microsoft" in " ".join(uname()).lower()
23 +
24 +
25 class _Backend(enum.Enum):
26 FLASH_ATTN = enum.auto()
27 FLASH_ATTN_VLLM_V1 = enum.auto()
@@ -221,6 +227,17 @@ class Platform:
227
228 return CpuArchEnum.OTHER if machine else CpuArchEnum.UNKNOWN
229
230 + @classmethod
231 + def is_pin_memory_available(cls) -> bool:
232 + """Checks whether pin memory is available on the current platform."""
233 + if in_wsl():
234 + # Pinning memory in WSL is not supported.
235 + # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
236 + logger.warning("Using 'pin_memory=False' as WSL is detected. "
237 + "This may slow down the performance.")
238 + return False
239 + return True
240 +
241
242 class UnspecifiedPlatform(Platform):
243 _enum = PlatformEnum.UNSPECIFIED
vllm/platforms/neuron.py
+9
@@ -1,5 +1,7 @@
1 from typing import TYPE_CHECKING, Optional
2
3 +from vllm.logger import init_logger
4 +
5 from .interface import Platform, PlatformEnum
6
7 if TYPE_CHECKING:
@@ -7,6 +9,8 @@ if TYPE_CHECKING:
9 else:
10 VllmConfig = None
11
12 +logger = init_logger(__name__)
13 +
14
15 class NeuronPlatform(Platform):
16 _enum = PlatformEnum.NEURON
@@ -28,3 +32,8 @@ class NeuronPlatform(Platform):
32 if parallel_config.worker_cls == "auto":
33 parallel_config.worker_cls = \
34 "vllm.worker.neuron_worker.NeuronWorker"
35 +
36 + @classmethod
37 + def is_pin_memory_available(cls) -> bool:
38 + logger.warning("Pin memory is not supported on Neuron.")
39 + return False
vllm/platforms/openvino.py
+5 -5
@@ -34,7 +34,7 @@ class OpenVinoPlatform(Platform):
34 return _Backend.OPENVINO
35
36 @classmethod
37 - def get_device_name(self, device_id: int = 0) -> str:
37 + def get_device_name(cls, device_id: int = 0) -> str:
38 return "openvino"
39
40 @classmethod
@@ -42,19 +42,19 @@ class OpenVinoPlatform(Platform):
42 return False
43
44 @classmethod
45 - def inference_mode(self):
45 + def inference_mode(cls):
46 return torch.inference_mode(mode=True)
47
48 @classmethod
49 - def is_openvino_cpu(self) -> bool:
49 + def is_openvino_cpu(cls) -> bool:
50 return "CPU" in envs.VLLM_OPENVINO_DEVICE
51
52 @classmethod
53 - def is_openvino_gpu(self) -> bool:
53 + def is_openvino_gpu(cls) -> bool:
54 return "GPU" in envs.VLLM_OPENVINO_DEVICE
55
56 @classmethod
57 - def is_pin_memory_available(self) -> bool:
57 + def is_pin_memory_available(cls) -> bool:
58 logger.warning("Pin memory is not supported on OpenViNO.")
59 return False
60
vllm/platforms/xpu.py
+5
@@ -78,3 +78,8 @@ class XPUPlatform(Platform):
78 parallel_config.distributed_executor_backend = "ray"
79 if parallel_config.worker_cls == "auto":
80 parallel_config.worker_cls = "vllm.worker.xpu_worker.XPUWorker"
81 +
82 + @classmethod
83 + def is_pin_memory_available(cls):
84 + logger.warning("Pin memory is not supported on XPU.")
85 + return False
vllm/utils.py
+1 -26
@@ -24,7 +24,6 @@ from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
24 from collections import UserDict, defaultdict
25 from collections.abc import Iterable, Mapping
26 from functools import lru_cache, partial, wraps
27 -from platform import uname
27 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
28 Dict, Generic, Hashable, List, Literal, Optional,
29 OrderedDict, Set, Tuple, Type, TypeVar, Union, overload)
@@ -344,12 +343,6 @@ def random_uuid() -> str:
343 return str(uuid.uuid4().hex)
344
345
347 -@lru_cache(maxsize=None)
348 -def in_wsl() -> bool:
349 - # Reference: https://github.com/microsoft/WSL/issues/4071
350 - return "microsoft" in " ".join(uname()).lower()
351 -
352 -
346 def make_async(
347 func: Callable[P, T],
348 executor: Optional[concurrent.futures.Executor] = None
@@ -729,25 +722,7 @@ def print_warning_once(msg: str) -> None:
722
723 @lru_cache(maxsize=None)
724 def is_pin_memory_available() -> bool:
732 -
733 - if in_wsl():
734 - # Pinning memory in WSL is not supported.
735 - # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
736 - print_warning_once("Using 'pin_memory=False' as WSL is detected. "
737 - "This may slow down the performance.")
738 - return False
739 - elif current_platform.is_xpu():
740 - print_warning_once("Pin memory is not supported on XPU.")
741 - return False
742 - elif current_platform.is_neuron():
743 - print_warning_once("Pin memory is not supported on Neuron.")
744 - return False
745 - elif current_platform.is_hpu():
746 - print_warning_once("Pin memory is not supported on HPU.")
747 - return False
748 - elif current_platform.is_cpu() or current_platform.is_openvino():
749 - return False
750 - return True
725 + return current_platform.is_pin_memory_available()
726
727
728 class DeviceMemoryProfiler: