A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Platform] Move `async output` check to platform (#10768)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Dec 10, 2024 at 01:24 UTC
aea2fc38c3b31b9a8ea7d1cffb8f37a2da6f6075
10 files changed
+66
-22
vllm/config.py
+3
-14
@@ -513,11 +513,10 @@ class ModelConfig:
513
514
# Reminder: Please update docs/source/usage/compatibility_matrix.rst
515
# If the feature combo become valid
516
- if device_config.device_type not in ("cuda", "tpu", "xpu", "hpu"):
516
+ if not current_platform.is_async_output_supported(self.enforce_eager):
517
logger.warning(
518
- "Async output processing is only supported for CUDA, TPU, XPU "
519
- "and HPU."
520
- "Disabling it for other platforms.")
518
+ "Async output processing is not supported on the "
519
+ "current platform type %s.", current_platform.device_type)
520
self.use_async_output_proc = False
521
return
522
@@ -527,16 +526,6 @@ class ModelConfig:
526
self.use_async_output_proc = False
527
return
528
530
- # Reminder: Please update docs/source/usage/compatibility_matrix.rst
531
- # If the feature combo become valid
532
- if device_config.device_type == "cuda" and self.enforce_eager:
533
- logger.warning(
534
- "To see benefits of async output processing, enable CUDA "
535
- "graph. Since, enforce-eager is enabled, async output "
536
- "processor cannot be used")
537
- self.use_async_output_proc = not self.enforce_eager
538
- return
539
-
529
# Async postprocessor is not necessary with embedding mode
530
# since there is no token generation
531
if self.task == "embedding":
vllm/platforms/cpu.py
+5
-1
@@ -1,4 +1,4 @@
1
-from typing import TYPE_CHECKING
1
+from typing import TYPE_CHECKING, Optional
2
3
import psutil
4
import torch
@@ -37,6 +37,10 @@ class CpuPlatform(Platform):
37
def get_device_total_memory(cls, device_id: int = 0) -> int:
38
return psutil.virtual_memory().total
39
40
+ @classmethod
41
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
42
+ return False
43
+
44
@classmethod
45
def inference_mode(cls):
46
return torch.no_grad()
vllm/platforms/cuda.py
+11
-1
@@ -4,7 +4,7 @@ pynvml. However, it should not initialize cuda context.
4
5
import os
6
from functools import lru_cache, wraps
7
-from typing import TYPE_CHECKING, Callable, List, TypeVar
7
+from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar
8
9
import pynvml
10
import torch
@@ -88,6 +88,16 @@ class CudaPlatformBase(Platform):
88
def get_device_total_memory(cls, device_id: int = 0) -> int:
89
raise NotImplementedError
90
91
+ @classmethod
92
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
93
+ if enforce_eager:
94
+ logger.warning(
95
+ "To see benefits of async output processing, enable CUDA "
96
+ "graph. Since, enforce-eager is enabled, async output "
97
+ "processor cannot be used")
98
+ return False
99
+ return True
100
+
101
@classmethod
102
def is_full_nvlink(cls, device_ids: List[int]) -> bool:
103
raise NotImplementedError
vllm/platforms/hpu.py
+5
-1
@@ -1,4 +1,4 @@
1
-from typing import TYPE_CHECKING
1
+from typing import TYPE_CHECKING, Optional
2
3
import torch
4
@@ -20,6 +20,10 @@ class HpuPlatform(Platform):
20
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
21
return _Backend.HPU_ATTN
22
23
+ @classmethod
24
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
25
+ return True
26
+
27
@staticmethod
28
def inference_mode():
29
return torch.no_grad()
vllm/platforms/interface.py
+11
@@ -6,11 +6,15 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
6
import numpy as np
7
import torch
8
9
+from vllm.logger import init_logger
10
+
11
if TYPE_CHECKING:
12
from vllm.config import VllmConfig
13
else:
14
VllmConfig = None
15
16
+logger = init_logger(__name__)
17
+
18
19
class _Backend(enum.Enum):
20
FLASH_ATTN = enum.auto()
@@ -147,6 +151,13 @@ class Platform:
151
"""Get the total memory of a device in bytes."""
152
raise NotImplementedError
153
154
+ @classmethod
155
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
156
+ """
157
+ Check if the current platform supports async output.
158
+ """
159
+ raise NotImplementedError
160
+
161
@classmethod
162
def inference_mode(cls):
163
"""A device-specific wrapper of `torch.inference_mode`.
vllm/platforms/neuron.py
+5
-1
@@ -1,4 +1,4 @@
1
-from typing import TYPE_CHECKING
1
+from typing import TYPE_CHECKING, Optional
2
3
from .interface import Platform, PlatformEnum
4
@@ -18,6 +18,10 @@ class NeuronPlatform(Platform):
18
def get_device_name(cls, device_id: int = 0) -> str:
19
return "neuron"
20
21
+ @classmethod
22
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
23
+ return False
24
+
25
@classmethod
26
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
27
parallel_config = vllm_config.parallel_config
vllm/platforms/openvino.py
+5
-1
@@ -1,4 +1,4 @@
1
-from typing import TYPE_CHECKING
1
+from typing import TYPE_CHECKING, Optional
2
3
import torch
4
@@ -37,6 +37,10 @@ class OpenVinoPlatform(Platform):
37
def get_device_name(self, device_id: int = 0) -> str:
38
return "openvino"
39
40
+ @classmethod
41
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
42
+ return False
43
+
44
@classmethod
45
def inference_mode(self):
46
return torch.inference_mode(mode=True)
vllm/platforms/rocm.py
+11
-1
@@ -1,6 +1,6 @@
1
import os
2
from functools import lru_cache
3
-from typing import TYPE_CHECKING
3
+from typing import TYPE_CHECKING, Optional
4
5
import torch
6
@@ -72,6 +72,16 @@ class RocmPlatform(Platform):
72
device_props = torch.cuda.get_device_properties(device_id)
73
return device_props.total_memory
74
75
+ @classmethod
76
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
77
+ if enforce_eager:
78
+ logger.warning(
79
+ "To see benefits of async output processing, enable CUDA "
80
+ "graph. Since, enforce-eager is enabled, async output "
81
+ "processor cannot be used")
82
+ return False
83
+ return True
84
+
85
@classmethod
86
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
87
parallel_config = vllm_config.parallel_config
vllm/platforms/tpu.py
+5
-1
@@ -1,4 +1,4 @@
1
-from typing import TYPE_CHECKING
1
+from typing import TYPE_CHECKING, Optional
2
3
import torch
4
@@ -35,6 +35,10 @@ class TpuPlatform(Platform):
35
def get_device_total_memory(cls, device_id: int = 0) -> int:
36
raise NotImplementedError
37
38
+ @classmethod
39
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
40
+ return True
41
+
42
@classmethod
43
def inference_mode(cls):
44
return torch.no_grad()
vllm/platforms/xpu.py
+5
-1
@@ -1,4 +1,4 @@
1
-from typing import TYPE_CHECKING
1
+from typing import TYPE_CHECKING, Optional
2
3
import torch
4
@@ -41,6 +41,10 @@ class XPUPlatform(Platform):
41
device_props = torch.xpu.get_device_properties(device_id)
42
return device_props.total_memory
43
44
+ @classmethod
45
+ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
46
+ return True
47
+
48
@staticmethod
49
def inference_mode():
50
return torch.no_grad()