@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aea2fc38c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Platform] Move `async output` check to platform (#10768)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Dec 10, 2024 at 01:24 UTC aea2fc38c3b31b9a8ea7d1cffb8f37a2da6f6075
10 files changed +66 -22
vllm/config.py
+3 -14
@@ -513,11 +513,10 @@ class ModelConfig:
513
514 # Reminder: Please update docs/source/usage/compatibility_matrix.rst
515 # If the feature combo become valid
516 - if device_config.device_type not in ("cuda", "tpu", "xpu", "hpu"):
516 + if not current_platform.is_async_output_supported(self.enforce_eager):
517 logger.warning(
518 - "Async output processing is only supported for CUDA, TPU, XPU "
519 - "and HPU."
520 - "Disabling it for other platforms.")
518 + "Async output processing is not supported on the "
519 + "current platform type %s.", current_platform.device_type)
520 self.use_async_output_proc = False
521 return
522
@@ -527,16 +526,6 @@ class ModelConfig:
526 self.use_async_output_proc = False
527 return
528
530 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
531 - # If the feature combo become valid
532 - if device_config.device_type == "cuda" and self.enforce_eager:
533 - logger.warning(
534 - "To see benefits of async output processing, enable CUDA "
535 - "graph. Since, enforce-eager is enabled, async output "
536 - "processor cannot be used")
537 - self.use_async_output_proc = not self.enforce_eager
538 - return
539 -
529 # Async postprocessor is not necessary with embedding mode
530 # since there is no token generation
531 if self.task == "embedding":
vllm/platforms/cpu.py
+5 -1
@@ -1,4 +1,4 @@
1 -from typing import TYPE_CHECKING
1 +from typing import TYPE_CHECKING, Optional
2
3 import psutil
4 import torch
@@ -37,6 +37,10 @@ class CpuPlatform(Platform):
37 def get_device_total_memory(cls, device_id: int = 0) -> int:
38 return psutil.virtual_memory().total
39
40 + @classmethod
41 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
42 + return False
43 +
44 @classmethod
45 def inference_mode(cls):
46 return torch.no_grad()
vllm/platforms/cuda.py
+11 -1
@@ -4,7 +4,7 @@ pynvml. However, it should not initialize cuda context.
4
5 import os
6 from functools import lru_cache, wraps
7 -from typing import TYPE_CHECKING, Callable, List, TypeVar
7 +from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar
8
9 import pynvml
10 import torch
@@ -88,6 +88,16 @@ class CudaPlatformBase(Platform):
88 def get_device_total_memory(cls, device_id: int = 0) -> int:
89 raise NotImplementedError
90
91 + @classmethod
92 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
93 + if enforce_eager:
94 + logger.warning(
95 + "To see benefits of async output processing, enable CUDA "
96 + "graph. Since, enforce-eager is enabled, async output "
97 + "processor cannot be used")
98 + return False
99 + return True
100 +
101 @classmethod
102 def is_full_nvlink(cls, device_ids: List[int]) -> bool:
103 raise NotImplementedError
vllm/platforms/hpu.py
+5 -1
@@ -1,4 +1,4 @@
1 -from typing import TYPE_CHECKING
1 +from typing import TYPE_CHECKING, Optional
2
3 import torch
4
@@ -20,6 +20,10 @@ class HpuPlatform(Platform):
20 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
21 return _Backend.HPU_ATTN
22
23 + @classmethod
24 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
25 + return True
26 +
27 @staticmethod
28 def inference_mode():
29 return torch.no_grad()
vllm/platforms/interface.py
+11
@@ -6,11 +6,15 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
6 import numpy as np
7 import torch
8
9 +from vllm.logger import init_logger
10 +
11 if TYPE_CHECKING:
12 from vllm.config import VllmConfig
13 else:
14 VllmConfig = None
15
16 +logger = init_logger(__name__)
17 +
18
19 class _Backend(enum.Enum):
20 FLASH_ATTN = enum.auto()
@@ -147,6 +151,13 @@ class Platform:
151 """Get the total memory of a device in bytes."""
152 raise NotImplementedError
153
154 + @classmethod
155 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
156 + """
157 + Check if the current platform supports async output.
158 + """
159 + raise NotImplementedError
160 +
161 @classmethod
162 def inference_mode(cls):
163 """A device-specific wrapper of `torch.inference_mode`.
vllm/platforms/neuron.py
+5 -1
@@ -1,4 +1,4 @@
1 -from typing import TYPE_CHECKING
1 +from typing import TYPE_CHECKING, Optional
2
3 from .interface import Platform, PlatformEnum
4
@@ -18,6 +18,10 @@ class NeuronPlatform(Platform):
18 def get_device_name(cls, device_id: int = 0) -> str:
19 return "neuron"
20
21 + @classmethod
22 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
23 + return False
24 +
25 @classmethod
26 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
27 parallel_config = vllm_config.parallel_config
vllm/platforms/openvino.py
+5 -1
@@ -1,4 +1,4 @@
1 -from typing import TYPE_CHECKING
1 +from typing import TYPE_CHECKING, Optional
2
3 import torch
4
@@ -37,6 +37,10 @@ class OpenVinoPlatform(Platform):
37 def get_device_name(self, device_id: int = 0) -> str:
38 return "openvino"
39
40 + @classmethod
41 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
42 + return False
43 +
44 @classmethod
45 def inference_mode(self):
46 return torch.inference_mode(mode=True)
vllm/platforms/rocm.py
+11 -1
@@ -1,6 +1,6 @@
1 import os
2 from functools import lru_cache
3 -from typing import TYPE_CHECKING
3 +from typing import TYPE_CHECKING, Optional
4
5 import torch
6
@@ -72,6 +72,16 @@ class RocmPlatform(Platform):
72 device_props = torch.cuda.get_device_properties(device_id)
73 return device_props.total_memory
74
75 + @classmethod
76 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
77 + if enforce_eager:
78 + logger.warning(
79 + "To see benefits of async output processing, enable CUDA "
80 + "graph. Since, enforce-eager is enabled, async output "
81 + "processor cannot be used")
82 + return False
83 + return True
84 +
85 @classmethod
86 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
87 parallel_config = vllm_config.parallel_config
vllm/platforms/tpu.py
+5 -1
@@ -1,4 +1,4 @@
1 -from typing import TYPE_CHECKING
1 +from typing import TYPE_CHECKING, Optional
2
3 import torch
4
@@ -35,6 +35,10 @@ class TpuPlatform(Platform):
35 def get_device_total_memory(cls, device_id: int = 0) -> int:
36 raise NotImplementedError
37
38 + @classmethod
39 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
40 + return True
41 +
42 @classmethod
43 def inference_mode(cls):
44 return torch.no_grad()
vllm/platforms/xpu.py
+5 -1
@@ -1,4 +1,4 @@
1 -from typing import TYPE_CHECKING
1 +from typing import TYPE_CHECKING, Optional
2
3 import torch
4
@@ -41,6 +41,10 @@ class XPUPlatform(Platform):
41 device_props = torch.xpu.get_device_properties(device_id)
42 return device_props.total_memory
43
44 + @classmethod
45 + def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
46 + return True
47 +
48 @staticmethod
49 def inference_mode():
50 return torch.no_grad()