A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Platforms] Refactor xpu code (#10468)
Signed-off-by: MengqingCao <cmq0113@163.com>
Mengqing Cao committed
Nov 20, 2024 at 14:52 UTC
d5b28447e005a79dec417a706900db0dad4e1a47
2 files changed
+21
-27
vllm/executor/xpu_executor.py
-27
@@ -1,8 +1,5 @@
1
from typing import Callable, List, Optional, Tuple, Type, Union
2
3
-import torch
4
-
5
-from vllm.config import ModelConfig, ParallelConfig
3
from vllm.executor.executor_base import ExecutorAsyncBase
4
from vllm.executor.gpu_executor import GPUExecutor
5
from vllm.logger import init_logger
@@ -23,7 +20,6 @@ class XPUExecutor(GPUExecutor):
20
assert self.speculative_config is None, (
21
"Speculative decoding not yet supported for XPU backend")
22
26
- self.model_config = _verify_and_get_model_config(self.model_config)
23
GPUExecutor._init_executor(self)
24
25
def _get_worker_module_and_class(
@@ -53,26 +49,3 @@ class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase):
49
output = await make_async(self.driver_worker.execute_model
50
)(execute_model_req=execute_model_req)
51
return output
56
-
57
-
58
-def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
59
- if config.dtype == torch.bfloat16:
60
- logger.warning(
61
- "bfloat16 is not fully supported on XPU, casting to float16.")
62
- config.dtype = torch.float16
63
- if not config.enforce_eager:
64
- logger.warning(
65
- "CUDA graph is not supported on XPU, fallback to the eager "
66
- "mode.")
67
- config.enforce_eager = True
68
- return config
69
-
70
-
71
-def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
72
- if (config.distributed_executor_backend is not None
73
- and config.distributed_executor_backend != "ray"):
74
- logger.warning(
75
- "%s is not supported on XPU, fallback to ray distributed executor "
76
- "backend.", config.distributed_executor_backend)
77
- config.distributed_executor_backend = "ray"
78
- return config
vllm/platforms/xpu.py
+21
@@ -1,9 +1,16 @@
1
+from typing import TYPE_CHECKING
2
+
3
import torch
4
5
from vllm.logger import init_logger
6
7
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
8
9
+if TYPE_CHECKING:
10
+ from vllm.config import VllmConfig
11
+else:
12
+ VllmConfig = None
13
+
14
logger = init_logger(__name__)
15
16
@@ -34,3 +41,17 @@ class XPUPlatform(Platform):
41
@staticmethod
42
def inference_mode():
43
return torch.no_grad()
44
+
45
+ @classmethod
46
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
47
+ # check and update model config
48
+ model_config = vllm_config.model_config
49
+ if model_config.dtype == torch.bfloat16:
50
+ logger.warning(
51
+ "bfloat16 is not fully supported on XPU, casting to float16.")
52
+ model_config.dtype = torch.float16
53
+ if not model_config.enforce_eager:
54
+ logger.warning(
55
+ "CUDA graph is not supported on XPU, fallback to the eager "
56
+ "mode.")
57
+ model_config.enforce_eager = True