@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d5b28447e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Platforms] Refactor xpu code (#10468)

Signed-off-by: MengqingCao <cmq0113@163.com>

Mengqing Cao committed Nov 20, 2024 at 14:52 UTC d5b28447e005a79dec417a706900db0dad4e1a47
2 files changed +21 -27
vllm/executor/xpu_executor.py
-27
@@ -1,8 +1,5 @@
1 from typing import Callable, List, Optional, Tuple, Type, Union
2
3 -import torch
4 -
5 -from vllm.config import ModelConfig, ParallelConfig
3 from vllm.executor.executor_base import ExecutorAsyncBase
4 from vllm.executor.gpu_executor import GPUExecutor
5 from vllm.logger import init_logger
@@ -23,7 +20,6 @@ class XPUExecutor(GPUExecutor):
20 assert self.speculative_config is None, (
21 "Speculative decoding not yet supported for XPU backend")
22
26 - self.model_config = _verify_and_get_model_config(self.model_config)
23 GPUExecutor._init_executor(self)
24
25 def _get_worker_module_and_class(
@@ -53,26 +49,3 @@ class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase):
49 output = await make_async(self.driver_worker.execute_model
50 )(execute_model_req=execute_model_req)
51 return output
56 -
57 -
58 -def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
59 - if config.dtype == torch.bfloat16:
60 - logger.warning(
61 - "bfloat16 is not fully supported on XPU, casting to float16.")
62 - config.dtype = torch.float16
63 - if not config.enforce_eager:
64 - logger.warning(
65 - "CUDA graph is not supported on XPU, fallback to the eager "
66 - "mode.")
67 - config.enforce_eager = True
68 - return config
69 -
70 -
71 -def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
72 - if (config.distributed_executor_backend is not None
73 - and config.distributed_executor_backend != "ray"):
74 - logger.warning(
75 - "%s is not supported on XPU, fallback to ray distributed executor "
76 - "backend.", config.distributed_executor_backend)
77 - config.distributed_executor_backend = "ray"
78 - return config
vllm/platforms/xpu.py
+21
@@ -1,9 +1,16 @@
1 +from typing import TYPE_CHECKING
2 +
3 import torch
4
5 from vllm.logger import init_logger
6
7 from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
8
9 +if TYPE_CHECKING:
10 + from vllm.config import VllmConfig
11 +else:
12 + VllmConfig = None
13 +
14 logger = init_logger(__name__)
15
16
@@ -34,3 +41,17 @@ class XPUPlatform(Platform):
41 @staticmethod
42 def inference_mode():
43 return torch.no_grad()
44 +
45 + @classmethod
46 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
47 + # check and update model config
48 + model_config = vllm_config.model_config
49 + if model_config.dtype == torch.bfloat16:
50 + logger.warning(
51 + "bfloat16 is not fully supported on XPU, casting to float16.")
52 + model_config.dtype = torch.float16
53 + if not model_config.enforce_eager:
54 + logger.warning(
55 + "CUDA graph is not supported on XPU, fallback to the eager "
56 + "mode.")
57 + model_config.enforce_eager = True