@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2394962d70
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Hardware][XPU] using current_platform.is_xpu (#9605)

Mengqing Cao committed Oct 23, 2024 at 16:28 UTC 2394962d7083f1c1001dba9efefadb674321e688
6 files changed +16 -38
vllm/attention/selector.py
+3 -3
@@ -10,7 +10,7 @@ import vllm.envs as envs
10 from vllm.attention.backends.abstract import AttentionBackend
11 from vllm.logger import init_logger
12 from vllm.platforms import current_platform
13 -from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino, is_xpu
13 +from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino
14
15 logger = init_logger(__name__)
16
@@ -136,7 +136,7 @@ def get_attn_backend(
136 from vllm.attention.backends.openvino import OpenVINOAttentionBackend
137 return OpenVINOAttentionBackend
138 elif backend == _Backend.IPEX:
139 - assert is_xpu(), RuntimeError(
139 + assert current_platform.is_xpu(), RuntimeError(
140 "IPEX attention backend is only used for the XPU device.")
141 logger.info("Using IPEX attention backend.")
142 from vllm.attention.backends.ipex_attn import IpexAttnBackend
@@ -198,7 +198,7 @@ def which_attn_to_use(
198 logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
199 return _Backend.OPENVINO
200
201 - if is_xpu():
201 + if current_platform.is_xpu():
202 if selected_backend != _Backend.IPEX:
203 logger.info("Cannot use %s backend on XPU.", selected_backend)
204 return _Backend.IPEX
vllm/config.py
+2 -2
@@ -17,7 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 get_hf_text_config)
19 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20 - is_hip, is_openvino, is_xpu, print_warning_once)
20 + is_hip, is_openvino, print_warning_once)
21
22 if TYPE_CHECKING:
23 from ray.util.placement_group import PlacementGroup
@@ -1121,7 +1121,7 @@ class DeviceConfig:
1121 self.device_type = "tpu"
1122 elif current_platform.is_cpu():
1123 self.device_type = "cpu"
1124 - elif is_xpu():
1124 + elif current_platform.is_xpu():
1125 self.device_type = "xpu"
1126 else:
1127 raise RuntimeError("Failed to infer device type")
vllm/executor/ray_utils.py
+2 -2
@@ -10,7 +10,7 @@ from vllm.executor.msgspec_utils import decode_hook, encode_hook
10 from vllm.logger import init_logger
11 from vllm.platforms import current_platform
12 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13 -from vllm.utils import get_ip, is_hip, is_xpu
13 +from vllm.utils import get_ip, is_hip
14 from vllm.worker.worker_base import WorkerWrapperBase
15
16 logger = init_logger(__name__)
@@ -231,7 +231,7 @@ def initialize_ray_cluster(
231 assert_ray_available()
232
233 # Connect to a ray cluster.
234 - if is_hip() or is_xpu():
234 + if is_hip() or current_platform.is_xpu():
235 ray.init(address=ray_address,
236 ignore_reinit_error=True,
237 num_gpus=parallel_config.world_size)
vllm/model_executor/custom_op.py
+2 -2
@@ -7,7 +7,7 @@ import vllm.envs as envs
7 from vllm.compilation.levels import CompilationLevel
8 from vllm.logger import init_logger
9 from vllm.platforms import current_platform
10 -from vllm.utils import is_hip, is_xpu, print_warning_once
10 +from vllm.utils import is_hip, print_warning_once
11
12 logger = init_logger(__name__)
13
@@ -78,7 +78,7 @@ class CustomOp(nn.Module):
78 return self.forward_cpu
79 elif current_platform.is_tpu():
80 return self.forward_tpu
81 - elif is_xpu():
81 + elif current_platform.is_xpu():
82 return self.forward_xpu
83 else:
84 return self.forward_cuda
vllm/utils.py
+3 -26
@@ -327,29 +327,6 @@ def is_openvino() -> bool:
327 return False
328
329
330 -@lru_cache(maxsize=None)
331 -def is_xpu() -> bool:
332 - from importlib.metadata import PackageNotFoundError, version
333 - try:
334 - is_xpu_flag = "xpu" in version("vllm")
335 - except PackageNotFoundError:
336 - return False
337 - # vllm is not build with xpu
338 - if not is_xpu_flag:
339 - return False
340 - try:
341 - import intel_extension_for_pytorch as ipex # noqa: F401
342 - _import_ipex = True
343 - except ImportError as e:
344 - logger.warning("Import Error for IPEX: %s", e.msg)
345 - _import_ipex = False
346 - # ipex dependency is not ready
347 - if not _import_ipex:
348 - logger.warning("not found ipex lib")
349 - return False
350 - return hasattr(torch, "xpu") and torch.xpu.is_available()
351 -
352 -
330 @lru_cache(maxsize=None)
331 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
332 """Returns the maximum shared memory per thread block in bytes."""
@@ -379,7 +356,7 @@ def seed_everything(seed: int) -> None:
356 if current_platform.is_cuda_alike():
357 torch.cuda.manual_seed_all(seed)
358
382 - if is_xpu():
359 + if current_platform.is_xpu():
360 torch.xpu.manual_seed_all(seed)
361
362
@@ -774,7 +751,7 @@ def is_pin_memory_available() -> bool:
751 print_warning_once("Using 'pin_memory=False' as WSL is detected. "
752 "This may slow down the performance.")
753 return False
777 - elif is_xpu():
754 + elif current_platform.is_xpu():
755 print_warning_once("Pin memory is not supported on XPU.")
756 return False
757 elif current_platform.is_neuron():
@@ -795,7 +772,7 @@ class DeviceMemoryProfiler:
772 if current_platform.is_cuda_alike():
773 torch.cuda.reset_peak_memory_stats(self.device)
774 mem = torch.cuda.max_memory_allocated(self.device)
798 - elif is_xpu():
775 + elif current_platform.is_xpu():
776 torch.xpu.reset_peak_memory_stats(self.device) # type: ignore
777 mem = torch.xpu.max_memory_allocated(self.device) # type: ignore
778 return mem
vllm/worker/xpu_worker.py
+4 -3
@@ -17,7 +17,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
17 from vllm.distributed.parallel_state import get_pp_group
18 from vllm.logger import init_logger
19 from vllm.model_executor import set_random_seed
20 -from vllm.utils import is_xpu
20 +from vllm.platforms import current_platform
21 from vllm.worker.cache_engine import CacheEngine
22 from vllm.worker.worker import Worker
23 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
@@ -53,7 +53,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
53 observability_config: Optional[ObservabilityConfig] = None,
54 ) -> None:
55 assert device_config.device_type == "xpu"
56 - assert is_xpu()
56 + assert current_platform.is_xpu()
57
58 self.model_config = model_config
59 self.parallel_config = parallel_config
@@ -91,7 +91,8 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
91 self.gpu_cache: Optional[List[List[torch.Tensor]]]
92
93 def init_device(self) -> None:
94 - if self.device_config.device.type == "xpu" and is_xpu():
94 + if self.device_config.device.type == "xpu" and current_platform.is_xpu(
95 + ):
96 self.device = torch.device(f"xpu:{self.local_rank}")
97 torch.xpu.set_device(self.device)
98 torch.xpu.empty_cache()