A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Hardware][XPU] using current_platform.is_xpu (#9605)
Mengqing Cao committed
Oct 23, 2024 at 16:28 UTC
2394962d7083f1c1001dba9efefadb674321e688
6 files changed
+16
-38
vllm/attention/selector.py
+3
-3
@@ -10,7 +10,7 @@ import vllm.envs as envs
10
from vllm.attention.backends.abstract import AttentionBackend
11
from vllm.logger import init_logger
12
from vllm.platforms import current_platform
13
-from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino, is_xpu
13
+from vllm.utils import STR_BACKEND_ENV_VAR, is_hip, is_openvino
14
15
logger = init_logger(__name__)
16
@@ -136,7 +136,7 @@ def get_attn_backend(
136
from vllm.attention.backends.openvino import OpenVINOAttentionBackend
137
return OpenVINOAttentionBackend
138
elif backend == _Backend.IPEX:
139
- assert is_xpu(), RuntimeError(
139
+ assert current_platform.is_xpu(), RuntimeError(
140
"IPEX attention backend is only used for the XPU device.")
141
logger.info("Using IPEX attention backend.")
142
from vllm.attention.backends.ipex_attn import IpexAttnBackend
@@ -198,7 +198,7 @@ def which_attn_to_use(
198
logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
199
return _Backend.OPENVINO
200
201
- if is_xpu():
201
+ if current_platform.is_xpu():
202
if selected_backend != _Backend.IPEX:
203
logger.info("Cannot use %s backend on XPU.", selected_backend)
204
return _Backend.IPEX
vllm/config.py
+2
-2
@@ -17,7 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
get_hf_image_processor_config,
18
get_hf_text_config)
19
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20
- is_hip, is_openvino, is_xpu, print_warning_once)
20
+ is_hip, is_openvino, print_warning_once)
21
22
if TYPE_CHECKING:
23
from ray.util.placement_group import PlacementGroup
@@ -1121,7 +1121,7 @@ class DeviceConfig:
1121
self.device_type = "tpu"
1122
elif current_platform.is_cpu():
1123
self.device_type = "cpu"
1124
- elif is_xpu():
1124
+ elif current_platform.is_xpu():
1125
self.device_type = "xpu"
1126
else:
1127
raise RuntimeError("Failed to infer device type")
vllm/executor/ray_utils.py
+2
-2
@@ -10,7 +10,7 @@ from vllm.executor.msgspec_utils import decode_hook, encode_hook
10
from vllm.logger import init_logger
11
from vllm.platforms import current_platform
12
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
13
-from vllm.utils import get_ip, is_hip, is_xpu
13
+from vllm.utils import get_ip, is_hip
14
from vllm.worker.worker_base import WorkerWrapperBase
15
16
logger = init_logger(__name__)
@@ -231,7 +231,7 @@ def initialize_ray_cluster(
231
assert_ray_available()
232
233
# Connect to a ray cluster.
234
- if is_hip() or is_xpu():
234
+ if is_hip() or current_platform.is_xpu():
235
ray.init(address=ray_address,
236
ignore_reinit_error=True,
237
num_gpus=parallel_config.world_size)
vllm/model_executor/custom_op.py
+2
-2
@@ -7,7 +7,7 @@ import vllm.envs as envs
7
from vllm.compilation.levels import CompilationLevel
8
from vllm.logger import init_logger
9
from vllm.platforms import current_platform
10
-from vllm.utils import is_hip, is_xpu, print_warning_once
10
+from vllm.utils import is_hip, print_warning_once
11
12
logger = init_logger(__name__)
13
@@ -78,7 +78,7 @@ class CustomOp(nn.Module):
78
return self.forward_cpu
79
elif current_platform.is_tpu():
80
return self.forward_tpu
81
- elif is_xpu():
81
+ elif current_platform.is_xpu():
82
return self.forward_xpu
83
else:
84
return self.forward_cuda
vllm/utils.py
+3
-26
@@ -327,29 +327,6 @@ def is_openvino() -> bool:
327
return False
328
329
330
-@lru_cache(maxsize=None)
331
-def is_xpu() -> bool:
332
- from importlib.metadata import PackageNotFoundError, version
333
- try:
334
- is_xpu_flag = "xpu" in version("vllm")
335
- except PackageNotFoundError:
336
- return False
337
- # vllm is not build with xpu
338
- if not is_xpu_flag:
339
- return False
340
- try:
341
- import intel_extension_for_pytorch as ipex # noqa: F401
342
- _import_ipex = True
343
- except ImportError as e:
344
- logger.warning("Import Error for IPEX: %s", e.msg)
345
- _import_ipex = False
346
- # ipex dependency is not ready
347
- if not _import_ipex:
348
- logger.warning("not found ipex lib")
349
- return False
350
- return hasattr(torch, "xpu") and torch.xpu.is_available()
351
-
352
-
330
@lru_cache(maxsize=None)
331
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
332
"""Returns the maximum shared memory per thread block in bytes."""
@@ -379,7 +356,7 @@ def seed_everything(seed: int) -> None:
356
if current_platform.is_cuda_alike():
357
torch.cuda.manual_seed_all(seed)
358
382
- if is_xpu():
359
+ if current_platform.is_xpu():
360
torch.xpu.manual_seed_all(seed)
361
362
@@ -774,7 +751,7 @@ def is_pin_memory_available() -> bool:
751
print_warning_once("Using 'pin_memory=False' as WSL is detected. "
752
"This may slow down the performance.")
753
return False
777
- elif is_xpu():
754
+ elif current_platform.is_xpu():
755
print_warning_once("Pin memory is not supported on XPU.")
756
return False
757
elif current_platform.is_neuron():
@@ -795,7 +772,7 @@ class DeviceMemoryProfiler:
772
if current_platform.is_cuda_alike():
773
torch.cuda.reset_peak_memory_stats(self.device)
774
mem = torch.cuda.max_memory_allocated(self.device)
798
- elif is_xpu():
775
+ elif current_platform.is_xpu():
776
torch.xpu.reset_peak_memory_stats(self.device) # type: ignore
777
mem = torch.xpu.max_memory_allocated(self.device) # type: ignore
778
return mem
vllm/worker/xpu_worker.py
+4
-3
@@ -17,7 +17,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
17
from vllm.distributed.parallel_state import get_pp_group
18
from vllm.logger import init_logger
19
from vllm.model_executor import set_random_seed
20
-from vllm.utils import is_xpu
20
+from vllm.platforms import current_platform
21
from vllm.worker.cache_engine import CacheEngine
22
from vllm.worker.worker import Worker
23
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
@@ -53,7 +53,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
53
observability_config: Optional[ObservabilityConfig] = None,
54
) -> None:
55
assert device_config.device_type == "xpu"
56
- assert is_xpu()
56
+ assert current_platform.is_xpu()
57
58
self.model_config = model_config
59
self.parallel_config = parallel_config
@@ -91,7 +91,8 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
91
self.gpu_cache: Optional[List[List[torch.Tensor]]]
92
93
def init_device(self) -> None:
94
- if self.device_config.device.type == "xpu" and is_xpu():
94
+ if self.device_config.device.type == "xpu" and current_platform.is_xpu(
95
+ ):
96
self.device = torch.device(f"xpu:{self.local_rank}")
97
torch.xpu.set_device(self.device)
98
torch.xpu.empty_cache()