A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[hardware] unify usage of is_tpu to current_platform.is_tpu() (#7102)
youkaichao committed
Aug 13, 2024 at 00:16 UTC
4d2dc5072b8cfaff0c02e504c79a72ee95d839a1
8 files changed
+29
-33
vllm/attention/selector.py
+2
-3
@@ -10,8 +10,7 @@ import vllm.envs as envs
10
from vllm.attention.backends.abstract import AttentionBackend
11
from vllm.logger import init_logger
12
from vllm.platforms import current_platform
13
-from vllm.utils import (STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino,
14
- is_tpu, is_xpu)
13
+from vllm.utils import STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino, is_xpu
14
15
logger = init_logger(__name__)
16
@@ -194,7 +193,7 @@ def which_attn_to_use(
193
logger.info("Cannot use %s backend on XPU.", selected_backend)
194
return _Backend.IPEX
195
197
- if is_tpu():
196
+ if current_platform.is_tpu():
197
if selected_backend != _Backend.PALLAS:
198
logger.info("Cannot use %s backend on TPU.", selected_backend)
199
return _Backend.PALLAS
vllm/config.py
+4
-3
@@ -10,11 +10,12 @@ import vllm.envs as envs
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
13
+from vllm.platforms import current_platform
14
from vllm.tracing import is_otel_installed
15
from vllm.transformers_utils.config import get_config, get_hf_text_config
16
from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
17
cuda_device_count_stateless, get_cpu_memory, is_cpu,
17
- is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
18
+ is_hip, is_neuron, is_openvino, is_xpu,
19
print_warning_once)
20
21
if TYPE_CHECKING:
@@ -282,7 +283,7 @@ class ModelConfig:
283
raise ValueError(
284
f"{self.quantization} quantization is currently not "
285
f"supported in ROCm.")
285
- if is_tpu(
286
+ if current_platform.is_tpu(
287
) and self.quantization not in tpu_supported_quantization:
288
raise ValueError(
289
f"{self.quantization} quantization is currently not "
@@ -910,7 +911,7 @@ class DeviceConfig:
911
self.device_type = "neuron"
912
elif is_openvino():
913
self.device_type = "openvino"
913
- elif is_tpu():
914
+ elif current_platform.is_tpu():
915
self.device_type = "tpu"
916
elif is_cpu():
917
self.device_type = "cpu"
vllm/executor/ray_utils.py
+3
-2
@@ -2,8 +2,9 @@ from typing import List, Optional, Tuple, Union
2
3
from vllm.config import ParallelConfig
4
from vllm.logger import init_logger
5
+from vllm.platforms import current_platform
6
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
6
-from vllm.utils import get_ip, is_hip, is_tpu, is_xpu
7
+from vllm.utils import get_ip, is_hip, is_xpu
8
from vllm.worker.worker_base import WorkerWrapperBase
9
10
logger = init_logger(__name__)
@@ -111,7 +112,7 @@ def initialize_ray_cluster(
112
# Placement group is already set.
113
return
114
114
- device_str = "GPU" if not is_tpu() else "TPU"
115
+ device_str = "GPU" if not current_platform.is_tpu() else "TPU"
116
# Create placement group for worker processes
117
current_placement_group = ray.util.get_current_placement_group()
118
if current_placement_group:
vllm/model_executor/custom_op.py
+3
-2
@@ -1,6 +1,7 @@
1
import torch.nn as nn
2
3
-from vllm.utils import is_cpu, is_hip, is_tpu, is_xpu
3
+from vllm.platforms import current_platform
4
+from vllm.utils import is_cpu, is_hip, is_xpu
5
6
7
class CustomOp(nn.Module):
@@ -54,7 +55,7 @@ class CustomOp(nn.Module):
55
return self.forward_hip
56
elif is_cpu():
57
return self.forward_cpu
57
- elif is_tpu():
58
+ elif current_platform.is_tpu():
59
return self.forward_tpu
60
elif is_xpu():
61
return self.forward_xpu
vllm/model_executor/layers/rotary_embedding.py
+2
-2
@@ -28,7 +28,7 @@ import torch
28
import torch.nn as nn
29
30
from vllm.model_executor.custom_op import CustomOp
31
-from vllm.utils import is_tpu
31
+from vllm.platforms import current_platform
32
33
34
def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -78,7 +78,7 @@ class RotaryEmbedding(CustomOp):
78
self.dtype = dtype
79
80
cache = self._compute_cos_sin_cache()
81
- self.use_native2 = is_tpu() and is_neox_style
81
+ self.use_native2 = current_platform.is_tpu() and is_neox_style
82
if not self.use_native2:
83
cache = cache.to(dtype)
84
self.register_buffer("cos_sin_cache", cache, persistent=False)
vllm/model_executor/model_loader/loader.py
+3
-3
@@ -41,7 +41,7 @@ from vllm.model_executor.models.interfaces import (has_inner_state,
41
supports_vision)
42
from vllm.model_executor.utils import set_weight_attrs
43
from vllm.platforms import current_platform
44
-from vllm.utils import is_pin_memory_available, is_tpu
44
+from vllm.utils import is_pin_memory_available
45
46
47
@contextmanager
@@ -94,7 +94,7 @@ def _get_quantization_config(
94
"""Get the quantization config."""
95
if model_config.quantization is not None:
96
quant_config = get_quant_config(model_config, load_config)
97
- if not is_tpu():
97
+ if not current_platform.is_tpu():
98
capability = current_platform.get_device_capability()
99
capability = capability[0] * 10 + capability[1]
100
if capability < quant_config.get_min_capability():
@@ -320,7 +320,7 @@ class DefaultModelLoader(BaseModelLoader):
320
else:
321
weights_iterator = pt_weights_iterator(hf_weights_files)
322
323
- if is_tpu():
323
+ if current_platform.is_tpu():
324
# In PyTorch XLA, we should call `xm.mark_step` frequently so that
325
# not too many ops are accumulated in the XLA program.
326
import torch_xla.core.xla_model as xm
vllm/platforms/__init__.py
+12
-9
@@ -1,22 +1,25 @@
1
-from typing import Optional
2
-
1
import torch
2
5
-from vllm.utils import is_tpu
6
-
3
from .interface import Platform, PlatformEnum, UnspecifiedPlatform
4
9
-current_platform: Optional[Platform]
5
+current_platform: Platform
6
11
-if torch.version.cuda is not None:
7
+try:
8
+ import libtpu
9
+except ImportError:
10
+ libtpu = None
11
+
12
+if libtpu is not None:
13
+ # people might install pytorch built with cuda but run on tpu
14
+ # so we need to check tpu first
15
+ from .tpu import TpuPlatform
16
+ current_platform = TpuPlatform()
17
+elif torch.version.cuda is not None:
18
from .cuda import CudaPlatform
19
current_platform = CudaPlatform()
20
elif torch.version.hip is not None:
21
from .rocm import RocmPlatform
22
current_platform = RocmPlatform()
17
-elif is_tpu():
18
- from .tpu import TpuPlatform
19
- current_platform = TpuPlatform()
23
else:
24
current_platform = UnspecifiedPlatform()
25
vllm/utils.py
-9
@@ -333,15 +333,6 @@ def is_neuron() -> bool:
333
return transformers_neuronx is not None
334
335
336
-@lru_cache(maxsize=None)
337
-def is_tpu() -> bool:
338
- try:
339
- import libtpu
340
- except ImportError:
341
- libtpu = None
342
- return libtpu is not None
343
-
344
-
336
@lru_cache(maxsize=None)
337
def is_xpu() -> bool:
338
from importlib.metadata import PackageNotFoundError, version