@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4d2dc5072b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[hardware] unify usage of is_tpu to current_platform.is_tpu() (#7102)

youkaichao committed Aug 13, 2024 at 00:16 UTC 4d2dc5072b8cfaff0c02e504c79a72ee95d839a1
8 files changed +29 -33
vllm/attention/selector.py
+2 -3
@@ -10,8 +10,7 @@ import vllm.envs as envs
10 from vllm.attention.backends.abstract import AttentionBackend
11 from vllm.logger import init_logger
12 from vllm.platforms import current_platform
13 -from vllm.utils import (STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino,
14 - is_tpu, is_xpu)
13 +from vllm.utils import STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino, is_xpu
14
15 logger = init_logger(__name__)
16
@@ -194,7 +193,7 @@ def which_attn_to_use(
193 logger.info("Cannot use %s backend on XPU.", selected_backend)
194 return _Backend.IPEX
195
197 - if is_tpu():
196 + if current_platform.is_tpu():
197 if selected_backend != _Backend.PALLAS:
198 logger.info("Cannot use %s backend on TPU.", selected_backend)
199 return _Backend.PALLAS
vllm/config.py
+4 -3
@@ -10,11 +10,12 @@ import vllm.envs as envs
10 from vllm.logger import init_logger
11 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
13 +from vllm.platforms import current_platform
14 from vllm.tracing import is_otel_installed
15 from vllm.transformers_utils.config import get_config, get_hf_text_config
16 from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
17 cuda_device_count_stateless, get_cpu_memory, is_cpu,
17 - is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
18 + is_hip, is_neuron, is_openvino, is_xpu,
19 print_warning_once)
20
21 if TYPE_CHECKING:
@@ -282,7 +283,7 @@ class ModelConfig:
283 raise ValueError(
284 f"{self.quantization} quantization is currently not "
285 f"supported in ROCm.")
285 - if is_tpu(
286 + if current_platform.is_tpu(
287 ) and self.quantization not in tpu_supported_quantization:
288 raise ValueError(
289 f"{self.quantization} quantization is currently not "
@@ -910,7 +911,7 @@ class DeviceConfig:
911 self.device_type = "neuron"
912 elif is_openvino():
913 self.device_type = "openvino"
913 - elif is_tpu():
914 + elif current_platform.is_tpu():
915 self.device_type = "tpu"
916 elif is_cpu():
917 self.device_type = "cpu"
vllm/executor/ray_utils.py
+3 -2
@@ -2,8 +2,9 @@ from typing import List, Optional, Tuple, Union
2
3 from vllm.config import ParallelConfig
4 from vllm.logger import init_logger
5 +from vllm.platforms import current_platform
6 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
6 -from vllm.utils import get_ip, is_hip, is_tpu, is_xpu
7 +from vllm.utils import get_ip, is_hip, is_xpu
8 from vllm.worker.worker_base import WorkerWrapperBase
9
10 logger = init_logger(__name__)
@@ -111,7 +112,7 @@ def initialize_ray_cluster(
112 # Placement group is already set.
113 return
114
114 - device_str = "GPU" if not is_tpu() else "TPU"
115 + device_str = "GPU" if not current_platform.is_tpu() else "TPU"
116 # Create placement group for worker processes
117 current_placement_group = ray.util.get_current_placement_group()
118 if current_placement_group:
vllm/model_executor/custom_op.py
+3 -2
@@ -1,6 +1,7 @@
1 import torch.nn as nn
2
3 -from vllm.utils import is_cpu, is_hip, is_tpu, is_xpu
3 +from vllm.platforms import current_platform
4 +from vllm.utils import is_cpu, is_hip, is_xpu
5
6
7 class CustomOp(nn.Module):
@@ -54,7 +55,7 @@ class CustomOp(nn.Module):
55 return self.forward_hip
56 elif is_cpu():
57 return self.forward_cpu
57 - elif is_tpu():
58 + elif current_platform.is_tpu():
59 return self.forward_tpu
60 elif is_xpu():
61 return self.forward_xpu
vllm/model_executor/layers/rotary_embedding.py
+2 -2
@@ -28,7 +28,7 @@ import torch
28 import torch.nn as nn
29
30 from vllm.model_executor.custom_op import CustomOp
31 -from vllm.utils import is_tpu
31 +from vllm.platforms import current_platform
32
33
34 def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -78,7 +78,7 @@ class RotaryEmbedding(CustomOp):
78 self.dtype = dtype
79
80 cache = self._compute_cos_sin_cache()
81 - self.use_native2 = is_tpu() and is_neox_style
81 + self.use_native2 = current_platform.is_tpu() and is_neox_style
82 if not self.use_native2:
83 cache = cache.to(dtype)
84 self.register_buffer("cos_sin_cache", cache, persistent=False)
vllm/model_executor/model_loader/loader.py
+3 -3
@@ -41,7 +41,7 @@ from vllm.model_executor.models.interfaces import (has_inner_state,
41 supports_vision)
42 from vllm.model_executor.utils import set_weight_attrs
43 from vllm.platforms import current_platform
44 -from vllm.utils import is_pin_memory_available, is_tpu
44 +from vllm.utils import is_pin_memory_available
45
46
47 @contextmanager
@@ -94,7 +94,7 @@ def _get_quantization_config(
94 """Get the quantization config."""
95 if model_config.quantization is not None:
96 quant_config = get_quant_config(model_config, load_config)
97 - if not is_tpu():
97 + if not current_platform.is_tpu():
98 capability = current_platform.get_device_capability()
99 capability = capability[0] * 10 + capability[1]
100 if capability < quant_config.get_min_capability():
@@ -320,7 +320,7 @@ class DefaultModelLoader(BaseModelLoader):
320 else:
321 weights_iterator = pt_weights_iterator(hf_weights_files)
322
323 - if is_tpu():
323 + if current_platform.is_tpu():
324 # In PyTorch XLA, we should call `xm.mark_step` frequently so that
325 # not too many ops are accumulated in the XLA program.
326 import torch_xla.core.xla_model as xm
vllm/platforms/__init__.py
+12 -9
@@ -1,22 +1,25 @@
1 -from typing import Optional
2 -
1 import torch
2
5 -from vllm.utils import is_tpu
6 -
3 from .interface import Platform, PlatformEnum, UnspecifiedPlatform
4
9 -current_platform: Optional[Platform]
5 +current_platform: Platform
6
11 -if torch.version.cuda is not None:
7 +try:
8 + import libtpu
9 +except ImportError:
10 + libtpu = None
11 +
12 +if libtpu is not None:
13 + # people might install pytorch built with cuda but run on tpu
14 + # so we need to check tpu first
15 + from .tpu import TpuPlatform
16 + current_platform = TpuPlatform()
17 +elif torch.version.cuda is not None:
18 from .cuda import CudaPlatform
19 current_platform = CudaPlatform()
20 elif torch.version.hip is not None:
21 from .rocm import RocmPlatform
22 current_platform = RocmPlatform()
17 -elif is_tpu():
18 - from .tpu import TpuPlatform
19 - current_platform = TpuPlatform()
23 else:
24 current_platform = UnspecifiedPlatform()
25
vllm/utils.py
-9
@@ -333,15 +333,6 @@ def is_neuron() -> bool:
333 return transformers_neuronx is not None
334
335
336 -@lru_cache(maxsize=None)
337 -def is_tpu() -> bool:
338 - try:
339 - import libtpu
340 - except ImportError:
341 - libtpu = None
342 - return libtpu is not None
343 -
344 -
336 @lru_cache(maxsize=None)
337 def is_xpu() -> bool:
338 from importlib.metadata import PackageNotFoundError, version