@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 482045ee77
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[hardware][misc] introduce platform abstraction (#6080)

youkaichao committed Jul 2, 2024 at 20:12 UTC 482045ee77a49d69ab9464d9d727960890d950f1
16 files changed +113 -29
tests/kernels/test_cutlass.py
+2 -2
@@ -8,13 +8,13 @@ import pytest
8 import torch
9
10 from vllm import _custom_ops as ops
11 -from vllm.utils import get_device_capability_stateless
11 +from vllm.platforms import current_platform
12
13 CUDA_DEVICES = [
14 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15 ]
16
17 -capability = get_device_capability_stateless()
17 +capability = current_platform.get_device_capability()
18 capability = capability[0] * 10 + capability[1]
19
20
tests/quantization/utils.py
+2 -2
@@ -1,7 +1,7 @@
1 import torch
2
3 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
4 -from vllm.utils import get_device_capability_stateless
4 +from vllm.platforms import current_platform
5
6
7 def is_quant_method_supported(quant_method: str) -> bool:
@@ -9,7 +9,7 @@ def is_quant_method_supported(quant_method: str) -> bool:
9 if not torch.cuda.is_available():
10 return False
11
12 - capability = get_device_capability_stateless()
12 + capability = current_platform.get_device_capability()
13 capability = capability[0] * 10 + capability[1]
14 return (capability >=
15 QUANTIZATION_METHODS[quant_method].get_min_capability())
vllm/attention/ops/blocksparse_attention/interface.py
+3 -2
@@ -2,13 +2,14 @@ import math
2
3 import torch
4
5 -from vllm.utils import get_device_capability_stateless, is_cpu, is_hip
5 +from vllm.platforms import current_platform
6 +from vllm.utils import is_cpu, is_hip
7
8 from .utils import (dense_to_crow_col, get_head_sliding_step,
9 get_sparse_attn_mask)
10
11 IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
11 - and get_device_capability_stateless()[0] >= 8)
12 + and current_platform.get_device_capability()[0] >= 8)
13
14 if IS_COMPUTE_8_OR_ABOVE:
15 from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
vllm/attention/ops/prefix_prefill.py
+2 -2
@@ -5,7 +5,7 @@ import torch
5 import triton
6 import triton.language as tl
7
8 -from vllm.utils import get_device_capability_stateless
8 +from vllm.platforms import current_platform
9
10 if triton.__version__ >= "2.1.0":
11
@@ -685,7 +685,7 @@ if triton.__version__ >= "2.1.0":
685 alibi_slopes=None,
686 sliding_window=None):
687
688 - cap = get_device_capability_stateless()
688 + cap = current_platform.get_device_capability()
689 BLOCK = 128 if cap[0] >= 8 else 64
690 # shape constraints
691 Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
vllm/lora/punica.py
+2 -2
@@ -5,14 +5,14 @@ from typing import Optional
5 import torch
6
7 from vllm import _custom_ops as ops
8 -from vllm.utils import get_device_capability_stateless
8 +from vllm.platforms import current_platform
9
10
11 def _check_punica_support():
12 if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
13 return
14
15 - if get_device_capability_stateless() < (8, 0):
15 + if current_platform.get_device_capability() < (8, 0):
16 raise ImportError(
17 "punica LoRA kernels require compute capability >= 8.0")
18 else:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2 -2
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
16 find_first_name_or_class_match)
17 -from vllm.utils import get_device_capability_stateless
17 +from vllm.platforms import current_platform
18
19
20 class CompressedTensorsConfig(QuantizationConfig):
@@ -85,7 +85,7 @@ class CompressedTensorsConfig(QuantizationConfig):
85 return []
86
87 def _check_gptq_and_marlin_can_run(self):
88 - capability = get_device_capability_stateless()
88 + capability = current_platform.get_device_capability()
89 capability = capability[0] * 10 + capability[1]
90 if capability < 80:
91 raise RuntimeError("The quantization config is not supported for ",
vllm/model_executor/layers/quantization/fp8.py
+3 -2
@@ -12,7 +12,8 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.utils import set_weight_attrs
15 -from vllm.utils import get_device_capability_stateless, print_warning_once
15 +from vllm.platforms import current_platform
16 +from vllm.utils import print_warning_once
17
18 ACTIVATION_SCHEMES = ["static", "dynamic"]
19
@@ -20,7 +21,7 @@ logger = init_logger(__name__)
21
22
23 def cutlass_fp8_supported() -> bool:
23 - capability = get_device_capability_stateless()
24 + capability = current_platform.get_device_capability()
25 capability = capability[0] * 10 + capability[1]
26
27 return ops.cutlass_scaled_mm_supports_fp8(capability)
vllm/model_executor/layers/quantization/gptq_marlin.py
+2 -2
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15 -from vllm.utils import get_device_capability_stateless
15 +from vllm.platforms import current_platform
16
17 logger = init_logger(__name__)
18
@@ -173,7 +173,7 @@ class GPTQMarlinConfig(QuantizationConfig):
173 return False
174
175 # If the capability of the device is too low, cannot convert.
176 - major, minor = get_device_capability_stateless()
176 + major, minor = current_platform.get_device_capability()
177 device_capability = major * 10 + minor
178 if device_capability < cls.get_min_capability():
179 return False
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+2 -2
@@ -12,9 +12,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
12 marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
13 from vllm.model_executor.layers.quantization.utils.quant_utils import (
14 get_pack_factor, quantize_weights, sort_weights)
15 -from vllm.utils import get_device_capability_stateless
15 +from vllm.platforms import current_platform
16
17 -__cuda_arch = get_device_capability_stateless()
17 +__cuda_arch = current_platform.get_device_capability()
18
19 MARLIN_TILE = 16
20
vllm/model_executor/model_loader/loader.py
+3 -2
@@ -35,7 +35,8 @@ from vllm.model_executor.model_loader.weight_utils import (
35 from vllm.model_executor.models.interfaces import (supports_lora,
36 supports_vision)
37 from vllm.model_executor.utils import set_weight_attrs
38 -from vllm.utils import get_device_capability_stateless, is_tpu
38 +from vllm.platforms import current_platform
39 +from vllm.utils import is_tpu
40
41 logger = init_logger(__name__)
42
@@ -46,7 +47,7 @@ def _get_quantization_config(
47 """Get the quantization config."""
48 if model_config.quantization is not None:
49 quant_config = get_quant_config(model_config, load_config)
49 - capability = get_device_capability_stateless()
50 + capability = current_platform.get_device_capability()
51 capability = capability[0] * 10 + capability[1]
52 if capability < quant_config.get_min_capability():
53 raise ValueError(
vllm/platforms/__init__.py new
+18
@@ -0,0 +1,18 @@
1 +from typing import Optional
2 +
3 +import torch
4 +
5 +from .interface import Platform, PlatformEnum
6 +
7 +current_platform: Optional[Platform]
8 +
9 +if torch.version.cuda is not None:
10 + from .cuda import CudaPlatform
11 + current_platform = CudaPlatform()
12 +elif torch.version.hip is not None:
13 + from .rocm import RocmPlatform
14 + current_platform = RocmPlatform()
15 +else:
16 + current_platform = None
17 +
18 +__all__ = ['Platform', 'PlatformEnum', 'current_platform']
vllm/platforms/cuda.py new
+34
@@ -0,0 +1,34 @@
1 +"""Code inside this file can safely assume cuda platform, e.g. importing
2 +pynvml. However, it should not initialize cuda context.
3 +"""
4 +
5 +from functools import lru_cache, wraps
6 +from typing import Tuple
7 +
8 +import pynvml
9 +
10 +from .interface import Platform, PlatformEnum
11 +
12 +
13 +def with_nvml_context(fn):
14 +
15 + @wraps(fn)
16 + def wrapper(*args, **kwargs):
17 + pynvml.nvmlInit()
18 + try:
19 + return fn(*args, **kwargs)
20 + finally:
21 + pynvml.nvmlShutdown()
22 +
23 + return wrapper
24 +
25 +
26 +class CudaPlatform(Platform):
27 + _enum = PlatformEnum.CUDA
28 +
29 + @staticmethod
30 + @lru_cache(maxsize=8)
31 + @with_nvml_context
32 + def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
33 + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
34 + return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
vllm/platforms/interface.py new
+21
@@ -0,0 +1,21 @@
1 +import enum
2 +from typing import Tuple
3 +
4 +
5 +class PlatformEnum(enum.Enum):
6 + CUDA = enum.auto()
7 + ROCM = enum.auto()
8 +
9 +
10 +class Platform:
11 + _enum: PlatformEnum
12 +
13 + def is_cuda(self) -> bool:
14 + return self._enum == PlatformEnum.CUDA
15 +
16 + def is_rocm(self) -> bool:
17 + return self._enum == PlatformEnum.ROCM
18 +
19 + @staticmethod
20 + def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
21 + raise NotImplementedError
vllm/platforms/rocm.py new
+15
@@ -0,0 +1,15 @@
1 +from functools import lru_cache
2 +from typing import Tuple
3 +
4 +import torch
5 +
6 +from .interface import Platform, PlatformEnum
7 +
8 +
9 +class RocmPlatform(Platform):
10 + _enum = PlatformEnum.ROCM
11 +
12 + @staticmethod
13 + @lru_cache(maxsize=8)
14 + def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
15 + return torch.cuda.get_device_capability(device_id)
vllm/utils.py
-7
@@ -866,13 +866,6 @@ def is_full_nvlink(device_ids: List[int]) -> bool:
866 return True
867
868
869 -@lru_cache(maxsize=8)
870 -@with_nvml_context
871 -def get_device_capability_stateless(device_id: int = 0) -> Tuple[int, int]:
872 - handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
873 - return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
874 -
875 -
869 #From: https://stackoverflow.com/a/4104188/2749989
870 def run_once(f):
871
vllm/worker/worker.py
+2 -2
@@ -15,8 +15,8 @@ from vllm.distributed import (ensure_model_parallel_initialized,
15 from vllm.lora.request import LoRARequest
16 from vllm.model_executor import set_random_seed
17 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
18 +from vllm.platforms import current_platform
19 from vllm.sequence import ExecuteModelRequest
19 -from vllm.utils import get_device_capability_stateless
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -333,7 +333,7 @@ def init_worker_distributed_environment(
333 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
334 # Check if the GPU supports the dtype.
335 if torch_dtype == torch.bfloat16:
336 - compute_capability = get_device_capability_stateless()
336 + compute_capability = current_platform.get_device_capability()
337 if compute_capability[0] < 8:
338 gpu_name = torch.cuda.get_device_name()
339 raise ValueError(