A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[hardware][misc] introduce platform abstraction (#6080)
youkaichao committed
Jul 2, 2024 at 20:12 UTC
482045ee77a49d69ab9464d9d727960890d950f1
16 files changed
+113
-29
tests/kernels/test_cutlass.py
+2
-2
@@ -8,13 +8,13 @@ import pytest
8
import torch
9
10
from vllm import _custom_ops as ops
11
-from vllm.utils import get_device_capability_stateless
11
+from vllm.platforms import current_platform
12
13
CUDA_DEVICES = [
14
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15
]
16
17
-capability = get_device_capability_stateless()
17
+capability = current_platform.get_device_capability()
18
capability = capability[0] * 10 + capability[1]
19
20
tests/quantization/utils.py
+2
-2
@@ -1,7 +1,7 @@
1
import torch
2
3
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
4
-from vllm.utils import get_device_capability_stateless
4
+from vllm.platforms import current_platform
5
6
7
def is_quant_method_supported(quant_method: str) -> bool:
@@ -9,7 +9,7 @@ def is_quant_method_supported(quant_method: str) -> bool:
9
if not torch.cuda.is_available():
10
return False
11
12
- capability = get_device_capability_stateless()
12
+ capability = current_platform.get_device_capability()
13
capability = capability[0] * 10 + capability[1]
14
return (capability >=
15
QUANTIZATION_METHODS[quant_method].get_min_capability())
vllm/attention/ops/blocksparse_attention/interface.py
+3
-2
@@ -2,13 +2,14 @@ import math
2
3
import torch
4
5
-from vllm.utils import get_device_capability_stateless, is_cpu, is_hip
5
+from vllm.platforms import current_platform
6
+from vllm.utils import is_cpu, is_hip
7
8
from .utils import (dense_to_crow_col, get_head_sliding_step,
9
get_sparse_attn_mask)
10
11
IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
11
- and get_device_capability_stateless()[0] >= 8)
12
+ and current_platform.get_device_capability()[0] >= 8)
13
14
if IS_COMPUTE_8_OR_ABOVE:
15
from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
vllm/attention/ops/prefix_prefill.py
+2
-2
@@ -5,7 +5,7 @@ import torch
5
import triton
6
import triton.language as tl
7
8
-from vllm.utils import get_device_capability_stateless
8
+from vllm.platforms import current_platform
9
10
if triton.__version__ >= "2.1.0":
11
@@ -685,7 +685,7 @@ if triton.__version__ >= "2.1.0":
685
alibi_slopes=None,
686
sliding_window=None):
687
688
- cap = get_device_capability_stateless()
688
+ cap = current_platform.get_device_capability()
689
BLOCK = 128 if cap[0] >= 8 else 64
690
# shape constraints
691
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
vllm/lora/punica.py
+2
-2
@@ -5,14 +5,14 @@ from typing import Optional
5
import torch
6
7
from vllm import _custom_ops as ops
8
-from vllm.utils import get_device_capability_stateless
8
+from vllm.platforms import current_platform
9
10
11
def _check_punica_support():
12
if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
13
return
14
15
- if get_device_capability_stateless() < (8, 0):
15
+ if current_platform.get_device_capability() < (8, 0):
16
raise ImportError(
17
"punica LoRA kernels require compute capability >= 8.0")
18
else:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2
-2
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
CompressionFormat, QuantizationArgs, QuantizationStrategy,
16
find_first_name_or_class_match)
17
-from vllm.utils import get_device_capability_stateless
17
+from vllm.platforms import current_platform
18
19
20
class CompressedTensorsConfig(QuantizationConfig):
@@ -85,7 +85,7 @@ class CompressedTensorsConfig(QuantizationConfig):
85
return []
86
87
def _check_gptq_and_marlin_can_run(self):
88
- capability = get_device_capability_stateless()
88
+ capability = current_platform.get_device_capability()
89
capability = capability[0] * 10 + capability[1]
90
if capability < 80:
91
raise RuntimeError("The quantization config is not supported for ",
vllm/model_executor/layers/quantization/fp8.py
+3
-2
@@ -12,7 +12,8 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
14
from vllm.model_executor.utils import set_weight_attrs
15
-from vllm.utils import get_device_capability_stateless, print_warning_once
15
+from vllm.platforms import current_platform
16
+from vllm.utils import print_warning_once
17
18
ACTIVATION_SCHEMES = ["static", "dynamic"]
19
@@ -20,7 +21,7 @@ logger = init_logger(__name__)
21
22
23
def cutlass_fp8_supported() -> bool:
23
- capability = get_device_capability_stateless()
24
+ capability = current_platform.get_device_capability()
25
capability = capability[0] * 10 + capability[1]
26
27
return ops.cutlass_scaled_mm_supports_fp8(capability)
vllm/model_executor/layers/quantization/gptq_marlin.py
+2
-2
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15
-from vllm.utils import get_device_capability_stateless
15
+from vllm.platforms import current_platform
16
17
logger = init_logger(__name__)
18
@@ -173,7 +173,7 @@ class GPTQMarlinConfig(QuantizationConfig):
173
return False
174
175
# If the capability of the device is too low, cannot convert.
176
- major, minor = get_device_capability_stateless()
176
+ major, minor = current_platform.get_device_capability()
177
device_capability = major * 10 + minor
178
if device_capability < cls.get_min_capability():
179
return False
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+2
-2
@@ -12,9 +12,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
12
marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
13
from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
get_pack_factor, quantize_weights, sort_weights)
15
-from vllm.utils import get_device_capability_stateless
15
+from vllm.platforms import current_platform
16
17
-__cuda_arch = get_device_capability_stateless()
17
+__cuda_arch = current_platform.get_device_capability()
18
19
MARLIN_TILE = 16
20
vllm/model_executor/model_loader/loader.py
+3
-2
@@ -35,7 +35,8 @@ from vllm.model_executor.model_loader.weight_utils import (
35
from vllm.model_executor.models.interfaces import (supports_lora,
36
supports_vision)
37
from vllm.model_executor.utils import set_weight_attrs
38
-from vllm.utils import get_device_capability_stateless, is_tpu
38
+from vllm.platforms import current_platform
39
+from vllm.utils import is_tpu
40
41
logger = init_logger(__name__)
42
@@ -46,7 +47,7 @@ def _get_quantization_config(
47
"""Get the quantization config."""
48
if model_config.quantization is not None:
49
quant_config = get_quant_config(model_config, load_config)
49
- capability = get_device_capability_stateless()
50
+ capability = current_platform.get_device_capability()
51
capability = capability[0] * 10 + capability[1]
52
if capability < quant_config.get_min_capability():
53
raise ValueError(
vllm/platforms/__init__.py
new
+18
@@ -0,0 +1,18 @@
1
+from typing import Optional
2
+
3
+import torch
4
+
5
+from .interface import Platform, PlatformEnum
6
+
7
+current_platform: Optional[Platform]
8
+
9
+if torch.version.cuda is not None:
10
+ from .cuda import CudaPlatform
11
+ current_platform = CudaPlatform()
12
+elif torch.version.hip is not None:
13
+ from .rocm import RocmPlatform
14
+ current_platform = RocmPlatform()
15
+else:
16
+ current_platform = None
17
+
18
+__all__ = ['Platform', 'PlatformEnum', 'current_platform']
vllm/platforms/cuda.py
new
+34
@@ -0,0 +1,34 @@
1
+"""Code inside this file can safely assume cuda platform, e.g. importing
2
+pynvml. However, it should not initialize cuda context.
3
+"""
4
+
5
+from functools import lru_cache, wraps
6
+from typing import Tuple
7
+
8
+import pynvml
9
+
10
+from .interface import Platform, PlatformEnum
11
+
12
+
13
+def with_nvml_context(fn):
14
+
15
+ @wraps(fn)
16
+ def wrapper(*args, **kwargs):
17
+ pynvml.nvmlInit()
18
+ try:
19
+ return fn(*args, **kwargs)
20
+ finally:
21
+ pynvml.nvmlShutdown()
22
+
23
+ return wrapper
24
+
25
+
26
+class CudaPlatform(Platform):
27
+ _enum = PlatformEnum.CUDA
28
+
29
+ @staticmethod
30
+ @lru_cache(maxsize=8)
31
+ @with_nvml_context
32
+ def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
33
+ handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
34
+ return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
vllm/platforms/interface.py
new
+21
@@ -0,0 +1,21 @@
1
+import enum
2
+from typing import Tuple
3
+
4
+
5
+class PlatformEnum(enum.Enum):
6
+ CUDA = enum.auto()
7
+ ROCM = enum.auto()
8
+
9
+
10
+class Platform:
11
+ _enum: PlatformEnum
12
+
13
+ def is_cuda(self) -> bool:
14
+ return self._enum == PlatformEnum.CUDA
15
+
16
+ def is_rocm(self) -> bool:
17
+ return self._enum == PlatformEnum.ROCM
18
+
19
+ @staticmethod
20
+ def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
21
+ raise NotImplementedError
vllm/platforms/rocm.py
new
+15
@@ -0,0 +1,15 @@
1
+from functools import lru_cache
2
+from typing import Tuple
3
+
4
+import torch
5
+
6
+from .interface import Platform, PlatformEnum
7
+
8
+
9
+class RocmPlatform(Platform):
10
+ _enum = PlatformEnum.ROCM
11
+
12
+ @staticmethod
13
+ @lru_cache(maxsize=8)
14
+ def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
15
+ return torch.cuda.get_device_capability(device_id)
vllm/utils.py
-7
@@ -866,13 +866,6 @@ def is_full_nvlink(device_ids: List[int]) -> bool:
866
return True
867
868
869
-@lru_cache(maxsize=8)
870
-@with_nvml_context
871
-def get_device_capability_stateless(device_id: int = 0) -> Tuple[int, int]:
872
- handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
873
- return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
874
-
875
-
869
#From: https://stackoverflow.com/a/4104188/2749989
870
def run_once(f):
871
vllm/worker/worker.py
+2
-2
@@ -15,8 +15,8 @@ from vllm.distributed import (ensure_model_parallel_initialized,
15
from vllm.lora.request import LoRARequest
16
from vllm.model_executor import set_random_seed
17
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
18
+from vllm.platforms import current_platform
19
from vllm.sequence import ExecuteModelRequest
19
-from vllm.utils import get_device_capability_stateless
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -333,7 +333,7 @@ def init_worker_distributed_environment(
333
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
334
# Check if the GPU supports the dtype.
335
if torch_dtype == torch.bfloat16:
336
- compute_capability = get_device_capability_stateless()
336
+ compute_capability = current_platform.get_device_capability()
337
if compute_capability[0] < 8:
338
gpu_name = torch.cuda.get_device_name()
339
raise ValueError(