@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 614aa51203
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[misc][cuda] use nvml to avoid accidentally cuda initialization (#6007)

youkaichao committed Jun 30, 2024 at 20:07 UTC 614aa5120303ab09be78fb1db669da198cc43b02
13 files changed +86 -68
tests/kernels/test_cutlass.py
+2 -1
@@ -8,12 +8,13 @@ import pytest
8 import torch
9
10 from vllm import _custom_ops as ops
11 +from vllm.utils import get_device_capability_stateless
12
13 CUDA_DEVICES = [
14 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15 ]
16
16 -capability = torch.cuda.get_device_capability()
17 +capability = get_device_capability_stateless()
18 capability = capability[0] * 10 + capability[1]
19
20
tests/quantization/utils.py
+2 -1
@@ -1,6 +1,7 @@
1 import torch
2
3 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
4 +from vllm.utils import get_device_capability_stateless
5
6
7 def is_quant_method_supported(quant_method: str) -> bool:
@@ -8,7 +9,7 @@ def is_quant_method_supported(quant_method: str) -> bool:
9 if not torch.cuda.is_available():
10 return False
11
11 - capability = torch.cuda.get_device_capability()
12 + capability = get_device_capability_stateless()
13 capability = capability[0] * 10 + capability[1]
14 return (capability >=
15 QUANTIZATION_METHODS[quant_method].get_min_capability())
vllm/attention/ops/blocksparse_attention/interface.py
+3 -3
@@ -2,13 +2,13 @@ import math
2
3 import torch
4
5 -from vllm.utils import is_cpu, is_hip
5 +from vllm.utils import get_device_capability_stateless, is_cpu, is_hip
6
7 from .utils import (dense_to_crow_col, get_head_sliding_step,
8 get_sparse_attn_mask)
9
10 IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
11 - and torch.cuda.get_device_capability()[0] >= 8)
11 + and get_device_capability_stateless()[0] >= 8)
12
13 if IS_COMPUTE_8_OR_ABOVE:
14 from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
@@ -235,4 +235,4 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
235 v,
236 cu_seqlens_k,
237 cu_seqlens_q=cu_seqlens_q,
238 - sm_scale=sm_scale)
\ No newline at end of file
238 + sm_scale=sm_scale)
vllm/attention/ops/prefix_prefill.py
+3 -1
@@ -5,6 +5,8 @@ import torch
5 import triton
6 import triton.language as tl
7
8 +from vllm.utils import get_device_capability_stateless
9 +
10 if triton.__version__ >= "2.1.0":
11
12 @triton.jit
@@ -683,7 +685,7 @@ if triton.__version__ >= "2.1.0":
685 alibi_slopes=None,
686 sliding_window=None):
687
686 - cap = torch.cuda.get_device_capability()
688 + cap = get_device_capability_stateless()
689 BLOCK = 128 if cap[0] >= 8 else 64
690 # shape constraints
691 Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
vllm/distributed/device_communicators/custom_all_reduce.py
+5 -53
@@ -11,66 +11,18 @@ from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11 gpu_p2p_access_check)
12 from vllm.distributed.parallel_state import is_in_the_same_node
13 from vllm.logger import init_logger
14 -from vllm.utils import cuda_device_count_stateless
14 +from vllm.utils import cuda_device_count_stateless, is_full_nvlink
15
16 try:
17 - import pynvml
18 -
19 - # Simulate ImportError if custom_ar ops are not supported.
20 - if not ops.is_custom_op_supported("_C_custom_ar::meta_size"):
21 - raise ImportError("custom_ar", __file__)
22 -
17 + assert ops.is_custom_op_supported("_C_custom_ar::meta_size")
18 custom_ar = True
24 -
25 - @contextmanager
26 - def _nvml():
27 - try:
28 - pynvml.nvmlInit()
29 - yield
30 - finally:
31 - pynvml.nvmlShutdown()
32 -
33 -except ImportError:
34 - # For AMD GPUs
19 +except Exception:
20 + # For AMD GPUs and CPUs
21 custom_ar = False
36 - pynvml = None
37 -
38 - @contextmanager
39 - def _nvml():
40 - try:
41 - yield
42 - finally:
43 - pass
44 -
22
23 logger = init_logger(__name__)
24
25
49 -@_nvml()
50 -def _is_full_nvlink(device_ids: List[int]) -> bool:
51 - """
52 - query if the set of gpus are fully connected by nvlink (1 hop)
53 - Note that `pynvml` is not affected by `CUDA_VISIBLE_DEVICES`,
54 - so it works on real physical device ids.
55 - """
56 - handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
57 - for i, handle in enumerate(handles):
58 - for j, peer_handle in enumerate(handles):
59 - if i < j:
60 - try:
61 - p2p_status = pynvml.nvmlDeviceGetP2PStatus(
62 - handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
63 - if p2p_status != pynvml.NVML_P2P_STATUS_OK:
64 - return False
65 - except pynvml.NVMLError as error:
66 - logger.error(
67 - "NVLink detection failed. This is normal if your"
68 - " machine has no NVLink equipped.",
69 - exc_info=error)
70 - return False
71 - return True
72 -
73 -
26 def _can_p2p(rank: int, world_size: int) -> bool:
27 for i in range(world_size):
28 if i == rank:
@@ -161,7 +113,7 @@ class CustomAllreduce:
113 # test nvlink first, this will filter out most of the cases
114 # where custom allreduce is not supported
115 # this checks hardware and driver support for NVLink
164 - full_nvlink = _is_full_nvlink(physical_device_ids)
116 + full_nvlink = is_full_nvlink(physical_device_ids)
117 if world_size > 2 and not full_nvlink:
118 logger.warning(
119 "Custom allreduce is disabled because it's not supported on"
vllm/lora/punica.py
+2 -1
@@ -5,13 +5,14 @@ from typing import Optional
5 import torch
6
7 from vllm import _custom_ops as ops
8 +from vllm.utils import get_device_capability_stateless
9
10
11 def _check_punica_support():
12 if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
13 return
14
14 - if torch.cuda.get_device_capability() < (8, 0):
15 + if get_device_capability_stateless() < (8, 0):
16 raise ImportError(
17 "punica LoRA kernels require compute capability >= 8.0")
18 else:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2 -1
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
14 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15 CompressionFormat, QuantizationArgs, QuantizationStrategy,
16 find_first_name_or_class_match)
17 +from vllm.utils import get_device_capability_stateless
18
19
20 class CompressedTensorsConfig(QuantizationConfig):
@@ -84,7 +85,7 @@ class CompressedTensorsConfig(QuantizationConfig):
85 return []
86
87 def _check_gptq_and_marlin_can_run(self):
87 - capability = torch.cuda.get_device_capability()
88 + capability = get_device_capability_stateless()
89 capability = capability[0] * 10 + capability[1]
90 if capability < 80:
91 raise RuntimeError("The quantization config is not supported for ",
vllm/model_executor/layers/quantization/fp8.py
+2 -2
@@ -10,7 +10,7 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig, QuantizeMethodBase)
12 from vllm.model_executor.utils import set_weight_attrs
13 -from vllm.utils import print_warning_once
13 +from vllm.utils import get_device_capability_stateless, print_warning_once
14
15 ACTIVATION_SCHEMES = ["static", "dynamic"]
16
@@ -18,7 +18,7 @@ logger = init_logger(__name__)
18
19
20 def cutlass_fp8_supported() -> bool:
21 - capability = torch.cuda.get_device_capability()
21 + capability = get_device_capability_stateless()
22 capability = capability[0] * 10 + capability[1]
23
24 return ops.cutlass_scaled_mm_supports_fp8(capability)
vllm/model_executor/layers/quantization/gptq_marlin.py
+2 -1
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 +from vllm.utils import get_device_capability_stateless
15
16 logger = init_logger(__name__)
17
@@ -165,7 +166,7 @@ class GPTQMarlinConfig(QuantizationConfig):
166 return False
167
168 # If the capability of the device is too low, cannot convert.
168 - major, minor = torch.cuda.get_device_capability()
169 + major, minor = get_device_capability_stateless()
170 device_capability = major * 10 + minor
171 if device_capability < cls.get_min_capability():
172 return False
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+2 -1
@@ -12,8 +12,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
12 marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
13 from vllm.model_executor.layers.quantization.utils.quant_utils import (
14 get_pack_factor, quantize_weights, sort_weights)
15 +from vllm.utils import get_device_capability_stateless
16
16 -__cuda_arch = torch.cuda.get_device_capability()
17 +__cuda_arch = get_device_capability_stateless()
18
19 MARLIN_TILE = 16
20
vllm/model_executor/model_loader/loader.py
+2 -2
@@ -35,7 +35,7 @@ from vllm.model_executor.model_loader.weight_utils import (
35 from vllm.model_executor.models.interfaces import (supports_lora,
36 supports_vision)
37 from vllm.model_executor.utils import set_weight_attrs
38 -from vllm.utils import is_tpu
38 +from vllm.utils import get_device_capability_stateless, is_tpu
39
40 logger = init_logger(__name__)
41
@@ -46,7 +46,7 @@ def _get_quantization_config(
46 """Get the quantization config."""
47 if model_config.quantization is not None:
48 quant_config = get_quant_config(model_config, load_config)
49 - capability = torch.cuda.get_device_capability()
49 + capability = get_device_capability_stateless()
50 capability = capability[0] * 10 + capability[1]
51 if capability < quant_config.get_min_capability():
52 raise ValueError(
vllm/utils.py
+57
@@ -816,6 +816,63 @@ def cuda_device_count_stateless() -> int:
816 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
817
818
819 +# NVML utils
820 +# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
821 +# all the related functions work on real physical device ids.
822 +# the major benefit of using NVML is that it will not initialize CUDA
823 +
824 +try:
825 + import pynvml
826 +except ImportError:
827 + # For non-NV devices
828 + pynvml = None
829 +
830 +
831 +def with_nvml_context(fn):
832 +
833 + @wraps(fn)
834 + def wrapper(*args, **kwargs):
835 + if pynvml is not None:
836 + pynvml.nvmlInit()
837 + try:
838 + return fn(*args, **kwargs)
839 + finally:
840 + if pynvml is not None:
841 + pynvml.nvmlShutdown()
842 +
843 + return wrapper
844 +
845 +
846 +@with_nvml_context
847 +def is_full_nvlink(device_ids: List[int]) -> bool:
848 + """
849 + query if the set of gpus are fully connected by nvlink (1 hop)
850 + """
851 + handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
852 + for i, handle in enumerate(handles):
853 + for j, peer_handle in enumerate(handles):
854 + if i < j:
855 + try:
856 + p2p_status = pynvml.nvmlDeviceGetP2PStatus(
857 + handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
858 + if p2p_status != pynvml.NVML_P2P_STATUS_OK:
859 + return False
860 + except pynvml.NVMLError as error:
861 + logger.error(
862 + "NVLink detection failed. This is normal if your"
863 + " machine has no NVLink equipped.",
864 + exc_info=error)
865 + return False
866 + return True
867 +
868 +
869 +@lru_cache(maxsize=8)
870 +@with_nvml_context
871 +def get_device_capability_stateless(device_id: int = 0) -> Tuple[int, int]:
872 + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
873 + return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
874 +
875 +
876 #From: https://stackoverflow.com/a/4104188/2749989
877 def run_once(f):
878
vllm/worker/worker.py
+2 -1
@@ -16,6 +16,7 @@ from vllm.lora.request import LoRARequest
16 from vllm.model_executor import set_random_seed
17 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
18 from vllm.sequence import ExecuteModelRequest
19 +from vllm.utils import get_device_capability_stateless
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -322,7 +323,7 @@ def init_worker_distributed_environment(
323 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
324 # Check if the GPU supports the dtype.
325 if torch_dtype == torch.bfloat16:
325 - compute_capability = torch.cuda.get_device_capability()
326 + compute_capability = get_device_capability_stateless()
327 if compute_capability[0] < 8:
328 gpu_name = torch.cuda.get_device_name()
329 raise ValueError(