A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[misc][cuda] use nvml to avoid accidentally cuda initialization (#6007)
youkaichao committed
Jun 30, 2024 at 20:07 UTC
614aa5120303ab09be78fb1db669da198cc43b02
13 files changed
+86
-68
tests/kernels/test_cutlass.py
+2
-1
@@ -8,12 +8,13 @@ import pytest
8
import torch
9
10
from vllm import _custom_ops as ops
11
+from vllm.utils import get_device_capability_stateless
12
13
CUDA_DEVICES = [
14
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15
]
16
16
-capability = torch.cuda.get_device_capability()
17
+capability = get_device_capability_stateless()
18
capability = capability[0] * 10 + capability[1]
19
20
tests/quantization/utils.py
+2
-1
@@ -1,6 +1,7 @@
1
import torch
2
3
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
4
+from vllm.utils import get_device_capability_stateless
5
6
7
def is_quant_method_supported(quant_method: str) -> bool:
@@ -8,7 +9,7 @@ def is_quant_method_supported(quant_method: str) -> bool:
9
if not torch.cuda.is_available():
10
return False
11
11
- capability = torch.cuda.get_device_capability()
12
+ capability = get_device_capability_stateless()
13
capability = capability[0] * 10 + capability[1]
14
return (capability >=
15
QUANTIZATION_METHODS[quant_method].get_min_capability())
vllm/attention/ops/blocksparse_attention/interface.py
+3
-3
@@ -2,13 +2,13 @@ import math
2
3
import torch
4
5
-from vllm.utils import is_cpu, is_hip
5
+from vllm.utils import get_device_capability_stateless, is_cpu, is_hip
6
7
from .utils import (dense_to_crow_col, get_head_sliding_step,
8
get_sparse_attn_mask)
9
10
IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
11
- and torch.cuda.get_device_capability()[0] >= 8)
11
+ and get_device_capability_stateless()[0] >= 8)
12
13
if IS_COMPUTE_8_OR_ABOVE:
14
from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
@@ -235,4 +235,4 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
235
v,
236
cu_seqlens_k,
237
cu_seqlens_q=cu_seqlens_q,
238
- sm_scale=sm_scale)
\ No newline at end of file
238
+ sm_scale=sm_scale)
vllm/attention/ops/prefix_prefill.py
+3
-1
@@ -5,6 +5,8 @@ import torch
5
import triton
6
import triton.language as tl
7
8
+from vllm.utils import get_device_capability_stateless
9
+
10
if triton.__version__ >= "2.1.0":
11
12
@triton.jit
@@ -683,7 +685,7 @@ if triton.__version__ >= "2.1.0":
685
alibi_slopes=None,
686
sliding_window=None):
687
686
- cap = torch.cuda.get_device_capability()
688
+ cap = get_device_capability_stateless()
689
BLOCK = 128 if cap[0] >= 8 else 64
690
# shape constraints
691
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
vllm/distributed/device_communicators/custom_all_reduce.py
+5
-53
@@ -11,66 +11,18 @@ from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
from vllm.distributed.parallel_state import is_in_the_same_node
13
from vllm.logger import init_logger
14
-from vllm.utils import cuda_device_count_stateless
14
+from vllm.utils import cuda_device_count_stateless, is_full_nvlink
15
16
try:
17
- import pynvml
18
-
19
- # Simulate ImportError if custom_ar ops are not supported.
20
- if not ops.is_custom_op_supported("_C_custom_ar::meta_size"):
21
- raise ImportError("custom_ar", __file__)
22
-
17
+ assert ops.is_custom_op_supported("_C_custom_ar::meta_size")
18
custom_ar = True
24
-
25
- @contextmanager
26
- def _nvml():
27
- try:
28
- pynvml.nvmlInit()
29
- yield
30
- finally:
31
- pynvml.nvmlShutdown()
32
-
33
-except ImportError:
34
- # For AMD GPUs
19
+except Exception:
20
+ # For AMD GPUs and CPUs
21
custom_ar = False
36
- pynvml = None
37
-
38
- @contextmanager
39
- def _nvml():
40
- try:
41
- yield
42
- finally:
43
- pass
44
-
22
23
logger = init_logger(__name__)
24
25
49
-@_nvml()
50
-def _is_full_nvlink(device_ids: List[int]) -> bool:
51
- """
52
- query if the set of gpus are fully connected by nvlink (1 hop)
53
- Note that `pynvml` is not affected by `CUDA_VISIBLE_DEVICES`,
54
- so it works on real physical device ids.
55
- """
56
- handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
57
- for i, handle in enumerate(handles):
58
- for j, peer_handle in enumerate(handles):
59
- if i < j:
60
- try:
61
- p2p_status = pynvml.nvmlDeviceGetP2PStatus(
62
- handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
63
- if p2p_status != pynvml.NVML_P2P_STATUS_OK:
64
- return False
65
- except pynvml.NVMLError as error:
66
- logger.error(
67
- "NVLink detection failed. This is normal if your"
68
- " machine has no NVLink equipped.",
69
- exc_info=error)
70
- return False
71
- return True
72
-
73
-
26
def _can_p2p(rank: int, world_size: int) -> bool:
27
for i in range(world_size):
28
if i == rank:
@@ -161,7 +113,7 @@ class CustomAllreduce:
113
# test nvlink first, this will filter out most of the cases
114
# where custom allreduce is not supported
115
# this checks hardware and driver support for NVLink
164
- full_nvlink = _is_full_nvlink(physical_device_ids)
116
+ full_nvlink = is_full_nvlink(physical_device_ids)
117
if world_size > 2 and not full_nvlink:
118
logger.warning(
119
"Custom allreduce is disabled because it's not supported on"
vllm/lora/punica.py
+2
-1
@@ -5,13 +5,14 @@ from typing import Optional
5
import torch
6
7
from vllm import _custom_ops as ops
8
+from vllm.utils import get_device_capability_stateless
9
10
11
def _check_punica_support():
12
if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
13
return
14
14
- if torch.cuda.get_device_capability() < (8, 0):
15
+ if get_device_capability_stateless() < (8, 0):
16
raise ImportError(
17
"punica LoRA kernels require compute capability >= 8.0")
18
else:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2
-1
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
14
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
CompressionFormat, QuantizationArgs, QuantizationStrategy,
16
find_first_name_or_class_match)
17
+from vllm.utils import get_device_capability_stateless
18
19
20
class CompressedTensorsConfig(QuantizationConfig):
@@ -84,7 +85,7 @@ class CompressedTensorsConfig(QuantizationConfig):
85
return []
86
87
def _check_gptq_and_marlin_can_run(self):
87
- capability = torch.cuda.get_device_capability()
88
+ capability = get_device_capability_stateless()
89
capability = capability[0] * 10 + capability[1]
90
if capability < 80:
91
raise RuntimeError("The quantization config is not supported for ",
vllm/model_executor/layers/quantization/fp8.py
+2
-2
@@ -10,7 +10,7 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig, QuantizeMethodBase)
12
from vllm.model_executor.utils import set_weight_attrs
13
-from vllm.utils import print_warning_once
13
+from vllm.utils import get_device_capability_stateless, print_warning_once
14
15
ACTIVATION_SCHEMES = ["static", "dynamic"]
16
@@ -18,7 +18,7 @@ logger = init_logger(__name__)
18
19
20
def cutlass_fp8_supported() -> bool:
21
- capability = torch.cuda.get_device_capability()
21
+ capability = get_device_capability_stateless()
22
capability = capability[0] * 10 + capability[1]
23
24
return ops.cutlass_scaled_mm_supports_fp8(capability)
vllm/model_executor/layers/quantization/gptq_marlin.py
+2
-1
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
+from vllm.utils import get_device_capability_stateless
15
16
logger = init_logger(__name__)
17
@@ -165,7 +166,7 @@ class GPTQMarlinConfig(QuantizationConfig):
166
return False
167
168
# If the capability of the device is too low, cannot convert.
168
- major, minor = torch.cuda.get_device_capability()
169
+ major, minor = get_device_capability_stateless()
170
device_capability = major * 10 + minor
171
if device_capability < cls.get_min_capability():
172
return False
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+2
-1
@@ -12,8 +12,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
12
marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
13
from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
get_pack_factor, quantize_weights, sort_weights)
15
+from vllm.utils import get_device_capability_stateless
16
16
-__cuda_arch = torch.cuda.get_device_capability()
17
+__cuda_arch = get_device_capability_stateless()
18
19
MARLIN_TILE = 16
20
vllm/model_executor/model_loader/loader.py
+2
-2
@@ -35,7 +35,7 @@ from vllm.model_executor.model_loader.weight_utils import (
35
from vllm.model_executor.models.interfaces import (supports_lora,
36
supports_vision)
37
from vllm.model_executor.utils import set_weight_attrs
38
-from vllm.utils import is_tpu
38
+from vllm.utils import get_device_capability_stateless, is_tpu
39
40
logger = init_logger(__name__)
41
@@ -46,7 +46,7 @@ def _get_quantization_config(
46
"""Get the quantization config."""
47
if model_config.quantization is not None:
48
quant_config = get_quant_config(model_config, load_config)
49
- capability = torch.cuda.get_device_capability()
49
+ capability = get_device_capability_stateless()
50
capability = capability[0] * 10 + capability[1]
51
if capability < quant_config.get_min_capability():
52
raise ValueError(
vllm/utils.py
+57
@@ -816,6 +816,63 @@ def cuda_device_count_stateless() -> int:
816
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
817
818
819
+# NVML utils
820
+# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
821
+# all the related functions work on real physical device ids.
822
+# the major benefit of using NVML is that it will not initialize CUDA
823
+
824
+try:
825
+ import pynvml
826
+except ImportError:
827
+ # For non-NV devices
828
+ pynvml = None
829
+
830
+
831
+def with_nvml_context(fn):
832
+
833
+ @wraps(fn)
834
+ def wrapper(*args, **kwargs):
835
+ if pynvml is not None:
836
+ pynvml.nvmlInit()
837
+ try:
838
+ return fn(*args, **kwargs)
839
+ finally:
840
+ if pynvml is not None:
841
+ pynvml.nvmlShutdown()
842
+
843
+ return wrapper
844
+
845
+
846
+@with_nvml_context
847
+def is_full_nvlink(device_ids: List[int]) -> bool:
848
+ """
849
+ query if the set of gpus are fully connected by nvlink (1 hop)
850
+ """
851
+ handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in device_ids]
852
+ for i, handle in enumerate(handles):
853
+ for j, peer_handle in enumerate(handles):
854
+ if i < j:
855
+ try:
856
+ p2p_status = pynvml.nvmlDeviceGetP2PStatus(
857
+ handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
858
+ if p2p_status != pynvml.NVML_P2P_STATUS_OK:
859
+ return False
860
+ except pynvml.NVMLError as error:
861
+ logger.error(
862
+ "NVLink detection failed. This is normal if your"
863
+ " machine has no NVLink equipped.",
864
+ exc_info=error)
865
+ return False
866
+ return True
867
+
868
+
869
+@lru_cache(maxsize=8)
870
+@with_nvml_context
871
+def get_device_capability_stateless(device_id: int = 0) -> Tuple[int, int]:
872
+ handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
873
+ return pynvml.nvmlDeviceGetCudaComputeCapability(handle)
874
+
875
+
876
#From: https://stackoverflow.com/a/4104188/2749989
877
def run_once(f):
878
vllm/worker/worker.py
+2
-1
@@ -16,6 +16,7 @@ from vllm.lora.request import LoRARequest
16
from vllm.model_executor import set_random_seed
17
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
18
from vllm.sequence import ExecuteModelRequest
19
+from vllm.utils import get_device_capability_stateless
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -322,7 +323,7 @@ def init_worker_distributed_environment(
323
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
324
# Check if the GPU supports the dtype.
325
if torch_dtype == torch.bfloat16:
325
- compute_capability = torch.cuda.get_device_capability()
326
+ compute_capability = get_device_capability_stateless()
327
if compute_capability[0] < 8:
328
gpu_name = torch.cuda.get_device_name()
329
raise ValueError(