@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ffa3f314c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[CI/Build] Avoid CUDA initialization (#8534)

Cyrus Leung committed Sep 18, 2024 at 18:38 UTC 6ffa3f314c59e42238f1c5f923ff2839e0af9698
55 files changed +256 -256
benchmarks/kernels/benchmark_layernorm.py
+3 -6
@@ -1,10 +1,10 @@
1 -import random
1 import time
2
3 import torch
4
5 from vllm.model_executor.layers.layernorm import RMSNorm
7 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser
6 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
7 + seed_everything)
8
9
10 @torch.inference_mode()
@@ -16,10 +16,7 @@ def main(num_tokens: int,
16 do_profile: bool = False,
17 num_warmup_iters: int = 5,
18 num_iters: int = 100) -> None:
19 - random.seed(seed)
20 - torch.random.manual_seed(seed)
21 - if torch.cuda.is_available():
22 - torch.cuda.manual_seed(seed)
19 + seed_everything(seed)
20 torch.set_default_device("cuda")
21
22 layer = RMSNorm(hidden_size).to(dtype=dtype)
benchmarks/kernels/benchmark_moe.py
+3 -3
@@ -10,7 +10,7 @@ from ray.experimental.tqdm_ray import tqdm
10 from transformers import AutoConfig
11
12 from vllm.model_executor.layers.fused_moe.fused_moe import *
13 -from vllm.utils import FlexibleArgumentParser
13 +from vllm.utils import FlexibleArgumentParser, seed_everything
14
15
16 class BenchmarkConfig(TypedDict):
@@ -166,7 +166,7 @@ class BenchmarkWorker:
166
167 def __init__(self, seed: int) -> None:
168 torch.set_default_device("cuda")
169 - torch.cuda.manual_seed_all(seed)
169 + seed_everything(seed)
170 self.seed = seed
171
172 def benchmark(
@@ -180,7 +180,7 @@ class BenchmarkWorker:
180 use_fp8_w8a8: bool,
181 use_int8_w8a16: bool,
182 ) -> Tuple[Dict[str, int], float]:
183 - torch.cuda.manual_seed_all(self.seed)
183 + seed_everything(self.seed)
184 dtype_str = get_config_dtype_str(dtype,
185 use_int8_w8a16=use_int8_w8a16,
186 use_fp8_w8a8=use_fp8_w8a8)
benchmarks/kernels/benchmark_paged_attention.py
+2 -5
@@ -6,7 +6,7 @@ import torch
6
7 from vllm import _custom_ops as ops
8 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
9 - create_kv_caches_with_random)
9 + create_kv_caches_with_random, seed_everything)
10
11 NUM_BLOCKS = 1024
12 PARTITION_SIZE = 512
@@ -28,10 +28,7 @@ def main(
28 device: str = "cuda",
29 kv_cache_dtype: Optional[str] = None,
30 ) -> None:
31 - random.seed(seed)
32 - torch.random.manual_seed(seed)
33 - if torch.cuda.is_available():
34 - torch.cuda.manual_seed(seed)
31 + seed_everything(seed)
32
33 scale = float(1.0 / (head_size**0.5))
34 query = torch.empty(num_seqs,
benchmarks/kernels/benchmark_quant.py
+3 -6
@@ -1,10 +1,10 @@
1 -import random
1 import time
2
3 import torch
4
5 from vllm import _custom_ops as ops
7 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser
6 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
7 + seed_everything)
8
9
10 @torch.inference_mode()
@@ -17,10 +17,7 @@ def main(num_tokens: int,
17 do_profile: bool = False,
18 num_warmup_iters: int = 5,
19 num_iters: int = 100) -> None:
20 - random.seed(seed)
21 - torch.random.manual_seed(seed)
22 - if torch.cuda.is_available():
23 - torch.cuda.manual_seed(seed)
20 + seed_everything(seed)
21 torch.set_default_device("cuda")
22
23 x = torch.randn(num_tokens, hidden_size, dtype=dtype)
benchmarks/kernels/benchmark_rope.py
+2 -4
@@ -6,7 +6,7 @@ import torch
6
7 from vllm.model_executor.layers.rotary_embedding import (RotaryEmbedding,
8 get_rope)
9 -from vllm.utils import FlexibleArgumentParser
9 +from vllm.utils import FlexibleArgumentParser, seed_everything
10
11
12 def benchmark_rope_kernels_multi_lora(
@@ -22,9 +22,7 @@ def benchmark_rope_kernels_multi_lora(
22 max_position: int = 8192,
23 base: int = 10000,
24 ) -> None:
25 - torch.random.manual_seed(seed)
26 - if torch.cuda.is_available():
27 - torch.cuda.manual_seed(seed)
25 + seed_everything(seed)
26 torch.set_default_device(device)
27 if rotary_dim is None:
28 rotary_dim = head_size
tests/kernels/test_activation.py
+3 -6
@@ -7,6 +7,7 @@ from tests.kernels.utils import opcheck
7 from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
8 NewGELU, QuickGELU,
9 SiluAndMul)
10 +from vllm.utils import seed_everything
11
12 from .allclose_default import get_default_atol, get_default_rtol
13
@@ -34,9 +35,7 @@ def test_act_and_mul(
35 seed: int,
36 device: str,
37 ) -> None:
37 - torch.random.manual_seed(seed)
38 - if torch.cuda.is_available():
39 - torch.cuda.manual_seed(seed)
38 + seed_everything(seed)
39 torch.set_default_device(device)
40 x = torch.randn(num_tokens, 2 * d, dtype=dtype)
41 if activation == "silu":
@@ -77,9 +76,7 @@ def test_activation(
76 seed: int,
77 device: str,
78 ) -> None:
80 - torch.random.manual_seed(seed)
81 - if torch.cuda.is_available():
82 - torch.cuda.manual_seed(seed)
79 + seed_everything(seed)
80 torch.set_default_device(device)
81 x = torch.randn(num_tokens, d, dtype=dtype)
82 layer = activation[0]()
tests/kernels/test_attention.py
+5 -13
@@ -6,7 +6,7 @@ import torch
6
7 from tests.kernels.utils import opcheck
8 from vllm import _custom_ops as ops
9 -from vllm.utils import get_max_shared_memory_bytes, is_hip
9 +from vllm.utils import get_max_shared_memory_bytes, is_hip, seed_everything
10
11 from .allclose_default import get_default_atol, get_default_rtol
12
@@ -139,10 +139,8 @@ def test_paged_attention(
139 ) -> None:
140 if kv_cache_dtype == "fp8" and head_size % 16:
141 pytest.skip()
142 - random.seed(seed)
143 - torch.random.manual_seed(seed)
144 - if torch.cuda.is_available():
145 - torch.cuda.manual_seed(seed)
142 +
143 + seed_everything(seed)
144 torch.set_default_device(device)
145 scale = float(1.0 / (head_size**0.5))
146 num_query_heads, num_kv_heads = num_heads
@@ -354,10 +352,7 @@ def test_paged_attention_rocm(
352 seed: int,
353 device: str,
354 ) -> None:
357 - random.seed(seed)
358 - torch.random.manual_seed(seed)
359 - if torch.cuda.is_available():
360 - torch.cuda.manual_seed(seed)
355 + seed_everything(seed)
356 torch.set_default_device(device)
357 scale = float(1.0 / (head_size**0.5))
358 num_query_heads, num_kv_heads = num_heads
@@ -506,10 +501,7 @@ def test_multi_query_kv_attention(
501 seed: int,
502 device: str,
503 ) -> None:
509 - random.seed(seed)
510 - torch.random.manual_seed(seed)
511 - if torch.cuda.is_available():
512 - torch.cuda.manual_seed(seed)
504 + seed_everything(seed)
505 torch.set_default_device(device)
506 # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
507 # As the xformers library is already tested with its own tests, we can use
tests/kernels/test_attention_selector.py
+1 -1
@@ -45,7 +45,7 @@ def test_flash_attn(monkeypatch):
45 override_backend_env_variable(monkeypatch, STR_FLASH_ATTN_VAL)
46
47 # Unsupported CUDA arch
48 - with patch("torch.cuda.get_device_capability", return_value=[7, 5]):
48 + with patch("torch.cuda.get_device_capability", return_value=(7, 5)):
49 backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
50 assert backend.name != STR_FLASH_ATTN_VAL
51
tests/kernels/test_awq_triton.py
+3 -2
@@ -7,6 +7,7 @@ import torch
7
8 from vllm.model_executor.layers.quantization.awq_triton import (
9 AWQ_TRITON_SUPPORTED_GROUP_SIZES, awq_dequantize_triton, awq_gemm_triton)
10 +from vllm.utils import seed_everything
11
12 device = "cuda"
13
@@ -79,7 +80,7 @@ def test_dequantize(qweight_rows, qweight_cols, group_size):
80 zeros_cols = qweight_cols
81 zeros_dtype = torch.int32
82
82 - torch.manual_seed(0)
83 + seed_everything(0)
84
85 qweight = torch.randint(0,
86 torch.iinfo(torch.int32).max,
@@ -133,7 +134,7 @@ def test_gemm(N, K, M, splitK, group_size):
134 qzeros_rows = scales_rows
135 qzeros_cols = qweight_cols
136
136 - torch.manual_seed(0)
137 + seed_everything(0)
138
139 input = torch.rand((input_rows, input_cols),
140 dtype=input_dtype,
tests/kernels/test_blocksparse_attention.py
+3 -9
@@ -7,7 +7,7 @@ import torch
7 from vllm import _custom_ops as ops
8 from vllm.attention.ops.blocksparse_attention.interface import (
9 LocalStridedBlockSparseAttn)
10 -from vllm.utils import get_max_shared_memory_bytes, is_hip
10 +from vllm.utils import get_max_shared_memory_bytes, is_hip, seed_everything
11
12 from .allclose_default import get_default_atol, get_default_rtol
13
@@ -172,10 +172,7 @@ def test_paged_attention(
172 blocksparse_block_size: int,
173 blocksparse_head_sliding_step: int,
174 ) -> None:
175 - random.seed(seed)
176 - torch.random.manual_seed(seed)
177 - if torch.cuda.is_available():
178 - torch.cuda.manual_seed(seed)
175 + seed_everything(seed)
176 torch.set_default_device(device)
177 scale = float(1.0 / (head_size**0.5))
178 num_query_heads, num_kv_heads = num_heads
@@ -386,10 +383,7 @@ def test_varlen_blocksparse_attention_prefill(
383 seed: int,
384 device: str,
385 ) -> None:
389 - random.seed(seed)
390 - torch.random.manual_seed(seed)
391 - if torch.cuda.is_available():
392 - torch.cuda.manual_seed(seed)
386 + seed_everything(seed)
387 torch.set_default_device(device)
388 # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
389 # As the xformers library is already tested with its own tests, we can use
tests/kernels/test_cache.py
+7 -18
@@ -6,6 +6,7 @@ import torch
6
7 from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
8 from vllm import _custom_ops as ops
9 +from vllm.utils import seed_everything
10
11 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
12 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -55,10 +56,7 @@ def test_copy_blocks(
56 ) -> None:
57 if kv_cache_dtype == "fp8" and head_size % 16:
58 pytest.skip()
58 - random.seed(seed)
59 - torch.random.manual_seed(seed)
60 - if torch.cuda.is_available():
61 - torch.cuda.manual_seed(seed)
59 + seed_everything(seed)
60 torch.set_default_device(device)
61 # Generate random block mappings where each source block is mapped to two
62 # destination blocks.
@@ -134,10 +132,7 @@ def test_reshape_and_cache(
132 ) -> None:
133 if kv_cache_dtype == "fp8" and head_size % 16:
134 pytest.skip()
137 - random.seed(seed)
138 - torch.random.manual_seed(seed)
139 - if torch.cuda.is_available():
140 - torch.cuda.manual_seed(seed)
135 + seed_everything(seed)
136 torch.set_default_device(device)
137 # Create a random slot mapping.
138 num_slots = block_size * num_blocks
@@ -229,9 +224,7 @@ def test_reshape_and_cache_flash(
224 device: str,
225 kv_cache_dtype: str,
226 ) -> None:
232 - random.seed(seed)
233 - torch.random.manual_seed(seed)
234 - torch.cuda.manual_seed(seed)
227 + seed_everything(seed)
228 torch.set_default_device(device)
229
230 # Create a random slot mapping.
@@ -345,10 +338,8 @@ def test_swap_blocks(
338 pytest.skip()
339 if kv_cache_dtype == "fp8" and head_size % 16:
340 pytest.skip()
348 - random.seed(seed)
349 - torch.random.manual_seed(seed)
350 - if torch.cuda.is_available():
351 - torch.cuda.manual_seed(seed)
341 +
342 + seed_everything(seed)
343
344 src_device = device if direction[0] == "cuda" else 'cpu'
345 dst_device = device if direction[1] == "cuda" else 'cpu'
@@ -417,9 +408,7 @@ def test_fp8_e4m3_conversion(
408 seed: int,
409 device: str,
410 ) -> None:
420 - random.seed(seed)
421 - torch.random.manual_seed(seed)
422 - torch.cuda.manual_seed(seed)
411 + seed_everything(seed)
412
413 low = -224.0
414 high = 224.0
tests/kernels/test_causal_conv1d.py
+3 -2
@@ -7,6 +7,7 @@ from einops import rearrange
7
8 from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
9 causal_conv1d_fn, causal_conv1d_update)
10 +from vllm.utils import seed_everything
11
12
13 def causal_conv1d_ref(
@@ -104,7 +105,7 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
105 if itype == torch.bfloat16:
106 rtol, atol = 1e-2, 5e-2
107 # set seed
107 - torch.random.manual_seed(0)
108 + seed_everything(0)
109 if not channel_last:
110 x = torch.randn(batch,
111 4096 + dim + 64,
@@ -175,7 +176,7 @@ def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
176 if itype == torch.bfloat16:
177 rtol, atol = 1e-2, 5e-2
178 # set seed
178 - torch.random.manual_seed(0)
179 + seed_everything(0)
180 batch = 2
181 x = torch.randn(batch, dim, device=device, dtype=itype)
182 conv_state = torch.randn(batch, dim, width, device=device, dtype=itype)
tests/kernels/test_cutlass.py
+4 -7
@@ -15,9 +15,6 @@ CUDA_DEVICES = [
15 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
16 ]
17
18 -capability = current_platform.get_device_capability()
19 -capability = capability[0] * 10 + capability[1]
20 -
18
19 def to_fp8(tensor: torch.Tensor):
20 finfo = torch.finfo(torch.float8_e4m3fn)
@@ -119,7 +116,7 @@ def cutlass_int8_gemm_helper(m: int,
116 @pytest.mark.parametrize("per_act_token", [True, False])
117 @pytest.mark.parametrize("per_out_ch", [True, False])
118 @pytest.mark.parametrize("use_bias", [True, False])
122 -@pytest.mark.skipif(capability < 89,
119 +@pytest.mark.skipif(not current_platform.has_device_capability(89),
120 reason="FP8 is not supported on this GPU type.")
121 def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
122 per_out_ch: bool, use_bias: bool):
@@ -157,7 +154,7 @@ def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
154 @pytest.mark.parametrize("per_out_ch", [True, False])
155 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
156 @pytest.mark.parametrize("use_bias", [True, False])
160 -@pytest.mark.skipif(capability < 89,
157 +@pytest.mark.skipif(not current_platform.has_device_capability(89),
158 reason="FP8 is not supported on this GPU type.")
159 def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
160 out_dtype: Type[torch.dtype],
@@ -175,7 +172,7 @@ def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
172 @pytest.mark.parametrize("per_out_ch", [True, False])
173 @pytest.mark.parametrize("use_bias", [True, False])
174 @pytest.mark.parametrize("device", CUDA_DEVICES)
178 -@pytest.mark.skipif(capability < 89,
175 +@pytest.mark.skipif(not current_platform.has_device_capability(89),
176 reason="FP8 is not supported on this GPU type.")
177 def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
178 use_bias: bool, device: str):
@@ -207,7 +204,7 @@ def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
204 @pytest.mark.parametrize("per_act_token", [True, False])
205 @pytest.mark.parametrize("per_out_ch", [True, False])
206 @pytest.mark.parametrize("use_bias", [True, False])
210 -@pytest.mark.skipif(capability < 89,
207 +@pytest.mark.skipif(not current_platform.has_device_capability(89),
208 reason="FP8 is not supported on this GPU type.")
209 def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
210 use_bias: bool):
tests/kernels/test_flash_attn.py
+3 -2
@@ -4,6 +4,7 @@ import pytest
4 import torch
5
6 import vllm.attention.backends.flash_attn # noqa: F401
7 +from vllm.utils import seed_everything
8
9 NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
10 HEAD_SIZES = [128, 256]
@@ -87,7 +88,7 @@ def test_flash_attn_with_paged_kv(
88 num_blocks: int,
89 ) -> None:
90 torch.set_default_device("cuda")
90 - torch.cuda.manual_seed_all(0)
91 + seed_everything(0)
92 num_seqs = len(kv_lens)
93 num_query_heads = num_heads[0]
94 num_kv_heads = num_heads[1]
@@ -174,7 +175,7 @@ def test_varlen_with_paged_kv(
175 num_blocks: int,
176 ) -> None:
177 torch.set_default_device("cuda")
177 - torch.cuda.manual_seed_all(0)
178 + seed_everything(0)
179 num_seqs = len(seq_lens)
180 query_lens = [x[0] for x in seq_lens]
181 kv_lens = [x[1] for x in seq_lens]
tests/kernels/test_flashinfer.py
+6 -4
@@ -4,6 +4,8 @@ import flashinfer
4 import pytest
5 import torch
6
7 +from vllm.utils import seed_everything
8 +
9 NUM_HEADS = [(16, 16), (32, 8), (64, 8), (6, 1)]
10 HEAD_SIZES = [128, 256]
11 BLOCK_SIZES = [16, 32]
@@ -82,7 +84,7 @@ def test_flashinfer_decode_with_paged_kv(
84 soft_cap: Optional[float],
85 ) -> None:
86 torch.set_default_device("cuda")
85 - torch.cuda.manual_seed_all(0)
87 + seed_everything(0)
88 num_seqs = len(kv_lens)
89 num_query_heads = num_heads[0]
90 num_kv_heads = num_heads[1]
@@ -168,7 +170,7 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
170 block_size: int,
171 soft_cap: Optional[float]) -> None:
172 torch.set_default_device("cuda")
171 - torch.cuda.manual_seed_all(0)
173 + seed_everything(0)
174 num_seqs = len(seq_lens)
175 query_lens = [x[0] for x in seq_lens]
176 kv_lens = [x[1] for x in seq_lens]
@@ -266,7 +268,7 @@ def test_flashinfer_prefill_with_paged_fp8_kv(
268 head_size: int, dtype: torch.dtype, block_size: int,
269 soft_cap: Optional[float]) -> None:
270 torch.set_default_device("cuda")
269 - torch.cuda.manual_seed_all(0)
271 + seed_everything(0)
272 num_seqs = len(seq_lens)
273 query_lens = [x[0] for x in seq_lens]
274 kv_lens = [x[1] for x in seq_lens]
@@ -379,7 +381,7 @@ def test_flashinfer_decode_with_paged_fp8_kv(
381 ) -> None:
382 # test doesn't work for num_heads = (16,16)
383 torch.set_default_device("cuda")
382 - torch.cuda.manual_seed_all(0)
384 + seed_everything(0)
385 num_seqs = len(kv_lens)
386 num_query_heads = num_heads[0]
387 num_kv_heads = num_heads[1]
tests/kernels/test_fp8_quant.py
+4 -6
@@ -5,6 +5,7 @@ import vllm._custom_ops as ops
5 from tests.kernels.quant_utils import (FP8_DTYPE,
6 ref_dynamic_per_tensor_fp8_quant,
7 ref_dynamic_per_token_quant)
8 +from vllm.utils import seed_everything
9
10 DTYPES = [torch.half, torch.bfloat16, torch.float]
11 HIDDEN_SIZES = [1, 2, 3, 4, 16, 67, 768, 2048, 5120, 5137, 8192,
@@ -24,8 +25,7 @@ SEEDS = [0]
25 def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
26 dtype: torch.dtype, scale_ub: bool,
27 seed: int) -> None:
27 - torch.random.manual_seed(seed)
28 - torch.cuda.manual_seed(seed)
28 + seed_everything(seed)
29
30 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
31 device="cuda") + 1e-6 # avoid nans
@@ -49,8 +49,7 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
49 @torch.inference_mode()
50 def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
51 dtype: torch.dtype, seed: int) -> None:
52 - torch.random.manual_seed(seed)
53 - torch.cuda.manual_seed(seed)
52 + seed_everything(seed)
53
54 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
55
@@ -67,8 +66,7 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
66 @torch.inference_mode()
67 @pytest.mark.parametrize("seed", SEEDS)
68 def test_fp8_quant_large(seed: int) -> None:
70 - torch.random.manual_seed(seed)
71 - torch.cuda.manual_seed(seed)
69 + seed_everything(seed)
70
71 num_tokens = 1024000 # Mistral-Nemo's max_position_embeddings
72 hidden_size = 1152 # Smallest hidden_size to reproduce the error
tests/kernels/test_gguf.py
+3 -2
@@ -7,6 +7,7 @@ from gguf import GGMLQuantizationType, GGUFReader, ReaderTensor, dequantize
7 from huggingface_hub import snapshot_download
8
9 import vllm._custom_ops as ops
10 +from vllm.utils import seed_everything
11
12 GGUF_SAMPLE = snapshot_download("Isotr0py/test-gguf-sample")
13
@@ -74,7 +75,7 @@ def test_dequantize(hidden_size: int, dtype: torch.dtype,
75 @torch.inference_mode()
76 def test_mmvq(hidden_size: int, dtype: torch.dtype,
77 quant_type: GGMLQuantizationType):
77 - torch.cuda.manual_seed_all(0)
78 + seed_everything(0)
79
80 tensors = get_gguf_sample_tensors(hidden_size, quant_type)
81 x = torch.rand((1, hidden_size), dtype=dtype, device="cuda")
@@ -110,7 +111,7 @@ def test_mmvq(hidden_size: int, dtype: torch.dtype,
111 @torch.inference_mode()
112 def test_mmq(num_tokens: int, hidden_size: int, dtype: torch.dtype,
113 quant_type: GGMLQuantizationType):
113 - torch.cuda.manual_seed_all(0)
114 + seed_everything(0)
115
116 tensors = get_gguf_sample_tensors(hidden_size, quant_type)
117 x = torch.rand((num_tokens, hidden_size), dtype=dtype, device="cuda")
tests/kernels/test_int8_quant.py
+5 -8
@@ -4,6 +4,7 @@ import torch
4 from tests.kernels.quant_utils import ref_dynamic_per_token_quant
5 from tests.kernels.utils import opcheck
6 from vllm._custom_ops import scaled_int8_quant
7 +from vllm.utils import seed_everything
8
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
10 HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
@@ -44,8 +45,7 @@ def opcheck_int8_quant_dynamic(output, input, symmetric=True):
45 @torch.inference_mode()
46 def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
47 dtype: torch.dtype, seed: int) -> None:
47 - torch.random.manual_seed(seed)
48 - torch.cuda.manual_seed(seed)
48 + seed_everything(seed)
49
50 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
51
@@ -68,8 +68,7 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
68 @torch.inference_mode()
69 def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
70 dtype: torch.dtype, seed: int) -> None:
71 - torch.random.manual_seed(seed)
72 - torch.cuda.manual_seed(seed)
71 + seed_everything(seed)
72 int8_traits = torch.iinfo(torch.int8)
73
74 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
@@ -113,8 +112,7 @@ def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
112 def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
113 dtype: torch.dtype, seed: int,
114 scale: float) -> None:
116 - torch.random.manual_seed(seed)
117 - torch.cuda.manual_seed(seed)
115 + seed_everything(seed)
116 int8_traits = torch.iinfo(torch.int8)
117
118 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
@@ -140,8 +138,7 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
138 def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
139 dtype: torch.dtype, seed: int,
140 scale: float, azp: int) -> None:
143 - torch.random.manual_seed(seed)
144 - torch.cuda.manual_seed(seed)
141 + seed_everything(seed)
142 int8_traits = torch.iinfo(torch.int8)
143
144 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
tests/kernels/test_layernorm.py
+2 -3
@@ -3,6 +3,7 @@ import torch
3
4 from tests.kernels.utils import opcheck
5 from vllm.model_executor.layers.layernorm import RMSNorm
6 +from vllm.utils import seed_everything
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
9 NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
@@ -30,9 +31,7 @@ def test_rms_norm(
31 seed: int,
32 device: str,
33 ) -> None:
33 - torch.random.manual_seed(seed)
34 - if torch.cuda.is_available():
35 - torch.cuda.manual_seed(seed)
34 + seed_everything(seed)
35 torch.set_default_device(device)
36 layer = RMSNorm(hidden_size).to(dtype=dtype)
37 layer.weight.data.normal_(mean=1.0, std=0.1)
tests/kernels/test_machete_gemm.py
+1 -1
@@ -48,7 +48,7 @@ WTYPE_ZEROPOINTS = [
48 # `is_quant_method_supported` conflates kernels with quantization methods
49 # an assumption which is breaking down as quantizations methods can have
50 # have kernels and some kernels support multiple quantization methods.
51 -IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
51 +IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
52
53
54 def rand_data(shape, dtype=torch.float16):
tests/kernels/test_mamba_ssm.py
+3 -2
@@ -5,6 +5,7 @@ from einops import rearrange, repeat
5
6 from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
7 selective_scan_fn, selective_state_update)
8 +from vllm.utils import seed_everything
9
10
11 def selective_state_update_ref(state,
@@ -186,7 +187,7 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
187 rtolw = max(rtolw, rtol)
188 atolw = max(atolw, atol)
189 # set seed
189 - torch.random.manual_seed(0)
190 + seed_everything(0)
191 batch_size = 2
192 dim = 4
193 dstate = 8
@@ -287,7 +288,7 @@ def test_selective_state_update(dim, dstate, has_z, itype):
288 if torch.version.hip:
289 atol *= 2
290 # set seed
290 - torch.random.manual_seed(0)
291 + seed_everything(0)
292 batch_size = 1
293 state = torch.randn(batch_size, dim, dstate, dtype=itype, device=device)
294 x = torch.randn(batch_size, dim, device=device, dtype=itype)
tests/kernels/test_moe.py
+2 -1
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
18 marlin_quantize)
19 from vllm.model_executor.models.mixtral import MixtralMoE
20 from vllm.scalar_type import scalar_types
21 +from vllm.utils import seed_everything
22
23
24 def torch_moe(a, w1, w2, score, topk):
@@ -151,7 +152,7 @@ def test_fused_marlin_moe(
152 act_order: bool,
153 num_bits: int,
154 ):
154 - torch.manual_seed(7)
155 + seed_everything(7)
156
157 if topk > e:
158 return
tests/kernels/test_pos_encoding.py
+5 -9
@@ -5,6 +5,7 @@ import pytest
5 import torch
6
7 from vllm.model_executor.layers.rotary_embedding import get_rope
8 +from vllm.utils import seed_everything
9
10 from .allclose_default import get_default_atol, get_default_rtol
11
@@ -46,9 +47,8 @@ def test_rotary_embedding(
47 ) -> None:
48 if rotary_dim is None:
49 rotary_dim = head_size
49 - torch.random.manual_seed(seed)
50 - if torch.cuda.is_available():
51 - torch.cuda.manual_seed(seed)
50 +
51 + seed_everything(seed)
52 torch.set_default_device(device)
53 if rotary_dim is None:
54 rotary_dim = head_size
@@ -100,9 +100,7 @@ def test_batched_rotary_embedding(
100 max_position: int = 8192,
101 base: int = 10000,
102 ) -> None:
103 - torch.random.manual_seed(seed)
104 - if torch.cuda.is_available():
105 - torch.cuda.manual_seed(seed)
103 + seed_everything(seed)
104 torch.set_default_device(device)
105 if rotary_dim is None:
106 rotary_dim = head_size
@@ -162,9 +160,7 @@ def test_batched_rotary_embedding_multi_lora(
160 max_position: int = 8192,
161 base: int = 10000,
162 ) -> None:
165 - torch.random.manual_seed(seed)
166 - if torch.cuda.is_available():
167 - torch.cuda.manual_seed(seed)
163 + seed_everything(seed)
164 torch.set_default_device(device)
165 if rotary_dim is None:
166 rotary_dim = head_size
tests/kernels/test_prefix_prefill.py
+3 -9
@@ -9,7 +9,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
10 from vllm.attention.backends.xformers import _make_alibi_bias
11 from vllm.attention.ops.prefix_prefill import context_attention_fwd
12 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
12 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, seed_everything
13
14 NUM_HEADS = [64]
15 NUM_QUERIES_PER_KV = [1, 8, 64]
@@ -39,10 +39,7 @@ def test_contexted_kv_attention(
39 kv_cache_dtype: str,
40 device: str,
41 ) -> None:
42 - random.seed(0)
43 - torch.manual_seed(0)
44 - if torch.cuda.is_available():
45 - torch.cuda.manual_seed(0)
42 + seed_everything(0)
43 torch.set_default_device(device)
44
45 # Need this, otherwise when we capture the graph the process
@@ -237,10 +234,7 @@ def test_contexted_kv_attention_alibi(
234 kv_cache_dtype: str,
235 device: str,
236 ) -> None:
240 - random.seed(0)
241 - torch.manual_seed(0)
242 - if torch.cuda.is_available():
243 - torch.cuda.manual_seed(0)
237 + seed_everything(0)
238 torch.set_default_device(device)
239
240 # Need this, otherwise when we capture the graph the process
tests/lora/test_layers.py
+2 -3
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 ParallelLMHead, VocabParallelEmbedding, get_masked_input_and_mask)
41 from vllm.model_executor.utils import set_random_seed
42 +from vllm.utils import seed_everything
43
44 from .utils import DummyLoRAManager
45
@@ -922,9 +923,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
923 seq_len) -> None:
924 dtype = torch.float16
925 seed = 0
925 - torch.random.manual_seed(seed)
926 - if torch.cuda.is_available():
927 - torch.cuda.manual_seed(seed)
926 + seed_everything(seed)
927 torch.set_default_device(device)
928 punica_wrapper = PunicaWrapper(8192, 256, device)
929 max_loras = 8
tests/lora/test_punica_sizes.py
+5 -13
@@ -4,7 +4,6 @@ hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4 whether the corresponding Triton kernel can run normally when tensor parallelism
5 is set to [1, 2, 4, 8, 16, 32, 64].
6 """
7 -import random
7 from unittest.mock import patch
8
9 import pytest
@@ -17,6 +16,7 @@ from vllm.lora.ops.sgmv_expand import sgmv_expand
16 from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
17 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
18 from vllm.triton_utils.libentry import LibEntry
19 +from vllm.utils import seed_everything
20
21 from .utils import (generate_data, generate_data_for_expand_nslices,
22 ref_torch_groupgemm)
@@ -145,11 +145,8 @@ def test_punica_sgmv(
145 seed: int,
146 device: str,
147 ):
148 - random.seed(seed)
148 torch.set_default_device(device)
150 - torch.random.manual_seed(seed)
151 - if torch.cuda.is_available():
152 - torch.cuda.manual_seed(seed)
149 + seed_everything(seed)
150
151 seq_length = 128
152 (
@@ -238,11 +235,8 @@ def test_punica_bgmv(
235 from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
236 from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
237
241 - random.seed(seed)
238 torch.set_default_device(device)
243 - torch.random.manual_seed(seed)
244 - if torch.cuda.is_available():
245 - torch.cuda.manual_seed(seed)
239 + seed_everything(seed)
240
241 seq_length = 1
242 (
@@ -329,11 +323,9 @@ def test_punica_expand_nslices(
323 ):
324 from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
325
332 - random.seed(seed)
326 torch.set_default_device(device)
334 - torch.random.manual_seed(seed)
335 - if torch.cuda.is_available():
336 - torch.cuda.manual_seed(seed)
327 + seed_everything(seed)
328 +
329 seq_length = 128 if op_type == "sgmv" else 1
330 (
331 inputs_tensor,
tests/lora/test_punica_variation.py
+5 -13
@@ -3,7 +3,6 @@ This script is mainly used to test whether trtion kernels can run normally
3 under different conditions, including various batches, numbers of LoRA , and
4 maximum ranks.
5 """
6 -import random
6 from unittest.mock import patch
7
8 import pytest
@@ -16,6 +15,7 @@ from vllm.lora.ops.sgmv_expand import sgmv_expand
15 from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
16 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
17 from vllm.triton_utils.libentry import LibEntry
18 +from vllm.utils import seed_everything
19
20 from .utils import (generate_data, generate_data_for_expand_nslices,
21 ref_torch_groupgemm)
@@ -60,11 +60,8 @@ def test_punica_sgmv(
60 seed: int,
61 device: str,
62 ):
63 - random.seed(seed)
63 torch.set_default_device(device)
65 - torch.random.manual_seed(seed)
66 - if torch.cuda.is_available():
67 - torch.cuda.manual_seed(seed)
64 + seed_everything(seed)
65
66 seq_length = 128
67 (
@@ -153,11 +150,8 @@ def test_punica_bgmv(
150 from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
151 from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
152
156 - random.seed(seed)
153 torch.set_default_device(device)
158 - torch.random.manual_seed(seed)
159 - if torch.cuda.is_available():
160 - torch.cuda.manual_seed(seed)
154 + seed_everything(seed)
155
156 seq_length = 1
157 (
@@ -244,11 +238,9 @@ def test_punica_expand_nslices(
238 ):
239 from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
240
247 - random.seed(seed)
241 torch.set_default_device(device)
249 - torch.random.manual_seed(seed)
250 - if torch.cuda.is_available():
251 - torch.cuda.manual_seed(seed)
242 + seed_everything(seed)
243 +
244 seq_length = 128 if op_type == "sgmv" else 1
245 (
246 inputs_tensor,
tests/models/decoder_only/language/test_granite.py
+2 -7
@@ -2,23 +2,18 @@
2
3 Run `pytest tests/models/test_granite.py`.
4 """
5 -import importlib.metadata
6 -
5 import pytest
6 +import transformers
7
8 from ...utils import check_logprobs_close
9
11 -TRANSFORMERS_VERSION = tuple(
12 - map(int,
13 - importlib.metadata.version("transformers").split(".")))
14 -
10 MODELS = [
11 "ibm/PowerLM-3b",
12 ]
13
14
15 # GraniteForCausalLM will be in transformers >= 4.45
21 -@pytest.mark.skipif(TRANSFORMERS_VERSION < (4, 45),
16 +@pytest.mark.skipif(transformers.__version__ < "4.45",
17 reason="granite model test requires transformers >= 4.45")
18 @pytest.mark.parametrize("model", MODELS)
19 @pytest.mark.parametrize("dtype", ["bfloat16"])
tests/quantization/test_fp8.py
+1 -3
@@ -86,9 +86,7 @@ def test_load_fp16_model(vllm_runner, kv_cache_dtype: str, force_marlin: bool,
86 assert attn._k_scale == 1.0
87 assert attn._v_scale == 1.0
88
89 - capability = current_platform.get_device_capability()
90 - capability = capability[0] * 10 + capability[1]
91 - if capability >= 89 and not force_marlin:
89 + if current_platform.has_device_capability(89) and not force_marlin:
90 # For GPUs with hardware support, we keep weights in fp8
91 assert fc1.weight.dtype == torch.float8_e4m3fn
92 else:
tests/quantization/utils.py
+5 -3
@@ -8,6 +8,8 @@ def is_quant_method_supported(quant_method: str) -> bool:
8 return False
9
10 capability = current_platform.get_device_capability()
11 - capability = capability[0] * 10 + capability[1]
12 - return (capability >=
13 - QUANTIZATION_METHODS[quant_method].get_min_capability())
11 + assert capability is not None
12 +
13 + min_capability = QUANTIZATION_METHODS[quant_method].get_min_capability()
14 +
15 + return capability.to_int() >= min_capability
vllm/attention/backends/rocm_flash_attn.py
+2 -1
@@ -13,6 +13,7 @@ from vllm.attention.backends.utils import (CommonAttentionState,
13 from vllm.attention.ops.paged_attn import (PagedAttention,
14 PagedAttentionMetadata)
15 from vllm.logger import init_logger
16 +from vllm.platforms import current_platform
17
18 logger = init_logger(__name__)
19
@@ -299,7 +300,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
300 else:
301 # if not using triton, navi3x/navi21/navi10 do not use flash-attn
302 # either
302 - if torch.cuda.get_device_capability()[0] != 9:
303 + if not current_platform.has_device_capability(90):
304 self.use_naive_attn = True
305 else:
306 try:
vllm/attention/ops/blocksparse_attention/interface.py
+2 -3
@@ -8,8 +8,7 @@ from vllm.utils import is_cpu, is_hip
8 from .utils import (dense_to_crow_col, get_head_sliding_step,
9 get_sparse_attn_mask)
10
11 -IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
12 - and current_platform.get_device_capability()[0] >= 8)
11 +IS_COMPUTE_8_OR_ABOVE = current_platform.has_device_capability(80)
12
13 if IS_COMPUTE_8_OR_ABOVE:
14 from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
@@ -36,7 +35,7 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
35 use_spda = is_hip() or is_cpu() or not \
36 IS_COMPUTE_8_OR_ABOVE
37 device = device or (torch.cuda.current_device()
39 - if torch.cuda.is_available() else "cpu")
38 + if current_platform.is_cuda_alike() else "cpu")
39 device = torch.device(device)
40 # NOTE: vllm CPU backend support BF16 instead of FP16.
41 dtype = dtype or (torch.bfloat16 if IS_COMPUTE_8_OR_ABOVE
vllm/attention/ops/prefix_prefill.py
+1 -2
@@ -709,8 +709,7 @@ if triton.__version__ >= "2.1.0":
709 alibi_slopes=None,
710 sliding_window=None):
711
712 - cap = current_platform.get_device_capability()
713 - BLOCK = 128 if cap[0] >= 8 else 64
712 + BLOCK = 128 if current_platform.has_device_capability(80) else 64
713 NUM_WARPS = 8
714
715 # need to reduce num. blocks when using fp32
vllm/attention/selector.py
+2 -2
@@ -203,7 +203,7 @@ def which_attn_to_use(
203 selected_backend = (_Backend.ROCM_FLASH if selected_backend
204 == _Backend.FLASH_ATTN else selected_backend)
205 if selected_backend == _Backend.ROCM_FLASH:
206 - if current_platform.get_device_capability()[0] != 9:
206 + if not current_platform.has_device_capability(90):
207 # not Instinct series GPUs.
208 logger.info("flash_attn is not supported on NAVI GPUs.")
209 else:
@@ -212,7 +212,7 @@ def which_attn_to_use(
212
213 # FlashAttn in NVIDIA GPUs.
214 if selected_backend == _Backend.FLASH_ATTN:
215 - if current_platform.get_device_capability()[0] < 8:
215 + if not current_platform.has_device_capability(80):
216 # Volta and Turing NVIDIA GPUs.
217 logger.info(
218 "Cannot use FlashAttention-2 backend for Volta and Turing "
vllm/config.py
+6 -6
@@ -17,7 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 get_hf_text_config)
19 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20 - is_cpu, is_hip, is_neuron, is_openvino, is_xpu,
20 + is_hip, is_neuron, is_openvino, is_xpu,
21 print_warning_once)
22
23 if TYPE_CHECKING:
@@ -1035,20 +1035,20 @@ class DeviceConfig:
1035 def __init__(self, device: str = "auto") -> None:
1036 if device == "auto":
1037 # Automated device type detection
1038 - if is_neuron():
1038 + if current_platform.is_cuda_alike():
1039 + self.device_type = "cuda"
1040 + elif is_neuron():
1041 self.device_type = "neuron"
1042 elif is_openvino():
1043 self.device_type = "openvino"
1044 elif current_platform.is_tpu():
1045 self.device_type = "tpu"
1044 - elif is_cpu():
1046 + elif current_platform.is_cpu():
1047 self.device_type = "cpu"
1048 elif is_xpu():
1049 self.device_type = "xpu"
1050 else:
1049 - # We don't call torch.cuda.is_available() here to
1050 - # avoid initializing CUDA before workers are forked
1051 - self.device_type = "cuda"
1051 + raise RuntimeError("Failed to infer device type")
1052 else:
1053 # Device type is assigned explicitly
1054 self.device_type = device
vllm/distributed/parallel_state.py
+2 -1
@@ -35,6 +35,7 @@ from torch.distributed import Backend, ProcessGroup
35
36 import vllm.envs as envs
37 from vllm.logger import init_logger
38 +from vllm.platforms import current_platform
39
40
41 @dataclass
@@ -191,7 +192,7 @@ class GroupCoordinator:
192 assert self.cpu_group is not None
193 assert self.device_group is not None
194
194 - if torch.cuda.is_available():
195 + if current_platform.is_cuda_alike():
196 self.device = torch.device(f"cuda:{local_rank}")
197 else:
198 self.device = torch.device("cpu")
vllm/envs.py
+1
@@ -60,6 +60,7 @@ if TYPE_CHECKING:
60 VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
61 VLLM_PLUGINS: Optional[List[str]] = None
62 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
63 + VLLM_USE_TRITON_AWQ: bool = False
64 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
65
66
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+3 -3
@@ -116,10 +116,10 @@ class CompressedTensorsConfig(QuantizationConfig):
116 def _check_scheme_supported(self,
117 min_capability: int,
118 error: bool = True) -> bool:
119 - capability = current_platform.get_device_capability() # type: ignore
119 + capability_tuple = current_platform.get_device_capability()
120
121 - if capability is not None:
122 - capability = capability[0] * 10 + capability[1]
121 + if capability_tuple is not None:
122 + capability = capability_tuple.to_int()
123 supported = capability >= min_capability
124 if error and not supported:
125 raise RuntimeError(
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+1 -3
@@ -32,9 +32,7 @@ class FBGEMMFp8Config(QuantizationConfig):
32
33 # For GPUs that lack FP8 hardware support, we can leverage the Marlin
34 # kernel for fast weight-only FP8 quantization
35 - capability = current_platform.get_device_capability()
36 - capability = capability[0] * 10 + capability[1]
37 - self.use_marlin = capability < 89
35 + self.use_marlin = not current_platform.has_device_capability(89)
36
37 @classmethod
38 def get_name(cls) -> str:
vllm/model_executor/layers/quantization/fp8.py
+2 -3
@@ -120,9 +120,8 @@ class Fp8LinearMethod(LinearMethodBase):
120
121 # For GPUs that lack FP8 hardware support, we can leverage the Marlin
122 # kernel for fast weight-only FP8 quantization
123 - capability = current_platform.get_device_capability()
124 - capability = capability[0] * 10 + capability[1]
125 - self.use_marlin = capability < 89 or envs.VLLM_TEST_FORCE_FP8_MARLIN
123 + self.use_marlin = (not current_platform.has_device_capability(89)
124 + or envs.VLLM_TEST_FORCE_FP8_MARLIN)
125 # Disable marlin for rocm
126 if is_hip():
127 self.use_marlin = False
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+6 -4
@@ -29,8 +29,9 @@ def query_marlin_supported_quant_types(has_zp: bool,
29 device_capability: Optional[int] = None
30 ):
31 if device_capability is None:
32 - major, minor = current_platform.get_device_capability()
33 - device_capability = major * 10 + minor
32 + capability_tuple = current_platform.get_device_capability()
33 + device_capability = (-1 if capability_tuple is None else
34 + capability_tuple.to_int())
35
36 if device_capability < 80:
37 return []
@@ -52,8 +53,9 @@ def _check_marlin_supported(
53 device_capability: Optional[int] = None) -> Tuple[bool, Optional[str]]:
54
55 if device_capability is None:
55 - major, minor = current_platform.get_device_capability()
56 - device_capability = major * 10 + minor
56 + capability_tuple = current_platform.get_device_capability()
57 + device_capability = (-1 if capability_tuple is None else
58 + capability_tuple.to_int())
59
60 supported_types = query_marlin_supported_quant_types(
61 has_zp, device_capability)
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+1 -2
@@ -10,8 +10,7 @@ from .marlin_utils import marlin_make_workspace, marlin_permute_scales
10
11
12 def is_fp8_marlin_supported():
13 - capability = current_platform.get_device_capability()
14 - return capability[0] >= 8
13 + return current_platform.has_device_capability(80)
14
15
16 def apply_fp8_marlin_linear(
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3 -2
@@ -17,8 +17,9 @@ def cutlass_fp8_supported() -> bool:
17 # cutlass is not supported on Rocm
18 if is_hip():
19 return False
20 - capability = current_platform.get_device_capability()
21 - capability = capability[0] * 10 + capability[1]
20 +
21 + capability_tuple = current_platform.get_device_capability()
22 + capability = -1 if capability_tuple is None else capability_tuple.to_int()
23
24 return ops.cutlass_scaled_mm_supports_fp8(capability)
25
vllm/model_executor/model_loader/loader.py
+3 -3
@@ -97,10 +97,10 @@ def _get_quantization_config(
97 """Get the quantization config."""
98 if model_config.quantization is not None:
99 quant_config = get_quant_config(model_config, load_config)
100 - capability = current_platform.get_device_capability() # type: ignore
100 + capability_tuple = current_platform.get_device_capability()
101
102 - if capability is not None:
103 - capability = capability[0] * 10 + capability[1]
102 + if capability_tuple is not None:
103 + capability = capability_tuple.to_int()
104 if capability < quant_config.get_min_capability():
105 raise ValueError(
106 f"The quantization method {model_config.quantization} "
vllm/model_executor/models/qwen2_vl.py
+1 -1
@@ -207,7 +207,7 @@ class Qwen2VisionAttention(nn.Module):
207 selected_backend = backend_name_to_enum(backend_by_env_var)
208 if selected_backend is None:
209 # For Volta and Turing GPUs, use xformers instead.
210 - device_available = current_platform.get_device_capability()[0] >= 8
210 + device_available = current_platform.has_device_capability(80)
211 if device_available:
212 from transformers.utils import is_flash_attn_2_available
213
vllm/model_executor/utils.py
+3 -7
@@ -1,17 +1,13 @@
1 """Utils for model executor."""
2 -import random
2 from typing import Any, Dict, Optional
3
5 -import numpy as np
4 import torch
5
6 +from vllm.utils import seed_everything
7 +
8
9 def set_random_seed(seed: int) -> None:
10 - random.seed(seed)
11 - np.random.seed(seed)
12 - torch.manual_seed(seed)
13 - if torch.cuda.is_available():
14 - torch.cuda.manual_seed_all(seed)
10 + seed_everything(seed)
11
12
13 def set_weight_attrs(
vllm/platforms/cpu.py
+4 -4
@@ -6,10 +6,10 @@ from .interface import Platform, PlatformEnum
6 class CpuPlatform(Platform):
7 _enum = PlatformEnum.CPU
8
9 - @staticmethod
10 - def get_device_name(device_id: int = 0) -> str:
9 + @classmethod
10 + def get_device_name(cls, device_id: int = 0) -> str:
11 return "cpu"
12
13 - @staticmethod
14 - def inference_mode():
13 + @classmethod
14 + def inference_mode(cls):
15 return torch.no_grad()
vllm/platforms/cuda.py
+9 -8
@@ -11,7 +11,7 @@ from typing_extensions import ParamSpec
11
12 from vllm.logger import init_logger
13
14 -from .interface import Platform, PlatformEnum
14 +from .interface import DeviceCapability, Platform, PlatformEnum
15
16 logger = init_logger(__name__)
17
@@ -96,19 +96,20 @@ def device_id_to_physical_device_id(device_id: int) -> int:
96 class CudaPlatform(Platform):
97 _enum = PlatformEnum.CUDA
98
99 - @staticmethod
100 - def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
99 + @classmethod
100 + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
101 physical_device_id = device_id_to_physical_device_id(device_id)
102 - return get_physical_device_capability(physical_device_id)
102 + major, minor = get_physical_device_capability(physical_device_id)
103 + return DeviceCapability(major=major, minor=minor)
104
104 - @staticmethod
105 - def get_device_name(device_id: int = 0) -> str:
105 + @classmethod
106 + def get_device_name(cls, device_id: int = 0) -> str:
107 physical_device_id = device_id_to_physical_device_id(device_id)
108 return get_physical_device_name(physical_device_id)
109
109 - @staticmethod
110 + @classmethod
111 @with_nvml_context
111 - def is_full_nvlink(physical_device_ids: List[int]) -> bool:
112 + def is_full_nvlink(cls, physical_device_ids: List[int]) -> bool:
113 """
114 query if the set of gpus are fully connected by nvlink (1 hop)
115 """
vllm/platforms/interface.py
+55 -7
@@ -1,5 +1,5 @@
1 import enum
2 -from typing import Optional, Tuple
2 +from typing import NamedTuple, Optional, Tuple, Union
3
4 import torch
5
@@ -12,6 +12,23 @@ class PlatformEnum(enum.Enum):
12 UNSPECIFIED = enum.auto()
13
14
15 +class DeviceCapability(NamedTuple):
16 + major: int
17 + minor: int
18 +
19 + def as_version_str(self) -> str:
20 + return f"{self.major}.{self.minor}"
21 +
22 + def to_int(self) -> int:
23 + """
24 + Express device capability as an integer ``<major><minor>``.
25 +
26 + It is assumed that the minor version is always a single digit.
27 + """
28 + assert 0 <= self.minor < 10
29 + return self.major * 10 + self.minor
30 +
31 +
32 class Platform:
33 _enum: PlatformEnum
34
@@ -27,16 +44,47 @@ class Platform:
44 def is_cpu(self) -> bool:
45 return self._enum == PlatformEnum.CPU
46
30 - @staticmethod
31 - def get_device_capability(device_id: int = 0) -> Optional[Tuple[int, int]]:
47 + def is_cuda_alike(self) -> bool:
48 + """Stateless version of :func:`torch.cuda.is_available`."""
49 + return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
50 +
51 + @classmethod
52 + def get_device_capability(
53 + cls,
54 + device_id: int = 0,
55 + ) -> Optional[DeviceCapability]:
56 + """Stateless version of :func:`torch.cuda.get_device_capability`."""
57 return None
58
34 - @staticmethod
35 - def get_device_name(device_id: int = 0) -> str:
59 + @classmethod
60 + def has_device_capability(
61 + cls,
62 + capability: Union[Tuple[int, int], int],
63 + device_id: int = 0,
64 + ) -> bool:
65 + """
66 + Test whether this platform is compatible with a device capability.
67 +
68 + The ``capability`` argument can either be:
69 +
70 + - A tuple ``(major, minor)``.
71 + - An integer ``<major><minor>``. (See :meth:`DeviceCapability.to_int`)
72 + """
73 + current_capability = cls.get_device_capability(device_id=device_id)
74 + if current_capability is None:
75 + return False
76 +
77 + if isinstance(capability, tuple):
78 + return current_capability >= capability
79 +
80 + return current_capability.to_int() >= capability
81 +
82 + @classmethod
83 + def get_device_name(cls, device_id: int = 0) -> str:
84 raise NotImplementedError
85
38 - @staticmethod
39 - def inference_mode():
86 + @classmethod
87 + def inference_mode(cls):
88 """A device-specific wrapper of `torch.inference_mode`.
89
90 This wrapper is recommended because some hardware backends such as TPU
vllm/platforms/rocm.py
+7 -7
@@ -1,12 +1,11 @@
1 import os
2 from functools import lru_cache
3 -from typing import Tuple
3
4 import torch
5
6 from vllm.logger import init_logger
7
9 -from .interface import Platform, PlatformEnum
8 +from .interface import DeviceCapability, Platform, PlatformEnum
9
10 logger = init_logger(__name__)
11
@@ -20,12 +19,13 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
19 class RocmPlatform(Platform):
20 _enum = PlatformEnum.ROCM
21
23 - @staticmethod
22 + @classmethod
23 @lru_cache(maxsize=8)
25 - def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
26 - return torch.cuda.get_device_capability(device_id)
24 + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
25 + major, minor = torch.cuda.get_device_capability(device_id)
26 + return DeviceCapability(major=major, minor=minor)
27
28 - @staticmethod
28 + @classmethod
29 @lru_cache(maxsize=8)
30 - def get_device_name(device_id: int = 0) -> str:
30 + def get_device_name(cls, device_id: int = 0) -> str:
31 return torch.cuda.get_device_name(device_id)
vllm/platforms/tpu.py
+6 -2
@@ -6,6 +6,10 @@ from .interface import Platform, PlatformEnum
6 class TpuPlatform(Platform):
7 _enum = PlatformEnum.TPU
8
9 - @staticmethod
10 - def inference_mode():
9 + @classmethod
10 + def get_device_name(cls, device_id: int = 0) -> str:
11 + raise NotImplementedError
12 +
13 + @classmethod
14 + def inference_mode(cls):
15 return torch.no_grad()
vllm/prompt_adapter/utils.py
+3 -1
@@ -8,13 +8,15 @@ from huggingface_hub import file_exists, hf_hub_download
8 from huggingface_hub.utils import EntryNotFoundError
9 from safetensors.torch import load_file as safe_load_file
10
11 +from vllm.platforms import current_platform
12 +
13 WEIGHTS_NAME = "adapter_model.bin"
14 SAFETENSORS_WEIGHTS_NAME = "adapter_model.safetensors"
15
16
17 # Get current device name based on available devices
18 def infer_device() -> str:
17 - if torch.cuda.is_available():
19 + if current_platform.is_cuda_alike():
20 return "cuda"
21 return "cpu"
22
vllm/usage/usage_lib.py
+2 -1
@@ -17,6 +17,7 @@ import torch
17
18 import vllm.envs as envs
19 from vllm.connections import global_http_connection
20 +from vllm.platforms import current_platform
21 from vllm.version import __version__ as VLLM_VERSION
22
23 _config_home = envs.VLLM_CONFIG_ROOT
@@ -151,7 +152,7 @@ class UsageMessage:
152 usage_context: UsageContext,
153 extra_kvs: Dict[str, Any]) -> None:
154 # Platform information
154 - if torch.cuda.is_available():
155 + if current_platform.is_cuda_alike():
156 device_property = torch.cuda.get_device_properties(0)
157 self.gpu_count = torch.cuda.device_count()
158 self.gpu_type = device_property.name
vllm/utils.py
+21 -7
@@ -5,6 +5,7 @@ import datetime
5 import enum
6 import gc
7 import os
8 +import random
9 import socket
10 import subprocess
11 import sys
@@ -32,6 +33,7 @@ from typing_extensions import ParamSpec, TypeIs, assert_never
33
34 import vllm.envs as envs
35 from vllm.logger import enable_trace_function_call, init_logger
36 +from vllm.platforms import current_platform
37
38 logger = init_logger(__name__)
39
@@ -373,6 +375,22 @@ def get_cpu_memory() -> int:
375 return psutil.virtual_memory().total
376
377
378 +def seed_everything(seed: int) -> None:
379 + """
380 + Set the seed of each random module.
381 +
382 + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20
383 + """
384 + random.seed(seed)
385 + np.random.seed(seed)
386 +
387 + if current_platform.is_cuda_alike():
388 + torch.cuda.manual_seed_all(seed)
389 +
390 + if is_xpu():
391 + torch.xpu.manual_seed_all(seed)
392 +
393 +
394 def random_uuid() -> str:
395 return str(uuid.uuid4().hex)
396
@@ -634,9 +652,7 @@ def create_kv_caches_with_random_flash(
652 seed: int = 0,
653 device: Optional[str] = "cuda",
654 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
637 - torch.random.manual_seed(seed)
638 - if torch.cuda.is_available():
639 - torch.cuda.manual_seed(seed)
655 + seed_everything(seed)
656
657 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
658 key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
@@ -678,9 +694,7 @@ def create_kv_caches_with_random(
694 f"Does not support key cache of type fp8 with head_size {head_size}"
695 )
696
681 - torch.random.manual_seed(seed)
682 - if torch.cuda.is_available():
683 - torch.cuda.manual_seed(seed)
697 + seed_everything(seed)
698
699 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
700
@@ -750,7 +764,7 @@ class CudaMemoryProfiler:
764
765 def current_memory_usage(self) -> float:
766 # Return the memory usage in bytes.
753 - if torch.cuda.is_available():
767 + if current_platform.is_cuda_alike():
768 torch.cuda.reset_peak_memory_stats(self.device)
769 mem = torch.cuda.max_memory_allocated(self.device)
770 elif is_xpu():
vllm/worker/worker.py
+11 -5
@@ -454,14 +454,20 @@ def init_worker_distributed_environment(
454
455 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
456 # Check if the GPU supports the dtype.
457 - if torch_dtype == torch.bfloat16:
458 - compute_capability = current_platform.get_device_capability()
459 - if compute_capability[0] < 8:
457 + if torch_dtype == torch.bfloat16: # noqa: SIM102
458 + if not current_platform.has_device_capability(80):
459 + capability = current_platform.get_device_capability()
460 gpu_name = current_platform.get_device_name()
461 +
462 + if capability is None:
463 + compute_str = "does not have a compute capability"
464 + else:
465 + version_str = capability.as_version_str()
466 + compute_str = f"has compute capability {version_str}"
467 +
468 raise ValueError(
469 "Bfloat16 is only supported on GPUs with compute capability "
463 - f"of at least 8.0. Your {gpu_name} GPU has compute capability "
464 - f"{compute_capability[0]}.{compute_capability[1]}. "
470 + f"of at least 8.0. Your {gpu_name} GPU {compute_str}. "
471 "You can use float16 instead by explicitly setting the"
472 "`dtype` flag in CLI, for example: --dtype=half.")
473