@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 622b7ab955
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Hardware] using current_platform.seed_everything (#9785)

Signed-off-by: wangshuai09 <391746016@qq.com>

wangshuai09 committed Oct 29, 2024 at 22:47 UTC 622b7ab955186f37879208d7a30e9faf985be220
27 files changed +104 -105
benchmarks/kernels/benchmark_layernorm.py
+3 -3
@@ -3,8 +3,8 @@ import time
3 import torch
4
5 from vllm.model_executor.layers.layernorm import RMSNorm
6 -from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
7 - seed_everything)
6 +from vllm.platforms import current_platform
7 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser
8
9
10 @torch.inference_mode()
@@ -16,7 +16,7 @@ def main(num_tokens: int,
16 do_profile: bool = False,
17 num_warmup_iters: int = 5,
18 num_iters: int = 100) -> None:
19 - seed_everything(seed)
19 + current_platform.seed_everything(seed)
20 torch.set_default_device("cuda")
21
22 layer = RMSNorm(hidden_size).to(dtype=dtype)
benchmarks/kernels/benchmark_moe.py
+4 -3
@@ -10,7 +10,8 @@ from ray.experimental.tqdm_ray import tqdm
10 from transformers import AutoConfig
11
12 from vllm.model_executor.layers.fused_moe.fused_moe import *
13 -from vllm.utils import FlexibleArgumentParser, seed_everything
13 +from vllm.platforms import current_platform
14 +from vllm.utils import FlexibleArgumentParser
15
16
17 class BenchmarkConfig(TypedDict):
@@ -167,7 +168,7 @@ class BenchmarkWorker:
168
169 def __init__(self, seed: int) -> None:
170 torch.set_default_device("cuda")
170 - seed_everything(seed)
171 + current_platform.seed_everything(seed)
172 self.seed = seed
173
174 def benchmark(
@@ -181,7 +182,7 @@ class BenchmarkWorker:
182 use_fp8_w8a8: bool,
183 use_int8_w8a16: bool,
184 ) -> Tuple[Dict[str, int], float]:
184 - seed_everything(self.seed)
185 + current_platform.seed_everything(self.seed)
186 dtype_str = get_config_dtype_str(dtype,
187 use_int8_w8a16=use_int8_w8a16,
188 use_fp8_w8a8=use_fp8_w8a8)
benchmarks/kernels/benchmark_paged_attention.py
+3 -2
@@ -5,8 +5,9 @@ from typing import List, Optional
5 import torch
6
7 from vllm import _custom_ops as ops
8 +from vllm.platforms import current_platform
9 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
9 - create_kv_caches_with_random, seed_everything)
10 + create_kv_caches_with_random)
11
12 NUM_BLOCKS = 1024
13 PARTITION_SIZE = 512
@@ -28,7 +29,7 @@ def main(
29 device: str = "cuda",
30 kv_cache_dtype: Optional[str] = None,
31 ) -> None:
31 - seed_everything(seed)
32 + current_platform.seed_everything(seed)
33
34 scale = float(1.0 / (head_size**0.5))
35 query = torch.empty(num_seqs,
benchmarks/kernels/benchmark_quant.py
+3 -3
@@ -3,8 +3,8 @@ import time
3 import torch
4
5 from vllm import _custom_ops as ops
6 -from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
7 - seed_everything)
6 +from vllm.platforms import current_platform
7 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser
8
9
10 @torch.inference_mode()
@@ -17,7 +17,7 @@ def main(num_tokens: int,
17 do_profile: bool = False,
18 num_warmup_iters: int = 5,
19 num_iters: int = 100) -> None:
20 - seed_everything(seed)
20 + current_platform.seed_everything(seed)
21 torch.set_default_device("cuda")
22
23 x = torch.randn(num_tokens, hidden_size, dtype=dtype)
benchmarks/kernels/benchmark_rope.py
+3 -2
@@ -6,7 +6,8 @@ import torch
6
7 from vllm.model_executor.layers.rotary_embedding import (RotaryEmbedding,
8 get_rope)
9 -from vllm.utils import FlexibleArgumentParser, seed_everything
9 +from vllm.platforms import current_platform
10 +from vllm.utils import FlexibleArgumentParser
11
12
13 def benchmark_rope_kernels_multi_lora(
@@ -22,7 +23,7 @@ def benchmark_rope_kernels_multi_lora(
23 max_position: int = 8192,
24 base: int = 10000,
25 ) -> None:
25 - seed_everything(seed)
26 + current_platform.seed_everything(seed)
27 torch.set_default_device(device)
28 if rotary_dim is None:
29 rotary_dim = head_size
tests/kernels/test_activation.py
+3 -3
@@ -8,7 +8,7 @@ from tests.kernels.utils import opcheck
8 from vllm.model_executor.layers.activation import (FastGELU, FatreluAndMul,
9 GeluAndMul, NewGELU,
10 QuickGELU, SiluAndMul)
11 -from vllm.utils import seed_everything
11 +from vllm.platforms import current_platform
12
13 from .allclose_default import get_default_atol, get_default_rtol
14
@@ -37,7 +37,7 @@ def test_act_and_mul(
37 seed: int,
38 device: str,
39 ) -> None:
40 - seed_everything(seed)
40 + current_platform.seed_everything(seed)
41 torch.set_default_device(device)
42 x = torch.randn(num_tokens, 2 * d, dtype=dtype)
43 if activation == "silu":
@@ -85,7 +85,7 @@ def test_activation(
85 seed: int,
86 device: str,
87 ) -> None:
88 - seed_everything(seed)
88 + current_platform.seed_everything(seed)
89 torch.set_default_device(device)
90 x = torch.randn(num_tokens, d, dtype=dtype)
91 layer = activation[0]()
tests/kernels/test_attention.py
+3 -3
@@ -7,7 +7,7 @@ import torch
7 from tests.kernels.utils import opcheck
8 from vllm import _custom_ops as ops
9 from vllm.platforms import current_platform
10 -from vllm.utils import get_max_shared_memory_bytes, seed_everything
10 +from vllm.utils import get_max_shared_memory_bytes
11
12 from .allclose_default import get_default_atol, get_default_rtol
13
@@ -144,7 +144,7 @@ def test_paged_attention(
144 or (version == "rocm" and head_size not in (64, 128))):
145 pytest.skip()
146
147 - seed_everything(seed)
147 + current_platform.seed_everything(seed)
148 torch.set_default_device(device)
149 scale = float(1.0 / (head_size**0.5))
150 num_query_heads, num_kv_heads = num_heads
@@ -382,7 +382,7 @@ def test_multi_query_kv_attention(
382 seed: int,
383 device: str,
384 ) -> None:
385 - seed_everything(seed)
385 + current_platform.seed_everything(seed)
386 torch.set_default_device(device)
387 # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
388 # As the xformers library is already tested with its own tests, we can use
tests/kernels/test_awq_triton.py
+3 -3
@@ -7,7 +7,7 @@ import torch
7
8 from vllm.model_executor.layers.quantization.awq_triton import (
9 AWQ_TRITON_SUPPORTED_GROUP_SIZES, awq_dequantize_triton, awq_gemm_triton)
10 -from vllm.utils import seed_everything
10 +from vllm.platforms import current_platform
11
12 device = "cuda"
13
@@ -80,7 +80,7 @@ def test_dequantize(qweight_rows, qweight_cols, group_size):
80 zeros_cols = qweight_cols
81 zeros_dtype = torch.int32
82
83 - seed_everything(0)
83 + current_platform.seed_everything(0)
84
85 qweight = torch.randint(0,
86 torch.iinfo(torch.int32).max,
@@ -134,7 +134,7 @@ def test_gemm(N, K, M, splitK, group_size):
134 qzeros_rows = scales_rows
135 qzeros_cols = qweight_cols
136
137 - seed_everything(0)
137 + current_platform.seed_everything(0)
138
139 input = torch.rand((input_rows, input_cols),
140 dtype=input_dtype,
tests/kernels/test_blocksparse_attention.py
+3 -3
@@ -8,7 +8,7 @@ from vllm import _custom_ops as ops
8 from vllm.attention.ops.blocksparse_attention.interface import (
9 LocalStridedBlockSparseAttn)
10 from vllm.platforms import current_platform
11 -from vllm.utils import get_max_shared_memory_bytes, seed_everything
11 +from vllm.utils import get_max_shared_memory_bytes
12
13 from .allclose_default import get_default_atol, get_default_rtol
14
@@ -173,7 +173,7 @@ def test_paged_attention(
173 blocksparse_block_size: int,
174 blocksparse_head_sliding_step: int,
175 ) -> None:
176 - seed_everything(seed)
176 + current_platform.seed_everything(seed)
177 torch.set_default_device(device)
178 scale = float(1.0 / (head_size**0.5))
179 num_query_heads, num_kv_heads = num_heads
@@ -384,7 +384,7 @@ def test_varlen_blocksparse_attention_prefill(
384 seed: int,
385 device: str,
386 ) -> None:
387 - seed_everything(seed)
387 + current_platform.seed_everything(seed)
388 torch.set_default_device(device)
389 # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
390 # As the xformers library is already tested with its own tests, we can use
tests/kernels/test_cache.py
+6 -6
@@ -6,7 +6,7 @@ import torch
6
7 from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
8 from vllm import _custom_ops as ops
9 -from vllm.utils import seed_everything
9 +from vllm.platforms import current_platform
10
11 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
12 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -56,7 +56,7 @@ def test_copy_blocks(
56 ) -> None:
57 if kv_cache_dtype == "fp8" and head_size % 16:
58 pytest.skip()
59 - seed_everything(seed)
59 + current_platform.seed_everything(seed)
60 torch.set_default_device(device)
61 # Generate random block mappings where each source block is mapped to two
62 # destination blocks.
@@ -132,7 +132,7 @@ def test_reshape_and_cache(
132 ) -> None:
133 if kv_cache_dtype == "fp8" and head_size % 16:
134 pytest.skip()
135 - seed_everything(seed)
135 + current_platform.seed_everything(seed)
136 torch.set_default_device(device)
137 # Create a random slot mapping.
138 num_slots = block_size * num_blocks
@@ -224,7 +224,7 @@ def test_reshape_and_cache_flash(
224 device: str,
225 kv_cache_dtype: str,
226 ) -> None:
227 - seed_everything(seed)
227 + current_platform.seed_everything(seed)
228 torch.set_default_device(device)
229
230 # Create a random slot mapping.
@@ -339,7 +339,7 @@ def test_swap_blocks(
339 if kv_cache_dtype == "fp8" and head_size % 16:
340 pytest.skip()
341
342 - seed_everything(seed)
342 + current_platform.seed_everything(seed)
343
344 src_device = device if direction[0] == "cuda" else 'cpu'
345 dst_device = device if direction[1] == "cuda" else 'cpu'
@@ -408,7 +408,7 @@ def test_fp8_e4m3_conversion(
408 seed: int,
409 device: str,
410 ) -> None:
411 - seed_everything(seed)
411 + current_platform.seed_everything(seed)
412
413 low = -224.0
414 high = 224.0
tests/kernels/test_causal_conv1d.py
+6 -6
@@ -9,7 +9,7 @@ from vllm import _custom_ops as ops # noqa: F401
9 from vllm.attention.backends.utils import PAD_SLOT_ID
10 from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
11 causal_conv1d_fn, causal_conv1d_update)
12 -from vllm.utils import seed_everything
12 +from vllm.platforms import current_platform
13
14
15 def causal_conv1d_ref(
@@ -70,7 +70,7 @@ def causal_conv1d_update_ref(x,
70 bias: (dim,)
71 cache_seqlens: (batch,), dtype int32.
72 If not None, the conv_state is treated as a circular buffer.
73 - The conv_state will be updated by copying x to the
73 + The conv_state will be updated by copying x to the
74 conv_state starting at the index
75 @cache_seqlens % state_len before performing the convolution.
76
@@ -161,7 +161,7 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
161 if itype == torch.bfloat16:
162 rtol, atol = 1e-2, 5e-2
163 # set seed
164 - seed_everything(0)
164 + current_platform.seed_everything(0)
165 x = torch.randn(batch, dim, seqlen, device=device,
166 dtype=itype).contiguous()
167
@@ -223,7 +223,7 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
223 if itype == torch.bfloat16:
224 rtol, atol = 1e-2, 5e-2
225 # set seed
226 - seed_everything(0)
226 + current_platform.seed_everything(0)
227 batch = 2
228 x = torch.randn(batch, dim, seqlen, device=device, dtype=itype)
229 x_ref = x.clone()
@@ -270,7 +270,7 @@ def test_causal_conv1d_update_with_batch_gather(with_padding, dim, width,
270 rtol, atol = 1e-2, 5e-2
271
272 # set seed
273 - seed_everything(0)
273 + current_platform.seed_everything(0)
274
275 batch_size = 3
276 padding = 5 if with_padding else 0
@@ -343,7 +343,7 @@ def test_causal_conv1d_varlen(with_padding, dim, seqlen, width, has_bias,
343 if itype == torch.bfloat16:
344 rtol, atol = 1e-2, 5e-2
345 # set seed
346 - seed_everything(0)
346 + current_platform.seed_everything(0)
347 seqlens = []
348 batch_size = 4
349 if seqlen < 10:
tests/kernels/test_flash_attn.py
+3 -3
@@ -3,7 +3,7 @@ from typing import List, Optional, Tuple
3 import pytest
4 import torch
5
6 -from vllm.utils import seed_everything
6 +from vllm.platforms import current_platform
7 from vllm.vllm_flash_attn import (flash_attn_varlen_func,
8 flash_attn_with_kvcache)
9
@@ -91,7 +91,7 @@ def test_flash_attn_with_paged_kv(
91 sliding_window: Optional[int],
92 ) -> None:
93 torch.set_default_device("cuda")
94 - seed_everything(0)
94 + current_platform.seed_everything(0)
95 num_seqs = len(kv_lens)
96 num_query_heads = num_heads[0]
97 num_kv_heads = num_heads[1]
@@ -161,7 +161,7 @@ def test_varlen_with_paged_kv(
161 num_blocks: int,
162 ) -> None:
163 torch.set_default_device("cuda")
164 - seed_everything(0)
164 + current_platform.seed_everything(0)
165 num_seqs = len(seq_lens)
166 query_lens = [x[0] for x in seq_lens]
167 kv_lens = [x[1] for x in seq_lens]
tests/kernels/test_flashinfer.py
+5 -5
@@ -4,7 +4,7 @@ import flashinfer
4 import pytest
5 import torch
6
7 -from vllm.utils import seed_everything
7 +from vllm.platforms import current_platform
8
9 NUM_HEADS = [(16, 16), (32, 8), (64, 8), (6, 1)]
10 HEAD_SIZES = [128, 256]
@@ -84,7 +84,7 @@ def test_flashinfer_decode_with_paged_kv(
84 soft_cap: Optional[float],
85 ) -> None:
86 torch.set_default_device("cuda")
87 - seed_everything(0)
87 + current_platform.seed_everything(0)
88 num_seqs = len(kv_lens)
89 num_query_heads = num_heads[0]
90 num_kv_heads = num_heads[1]
@@ -170,7 +170,7 @@ def test_flashinfer_prefill_with_paged_kv(seq_lens: List[Tuple[int, int]],
170 block_size: int,
171 soft_cap: Optional[float]) -> None:
172 torch.set_default_device("cuda")
173 - seed_everything(0)
173 + current_platform.seed_everything(0)
174 num_seqs = len(seq_lens)
175 query_lens = [x[0] for x in seq_lens]
176 kv_lens = [x[1] for x in seq_lens]
@@ -268,7 +268,7 @@ def test_flashinfer_prefill_with_paged_fp8_kv(
268 head_size: int, dtype: torch.dtype, block_size: int,
269 soft_cap: Optional[float]) -> None:
270 torch.set_default_device("cuda")
271 - seed_everything(0)
271 + current_platform.seed_everything(0)
272 num_seqs = len(seq_lens)
273 query_lens = [x[0] for x in seq_lens]
274 kv_lens = [x[1] for x in seq_lens]
@@ -381,7 +381,7 @@ def test_flashinfer_decode_with_paged_fp8_kv(
381 ) -> None:
382 # test doesn't work for num_heads = (16,16)
383 torch.set_default_device("cuda")
384 - seed_everything(0)
384 + current_platform.seed_everything(0)
385 num_seqs = len(kv_lens)
386 num_query_heads = num_heads[0]
387 num_kv_heads = num_heads[1]
tests/kernels/test_fp8_quant.py
+4 -4
@@ -6,7 +6,7 @@ from tests.kernels.quant_utils import (FP8_DTYPE,
6 ref_dynamic_per_tensor_fp8_quant,
7 ref_dynamic_per_token_quant)
8 from tests.kernels.utils import opcheck
9 -from vllm.utils import seed_everything
9 +from vllm.platforms import current_platform
10
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
12 HIDDEN_SIZES = [1, 2, 3, 4, 16, 67, 768, 2048, 5120, 5137, 8192,
@@ -46,7 +46,7 @@ def opcheck_fp8_quant(output,
46 def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
47 dtype: torch.dtype, scale_ub: bool,
48 seed: int) -> None:
49 - seed_everything(seed)
49 + current_platform.seed_everything(seed)
50
51 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
52 device="cuda") + 1e-6 # avoid nans
@@ -76,7 +76,7 @@ def test_dynamic_per_token_fp8_quant(num_tokens: int, hidden_size: int,
76 @torch.inference_mode()
77 def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
78 dtype: torch.dtype, seed: int) -> None:
79 - seed_everything(seed)
79 + current_platform.seed_everything(seed)
80
81 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
82
@@ -95,7 +95,7 @@ def test_dynamic_per_tensor_fp8_quant(num_tokens: int, hidden_size: int,
95 @torch.inference_mode()
96 @pytest.mark.parametrize("seed", SEEDS)
97 def test_fp8_quant_large(seed: int) -> None:
98 - seed_everything(seed)
98 + current_platform.seed_everything(seed)
99
100 num_tokens = 1024000 # Mistral-Nemo's max_position_embeddings
101 hidden_size = 1152 # Smallest hidden_size to reproduce the error
tests/kernels/test_gguf.py
+3 -3
@@ -7,7 +7,7 @@ from gguf import GGMLQuantizationType, GGUFReader, ReaderTensor, dequantize
7 from huggingface_hub import snapshot_download
8
9 import vllm._custom_ops as ops
10 -from vllm.utils import seed_everything
10 +from vllm.platforms import current_platform
11
12 GGUF_SAMPLE = snapshot_download("Isotr0py/test-gguf-sample")
13
@@ -75,7 +75,7 @@ def test_dequantize(hidden_size: int, dtype: torch.dtype,
75 @torch.inference_mode()
76 def test_mmvq(hidden_size: int, dtype: torch.dtype,
77 quant_type: GGMLQuantizationType):
78 - seed_everything(0)
78 + current_platform.seed_everything(0)
79
80 tensors = get_gguf_sample_tensors(hidden_size, quant_type)
81 x = torch.rand((1, hidden_size), dtype=dtype, device="cuda")
@@ -111,7 +111,7 @@ def test_mmvq(hidden_size: int, dtype: torch.dtype,
111 @torch.inference_mode()
112 def test_mmq(num_tokens: int, hidden_size: int, dtype: torch.dtype,
113 quant_type: GGMLQuantizationType):
114 - seed_everything(0)
114 + current_platform.seed_everything(0)
115
116 tensors = get_gguf_sample_tensors(hidden_size, quant_type)
117 x = torch.rand((num_tokens, hidden_size), dtype=dtype, device="cuda")
tests/kernels/test_int8_quant.py
+5 -5
@@ -4,7 +4,7 @@ import torch
4 from tests.kernels.quant_utils import ref_dynamic_per_token_quant
5 from tests.kernels.utils import opcheck
6 from vllm._custom_ops import scaled_int8_quant
7 -from vllm.utils import seed_everything
7 +from vllm.platforms import current_platform
8
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
10 HIDDEN_SIZES = [16, 67, 768, 2048, 5120, 5137, 8192,
@@ -45,7 +45,7 @@ def opcheck_int8_quant_dynamic(output, input, symmetric=True):
45 @torch.inference_mode()
46 def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
47 dtype: torch.dtype, seed: int) -> None:
48 - seed_everything(seed)
48 + current_platform.seed_everything(seed)
49
50 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
51
@@ -68,7 +68,7 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
68 @torch.inference_mode()
69 def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
70 dtype: torch.dtype, seed: int) -> None:
71 - seed_everything(seed)
71 + current_platform.seed_everything(seed)
72 int8_traits = torch.iinfo(torch.int8)
73
74 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
@@ -112,7 +112,7 @@ def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
112 def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
113 dtype: torch.dtype, seed: int,
114 scale: float) -> None:
115 - seed_everything(seed)
115 + current_platform.seed_everything(seed)
116 int8_traits = torch.iinfo(torch.int8)
117
118 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
@@ -138,7 +138,7 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
138 def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
139 dtype: torch.dtype, seed: int,
140 scale: float, azp: int) -> None:
141 - seed_everything(seed)
141 + current_platform.seed_everything(seed)
142 int8_traits = torch.iinfo(torch.int8)
143
144 x = torch.rand(num_tokens, hidden_size, dtype=dtype,
tests/kernels/test_layernorm.py
+2 -2
@@ -3,7 +3,7 @@ import torch
3
4 from tests.kernels.utils import opcheck
5 from vllm.model_executor.layers.layernorm import RMSNorm
6 -from vllm.utils import seed_everything
6 +from vllm.platforms import current_platform
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
9 NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
@@ -31,7 +31,7 @@ def test_rms_norm(
31 seed: int,
32 device: str,
33 ) -> None:
34 - seed_everything(seed)
34 + current_platform.seed_everything(seed)
35 torch.set_default_device(device)
36 layer = RMSNorm(hidden_size).to(dtype=dtype)
37 layer.weight.data.normal_(mean=1.0, std=0.1)
tests/kernels/test_mamba_ssm.py
+3 -3
@@ -8,7 +8,7 @@ from vllm import _custom_ops as ops # noqa: F401
8 from vllm.attention.backends.utils import PAD_SLOT_ID
9 from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
10 selective_scan_fn, selective_state_update)
11 -from vllm.utils import seed_everything
11 +from vllm.platforms import current_platform
12
13
14 def selective_state_update_ref(state,
@@ -235,7 +235,7 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
235 rtolw = max(rtolw, rtol)
236 atolw = max(atolw, atol)
237 # set seed
238 - seed_everything(0)
238 + current_platform.seed_everything(0)
239 batch_size = 1
240 dim = 4
241 dstate = 8
@@ -358,7 +358,7 @@ def test_selective_state_update(dim, dstate, has_z, itype):
358 if torch.version.hip:
359 atol *= 2
360 # set seed
361 - seed_everything(0)
361 + current_platform.seed_everything(0)
362 batch_size = 1
363 state = torch.randn(batch_size, dim, dstate, dtype=itype, device=device)
364 x = torch.randn(batch_size, dim, device=device, dtype=itype)
tests/kernels/test_moe.py
+1 -2
@@ -19,7 +19,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
19 from vllm.model_executor.models.mixtral import MixtralMoE
20 from vllm.platforms import current_platform
21 from vllm.scalar_type import scalar_types
22 -from vllm.utils import seed_everything
22
23
24 @pytest.mark.parametrize("m", [1024 * 128, 512, 222, 33, 1])
@@ -115,7 +114,7 @@ def test_fused_marlin_moe(
114 num_bits: int,
115 is_k_full: bool,
116 ):
118 - seed_everything(7)
117 + current_platform.seed_everything(7)
118
119 # Filter act_order
120 if act_order:
tests/kernels/test_pos_encoding.py
+4 -4
@@ -5,7 +5,7 @@ import pytest
5 import torch
6
7 from vllm.model_executor.layers.rotary_embedding import get_rope
8 -from vllm.utils import seed_everything
8 +from vllm.platforms import current_platform
9
10 from .allclose_default import get_default_atol, get_default_rtol
11
@@ -48,7 +48,7 @@ def test_rotary_embedding(
48 if rotary_dim is None:
49 rotary_dim = head_size
50
51 - seed_everything(seed)
51 + current_platform.seed_everything(seed)
52 torch.set_default_device(device)
53 if rotary_dim is None:
54 rotary_dim = head_size
@@ -100,7 +100,7 @@ def test_batched_rotary_embedding(
100 max_position: int = 8192,
101 base: int = 10000,
102 ) -> None:
103 - seed_everything(seed)
103 + current_platform.seed_everything(seed)
104 torch.set_default_device(device)
105 if rotary_dim is None:
106 rotary_dim = head_size
@@ -160,7 +160,7 @@ def test_batched_rotary_embedding_multi_lora(
160 max_position: int = 8192,
161 base: int = 10000,
162 ) -> None:
163 - seed_everything(seed)
163 + current_platform.seed_everything(seed)
164 torch.set_default_device(device)
165 if rotary_dim is None:
166 rotary_dim = head_size
tests/kernels/test_prefix_prefill.py
+4 -3
@@ -9,7 +9,8 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
10 from vllm.attention.backends.xformers import _make_alibi_bias
11 from vllm.attention.ops.prefix_prefill import context_attention_fwd
12 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, seed_everything
12 +from vllm.platforms import current_platform
13 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
14
15 NUM_HEADS = [64]
16 NUM_QUERIES_PER_KV = [1, 8, 64]
@@ -39,7 +40,7 @@ def test_contexted_kv_attention(
40 kv_cache_dtype: str,
41 device: str,
42 ) -> None:
42 - seed_everything(0)
43 + current_platform.seed_everything(0)
44 torch.set_default_device(device)
45
46 # Need this, otherwise when we capture the graph the process
@@ -234,7 +235,7 @@ def test_contexted_kv_attention_alibi(
235 kv_cache_dtype: str,
236 device: str,
237 ) -> None:
237 - seed_everything(0)
238 + current_platform.seed_everything(0)
239 torch.set_default_device(device)
240
241 # Need this, otherwise when we capture the graph the process
tests/lora/test_layers.py
+2 -2
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 ParallelLMHead, VocabParallelEmbedding, get_masked_input_and_mask)
41 from vllm.model_executor.utils import set_random_seed
42 -from vllm.utils import seed_everything
42 +from vllm.platforms import current_platform
43
44 from .utils import DummyLoRAManager
45
@@ -923,7 +923,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
923 seq_len) -> None:
924 dtype = torch.float16
925 seed = 0
926 - seed_everything(seed)
926 + current_platform.seed_everything(seed)
927 torch.set_default_device(device)
928 punica_wrapper = PunicaWrapper(8192, 256, device)
929 max_loras = 8
tests/lora/test_punica_sizes.py
+5 -5
@@ -1,5 +1,5 @@
1 """
2 -This script is mainly used to tests various hidden_sizes. We have collected the
2 +This script is mainly used to tests various hidden_sizes. We have collected the
3 hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4 whether the corresponding Triton kernel can run normally when tensor parallelism
5 is set to [1, 2, 4, 8, 16, 32, 64].
@@ -15,8 +15,8 @@ from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15 from vllm.lora.ops.sgmv_expand import sgmv_expand
16 from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
17 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
18 +from vllm.platforms import current_platform
19 from vllm.triton_utils.libentry import LibEntry
19 -from vllm.utils import seed_everything
20
21 from .utils import (generate_data, generate_data_for_expand_nslices,
22 ref_torch_groupgemm)
@@ -146,7 +146,7 @@ def test_punica_sgmv(
146 device: str,
147 ):
148 torch.set_default_device(device)
149 - seed_everything(seed)
149 + current_platform.seed_everything(seed)
150
151 seq_length = 128
152 (
@@ -239,7 +239,7 @@ def test_punica_bgmv(
239 from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
240
241 torch.set_default_device(device)
242 - seed_everything(seed)
242 + current_platform.seed_everything(seed)
243
244 seq_length = 1
245 (
@@ -327,7 +327,7 @@ def test_punica_expand_nslices(
327 from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
328
329 torch.set_default_device(device)
330 - seed_everything(seed)
330 + current_platform.seed_everything(seed)
331
332 seq_length = 128 if op_type == "sgmv" else 1
333 (
tests/lora/test_punica_variation.py
+6 -6
@@ -1,6 +1,6 @@
1 """
2 -This script is mainly used to test whether trtion kernels can run normally
3 -under different conditions, including various batches, numbers of LoRA , and
2 +This script is mainly used to test whether trtion kernels can run normally
3 +under different conditions, including various batches, numbers of LoRA , and
4 maximum ranks.
5 """
6 from unittest.mock import patch
@@ -14,8 +14,8 @@ from vllm.lora.ops.bgmv_shrink import bgmv_shrink
14 from vllm.lora.ops.sgmv_expand import sgmv_expand
15 from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
16 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
17 +from vllm.platforms import current_platform
18 from vllm.triton_utils.libentry import LibEntry
18 -from vllm.utils import seed_everything
19
20 from .utils import (generate_data, generate_data_for_expand_nslices,
21 ref_torch_groupgemm)
@@ -61,7 +61,7 @@ def test_punica_sgmv(
61 device: str,
62 ):
63 torch.set_default_device(device)
64 - seed_everything(seed)
64 + current_platform.seed_everything(seed)
65
66 seq_length = 128
67 (
@@ -154,7 +154,7 @@ def test_punica_bgmv(
154 from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
155
156 torch.set_default_device(device)
157 - seed_everything(seed)
157 + current_platform.seed_everything(seed)
158
159 seq_length = 1
160 (
@@ -242,7 +242,7 @@ def test_punica_expand_nslices(
242 from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
243
244 torch.set_default_device(device)
245 - seed_everything(seed)
245 + current_platform.seed_everything(seed)
246
247 seq_length = 128 if op_type == "sgmv" else 1
248 (
vllm/model_executor/utils.py
+1 -2
@@ -4,11 +4,10 @@ from typing import Any, Dict, Optional
4 import torch
5
6 from vllm.platforms import current_platform
7 -from vllm.utils import seed_everything
7
8
9 def set_random_seed(seed: int) -> None:
11 - seed_everything(seed)
10 + current_platform.seed_everything(seed)
11
12
13 def set_weight_attrs(
vllm/platforms/interface.py
+14
@@ -1,6 +1,8 @@
1 import enum
2 +import random
3 from typing import NamedTuple, Optional, Tuple, Union
4
5 +import numpy as np
6 import torch
7
8
@@ -111,6 +113,18 @@ class Platform:
113 """
114 return torch.inference_mode(mode=True)
115
116 + @classmethod
117 + def seed_everything(cls, seed: int) -> None:
118 + """
119 + Set the seed of each random module.
120 + `torch.manual_seed` will set seed on all devices.
121 +
122 + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20
123 + """
124 + random.seed(seed)
125 + np.random.seed(seed)
126 + torch.manual_seed(seed)
127 +
128
129 class UnspecifiedPlatform(Platform):
130 _enum = PlatformEnum.UNSPECIFIED
vllm/utils.py
+2 -19
@@ -7,7 +7,6 @@ import gc
7 import inspect
8 import ipaddress
9 import os
10 -import random
10 import socket
11 import subprocess
12 import sys
@@ -331,22 +330,6 @@ def get_cpu_memory() -> int:
330 return psutil.virtual_memory().total
331
332
334 -def seed_everything(seed: int) -> None:
335 - """
336 - Set the seed of each random module.
337 -
338 - Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20
339 - """
340 - random.seed(seed)
341 - np.random.seed(seed)
342 -
343 - if current_platform.is_cuda_alike():
344 - torch.cuda.manual_seed_all(seed)
345 -
346 - if current_platform.is_xpu():
347 - torch.xpu.manual_seed_all(seed)
348 -
349 -
333 def random_uuid() -> str:
334 return str(uuid.uuid4().hex)
335
@@ -643,7 +626,7 @@ def create_kv_caches_with_random_flash(
626 seed: int = 0,
627 device: Optional[str] = "cuda",
628 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
646 - seed_everything(seed)
629 + current_platform.seed_everything(seed)
630
631 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
632 key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
@@ -685,7 +668,7 @@ def create_kv_caches_with_random(
668 f"Does not support key cache of type fp8 with head_size {head_size}"
669 )
670
688 - seed_everything(seed)
671 + current_platform.seed_everything(seed)
672
673 torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
674