@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56f738ae9b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] Fix some kernels failed unit tests (#2498)

Hongxia Yang committed Feb 5, 2024 at 17:25 UTC 56f738ae9b631189e67795b397258afbed59b042
5 files changed +62 -12
tests/kernels/allclose_default.py new
+18
@@ -0,0 +1,18 @@
1 +import torch
2 +
3 +# Reference default values of atol and rtol are from
4 +# https://github.com/pytorch/pytorch/blob/6d96beb6bec24d73ee3f080bac54d2104068f675/test/test_transformers.py#L67
5 +default_atol = {torch.float16: 1e-3, torch.bfloat16: 1e-3, torch.float: 1e-5}
6 +default_rtol = {
7 + torch.float16: 1e-3,
8 + torch.bfloat16: 1.6e-2,
9 + torch.float: 1.3e-6
10 +}
11 +
12 +
13 +def get_default_atol(output) -> float:
14 + return default_atol[output.dtype]
15 +
16 +
17 +def get_default_rtol(output) -> float:
18 + return default_rtol[output.dtype]
tests/kernels/test_activation.py
+13 -3
@@ -2,6 +2,7 @@ import pytest
2 import torch
3
4 from vllm.model_executor.layers.activation import FastGELU, NewGELU, SiluAndMul
5 +from allclose_default import get_default_atol, get_default_rtol
6
7 DTYPES = [torch.half, torch.bfloat16, torch.float]
8 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
@@ -33,7 +34,10 @@ def test_silu_and_mul(
34 layer = SiluAndMul()
35 out = layer(x)
36 ref_out = layer._forward(x)
36 - assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
37 + assert torch.allclose(out,
38 + ref_out,
39 + atol=get_default_atol(out),
40 + rtol=get_default_rtol(out))
41
42
43 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -57,7 +61,10 @@ def test_gelu_new(
61 layer = NewGELU()
62 out = layer(x)
63 ref_out = layer._forward(x)
60 - assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
64 + assert torch.allclose(out,
65 + ref_out,
66 + atol=get_default_atol(out),
67 + rtol=get_default_rtol(out))
68
69
70 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -80,4 +87,7 @@ def test_gelu_fast(
87 layer = FastGELU()
88 out = layer(x)
89 ref_out = layer._forward(x)
83 - assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
90 + assert torch.allclose(out,
91 + ref_out,
92 + atol=get_default_atol(out),
93 + rtol=get_default_rtol(out))
tests/kernels/test_attention.py
+17 -5
@@ -8,6 +8,8 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9 from vllm._C import ops, cache_ops
10 from vllm.utils import get_max_shared_memory_bytes
11 +from vllm.utils import is_hip
12 +from allclose_default import get_default_atol, get_default_rtol
13
14 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
15 # This will change depending on the compute capability.
@@ -17,12 +19,18 @@ MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
19 # Reduce NUM_BLOCKS when it happens.
20 NUM_BLOCKS = 4321 # Arbitrary values for testing
21 PARTITION_SIZE = 512
20 -
21 -DTYPES = [torch.half, torch.bfloat16, torch.float]
22 +# flshattF and tritonflashattF supported: {torch.float16, torch.bfloat16}
23 +DTYPES = [torch.half, torch.bfloat16, torch.float
24 + ] if not is_hip() else [torch.half, torch.bfloat16]
25 NUM_GEN_SEQS = [7] # Arbitrary values for testing
26 NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
27 NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
25 -HEAD_SIZES = [64, 80, 96, 112, 128, 256]
28 +
29 +# FlashAttention forward only supports head dimension at most 128
30 +# https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
31 +HEAD_SIZES = [64, 80, 96, 112, 128, 256
32 + ] if not is_hip() else [64, 80, 96, 112, 128]
33 +
34 BLOCK_SIZES = [16, 32]
35 USE_ALIBI = [False, True]
36 KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
@@ -251,9 +259,11 @@ def test_paged_attention(
259 # NOTE(woosuk): Due to the kernel-level differences in the two
260 # implementations, there is a small numerical difference in the two
261 # outputs. Thus, we use a relaxed tolerance for the test.
262 + atol = get_default_atol(output) if is_hip() else 1e-3
263 + rtol = get_default_rtol(output) if is_hip() else 1e-5
264 +
265 # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
266 # so we use a relaxed tolerance for the test.
256 - atol, rtol = 1e-3, 1e-5
267 if kv_cache_dtype == "fp8_e5m2":
268 atol, rtol = 1e-2, 1e-5
269 assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
@@ -357,4 +367,6 @@ def test_multi_query_kv_attention(
367 scale,
368 dtype,
369 )
360 - assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
370 + atol = get_default_atol(output) if is_hip() else 1e-3
371 + rtol = get_default_rtol(output) if is_hip() else 1e-5
372 + assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
tests/kernels/test_cache.py
+5 -1
@@ -6,6 +6,7 @@ import torch
6 from typing import Tuple
7
8 from vllm._C import cache_ops
9 +from vllm.utils import is_hip
10
11 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
12 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -14,7 +15,10 @@ NUM_LAYERS = [1] # Arbitrary values for testing
15 NUM_HEADS = [8] # Arbitrary values for testing
16 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
17 BLOCK_SIZES = [8, 16, 32]
17 -NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
18 +# reduce the size for ROCm test to avoid HIP OOM
19 +NUM_BLOCKS = [1024, 36000] if not is_hip else [
20 + 1024, 10000
21 +] # Arbitrary values for testing
22 NUM_MAPPINGS = [256] # Arbitrary values for testing
23 SEEDS = [0]
24 CUDA_DEVICES = [
tests/kernels/test_pos_encoding.py
+9 -3
@@ -2,7 +2,7 @@ from typing import Optional
2
3 import pytest
4 import torch
5 -
5 +from allclose_default import get_default_atol, get_default_rtol
6 from vllm.model_executor.layers.rotary_embedding import get_rope
7
8 IS_NEOX_STYLE = [True, False]
@@ -64,5 +64,11 @@ def test_rotary_embedding(
64 ref_query, ref_key = rope._forward(positions, query, key)
65 out_query, out_key = rope.forward(positions, query, key)
66 # Compare the results.
67 - assert torch.allclose(out_query, ref_query, atol=1e-5, rtol=1e-5)
68 - assert torch.allclose(out_key, ref_key, atol=1e-5, rtol=1e-5)
67 + assert torch.allclose(out_query,
68 + ref_query,
69 + atol=get_default_atol(out_query),
70 + rtol=get_default_rtol(out_query))
71 + assert torch.allclose(out_key,
72 + ref_key,
73 + atol=get_default_atol(out_key),
74 + rtol=get_default_rtol(out_key))