A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[CI] typo fix: is_hip --> is_hip() (#3595)
youkaichao committed
Mar 24, 2024 at 16:03 UTC
8b268a46a7c955bb1f2b25bbed02f9b1e3a4ff05
1 file changed
+5
-5
tests/kernels/test_cache.py
+5
-5
@@ -6,7 +6,6 @@ import torch
6
from typing import Tuple
7
8
from vllm._C import cache_ops
9
-from vllm.utils import is_hip
9
10
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
11
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -15,10 +14,11 @@ NUM_LAYERS = [1] # Arbitrary values for testing
14
NUM_HEADS = [8] # Arbitrary values for testing
15
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
16
BLOCK_SIZES = [8, 16, 32]
18
-# reduce the size for ROCm test to avoid HIP OOM
19
-NUM_BLOCKS = [1024, 36000] if not is_hip else [
20
- 1024, 10000
21
-] # Arbitrary values for testing
17
+
18
+# Arbitrary values for testing
19
+# don't make it too large. e.g. [1024, 36000] will OOM
20
+NUM_BLOCKS = [1024, 10000]
21
+
22
NUM_MAPPINGS = [256] # Arbitrary values for testing
23
SEEDS = [0]
24
CUDA_DEVICES = [