@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b268a46a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[CI] typo fix: is_hip --> is_hip() (#3595)

youkaichao committed Mar 24, 2024 at 16:03 UTC 8b268a46a7c955bb1f2b25bbed02f9b1e3a4ff05
1 file changed +5 -5
tests/kernels/test_cache.py
+5 -5
@@ -6,7 +6,6 @@ import torch
6 from typing import Tuple
7
8 from vllm._C import cache_ops
9 -from vllm.utils import is_hip
9
10 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -15,10 +14,11 @@ NUM_LAYERS = [1] # Arbitrary values for testing
14 NUM_HEADS = [8] # Arbitrary values for testing
15 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
16 BLOCK_SIZES = [8, 16, 32]
18 -# reduce the size for ROCm test to avoid HIP OOM
19 -NUM_BLOCKS = [1024, 36000] if not is_hip else [
20 - 1024, 10000
21 -] # Arbitrary values for testing
17 +
18 +# Arbitrary values for testing
19 +# don't make it too large. e.g. [1024, 36000] will OOM
20 +NUM_BLOCKS = [1024, 10000]
21 +
22 NUM_MAPPINGS = [256] # Arbitrary values for testing
23 SEEDS = [0]
24 CUDA_DEVICES = [