@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6f88f762bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix OOM in attention kernel test (#1223)

Woosuk Kwon committed Sep 28, 2023 at 14:33 UTC 6f88f762bf907bccdadc52948001b21ccb616a01
1 file changed +5 -2
tests/kernels/test_attention.py
+5 -2
@@ -247,8 +247,11 @@ def test_multi_query_kv_attention(
247 torch.random.manual_seed(seed)
248 torch.cuda.manual_seed(seed)
249
250 - seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
251 - seq_lens[-1] = MAX_SEQ_LEN
250 + # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
251 + # As the xformers library is already tested with its own tests, we can use
252 + # a smaller MAX_SEQ_LEN here.
253 + max_len = min(MAX_SEQ_LEN, 4096)
254 + seq_lens = random.sample(range(1, max_len), num_seqs)
255 num_tokens = sum(seq_lens)
256
257 scale = float(1.0 / (head_size**0.5))