A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix OOM in attention kernel test (#1223)
Woosuk Kwon committed
Sep 28, 2023 at 14:33 UTC
6f88f762bf907bccdadc52948001b21ccb616a01
1 file changed
+5
-2
tests/kernels/test_attention.py
+5
-2
@@ -247,8 +247,11 @@ def test_multi_query_kv_attention(
247
torch.random.manual_seed(seed)
248
torch.cuda.manual_seed(seed)
249
250
- seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
251
- seq_lens[-1] = MAX_SEQ_LEN
250
+ # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
251
+ # As the xformers library is already tested with its own tests, we can use
252
+ # a smaller MAX_SEQ_LEN here.
253
+ max_len = min(MAX_SEQ_LEN, 4096)
254
+ seq_lens = random.sample(range(1, max_len), num_seqs)
255
num_tokens = sum(seq_lens)
256
257
scale = float(1.0 / (head_size**0.5))