@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 230c4b38c1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI/Test] fix swap test for multi gpu (#4689)

youkaichao committed May 8, 2024 at 13:14 UTC 230c4b38c10148c3fbbbb67cd1046766d73c865a
1 file changed +3 -1
tests/kernels/test_cache.py
+3 -1
@@ -222,11 +222,12 @@ def test_reshape_and_cache_flash(
222 random.seed(seed)
223 torch.random.manual_seed(seed)
224 torch.cuda.manual_seed(seed)
225 + torch.set_default_device(device)
226
227 # Create a random slot mapping.
228 num_slots = block_size * num_blocks
229 slot_mapping = random.sample(range(num_slots), num_tokens)
229 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device='cuda')
230 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=device)
231
232 qkv = torch.randn(num_tokens,
233 3,
@@ -245,6 +246,7 @@ def test_reshape_and_cache_flash(
246 head_size,
247 kv_cache_dtype,
248 dtype,
249 + device=device,
250 )
251 key_cache, value_cache = key_caches[0], value_caches[0]
252