A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CI/Test] fix swap test for multi gpu (#4689)
youkaichao committed
May 8, 2024 at 13:14 UTC
230c4b38c10148c3fbbbb67cd1046766d73c865a
1 file changed
+3
-1
tests/kernels/test_cache.py
+3
-1
@@ -222,11 +222,12 @@ def test_reshape_and_cache_flash(
222
random.seed(seed)
223
torch.random.manual_seed(seed)
224
torch.cuda.manual_seed(seed)
225
+ torch.set_default_device(device)
226
227
# Create a random slot mapping.
228
num_slots = block_size * num_blocks
229
slot_mapping = random.sample(range(num_slots), num_tokens)
229
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device='cuda')
230
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=device)
231
232
qkv = torch.randn(num_tokens,
233
3,
@@ -245,6 +246,7 @@ def test_reshape_and_cache_flash(
246
head_size,
247
kv_cache_dtype,
248
dtype,
249
+ device=device,
250
)
251
key_cache, value_cache = key_caches[0], value_caches[0]
252