@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5265631d15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

use a correct device when creating OptionalCUDAGuard (#2583)

Vladimir committed Jan 26, 2024 at 08:48 UTC 5265631d15d59735152c8b72b38d960110987f10
1 file changed +1 -1
csrc/cache_kernels.cu
+1 -1
@@ -34,7 +34,7 @@ void swap_blocks(
34 char *dst_ptr = static_cast<char*>(dst.data_ptr());
35
36 const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
37 - const at::cuda::OptionalCUDAGuard device_guard(src_device);
37 + const at::cuda::OptionalCUDAGuard device_guard(src_device.is_cuda() ? src_device : dst_device);
38 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
39 // NOTE(woosuk): This can be slow if the number of blocks is large.
40 for (const auto& pair : block_mapping) {