A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 04:30 UTC
[ROCm] fix sleep mode not releasing GPU memory problem on ROCm (#37533)
Signed-off-by: bingzhaodong <aaab8b@gmail.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Zhaodong Bing committed
Mar 23, 2026 at 21:07 UTC
10a1018c127ac34ad0f255ae9fffdc452d0cf4d7
1 file changed
+22
csrc/cumem_allocator.cpp
+22
@@ -232,6 +232,28 @@ void unmap_and_release(unsigned long long device, ssize_t size,
232
}
233
}
234
235
+ // ROCm workaround: hipMemRelease does not return physical VRAM to the
236
+ // free pool while the virtual-address reservation is still held.
237
+ // Cycling cuMemAddressFree → cuMemAddressReserve (at the same address)
238
+ // forces the driver to actually release the physical pages while keeping
239
+ // the same VA available for a later create_and_map.
240
+ if (first_error == no_error) {
241
+ first_error = cuMemAddressFree(d_mem, size);
242
+ if (first_error == no_error) {
243
+ CUdeviceptr d_mem_new = 0;
244
+ first_error = cuMemAddressReserve(&d_mem_new, size, 0, d_mem, 0);
245
+ if (first_error == no_error && d_mem_new != d_mem) {
246
+ cuMemAddressFree(d_mem_new, size);
247
+ snprintf(error_msg, sizeof(error_msg),
248
+ "ROCm: VA re-reserve got %p instead of %p", (void*)d_mem_new,
249
+ (void*)d_mem);
250
+ error_code = CUresult(1);
251
+ std::cerr << error_msg << std::endl;
252
+ return;
253
+ }
254
+ }
255
+ }
256
+
257
if (first_error != no_error) {
258
CUDA_CHECK(first_error);
259
}