@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59fff4a01a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[core] improve error handling when wake up from sleep mode (#12981)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 10, 2025 at 09:38 UTC 59fff4a01ae0f5c887cc547af6b49a9b028b4c70
2 files changed +78 -12
csrc/cumem_allocator.cpp
+51 -12
@@ -12,15 +12,21 @@ extern "C" {
12 #include <cuda_runtime_api.h>
13 #include <cuda.h>
14
15 -#define CUDA_CHECK(condition) \
16 - do { \
17 - CUresult error = condition; \
18 - if (error != 0) { \
19 - char* error_string; \
20 - cuGetErrorString(error, (const char**)&error_string); \
21 - std::cerr << "CUDA Error: " << error_string << " at " << __FILE__ << ":" \
22 - << __LINE__ << std::endl; \
23 - } \
15 +char error_msg[10240]; // 10KB buffer to store error messages
16 +CUresult no_error = CUresult(0);
17 +CUresult error_code = no_error; // store error code
18 +
19 +#define CUDA_CHECK(condition) \
20 + do { \
21 + CUresult error = condition; \
22 + if (error != 0) { \
23 + error_code = error; \
24 + char* error_string; \
25 + cuGetErrorString(error, (const char**)&error_string); \
26 + snprintf(error_msg, sizeof(error_msg), "CUDA Error: %s at %s:%d", \
27 + error_string, __FILE__, __LINE__); \
28 + std::cerr << error_msg << std::endl; \
29 + } \
30 } while (0)
31
32 // Global references to Python callables
@@ -54,14 +60,22 @@ void create_and_map(unsigned long long device, ssize_t size, CUdeviceptr d_mem,
60
61 // Allocate memory using cuMemCreate
62 CUDA_CHECK(cuMemCreate(p_memHandle, size, &prop, 0));
63 + if (error_code != 0) {
64 + return;
65 + }
66 CUDA_CHECK(cuMemMap(d_mem, size, 0, *p_memHandle, 0));
58 -
67 + if (error_code != 0) {
68 + return;
69 + }
70 CUmemAccessDesc accessDesc = {};
71 accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
72 accessDesc.location.id = device;
73 accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
74
75 CUDA_CHECK(cuMemSetAccess(d_mem, size, &accessDesc, 1));
76 + if (error_code != 0) {
77 + return;
78 + }
79 // std::cout << "create_and_map: device=" << device << ", size=" << size << ",
80 // d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
81 }
@@ -73,7 +87,13 @@ void unmap_and_release(unsigned long long device, ssize_t size,
87 // ", d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
88 ensure_context(device);
89 CUDA_CHECK(cuMemUnmap(d_mem, size));
90 + if (error_code != 0) {
91 + return;
92 + }
93 CUDA_CHECK(cuMemRelease(*p_memHandle));
94 + if (error_code != 0) {
95 + return;
96 + }
97 }
98
99 PyObject* create_tuple_from_c_integers(unsigned long long a,
@@ -121,12 +141,16 @@ void* my_malloc(ssize_t size, int device, CUstream stream) {
141 size_t granularity;
142 CUDA_CHECK(cuMemGetAllocationGranularity(&granularity, &prop,
143 CU_MEM_ALLOC_GRANULARITY_MINIMUM));
124 -
144 + if (error_code != 0) {
145 + return nullptr;
146 + }
147 size_t alignedSize = ((size + granularity - 1) / granularity) * granularity;
148
149 CUdeviceptr d_mem;
150 CUDA_CHECK(cuMemAddressReserve(&d_mem, alignedSize, 0, 0, 0));
129 -
151 + if (error_code != 0) {
152 + return nullptr;
153 + }
154 // allocate the CUmemGenericAllocationHandle
155 CUmemGenericAllocationHandle* p_memHandle =
156 (CUmemGenericAllocationHandle*)malloc(
@@ -208,6 +232,9 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
232
233 // free address and the handle
234 CUDA_CHECK(cuMemAddressFree(d_mem, size));
235 + if (error_code != 0) {
236 + return;
237 + }
238 free(p_memHandle);
239 }
240
@@ -258,6 +285,12 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
285
286 unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle);
287
288 + if (error_code != 0) {
289 + error_code = no_error;
290 + PyErr_SetString(PyExc_RuntimeError, error_msg);
291 + return nullptr;
292 + }
293 +
294 Py_RETURN_NONE;
295 }
296
@@ -282,6 +315,12 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
315
316 create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle);
317
318 + if (error_code != 0) {
319 + error_code = no_error;
320 + PyErr_SetString(PyExc_RuntimeError, error_msg);
321 + return nullptr;
322 + }
323 +
324 Py_RETURN_NONE;
325 }
326
tests/basic_correctness/test_cumem.py
+27
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +import pytest
4 import torch
5
6 from vllm import LLM, SamplingParams
@@ -9,6 +10,32 @@ from vllm.utils import GiB_bytes
10 from ..utils import fork_new_process_for_each_test
11
12
13 +@fork_new_process_for_each_test
14 +def test_python_error():
15 + """
16 + Test if Python error occurs when there's low-level
17 + error happening from the C++ side.
18 + """
19 + allocator = CuMemAllocator.get_instance()
20 + total_bytes = torch.cuda.mem_get_info()[1]
21 + alloc_bytes = int(total_bytes * 0.7)
22 + tensors = []
23 + with allocator.use_memory_pool():
24 + # allocate 70% of the total memory
25 + x = torch.empty(alloc_bytes, dtype=torch.uint8, device='cuda')
26 + tensors.append(x)
27 + # release the memory
28 + allocator.sleep()
29 +
30 + # allocate more memory than the total memory
31 + y = torch.empty(alloc_bytes, dtype=torch.uint8, device='cuda')
32 + tensors.append(y)
33 + with pytest.raises(RuntimeError):
34 + # when the allocator is woken up, it should raise an error
35 + # because we don't have enough memory
36 + allocator.wake_up()
37 +
38 +
39 @fork_new_process_for_each_test
40 def test_basic_cumem():
41 # some tensors from default memory pool