A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[core] improve error handling when wake up from sleep mode (#12981)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 10, 2025 at 09:38 UTC
59fff4a01ae0f5c887cc547af6b49a9b028b4c70
2 files changed
+78
-12
csrc/cumem_allocator.cpp
+51
-12
@@ -12,15 +12,21 @@ extern "C" {
12
#include <cuda_runtime_api.h>
13
#include <cuda.h>
14
15
-#define CUDA_CHECK(condition) \
16
- do { \
17
- CUresult error = condition; \
18
- if (error != 0) { \
19
- char* error_string; \
20
- cuGetErrorString(error, (const char**)&error_string); \
21
- std::cerr << "CUDA Error: " << error_string << " at " << __FILE__ << ":" \
22
- << __LINE__ << std::endl; \
23
- } \
15
+char error_msg[10240]; // 10KB buffer to store error messages
16
+CUresult no_error = CUresult(0);
17
+CUresult error_code = no_error; // store error code
18
+
19
+#define CUDA_CHECK(condition) \
20
+ do { \
21
+ CUresult error = condition; \
22
+ if (error != 0) { \
23
+ error_code = error; \
24
+ char* error_string; \
25
+ cuGetErrorString(error, (const char**)&error_string); \
26
+ snprintf(error_msg, sizeof(error_msg), "CUDA Error: %s at %s:%d", \
27
+ error_string, __FILE__, __LINE__); \
28
+ std::cerr << error_msg << std::endl; \
29
+ } \
30
} while (0)
31
32
// Global references to Python callables
@@ -54,14 +60,22 @@ void create_and_map(unsigned long long device, ssize_t size, CUdeviceptr d_mem,
60
61
// Allocate memory using cuMemCreate
62
CUDA_CHECK(cuMemCreate(p_memHandle, size, &prop, 0));
63
+ if (error_code != 0) {
64
+ return;
65
+ }
66
CUDA_CHECK(cuMemMap(d_mem, size, 0, *p_memHandle, 0));
58
-
67
+ if (error_code != 0) {
68
+ return;
69
+ }
70
CUmemAccessDesc accessDesc = {};
71
accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
72
accessDesc.location.id = device;
73
accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
74
75
CUDA_CHECK(cuMemSetAccess(d_mem, size, &accessDesc, 1));
76
+ if (error_code != 0) {
77
+ return;
78
+ }
79
// std::cout << "create_and_map: device=" << device << ", size=" << size << ",
80
// d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
81
}
@@ -73,7 +87,13 @@ void unmap_and_release(unsigned long long device, ssize_t size,
87
// ", d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
88
ensure_context(device);
89
CUDA_CHECK(cuMemUnmap(d_mem, size));
90
+ if (error_code != 0) {
91
+ return;
92
+ }
93
CUDA_CHECK(cuMemRelease(*p_memHandle));
94
+ if (error_code != 0) {
95
+ return;
96
+ }
97
}
98
99
PyObject* create_tuple_from_c_integers(unsigned long long a,
@@ -121,12 +141,16 @@ void* my_malloc(ssize_t size, int device, CUstream stream) {
141
size_t granularity;
142
CUDA_CHECK(cuMemGetAllocationGranularity(&granularity, &prop,
143
CU_MEM_ALLOC_GRANULARITY_MINIMUM));
124
-
144
+ if (error_code != 0) {
145
+ return nullptr;
146
+ }
147
size_t alignedSize = ((size + granularity - 1) / granularity) * granularity;
148
149
CUdeviceptr d_mem;
150
CUDA_CHECK(cuMemAddressReserve(&d_mem, alignedSize, 0, 0, 0));
129
-
151
+ if (error_code != 0) {
152
+ return nullptr;
153
+ }
154
// allocate the CUmemGenericAllocationHandle
155
CUmemGenericAllocationHandle* p_memHandle =
156
(CUmemGenericAllocationHandle*)malloc(
@@ -208,6 +232,9 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
232
233
// free address and the handle
234
CUDA_CHECK(cuMemAddressFree(d_mem, size));
235
+ if (error_code != 0) {
236
+ return;
237
+ }
238
free(p_memHandle);
239
}
240
@@ -258,6 +285,12 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
285
286
unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle);
287
288
+ if (error_code != 0) {
289
+ error_code = no_error;
290
+ PyErr_SetString(PyExc_RuntimeError, error_msg);
291
+ return nullptr;
292
+ }
293
+
294
Py_RETURN_NONE;
295
}
296
@@ -282,6 +315,12 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
315
316
create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle);
317
318
+ if (error_code != 0) {
319
+ error_code = no_error;
320
+ PyErr_SetString(PyExc_RuntimeError, error_msg);
321
+ return nullptr;
322
+ }
323
+
324
Py_RETURN_NONE;
325
}
326
tests/basic_correctness/test_cumem.py
+27
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import pytest
4
import torch
5
6
from vllm import LLM, SamplingParams
@@ -9,6 +10,32 @@ from vllm.utils import GiB_bytes
10
from ..utils import fork_new_process_for_each_test
11
12
13
+@fork_new_process_for_each_test
14
+def test_python_error():
15
+ """
16
+ Test if Python error occurs when there's low-level
17
+ error happening from the C++ side.
18
+ """
19
+ allocator = CuMemAllocator.get_instance()
20
+ total_bytes = torch.cuda.mem_get_info()[1]
21
+ alloc_bytes = int(total_bytes * 0.7)
22
+ tensors = []
23
+ with allocator.use_memory_pool():
24
+ # allocate 70% of the total memory
25
+ x = torch.empty(alloc_bytes, dtype=torch.uint8, device='cuda')
26
+ tensors.append(x)
27
+ # release the memory
28
+ allocator.sleep()
29
+
30
+ # allocate more memory than the total memory
31
+ y = torch.empty(alloc_bytes, dtype=torch.uint8, device='cuda')
32
+ tensors.append(y)
33
+ with pytest.raises(RuntimeError):
34
+ # when the allocator is woken up, it should raise an error
35
+ # because we don't have enough memory
36
+ allocator.wake_up()
37
+
38
+
39
@fork_new_process_for_each_test
40
def test_basic_cumem():
41
# some tensors from default memory pool