A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Core] Support fully transparent sleep mode (#11743)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 22, 2025 at 14:39 UTC
68ad4e3a8d8a66fb2a43be57471ee13a8bec4ec0
14 files changed
+877
-40
.buildkite/test-pipeline.yaml
+2
@@ -76,7 +76,9 @@ steps:
76
- tests/basic_correctness/test_basic_correctness
77
- tests/basic_correctness/test_cpu_offload
78
- tests/basic_correctness/test_preemption
79
+ - tests/basic_correctness/test_cumem.py
80
commands:
81
+ - pytest -v -s basic_correctness/test_cumem.py
82
- pytest -v -s basic_correctness/test_basic_correctness.py
83
- pytest -v -s basic_correctness/test_cpu_offload.py
84
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
CMakeLists.txt
+25
@@ -181,6 +181,31 @@ message(STATUS "FetchContent base directory: ${FETCHCONTENT_BASE_DIR}")
181
# Define other extension targets
182
#
183
184
+#
185
+# cumem_allocator extension
186
+#
187
+
188
+set(VLLM_CUMEM_EXT_SRC
189
+ "csrc/cumem_allocator.cpp")
190
+
191
+set_gencode_flags_for_srcs(
192
+ SRCS "${VLLM_CUMEM_EXT_SRC}"
193
+ CUDA_ARCHS "${CUDA_ARCHS}")
194
+
195
+if(VLLM_GPU_LANG STREQUAL "CUDA")
196
+ message(STATUS "Enabling cumem allocator extension.")
197
+ # link against cuda driver library
198
+ list(APPEND CUMEM_LIBS cuda)
199
+ define_gpu_extension_target(
200
+ cumem_allocator
201
+ DESTINATION vllm
202
+ LANGUAGE CXX
203
+ SOURCES ${VLLM_CUMEM_EXT_SRC}
204
+ LIBRARIES ${CUMEM_LIBS}
205
+ USE_SABI 3.8
206
+ WITH_SOABI)
207
+endif()
208
+
209
#
210
# _C extension
211
#
csrc/cumem_allocator.cpp
new
+310
@@ -0,0 +1,310 @@
1
+// A CUDAPluggableAllocator based on cumem* APIs.
2
+// Important: allocation size, CUdeviceptr and CUmemGenericAllocationHandle*
3
+// need to be unsigned long long
4
+#include <iostream>
5
+
6
+extern "C" {
7
+
8
+#define PY_SSIZE_T_CLEAN
9
+#include <Python.h>
10
+
11
+#include <sys/types.h>
12
+#include <cuda_runtime_api.h>
13
+#include <cuda.h>
14
+
15
+#define CUDA_CHECK(condition) \
16
+ do { \
17
+ CUresult error = condition; \
18
+ if (error != 0) { \
19
+ char* error_string; \
20
+ cuGetErrorString(error, (const char**)&error_string); \
21
+ std::cerr << "CUDA Error: " << error_string << " at " << __FILE__ << ":" \
22
+ << __LINE__ << std::endl; \
23
+ } \
24
+ } while (0)
25
+
26
+// Global references to Python callables
27
+// NOTE: this is borrowed reference, so we don't need to DECREF them.
28
+// This brings the limitation that the allocator needs to be singleton.
29
+static PyObject* g_python_malloc_callback = nullptr;
30
+static PyObject* g_python_free_callback = nullptr;
31
+
32
+// ---------------------------------------------------------------------------
33
+// Helper functions:
34
+
35
+void ensure_context(unsigned long long device) {
36
+ CUcontext pctx;
37
+ CUDA_CHECK(cuCtxGetCurrent(&pctx));
38
+ if (!pctx) {
39
+ // Ensure device context.
40
+ CUDA_CHECK(cuDevicePrimaryCtxRetain(&pctx, device));
41
+ CUDA_CHECK(cuCtxSetCurrent(pctx));
42
+ }
43
+}
44
+
45
+void create_and_map(unsigned long long device, ssize_t size, CUdeviceptr d_mem,
46
+ CUmemGenericAllocationHandle* p_memHandle) {
47
+ ensure_context(device);
48
+ // Define memory allocation properties
49
+ CUmemAllocationProp prop = {};
50
+ prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
51
+ prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
52
+ prop.location.id = device;
53
+ prop.allocFlags.compressionType = CU_MEM_ALLOCATION_COMP_NONE;
54
+
55
+ // Allocate memory using cuMemCreate
56
+ CUDA_CHECK(cuMemCreate(p_memHandle, size, &prop, 0));
57
+ CUDA_CHECK(cuMemMap(d_mem, size, 0, *p_memHandle, 0));
58
+
59
+ CUmemAccessDesc accessDesc = {};
60
+ accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
61
+ accessDesc.location.id = device;
62
+ accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
63
+
64
+ CUDA_CHECK(cuMemSetAccess(d_mem, size, &accessDesc, 1));
65
+ // std::cout << "create_and_map: device=" << device << ", size=" << size << ",
66
+ // d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
67
+}
68
+
69
+void unmap_and_release(unsigned long long device, ssize_t size,
70
+ CUdeviceptr d_mem,
71
+ CUmemGenericAllocationHandle* p_memHandle) {
72
+ // std::cout << "unmap_and_release: device=" << device << ", size=" << size <<
73
+ // ", d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
74
+ ensure_context(device);
75
+ CUDA_CHECK(cuMemUnmap(d_mem, size));
76
+ CUDA_CHECK(cuMemRelease(*p_memHandle));
77
+}
78
+
79
+PyObject* create_tuple_from_c_integers(unsigned long long a,
80
+ unsigned long long b,
81
+ unsigned long long c,
82
+ unsigned long long d) {
83
+ // Create a new tuple of size 4
84
+ PyObject* tuple = PyTuple_New(4);
85
+ if (!tuple) {
86
+ return NULL; // Return NULL on failure
87
+ }
88
+
89
+ // Convert integers to Python objects and set them in the tuple
90
+ PyTuple_SetItem(
91
+ tuple, 0,
92
+ PyLong_FromUnsignedLongLong(a)); // Steals reference to the PyLong
93
+ PyTuple_SetItem(tuple, 1, PyLong_FromUnsignedLongLong(b));
94
+ PyTuple_SetItem(tuple, 2, PyLong_FromUnsignedLongLong(c));
95
+ PyTuple_SetItem(tuple, 3, PyLong_FromUnsignedLongLong(d));
96
+
97
+ // Note: PyTuple_SetItem "steals" a reference to each object,
98
+ // so we do not need to Py_DECREF the PyLong objects explicitly.
99
+
100
+ return tuple; // Return the created tuple
101
+}
102
+
103
+// ---------------------------------------------------------------------------
104
+// Our exported C functions that call Python:
105
+
106
+// use CUstream instead of cudaStream_t, to avoid including cuda_runtime_api.h
107
+void* my_malloc(ssize_t size, int device, CUstream stream) {
108
+ ensure_context(device);
109
+
110
+ // first allocation, align the size, and reserve an address, and also allocate
111
+ // a CUmemGenericAllocationHandle
112
+
113
+ // Define memory allocation properties
114
+ CUmemAllocationProp prop = {};
115
+ prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
116
+ prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
117
+ prop.location.id = device;
118
+ prop.allocFlags.compressionType = CU_MEM_ALLOCATION_COMP_NONE;
119
+
120
+ // Check if the allocation is supported
121
+ size_t granularity;
122
+ CUDA_CHECK(cuMemGetAllocationGranularity(&granularity, &prop,
123
+ CU_MEM_ALLOC_GRANULARITY_MINIMUM));
124
+
125
+ size_t alignedSize = ((size + granularity - 1) / granularity) * granularity;
126
+
127
+ CUdeviceptr d_mem;
128
+ CUDA_CHECK(cuMemAddressReserve(&d_mem, alignedSize, 0, 0, 0));
129
+
130
+ // allocate the CUmemGenericAllocationHandle
131
+ CUmemGenericAllocationHandle* p_memHandle =
132
+ (CUmemGenericAllocationHandle*)malloc(
133
+ sizeof(CUmemGenericAllocationHandle));
134
+
135
+ if (!g_python_malloc_callback) {
136
+ std::cerr << "ERROR: g_python_malloc_callback not set.\n";
137
+ return nullptr;
138
+ }
139
+
140
+ // Acquire GIL (not in stable ABI officially, but often works)
141
+ PyGILState_STATE gstate = PyGILState_Ensure();
142
+
143
+ PyObject* arg_tuple = create_tuple_from_c_integers(
144
+ (unsigned long long)device, (unsigned long long)alignedSize,
145
+ (unsigned long long)d_mem, (unsigned long long)p_memHandle);
146
+
147
+ // Call g_python_malloc_callback
148
+ PyObject* py_result =
149
+ PyObject_CallFunctionObjArgs(g_python_malloc_callback, arg_tuple, NULL);
150
+ Py_DECREF(arg_tuple);
151
+
152
+ if (!py_result) {
153
+ PyErr_Print();
154
+ PyGILState_Release(gstate);
155
+ return nullptr;
156
+ }
157
+
158
+ PyGILState_Release(gstate);
159
+
160
+ // do the final mapping
161
+ create_and_map(device, alignedSize, d_mem, p_memHandle);
162
+
163
+ return (void*)d_mem;
164
+}
165
+
166
+// use CUstream instead of cudaStream_t, to avoid including cuda_runtime_api.h
167
+void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
168
+ // get memory handle from the pointer
169
+ if (!g_python_free_callback) {
170
+ std::cerr << "ERROR: g_python_free_callback not set.\n";
171
+ return;
172
+ }
173
+
174
+ // Acquire GIL (not in stable ABI officially, but often works)
175
+ PyGILState_STATE gstate = PyGILState_Ensure();
176
+
177
+ PyObject* py_ptr =
178
+ PyLong_FromUnsignedLongLong(reinterpret_cast<unsigned long long>(ptr));
179
+
180
+ PyObject* py_result =
181
+ PyObject_CallFunctionObjArgs(g_python_free_callback, py_ptr, NULL);
182
+
183
+ if (!py_result || !PyTuple_Check(py_result) || PyTuple_Size(py_result) != 4) {
184
+ PyErr_SetString(PyExc_TypeError, "Expected a tuple of size 4");
185
+ return;
186
+ }
187
+
188
+ unsigned long long recv_device, recv_size;
189
+ unsigned long long recv_d_mem, recv_p_memHandle;
190
+ // Unpack the tuple into four C integers
191
+ if (!PyArg_ParseTuple(py_result, "KKKK", &recv_device, &recv_size,
192
+ &recv_d_mem, &recv_p_memHandle)) {
193
+ // PyArg_ParseTuple sets an error if it fails
194
+ return;
195
+ }
196
+
197
+ PyGILState_Release(gstate);
198
+
199
+ // recv_size == size
200
+ // recv_device == device
201
+
202
+ // Free memory
203
+
204
+ CUdeviceptr d_mem = (CUdeviceptr)recv_d_mem;
205
+ CUmemGenericAllocationHandle* p_memHandle =
206
+ (CUmemGenericAllocationHandle*)recv_p_memHandle;
207
+ unmap_and_release(device, size, d_mem, p_memHandle);
208
+
209
+ // free address and the handle
210
+ CUDA_CHECK(cuMemAddressFree(d_mem, size));
211
+ free(p_memHandle);
212
+}
213
+
214
+// ---------------------------------------------------------------------------
215
+// Python extension boilerplate:
216
+
217
+// Python-exposed function: init_module(python_malloc, python_free)
218
+static PyObject* py_init_module(PyObject* self, PyObject* args) {
219
+ PyObject* malloc_callback = nullptr;
220
+ PyObject* free_callback = nullptr;
221
+
222
+ if (!PyArg_ParseTuple(args, "OO", &malloc_callback, &free_callback)) {
223
+ return nullptr;
224
+ }
225
+
226
+ if (!PyCallable_Check(malloc_callback) || !PyCallable_Check(free_callback)) {
227
+ PyErr_SetString(PyExc_TypeError, "Both arguments must be callables");
228
+ return nullptr;
229
+ }
230
+
231
+ // Save the Python callables
232
+ // This module does not handle GC of these objects, so they must be kept alive
233
+ // outside of this module.
234
+ g_python_malloc_callback = malloc_callback;
235
+ g_python_free_callback = free_callback;
236
+
237
+ Py_RETURN_NONE;
238
+}
239
+
240
+static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
241
+ if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != 4) {
242
+ PyErr_SetString(PyExc_TypeError, "Expected a tuple of size 4");
243
+ return nullptr;
244
+ }
245
+
246
+ unsigned long long recv_device, recv_size;
247
+ unsigned long long recv_d_mem, recv_p_memHandle;
248
+ // Unpack the tuple into four C integers
249
+ if (!PyArg_ParseTuple(args, "KKKK", &recv_device, &recv_size, &recv_d_mem,
250
+ &recv_p_memHandle)) {
251
+ // PyArg_ParseTuple sets an error if it fails
252
+ return nullptr;
253
+ }
254
+
255
+ CUdeviceptr d_mem_ptr = (CUdeviceptr)recv_d_mem;
256
+ CUmemGenericAllocationHandle* p_memHandle =
257
+ (CUmemGenericAllocationHandle*)recv_p_memHandle;
258
+
259
+ unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle);
260
+
261
+ Py_RETURN_NONE;
262
+}
263
+
264
+static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
265
+ if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != 4) {
266
+ PyErr_SetString(PyExc_TypeError, "Expected a tuple of size 4");
267
+ return nullptr;
268
+ }
269
+
270
+ unsigned long long recv_device, recv_size;
271
+ unsigned long long recv_d_mem, recv_p_memHandle;
272
+ // Unpack the tuple into four C integers
273
+ if (!PyArg_ParseTuple(args, "KKKK", &recv_device, &recv_size, &recv_d_mem,
274
+ &recv_p_memHandle)) {
275
+ // PyArg_ParseTuple sets an error if it fails
276
+ return nullptr;
277
+ }
278
+
279
+ CUdeviceptr d_mem_ptr = (CUdeviceptr)recv_d_mem;
280
+ CUmemGenericAllocationHandle* p_memHandle =
281
+ (CUmemGenericAllocationHandle*)recv_p_memHandle;
282
+
283
+ create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle);
284
+
285
+ Py_RETURN_NONE;
286
+}
287
+
288
+static PyMethodDef module_methods[] = {
289
+ {"init_module", (PyCFunction)py_init_module, METH_VARARGS,
290
+ "Initialize module with python_malloc and python_free callables."},
291
+ {"python_create_and_map", (PyCFunction)python_create_and_map, METH_VARARGS,
292
+ "Create and map memory on the device."},
293
+ {"python_unmap_and_release", (PyCFunction)python_unmap_and_release,
294
+ METH_VARARGS, "Unmap and release memory on the device."},
295
+ {NULL, NULL, 0, NULL} // sentinel
296
+};
297
+
298
+static struct PyModuleDef cumem_allocator_module = {
299
+ PyModuleDef_HEAD_INIT, "cumem_allocator",
300
+ "cumem-based allocator for CUDAPluggableAllocator", -1, module_methods};
301
+
302
+PyMODINIT_FUNC PyInit_cumem_allocator(void) {
303
+ // Initialize the module
304
+ PyObject* module = PyModule_Create(&cumem_allocator_module);
305
+ if (!module) {
306
+ return NULL;
307
+ }
308
+ return module;
309
+}
310
+} // extern "C"
setup.py
+2
@@ -301,6 +301,7 @@ class repackage_wheel(build_ext):
301
"vllm/vllm_flash_attn/vllm_flash_attn_c.abi3.so",
302
"vllm/vllm_flash_attn/flash_attn_interface.py",
303
"vllm/vllm_flash_attn/__init__.py",
304
+ "vllm/cumem_allocator.abi3.so",
305
# "vllm/_version.py", # not available in nightly wheels yet
306
]
307
file_members = filter(lambda x: x.filename in files_to_copy,
@@ -594,6 +595,7 @@ if _is_hip():
595
if _is_cuda():
596
ext_modules.append(
597
CMakeExtension(name="vllm.vllm_flash_attn.vllm_flash_attn_c"))
598
+ ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
599
600
if _build_custom_ops():
601
ext_modules.append(CMakeExtension(name="vllm._C"))
tests/basic_correctness/test_cumem.py
new
+112
@@ -0,0 +1,112 @@
1
+import torch
2
+
3
+from vllm import LLM, SamplingParams
4
+from vllm.device_allocator.cumem import CuMemAllocator
5
+from vllm.utils import GiB_bytes
6
+
7
+from ..utils import fork_new_process_for_each_test
8
+
9
+
10
+@fork_new_process_for_each_test
11
+def test_basic_cumem():
12
+ # some tensors from default memory pool
13
+ shape = (1024, 1024)
14
+ x = torch.empty(shape, device='cuda')
15
+ x.zero_()
16
+
17
+ # some tensors from custom memory pool
18
+ allocator = CuMemAllocator.get_instance()
19
+ with allocator.use_memory_pool():
20
+ # custom memory pool
21
+ y = torch.empty(shape, device='cuda')
22
+ y.zero_()
23
+ y += 1
24
+ z = torch.empty(shape, device='cuda')
25
+ z.zero_()
26
+ z += 2
27
+
28
+ # they can be used together
29
+ output = x + y + z
30
+ assert torch.allclose(output, torch.ones_like(output) * 3)
31
+
32
+ free_bytes = torch.cuda.mem_get_info()[0]
33
+ allocator.sleep()
34
+ free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
35
+ assert free_bytes_after_sleep > free_bytes
36
+ allocator.wake_up()
37
+
38
+ # they can be used together
39
+ output = x + y + z
40
+ assert torch.allclose(output, torch.ones_like(output) * 3)
41
+
42
+
43
+@fork_new_process_for_each_test
44
+def test_cumem_with_cudagraph():
45
+ allocator = CuMemAllocator.get_instance()
46
+ with allocator.use_memory_pool():
47
+ weight = torch.eye(1024, device='cuda')
48
+ with allocator.use_memory_pool(tag="discard"):
49
+ cache = torch.empty(1024, 1024, device='cuda')
50
+
51
+ def model(x):
52
+ out = x @ weight
53
+ cache[:out.size(0)].copy_(out)
54
+ return out + 1
55
+
56
+ x = torch.empty(128, 1024, device='cuda')
57
+
58
+ # warmup
59
+ model(x)
60
+
61
+ # capture cudagraph
62
+ model_graph = torch.cuda.CUDAGraph()
63
+ with torch.cuda.graph(model_graph):
64
+ y = model(x)
65
+
66
+ free_bytes = torch.cuda.mem_get_info()[0]
67
+ allocator.sleep()
68
+ free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
69
+ assert free_bytes_after_sleep > free_bytes
70
+ allocator.wake_up()
71
+
72
+ # after waking up, the content in the weight tensor
73
+ # should be restored, but the content in the cache tensor
74
+ # should be discarded
75
+
76
+ # this operation is also compatible with cudagraph
77
+
78
+ x.random_()
79
+ model_graph.replay()
80
+
81
+ # cache content is as expected
82
+ assert torch.allclose(x, cache[:x.size(0)])
83
+
84
+ # output content is as expected
85
+ assert torch.allclose(y, x + 1)
86
+
87
+
88
+@fork_new_process_for_each_test
89
+def test_end_to_end():
90
+ free, total = torch.cuda.mem_get_info()
91
+ used_bytes_baseline = total - free # in case other process is running
92
+ llm = LLM("meta-llama/Llama-3.2-1B", enable_sleep_mode=True)
93
+ prompt = "How are you?"
94
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
95
+ output = llm.generate(prompt, sampling_params)
96
+
97
+ # the benefit of `llm.sleep(level=2)` is mainly CPU memory usage,
98
+ # which is difficult to measure in the test. therefore, we only
99
+ # test sleep level 1 here.
100
+ llm.sleep(level=1)
101
+
102
+ free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
103
+ used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
104
+ # now the memory usage is mostly cudagraph memory pool,
105
+ # and it should be less than the model weights (1B model, 2GiB weights)
106
+ assert used_bytes < 2 * GiB_bytes
107
+
108
+ llm.wake_up()
109
+ output2 = llm.generate(prompt, sampling_params)
110
+
111
+ # cmp output
112
+ assert output[0].outputs[0].text == output2[0].outputs[0].text
vllm/config.py
+43
-35
@@ -195,40 +195,43 @@ class ModelConfig:
195
factors.append(self.rope_theta)
196
return hashlib.sha256(str(factors).encode()).hexdigest()
197
198
- def __init__(self,
199
- model: str,
200
- task: Union[TaskOption, Literal["draft"]],
201
- tokenizer: str,
202
- tokenizer_mode: str,
203
- trust_remote_code: bool,
204
- dtype: Union[str, torch.dtype],
205
- seed: int,
206
- allowed_local_media_path: str = "",
207
- revision: Optional[str] = None,
208
- code_revision: Optional[str] = None,
209
- rope_scaling: Optional[Dict[str, Any]] = None,
210
- rope_theta: Optional[float] = None,
211
- tokenizer_revision: Optional[str] = None,
212
- max_model_len: Optional[int] = None,
213
- spec_target_max_model_len: Optional[int] = None,
214
- quantization: Optional[str] = None,
215
- quantization_param_path: Optional[str] = None,
216
- enforce_eager: Optional[bool] = None,
217
- max_seq_len_to_capture: Optional[int] = None,
218
- max_logprobs: int = 20,
219
- disable_sliding_window: bool = False,
220
- skip_tokenizer_init: bool = False,
221
- served_model_name: Optional[Union[str, List[str]]] = None,
222
- limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
223
- use_async_output_proc: bool = True,
224
- config_format: ConfigFormat = ConfigFormat.AUTO,
225
- hf_overrides: Optional[HfOverrides] = None,
226
- mm_processor_kwargs: Optional[Dict[str, Any]] = None,
227
- disable_mm_preprocessor_cache: bool = False,
228
- override_neuron_config: Optional[Dict[str, Any]] = None,
229
- override_pooler_config: Optional["PoolerConfig"] = None,
230
- logits_processor_pattern: Optional[str] = None,
231
- generation_config: Optional[str] = None) -> None:
198
+ def __init__(
199
+ self,
200
+ model: str,
201
+ task: Union[TaskOption, Literal["draft"]],
202
+ tokenizer: str,
203
+ tokenizer_mode: str,
204
+ trust_remote_code: bool,
205
+ dtype: Union[str, torch.dtype],
206
+ seed: int,
207
+ allowed_local_media_path: str = "",
208
+ revision: Optional[str] = None,
209
+ code_revision: Optional[str] = None,
210
+ rope_scaling: Optional[Dict[str, Any]] = None,
211
+ rope_theta: Optional[float] = None,
212
+ tokenizer_revision: Optional[str] = None,
213
+ max_model_len: Optional[int] = None,
214
+ spec_target_max_model_len: Optional[int] = None,
215
+ quantization: Optional[str] = None,
216
+ quantization_param_path: Optional[str] = None,
217
+ enforce_eager: Optional[bool] = None,
218
+ max_seq_len_to_capture: Optional[int] = None,
219
+ max_logprobs: int = 20,
220
+ disable_sliding_window: bool = False,
221
+ skip_tokenizer_init: bool = False,
222
+ served_model_name: Optional[Union[str, List[str]]] = None,
223
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
224
+ use_async_output_proc: bool = True,
225
+ config_format: ConfigFormat = ConfigFormat.AUTO,
226
+ hf_overrides: Optional[HfOverrides] = None,
227
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
228
+ disable_mm_preprocessor_cache: bool = False,
229
+ override_neuron_config: Optional[Dict[str, Any]] = None,
230
+ override_pooler_config: Optional["PoolerConfig"] = None,
231
+ logits_processor_pattern: Optional[str] = None,
232
+ generation_config: Optional[str] = None,
233
+ enable_sleep_mode: bool = False,
234
+ ) -> None:
235
self.model = model
236
self.tokenizer = tokenizer
237
self.tokenizer_mode = tokenizer_mode
@@ -277,6 +280,12 @@ class ModelConfig:
280
self.max_logprobs = max_logprobs
281
self.disable_sliding_window = disable_sliding_window
282
self.skip_tokenizer_init = skip_tokenizer_init
283
+ self.enable_sleep_mode = enable_sleep_mode
284
+
285
+ from vllm.platforms import current_platform
286
+
287
+ if self.enable_sleep_mode and not current_platform.is_cuda():
288
+ raise ValueError("Sleep mode is only supported on CUDA devices.")
289
290
hf_config = get_config(self.model, trust_remote_code, revision,
291
code_revision, config_format)
@@ -348,7 +357,6 @@ class ModelConfig:
357
self.is_hybrid = self._init_is_hybrid()
358
self.has_inner_state = self._init_has_inner_state()
359
351
- from vllm.platforms import current_platform
360
if current_platform.is_neuron():
361
self.override_neuron_config = override_neuron_config
362
else:
vllm/device_allocator/__init__.py
vllm/device_allocator/cumem.py
new
+254
@@ -0,0 +1,254 @@
1
+# cumem-based pytorch pluggable allocator to implement sleep mode.
2
+# other approaches tried but failed:
3
+# - cuda-python package binding
4
+# - custom libcuda driver ctypes wrapper
5
+# both of them failed because of cuda context mismatch.
6
+# not sure why, they are created from a different context.
7
+# the only successful approach is to call cuda driver API in C.
8
+import dataclasses
9
+from contextlib import contextmanager
10
+from typing import Callable, Dict, Optional, Tuple, Union
11
+
12
+import torch
13
+
14
+from vllm.utils import is_pin_memory_available
15
+
16
+
17
+def find_loaded_library(lib_name) -> Optional[str]:
18
+ """
19
+ According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
20
+ the file `/proc/self/maps` contains the memory maps of the process, which includes the
21
+ shared libraries loaded by the process. We can use this file to find the path of the
22
+ a loaded library.
23
+ """ # noqa
24
+ found_line = None
25
+ with open("/proc/self/maps") as f:
26
+ for line in f:
27
+ if lib_name in line:
28
+ found_line = line
29
+ break
30
+ if found_line is None:
31
+ # the library is not loaded in the current process
32
+ return None
33
+ # if lib_name is libcudart, we need to match a line with:
34
+ # address /path/to/libcudart-hash.so.11.0
35
+ start = found_line.index("/")
36
+ path = found_line[start:].strip()
37
+ filename = path.split("/")[-1]
38
+ assert filename.rpartition(".so")[0].startswith(lib_name), \
39
+ f"Unexpected filename: {filename} for library {lib_name}"
40
+ return path
41
+
42
+
43
+cumem_available = False
44
+try:
45
+ from vllm.cumem_allocator import (init_module, python_create_and_map,
46
+ python_unmap_and_release)
47
+ from vllm.distributed.device_communicators.cuda_wrapper import (
48
+ CudaRTLibrary)
49
+ lib_name = find_loaded_library("cumem_allocator")
50
+ libcudart = CudaRTLibrary()
51
+ cumem_available = True
52
+except ModuleNotFoundError:
53
+ # rocm platform does not support cumem allocator
54
+ init_module = None
55
+ python_create_and_map = None
56
+ python_unmap_and_release = None
57
+ CudaRTLibrary = None
58
+ lib_name = None
59
+ libcudart = None
60
+
61
+# py_device, py_alignedSize, py_d_mem, py_p_memHandle
62
+HandleType = Tuple[int, int, int, int]
63
+
64
+
65
+@dataclasses.dataclass
66
+class AllocationData:
67
+ handle: HandleType
68
+ tag: str
69
+ cpu_backup_tensor: Optional[torch.Tensor] = None
70
+
71
+
72
+def create_and_map(allocation_handle: HandleType) -> None:
73
+ python_create_and_map(*allocation_handle)
74
+
75
+
76
+def unmap_and_release(allocation_handle: HandleType) -> None:
77
+ python_unmap_and_release(*allocation_handle)
78
+
79
+
80
+def get_pluggable_allocator(
81
+ python_malloc_fn: Callable[[int],
82
+ int], python_free_func: Callable[[int, int],
83
+ None]
84
+) -> torch.cuda.memory.CUDAPluggableAllocator:
85
+ init_module(python_malloc_fn, python_free_func)
86
+ new_alloc = torch.cuda.memory.CUDAPluggableAllocator(
87
+ lib_name, 'my_malloc', 'my_free')
88
+ return new_alloc
89
+
90
+
91
+@contextmanager
92
+def use_memory_pool_with_allocator(
93
+ python_malloc_fn: Callable[[int], int],
94
+ python_free_func: Callable[[int, int], None]) -> None:
95
+ new_alloc = get_pluggable_allocator(python_malloc_fn, python_free_func)
96
+ mem_pool = torch.cuda.memory.MemPool(new_alloc._allocator)
97
+ with torch.cuda.memory.use_mem_pool(mem_pool):
98
+ yield mem_pool
99
+
100
+
101
+class CuMemAllocator:
102
+ """
103
+ A singleton class that manages a memory pool for CUDA tensors.
104
+ The memory in this pool can be offloaded or discarded when the
105
+ allocator sleeps.
106
+
107
+ Inside the `use_memory_pool(tag)` context, all tensors created will
108
+ be allocated in the memory pool, and has the same tag as the
109
+ tag passed to the context.
110
+
111
+ When we call `sleep`, all tensors with the specified tag will be
112
+ offloaded to CPU memory, and the rest of the tensors will be discarded.
113
+ When we call `wake_up`, all tensors that are previously offloaded
114
+ will be loaded back to GPU memory, and the rest of the tensors will
115
+ have empty memory.
116
+
117
+ Why it needs to be a singleton?
118
+ When allocated tensors are garbage collected, PyTorch will call
119
+ the free callback, which will call the `python_free_callback` method.
120
+ The C-extension uses a global variable to store the function of an
121
+ instance of this class. If we create multiple instances of this class,
122
+ the global variable will be overwritten and the free callback will
123
+ not work as expected.
124
+ """
125
+ instance: "CuMemAllocator" = None
126
+ default_tag: str = "default"
127
+
128
+ @staticmethod
129
+ def get_instance() -> "CuMemAllocator":
130
+ """
131
+ CuMemAllocator is a singleton class.
132
+ We cannot call the constructor directly.
133
+ Call this method to get the instance.
134
+ """
135
+ assert cumem_available, "cumem allocator is not available"
136
+ if CuMemAllocator.instance is None:
137
+ CuMemAllocator.instance = CuMemAllocator()
138
+ return CuMemAllocator.instance
139
+
140
+ def __init__(self):
141
+ self.pointer_to_data: Dict[int, AllocationData] = {}
142
+ self.current_tag: str = CuMemAllocator.default_tag
143
+
144
+ def python_malloc_callback(self, allocation_handle: HandleType) -> None:
145
+ """
146
+ Internal method to store the allocation data
147
+ when memory is allocated in the memory pool."""
148
+ py_d_mem = allocation_handle[2]
149
+ self.pointer_to_data[py_d_mem] = AllocationData(
150
+ allocation_handle, self.current_tag)
151
+ return
152
+
153
+ def python_free_callback(self, ptr: int) -> HandleType:
154
+ """
155
+ Internal method to look up the allocation data
156
+ when memory is freed in the memory pool."""
157
+ data = self.pointer_to_data.pop(ptr)
158
+ if data.cpu_backup_tensor is not None:
159
+ data.cpu_backup_tensor = None
160
+ return data.handle
161
+
162
+ def sleep(
163
+ self,
164
+ offload_tags: Optional[Union[Tuple[str, ...],
165
+ str]] = None) -> None:
166
+ """
167
+ Put the allocator in sleep mode.
168
+ All data in the memory allocation with the specified tag will be
169
+ offloaded to CPU memory, and others will be discarded.
170
+
171
+ :param offload_tags: The tags of the memory allocation that will be
172
+ offloaded. The rest of the memory allocation will be discarded.
173
+ """
174
+ if offload_tags is None:
175
+ # by default, allocated tensors are offloaded
176
+ # when the allocator sleeps
177
+ offload_tags = (CuMemAllocator.default_tag, )
178
+ elif isinstance(offload_tags, str):
179
+ offload_tags = (offload_tags, )
180
+
181
+ assert isinstance(offload_tags, tuple)
182
+
183
+ for ptr, data in self.pointer_to_data.items():
184
+ handle = data.handle
185
+ if data.tag in offload_tags:
186
+ size_in_bytes = handle[1]
187
+ cpu_backup_tensor = torch.empty(
188
+ size_in_bytes,
189
+ dtype=torch.uint8,
190
+ device='cpu',
191
+ pin_memory=is_pin_memory_available())
192
+ cpu_ptr = cpu_backup_tensor.data_ptr()
193
+ libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
194
+ data.cpu_backup_tensor = cpu_backup_tensor
195
+ unmap_and_release(handle)
196
+
197
+ def wake_up(self):
198
+ """
199
+ Wake up the allocator from sleep mode.
200
+ All data that is previously offloaded will be loaded back to GPU
201
+ memory, and the rest of the data will have empty memory."""
202
+ for ptr, data in self.pointer_to_data.items():
203
+ handle = data.handle
204
+ create_and_map(handle)
205
+ if data.cpu_backup_tensor is not None:
206
+ cpu_backup_tensor = data.cpu_backup_tensor
207
+ if cpu_backup_tensor is not None:
208
+ size_in_bytes = cpu_backup_tensor.numel(
209
+ ) * cpu_backup_tensor.element_size()
210
+ cpu_ptr = cpu_backup_tensor.data_ptr()
211
+ libcudart.cudaMemcpy(ptr, cpu_ptr, size_in_bytes)
212
+ data.cpu_backup_tensor = None
213
+
214
+ @contextmanager
215
+ def use_memory_pool(self, tag: Optional[str] = None):
216
+ """
217
+ A context manager to use the memory pool.
218
+ All memory allocation created inside the context will be allocated
219
+ in the memory pool, and has the specified tag.
220
+
221
+ :param tag: The tag of the memory allocation. If None, the default tag
222
+ will be used.
223
+ """
224
+ if tag is None:
225
+ tag = CuMemAllocator.default_tag
226
+
227
+ assert isinstance(tag, str)
228
+
229
+ old_tag = self.current_tag
230
+ self.current_tag = tag
231
+ with use_memory_pool_with_allocator(self.python_malloc_callback,
232
+ self.python_free_callback):
233
+ yield
234
+ # PyTorch's bug, calling torch.cuda.empty_cache() will error
235
+ # when using pluggable allocator, see
236
+ # https://github.com/pytorch/pytorch/issues/145168 .
237
+ # if we have some memory allocated and then freed,
238
+ # the memory will not be released.
239
+ # right now it is fine, because we only use this allocator
240
+ # during weight loading and kv cache creation, where we only
241
+ # allocate memory.
242
+ # TODO: we need to find a way to release the memory,
243
+ # i.e. calling torch.cuda.empty_cache()
244
+ self.current_tag = old_tag
245
+
246
+ def get_current_usage(self) -> int:
247
+ """
248
+ Get the total number of bytes allocated in the memory pool.
249
+ """
250
+ sum_bytes: int = 0
251
+ for ptr, data in self.pointer_to_data.items():
252
+ handle = data.handle
253
+ sum_bytes += handle[1]
254
+ return sum_bytes
vllm/engine/arg_utils.py
+10
-1
@@ -197,6 +197,7 @@ class EngineArgs:
197
kv_transfer_config: Optional[KVTransferConfig] = None
198
199
generation_config: Optional[str] = None
200
+ enable_sleep_mode: bool = False
201
202
def __post_init__(self):
203
if not self.tokenizer:
@@ -955,6 +956,12 @@ class EngineArgs:
956
"loaded from model. If set to a folder path, the generation config "
957
"will be loaded from the specified folder path.")
958
959
+ parser.add_argument("--enable-sleep-mode",
960
+ action="store_true",
961
+ default=False,
962
+ help="Enable sleep mode for the engine. "
963
+ "(only cuda platform is supported)")
964
+
965
return parser
966
967
@classmethod
@@ -999,7 +1006,9 @@ class EngineArgs:
1006
override_neuron_config=self.override_neuron_config,
1007
override_pooler_config=self.override_pooler_config,
1008
logits_processor_pattern=self.logits_processor_pattern,
1002
- generation_config=self.generation_config)
1009
+ generation_config=self.generation_config,
1010
+ enable_sleep_mode=self.enable_sleep_mode,
1011
+ )
1012
1013
def create_load_config(self) -> LoadConfig:
1014
return LoadConfig(
vllm/engine/llm_engine.py
+10
@@ -1818,6 +1818,16 @@ class LLMEngine:
1818
def stop_profile(self) -> None:
1819
self.model_executor.stop_profile()
1820
1821
+ def sleep(self, level: int = 1) -> None:
1822
+ assert self.vllm_config.model_config.enable_sleep_mode, (
1823
+ "Sleep mode is not enabled in the model config")
1824
+ self.model_executor.sleep(level=level)
1825
+
1826
+ def wake_up(self) -> None:
1827
+ assert self.vllm_config.model_config.enable_sleep_mode, (
1828
+ "Sleep mode is not enabled in the model config")
1829
+ self.model_executor.wake_up()
1830
+
1831
def check_health(self) -> None:
1832
if self.tokenizer:
1833
self.tokenizer.check_health()
vllm/entrypoints/llm.py
+23
@@ -1132,6 +1132,29 @@ class LLM:
1132
def stop_profile(self) -> None:
1133
self.llm_engine.stop_profile()
1134
1135
+ def sleep(self, level: int = 1):
1136
+ """
1137
+ Put the engine to sleep. The engine should not process any requests.
1138
+ The caller should guarantee that no requests are being processed
1139
+ during the sleep period, before `wake_up` is called.
1140
+
1141
+ :param level: The sleep level. Level 1 sleep will offload the model
1142
+ weights and discard the kv cache. The content of kv cache is
1143
+ forgotten. Level 1 sleep is good for sleeping and waking up the
1144
+ engine to run the same model again. The model weights are backed
1145
+ up in CPU memory. Please make sure there's enough CPU memory to
1146
+ store the model weights. Level 2 sleep will discard both the model
1147
+ weights and the kv cache. The content of both the model weights
1148
+ and kv cache is forgotten. Level 2 sleep is good for sleeping and
1149
+ waking up the engine to run a different model or update the model,
1150
+ where previous model weights are not needed. It reduces CPU memory
1151
+ pressure.
1152
+ """
1153
+ self.llm_engine.sleep(level=level)
1154
+
1155
+ def wake_up(self):
1156
+ self.llm_engine.wake_up()
1157
+
1158
# LEGACY
1159
def _convert_v1_inputs(
1160
self,
vllm/executor/executor_base.py
+11
@@ -193,6 +193,17 @@ class ExecutorBase(ABC):
193
def stop_profile(self) -> None:
194
self.collective_rpc("stop_profile")
195
196
+ def sleep(self, level: int = 1):
197
+ if self.cache_config.enable_prefix_caching:
198
+ # TODO: support sleep with prefix caching
199
+ # by resetting the prefix cache state,
200
+ # after https://github.com/vllm-project/vllm/pull/12284
201
+ raise ValueError("Cannot sleep when prefix caching is enabled.")
202
+ self.collective_rpc("sleep", kwargs=dict(level=level))
203
+
204
+ def wake_up(self):
205
+ self.collective_rpc("wake_up")
206
+
207
def save_sharded_state(
208
self,
209
path: str,
vllm/v1/worker/gpu_worker.py
+38
-2
@@ -9,12 +9,14 @@ import torch.nn as nn
9
10
import vllm.envs as envs
11
from vllm.config import ParallelConfig, VllmConfig
12
+from vllm.device_allocator.cumem import CuMemAllocator
13
from vllm.distributed import (ensure_model_parallel_initialized,
14
init_distributed_environment,
15
set_custom_all_reduce)
16
from vllm.logger import init_logger
17
from vllm.model_executor import set_random_seed
18
from vllm.platforms import current_platform
19
+from vllm.utils import GiB_bytes
20
from vllm.v1.core.scheduler import SchedulerOutput
21
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
22
from vllm.v1.outputs import ModelRunnerOutput
@@ -77,6 +79,23 @@ class Worker:
79
else:
80
self.profiler = None
81
82
+ def sleep(self, level: int = 1) -> None:
83
+ free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
84
+ allocator = CuMemAllocator.get_instance()
85
+ allocator.sleep(offload_tags=("weights", ) if level == 1 else tuple())
86
+ free_bytes_after_sleep, total = torch.cuda.mem_get_info()
87
+ freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
88
+ used_bytes = total - free_bytes_after_sleep
89
+ assert freed_bytes >= 0, "Memory usage increased after sleeping."
90
+ logger.info(
91
+ "Sleep mode freed %.2f GiB memory, "
92
+ "%.2f GiB memory is still in use.", freed_bytes / GiB_bytes,
93
+ used_bytes / GiB_bytes)
94
+
95
+ def wake_up(self) -> None:
96
+ allocator = CuMemAllocator.get_instance()
97
+ allocator.wake_up()
98
+
99
def init_device(self):
100
if self.device_config.device.type == "cuda":
101
# torch.distributed.all_reduce does not free the input tensor until
@@ -110,7 +129,17 @@ class Worker:
129
self.model_runner = GPUModelRunner(self.vllm_config, self.device)
130
131
def load_model(self) -> None:
113
- self.model_runner.load_model()
132
+ if self.vllm_config.model_config.enable_sleep_mode:
133
+ allocator = CuMemAllocator.get_instance()
134
+ assert allocator.get_current_usage() == 0, (
135
+ "Sleep mode can only be "
136
+ "used for one instance per process.")
137
+ context = allocator.use_memory_pool(tag="weights")
138
+ else:
139
+ from contextlib import nullcontext
140
+ context = nullcontext()
141
+ with context:
142
+ self.model_runner.load_model()
143
144
@torch.inference_mode()
145
def determine_available_memory(self) -> int:
@@ -167,7 +196,14 @@ class Worker:
196
197
def initialize_cache(self, kv_cache_config: KVCacheConfig) -> None:
198
"""Allocate GPU KV cache with the specified kv_cache_config."""
170
- self.model_runner.initialize_kv_cache(kv_cache_config)
199
+ if self.vllm_config.model_config.enable_sleep_mode:
200
+ allocator = CuMemAllocator.get_instance()
201
+ context = allocator.use_memory_pool(tag="kv_cache")
202
+ else:
203
+ from contextlib import nullcontext
204
+ context = nullcontext()
205
+ with context:
206
+ self.model_runner.initialize_kv_cache(kv_cache_config)
207
208
def compile_or_warm_up_model(self) -> None:
209
if not self.model_config.enforce_eager:
vllm/worker/worker.py
+37
-2
@@ -8,6 +8,7 @@ import torch.distributed
8
9
import vllm.envs as envs
10
from vllm.config import VllmConfig
11
+from vllm.device_allocator.cumem import CuMemAllocator
12
from vllm.distributed import (ensure_kv_transfer_initialized,
13
ensure_model_parallel_initialized,
14
init_distributed_environment,
@@ -120,6 +121,23 @@ class Worker(LocalOrDistributedWorkerBase):
121
raise RuntimeError("Profiler is not enabled.")
122
self.profiler.stop()
123
124
+ def sleep(self, level: int = 1) -> None:
125
+ free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
126
+ allocator = CuMemAllocator.get_instance()
127
+ allocator.sleep(offload_tags=("weights", ) if level == 1 else tuple())
128
+ free_bytes_after_sleep, total = torch.cuda.mem_get_info()
129
+ freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
130
+ used_bytes = total - free_bytes_after_sleep
131
+ assert freed_bytes >= 0, "Memory usage increased after sleeping."
132
+ logger.info(
133
+ "Sleep mode freed %.2f GiB memory, "
134
+ "%.2f GiB memory is still in use.", freed_bytes / GiB_bytes,
135
+ used_bytes / GiB_bytes)
136
+
137
+ def wake_up(self) -> None:
138
+ allocator = CuMemAllocator.get_instance()
139
+ allocator.wake_up()
140
+
141
def init_device(self) -> None:
142
if self.device_config.device.type == "cuda":
143
# torch.distributed.all_reduce does not free the input tensor until
@@ -151,7 +169,17 @@ class Worker(LocalOrDistributedWorkerBase):
169
set_random_seed(self.model_config.seed)
170
171
def load_model(self):
154
- self.model_runner.load_model()
172
+ if self.vllm_config.model_config.enable_sleep_mode:
173
+ allocator = CuMemAllocator.get_instance()
174
+ assert allocator.get_current_usage() == 0, (
175
+ "Sleep mode can only be "
176
+ "used for one instance per process.")
177
+ context = allocator.use_memory_pool(tag="weights")
178
+ else:
179
+ from contextlib import nullcontext
180
+ context = nullcontext()
181
+ with context:
182
+ self.model_runner.load_model()
183
184
def save_sharded_state(
185
self,
@@ -270,7 +298,14 @@ class Worker(LocalOrDistributedWorkerBase):
298
self.cache_config.num_gpu_blocks = num_gpu_blocks
299
self.cache_config.num_cpu_blocks = num_cpu_blocks
300
273
- self._init_cache_engine()
301
+ if self.vllm_config.model_config.enable_sleep_mode:
302
+ allocator = CuMemAllocator.get_instance()
303
+ context = allocator.use_memory_pool(tag="kv_cache")
304
+ else:
305
+ from contextlib import nullcontext
306
+ context = nullcontext()
307
+ with context:
308
+ self._init_cache_engine()
309
self._warm_up_model()
310
311
def _init_cache_engine(self):