@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 68ad4e3a8d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Core] Support fully transparent sleep mode (#11743)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 22, 2025 at 14:39 UTC 68ad4e3a8d8a66fb2a43be57471ee13a8bec4ec0
14 files changed +877 -40
.buildkite/test-pipeline.yaml
+2
@@ -76,7 +76,9 @@ steps:
76 - tests/basic_correctness/test_basic_correctness
77 - tests/basic_correctness/test_cpu_offload
78 - tests/basic_correctness/test_preemption
79 + - tests/basic_correctness/test_cumem.py
80 commands:
81 + - pytest -v -s basic_correctness/test_cumem.py
82 - pytest -v -s basic_correctness/test_basic_correctness.py
83 - pytest -v -s basic_correctness/test_cpu_offload.py
84 - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
CMakeLists.txt
+25
@@ -181,6 +181,31 @@ message(STATUS "FetchContent base directory: ${FETCHCONTENT_BASE_DIR}")
181 # Define other extension targets
182 #
183
184 +#
185 +# cumem_allocator extension
186 +#
187 +
188 +set(VLLM_CUMEM_EXT_SRC
189 + "csrc/cumem_allocator.cpp")
190 +
191 +set_gencode_flags_for_srcs(
192 + SRCS "${VLLM_CUMEM_EXT_SRC}"
193 + CUDA_ARCHS "${CUDA_ARCHS}")
194 +
195 +if(VLLM_GPU_LANG STREQUAL "CUDA")
196 + message(STATUS "Enabling cumem allocator extension.")
197 + # link against cuda driver library
198 + list(APPEND CUMEM_LIBS cuda)
199 + define_gpu_extension_target(
200 + cumem_allocator
201 + DESTINATION vllm
202 + LANGUAGE CXX
203 + SOURCES ${VLLM_CUMEM_EXT_SRC}
204 + LIBRARIES ${CUMEM_LIBS}
205 + USE_SABI 3.8
206 + WITH_SOABI)
207 +endif()
208 +
209 #
210 # _C extension
211 #
csrc/cumem_allocator.cpp new
+310
@@ -0,0 +1,310 @@
1 +// A CUDAPluggableAllocator based on cumem* APIs.
2 +// Important: allocation size, CUdeviceptr and CUmemGenericAllocationHandle*
3 +// need to be unsigned long long
4 +#include <iostream>
5 +
6 +extern "C" {
7 +
8 +#define PY_SSIZE_T_CLEAN
9 +#include <Python.h>
10 +
11 +#include <sys/types.h>
12 +#include <cuda_runtime_api.h>
13 +#include <cuda.h>
14 +
15 +#define CUDA_CHECK(condition) \
16 + do { \
17 + CUresult error = condition; \
18 + if (error != 0) { \
19 + char* error_string; \
20 + cuGetErrorString(error, (const char**)&error_string); \
21 + std::cerr << "CUDA Error: " << error_string << " at " << __FILE__ << ":" \
22 + << __LINE__ << std::endl; \
23 + } \
24 + } while (0)
25 +
26 +// Global references to Python callables
27 +// NOTE: this is borrowed reference, so we don't need to DECREF them.
28 +// This brings the limitation that the allocator needs to be singleton.
29 +static PyObject* g_python_malloc_callback = nullptr;
30 +static PyObject* g_python_free_callback = nullptr;
31 +
32 +// ---------------------------------------------------------------------------
33 +// Helper functions:
34 +
35 +void ensure_context(unsigned long long device) {
36 + CUcontext pctx;
37 + CUDA_CHECK(cuCtxGetCurrent(&pctx));
38 + if (!pctx) {
39 + // Ensure device context.
40 + CUDA_CHECK(cuDevicePrimaryCtxRetain(&pctx, device));
41 + CUDA_CHECK(cuCtxSetCurrent(pctx));
42 + }
43 +}
44 +
45 +void create_and_map(unsigned long long device, ssize_t size, CUdeviceptr d_mem,
46 + CUmemGenericAllocationHandle* p_memHandle) {
47 + ensure_context(device);
48 + // Define memory allocation properties
49 + CUmemAllocationProp prop = {};
50 + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
51 + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
52 + prop.location.id = device;
53 + prop.allocFlags.compressionType = CU_MEM_ALLOCATION_COMP_NONE;
54 +
55 + // Allocate memory using cuMemCreate
56 + CUDA_CHECK(cuMemCreate(p_memHandle, size, &prop, 0));
57 + CUDA_CHECK(cuMemMap(d_mem, size, 0, *p_memHandle, 0));
58 +
59 + CUmemAccessDesc accessDesc = {};
60 + accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
61 + accessDesc.location.id = device;
62 + accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
63 +
64 + CUDA_CHECK(cuMemSetAccess(d_mem, size, &accessDesc, 1));
65 + // std::cout << "create_and_map: device=" << device << ", size=" << size << ",
66 + // d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
67 +}
68 +
69 +void unmap_and_release(unsigned long long device, ssize_t size,
70 + CUdeviceptr d_mem,
71 + CUmemGenericAllocationHandle* p_memHandle) {
72 + // std::cout << "unmap_and_release: device=" << device << ", size=" << size <<
73 + // ", d_mem=" << d_mem << ", p_memHandle=" << p_memHandle << std::endl;
74 + ensure_context(device);
75 + CUDA_CHECK(cuMemUnmap(d_mem, size));
76 + CUDA_CHECK(cuMemRelease(*p_memHandle));
77 +}
78 +
79 +PyObject* create_tuple_from_c_integers(unsigned long long a,
80 + unsigned long long b,
81 + unsigned long long c,
82 + unsigned long long d) {
83 + // Create a new tuple of size 4
84 + PyObject* tuple = PyTuple_New(4);
85 + if (!tuple) {
86 + return NULL; // Return NULL on failure
87 + }
88 +
89 + // Convert integers to Python objects and set them in the tuple
90 + PyTuple_SetItem(
91 + tuple, 0,
92 + PyLong_FromUnsignedLongLong(a)); // Steals reference to the PyLong
93 + PyTuple_SetItem(tuple, 1, PyLong_FromUnsignedLongLong(b));
94 + PyTuple_SetItem(tuple, 2, PyLong_FromUnsignedLongLong(c));
95 + PyTuple_SetItem(tuple, 3, PyLong_FromUnsignedLongLong(d));
96 +
97 + // Note: PyTuple_SetItem "steals" a reference to each object,
98 + // so we do not need to Py_DECREF the PyLong objects explicitly.
99 +
100 + return tuple; // Return the created tuple
101 +}
102 +
103 +// ---------------------------------------------------------------------------
104 +// Our exported C functions that call Python:
105 +
106 +// use CUstream instead of cudaStream_t, to avoid including cuda_runtime_api.h
107 +void* my_malloc(ssize_t size, int device, CUstream stream) {
108 + ensure_context(device);
109 +
110 + // first allocation, align the size, and reserve an address, and also allocate
111 + // a CUmemGenericAllocationHandle
112 +
113 + // Define memory allocation properties
114 + CUmemAllocationProp prop = {};
115 + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
116 + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
117 + prop.location.id = device;
118 + prop.allocFlags.compressionType = CU_MEM_ALLOCATION_COMP_NONE;
119 +
120 + // Check if the allocation is supported
121 + size_t granularity;
122 + CUDA_CHECK(cuMemGetAllocationGranularity(&granularity, &prop,
123 + CU_MEM_ALLOC_GRANULARITY_MINIMUM));
124 +
125 + size_t alignedSize = ((size + granularity - 1) / granularity) * granularity;
126 +
127 + CUdeviceptr d_mem;
128 + CUDA_CHECK(cuMemAddressReserve(&d_mem, alignedSize, 0, 0, 0));
129 +
130 + // allocate the CUmemGenericAllocationHandle
131 + CUmemGenericAllocationHandle* p_memHandle =
132 + (CUmemGenericAllocationHandle*)malloc(
133 + sizeof(CUmemGenericAllocationHandle));
134 +
135 + if (!g_python_malloc_callback) {
136 + std::cerr << "ERROR: g_python_malloc_callback not set.\n";
137 + return nullptr;
138 + }
139 +
140 + // Acquire GIL (not in stable ABI officially, but often works)
141 + PyGILState_STATE gstate = PyGILState_Ensure();
142 +
143 + PyObject* arg_tuple = create_tuple_from_c_integers(
144 + (unsigned long long)device, (unsigned long long)alignedSize,
145 + (unsigned long long)d_mem, (unsigned long long)p_memHandle);
146 +
147 + // Call g_python_malloc_callback
148 + PyObject* py_result =
149 + PyObject_CallFunctionObjArgs(g_python_malloc_callback, arg_tuple, NULL);
150 + Py_DECREF(arg_tuple);
151 +
152 + if (!py_result) {
153 + PyErr_Print();
154 + PyGILState_Release(gstate);
155 + return nullptr;
156 + }
157 +
158 + PyGILState_Release(gstate);
159 +
160 + // do the final mapping
161 + create_and_map(device, alignedSize, d_mem, p_memHandle);
162 +
163 + return (void*)d_mem;
164 +}
165 +
166 +// use CUstream instead of cudaStream_t, to avoid including cuda_runtime_api.h
167 +void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
168 + // get memory handle from the pointer
169 + if (!g_python_free_callback) {
170 + std::cerr << "ERROR: g_python_free_callback not set.\n";
171 + return;
172 + }
173 +
174 + // Acquire GIL (not in stable ABI officially, but often works)
175 + PyGILState_STATE gstate = PyGILState_Ensure();
176 +
177 + PyObject* py_ptr =
178 + PyLong_FromUnsignedLongLong(reinterpret_cast<unsigned long long>(ptr));
179 +
180 + PyObject* py_result =
181 + PyObject_CallFunctionObjArgs(g_python_free_callback, py_ptr, NULL);
182 +
183 + if (!py_result || !PyTuple_Check(py_result) || PyTuple_Size(py_result) != 4) {
184 + PyErr_SetString(PyExc_TypeError, "Expected a tuple of size 4");
185 + return;
186 + }
187 +
188 + unsigned long long recv_device, recv_size;
189 + unsigned long long recv_d_mem, recv_p_memHandle;
190 + // Unpack the tuple into four C integers
191 + if (!PyArg_ParseTuple(py_result, "KKKK", &recv_device, &recv_size,
192 + &recv_d_mem, &recv_p_memHandle)) {
193 + // PyArg_ParseTuple sets an error if it fails
194 + return;
195 + }
196 +
197 + PyGILState_Release(gstate);
198 +
199 + // recv_size == size
200 + // recv_device == device
201 +
202 + // Free memory
203 +
204 + CUdeviceptr d_mem = (CUdeviceptr)recv_d_mem;
205 + CUmemGenericAllocationHandle* p_memHandle =
206 + (CUmemGenericAllocationHandle*)recv_p_memHandle;
207 + unmap_and_release(device, size, d_mem, p_memHandle);
208 +
209 + // free address and the handle
210 + CUDA_CHECK(cuMemAddressFree(d_mem, size));
211 + free(p_memHandle);
212 +}
213 +
214 +// ---------------------------------------------------------------------------
215 +// Python extension boilerplate:
216 +
217 +// Python-exposed function: init_module(python_malloc, python_free)
218 +static PyObject* py_init_module(PyObject* self, PyObject* args) {
219 + PyObject* malloc_callback = nullptr;
220 + PyObject* free_callback = nullptr;
221 +
222 + if (!PyArg_ParseTuple(args, "OO", &malloc_callback, &free_callback)) {
223 + return nullptr;
224 + }
225 +
226 + if (!PyCallable_Check(malloc_callback) || !PyCallable_Check(free_callback)) {
227 + PyErr_SetString(PyExc_TypeError, "Both arguments must be callables");
228 + return nullptr;
229 + }
230 +
231 + // Save the Python callables
232 + // This module does not handle GC of these objects, so they must be kept alive
233 + // outside of this module.
234 + g_python_malloc_callback = malloc_callback;
235 + g_python_free_callback = free_callback;
236 +
237 + Py_RETURN_NONE;
238 +}
239 +
240 +static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
241 + if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != 4) {
242 + PyErr_SetString(PyExc_TypeError, "Expected a tuple of size 4");
243 + return nullptr;
244 + }
245 +
246 + unsigned long long recv_device, recv_size;
247 + unsigned long long recv_d_mem, recv_p_memHandle;
248 + // Unpack the tuple into four C integers
249 + if (!PyArg_ParseTuple(args, "KKKK", &recv_device, &recv_size, &recv_d_mem,
250 + &recv_p_memHandle)) {
251 + // PyArg_ParseTuple sets an error if it fails
252 + return nullptr;
253 + }
254 +
255 + CUdeviceptr d_mem_ptr = (CUdeviceptr)recv_d_mem;
256 + CUmemGenericAllocationHandle* p_memHandle =
257 + (CUmemGenericAllocationHandle*)recv_p_memHandle;
258 +
259 + unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle);
260 +
261 + Py_RETURN_NONE;
262 +}
263 +
264 +static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
265 + if (!args || !PyTuple_Check(args) || PyTuple_Size(args) != 4) {
266 + PyErr_SetString(PyExc_TypeError, "Expected a tuple of size 4");
267 + return nullptr;
268 + }
269 +
270 + unsigned long long recv_device, recv_size;
271 + unsigned long long recv_d_mem, recv_p_memHandle;
272 + // Unpack the tuple into four C integers
273 + if (!PyArg_ParseTuple(args, "KKKK", &recv_device, &recv_size, &recv_d_mem,
274 + &recv_p_memHandle)) {
275 + // PyArg_ParseTuple sets an error if it fails
276 + return nullptr;
277 + }
278 +
279 + CUdeviceptr d_mem_ptr = (CUdeviceptr)recv_d_mem;
280 + CUmemGenericAllocationHandle* p_memHandle =
281 + (CUmemGenericAllocationHandle*)recv_p_memHandle;
282 +
283 + create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle);
284 +
285 + Py_RETURN_NONE;
286 +}
287 +
288 +static PyMethodDef module_methods[] = {
289 + {"init_module", (PyCFunction)py_init_module, METH_VARARGS,
290 + "Initialize module with python_malloc and python_free callables."},
291 + {"python_create_and_map", (PyCFunction)python_create_and_map, METH_VARARGS,
292 + "Create and map memory on the device."},
293 + {"python_unmap_and_release", (PyCFunction)python_unmap_and_release,
294 + METH_VARARGS, "Unmap and release memory on the device."},
295 + {NULL, NULL, 0, NULL} // sentinel
296 +};
297 +
298 +static struct PyModuleDef cumem_allocator_module = {
299 + PyModuleDef_HEAD_INIT, "cumem_allocator",
300 + "cumem-based allocator for CUDAPluggableAllocator", -1, module_methods};
301 +
302 +PyMODINIT_FUNC PyInit_cumem_allocator(void) {
303 + // Initialize the module
304 + PyObject* module = PyModule_Create(&cumem_allocator_module);
305 + if (!module) {
306 + return NULL;
307 + }
308 + return module;
309 +}
310 +} // extern "C"
setup.py
+2
@@ -301,6 +301,7 @@ class repackage_wheel(build_ext):
301 "vllm/vllm_flash_attn/vllm_flash_attn_c.abi3.so",
302 "vllm/vllm_flash_attn/flash_attn_interface.py",
303 "vllm/vllm_flash_attn/__init__.py",
304 + "vllm/cumem_allocator.abi3.so",
305 # "vllm/_version.py", # not available in nightly wheels yet
306 ]
307 file_members = filter(lambda x: x.filename in files_to_copy,
@@ -594,6 +595,7 @@ if _is_hip():
595 if _is_cuda():
596 ext_modules.append(
597 CMakeExtension(name="vllm.vllm_flash_attn.vllm_flash_attn_c"))
598 + ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
599
600 if _build_custom_ops():
601 ext_modules.append(CMakeExtension(name="vllm._C"))
tests/basic_correctness/test_cumem.py new
+112
@@ -0,0 +1,112 @@
1 +import torch
2 +
3 +from vllm import LLM, SamplingParams
4 +from vllm.device_allocator.cumem import CuMemAllocator
5 +from vllm.utils import GiB_bytes
6 +
7 +from ..utils import fork_new_process_for_each_test
8 +
9 +
10 +@fork_new_process_for_each_test
11 +def test_basic_cumem():
12 + # some tensors from default memory pool
13 + shape = (1024, 1024)
14 + x = torch.empty(shape, device='cuda')
15 + x.zero_()
16 +
17 + # some tensors from custom memory pool
18 + allocator = CuMemAllocator.get_instance()
19 + with allocator.use_memory_pool():
20 + # custom memory pool
21 + y = torch.empty(shape, device='cuda')
22 + y.zero_()
23 + y += 1
24 + z = torch.empty(shape, device='cuda')
25 + z.zero_()
26 + z += 2
27 +
28 + # they can be used together
29 + output = x + y + z
30 + assert torch.allclose(output, torch.ones_like(output) * 3)
31 +
32 + free_bytes = torch.cuda.mem_get_info()[0]
33 + allocator.sleep()
34 + free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
35 + assert free_bytes_after_sleep > free_bytes
36 + allocator.wake_up()
37 +
38 + # they can be used together
39 + output = x + y + z
40 + assert torch.allclose(output, torch.ones_like(output) * 3)
41 +
42 +
43 +@fork_new_process_for_each_test
44 +def test_cumem_with_cudagraph():
45 + allocator = CuMemAllocator.get_instance()
46 + with allocator.use_memory_pool():
47 + weight = torch.eye(1024, device='cuda')
48 + with allocator.use_memory_pool(tag="discard"):
49 + cache = torch.empty(1024, 1024, device='cuda')
50 +
51 + def model(x):
52 + out = x @ weight
53 + cache[:out.size(0)].copy_(out)
54 + return out + 1
55 +
56 + x = torch.empty(128, 1024, device='cuda')
57 +
58 + # warmup
59 + model(x)
60 +
61 + # capture cudagraph
62 + model_graph = torch.cuda.CUDAGraph()
63 + with torch.cuda.graph(model_graph):
64 + y = model(x)
65 +
66 + free_bytes = torch.cuda.mem_get_info()[0]
67 + allocator.sleep()
68 + free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
69 + assert free_bytes_after_sleep > free_bytes
70 + allocator.wake_up()
71 +
72 + # after waking up, the content in the weight tensor
73 + # should be restored, but the content in the cache tensor
74 + # should be discarded
75 +
76 + # this operation is also compatible with cudagraph
77 +
78 + x.random_()
79 + model_graph.replay()
80 +
81 + # cache content is as expected
82 + assert torch.allclose(x, cache[:x.size(0)])
83 +
84 + # output content is as expected
85 + assert torch.allclose(y, x + 1)
86 +
87 +
88 +@fork_new_process_for_each_test
89 +def test_end_to_end():
90 + free, total = torch.cuda.mem_get_info()
91 + used_bytes_baseline = total - free # in case other process is running
92 + llm = LLM("meta-llama/Llama-3.2-1B", enable_sleep_mode=True)
93 + prompt = "How are you?"
94 + sampling_params = SamplingParams(temperature=0, max_tokens=10)
95 + output = llm.generate(prompt, sampling_params)
96 +
97 + # the benefit of `llm.sleep(level=2)` is mainly CPU memory usage,
98 + # which is difficult to measure in the test. therefore, we only
99 + # test sleep level 1 here.
100 + llm.sleep(level=1)
101 +
102 + free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
103 + used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
104 + # now the memory usage is mostly cudagraph memory pool,
105 + # and it should be less than the model weights (1B model, 2GiB weights)
106 + assert used_bytes < 2 * GiB_bytes
107 +
108 + llm.wake_up()
109 + output2 = llm.generate(prompt, sampling_params)
110 +
111 + # cmp output
112 + assert output[0].outputs[0].text == output2[0].outputs[0].text
vllm/config.py
+43 -35
@@ -195,40 +195,43 @@ class ModelConfig:
195 factors.append(self.rope_theta)
196 return hashlib.sha256(str(factors).encode()).hexdigest()
197
198 - def __init__(self,
199 - model: str,
200 - task: Union[TaskOption, Literal["draft"]],
201 - tokenizer: str,
202 - tokenizer_mode: str,
203 - trust_remote_code: bool,
204 - dtype: Union[str, torch.dtype],
205 - seed: int,
206 - allowed_local_media_path: str = "",
207 - revision: Optional[str] = None,
208 - code_revision: Optional[str] = None,
209 - rope_scaling: Optional[Dict[str, Any]] = None,
210 - rope_theta: Optional[float] = None,
211 - tokenizer_revision: Optional[str] = None,
212 - max_model_len: Optional[int] = None,
213 - spec_target_max_model_len: Optional[int] = None,
214 - quantization: Optional[str] = None,
215 - quantization_param_path: Optional[str] = None,
216 - enforce_eager: Optional[bool] = None,
217 - max_seq_len_to_capture: Optional[int] = None,
218 - max_logprobs: int = 20,
219 - disable_sliding_window: bool = False,
220 - skip_tokenizer_init: bool = False,
221 - served_model_name: Optional[Union[str, List[str]]] = None,
222 - limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
223 - use_async_output_proc: bool = True,
224 - config_format: ConfigFormat = ConfigFormat.AUTO,
225 - hf_overrides: Optional[HfOverrides] = None,
226 - mm_processor_kwargs: Optional[Dict[str, Any]] = None,
227 - disable_mm_preprocessor_cache: bool = False,
228 - override_neuron_config: Optional[Dict[str, Any]] = None,
229 - override_pooler_config: Optional["PoolerConfig"] = None,
230 - logits_processor_pattern: Optional[str] = None,
231 - generation_config: Optional[str] = None) -> None:
198 + def __init__(
199 + self,
200 + model: str,
201 + task: Union[TaskOption, Literal["draft"]],
202 + tokenizer: str,
203 + tokenizer_mode: str,
204 + trust_remote_code: bool,
205 + dtype: Union[str, torch.dtype],
206 + seed: int,
207 + allowed_local_media_path: str = "",
208 + revision: Optional[str] = None,
209 + code_revision: Optional[str] = None,
210 + rope_scaling: Optional[Dict[str, Any]] = None,
211 + rope_theta: Optional[float] = None,
212 + tokenizer_revision: Optional[str] = None,
213 + max_model_len: Optional[int] = None,
214 + spec_target_max_model_len: Optional[int] = None,
215 + quantization: Optional[str] = None,
216 + quantization_param_path: Optional[str] = None,
217 + enforce_eager: Optional[bool] = None,
218 + max_seq_len_to_capture: Optional[int] = None,
219 + max_logprobs: int = 20,
220 + disable_sliding_window: bool = False,
221 + skip_tokenizer_init: bool = False,
222 + served_model_name: Optional[Union[str, List[str]]] = None,
223 + limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
224 + use_async_output_proc: bool = True,
225 + config_format: ConfigFormat = ConfigFormat.AUTO,
226 + hf_overrides: Optional[HfOverrides] = None,
227 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
228 + disable_mm_preprocessor_cache: bool = False,
229 + override_neuron_config: Optional[Dict[str, Any]] = None,
230 + override_pooler_config: Optional["PoolerConfig"] = None,
231 + logits_processor_pattern: Optional[str] = None,
232 + generation_config: Optional[str] = None,
233 + enable_sleep_mode: bool = False,
234 + ) -> None:
235 self.model = model
236 self.tokenizer = tokenizer
237 self.tokenizer_mode = tokenizer_mode
@@ -277,6 +280,12 @@ class ModelConfig:
280 self.max_logprobs = max_logprobs
281 self.disable_sliding_window = disable_sliding_window
282 self.skip_tokenizer_init = skip_tokenizer_init
283 + self.enable_sleep_mode = enable_sleep_mode
284 +
285 + from vllm.platforms import current_platform
286 +
287 + if self.enable_sleep_mode and not current_platform.is_cuda():
288 + raise ValueError("Sleep mode is only supported on CUDA devices.")
289
290 hf_config = get_config(self.model, trust_remote_code, revision,
291 code_revision, config_format)
@@ -348,7 +357,6 @@ class ModelConfig:
357 self.is_hybrid = self._init_is_hybrid()
358 self.has_inner_state = self._init_has_inner_state()
359
351 - from vllm.platforms import current_platform
360 if current_platform.is_neuron():
361 self.override_neuron_config = override_neuron_config
362 else:
vllm/device_allocator/__init__.py
vllm/device_allocator/cumem.py new
+254
@@ -0,0 +1,254 @@
1 +# cumem-based pytorch pluggable allocator to implement sleep mode.
2 +# other approaches tried but failed:
3 +# - cuda-python package binding
4 +# - custom libcuda driver ctypes wrapper
5 +# both of them failed because of cuda context mismatch.
6 +# not sure why, they are created from a different context.
7 +# the only successful approach is to call cuda driver API in C.
8 +import dataclasses
9 +from contextlib import contextmanager
10 +from typing import Callable, Dict, Optional, Tuple, Union
11 +
12 +import torch
13 +
14 +from vllm.utils import is_pin_memory_available
15 +
16 +
17 +def find_loaded_library(lib_name) -> Optional[str]:
18 + """
19 + According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
20 + the file `/proc/self/maps` contains the memory maps of the process, which includes the
21 + shared libraries loaded by the process. We can use this file to find the path of the
22 + a loaded library.
23 + """ # noqa
24 + found_line = None
25 + with open("/proc/self/maps") as f:
26 + for line in f:
27 + if lib_name in line:
28 + found_line = line
29 + break
30 + if found_line is None:
31 + # the library is not loaded in the current process
32 + return None
33 + # if lib_name is libcudart, we need to match a line with:
34 + # address /path/to/libcudart-hash.so.11.0
35 + start = found_line.index("/")
36 + path = found_line[start:].strip()
37 + filename = path.split("/")[-1]
38 + assert filename.rpartition(".so")[0].startswith(lib_name), \
39 + f"Unexpected filename: {filename} for library {lib_name}"
40 + return path
41 +
42 +
43 +cumem_available = False
44 +try:
45 + from vllm.cumem_allocator import (init_module, python_create_and_map,
46 + python_unmap_and_release)
47 + from vllm.distributed.device_communicators.cuda_wrapper import (
48 + CudaRTLibrary)
49 + lib_name = find_loaded_library("cumem_allocator")
50 + libcudart = CudaRTLibrary()
51 + cumem_available = True
52 +except ModuleNotFoundError:
53 + # rocm platform does not support cumem allocator
54 + init_module = None
55 + python_create_and_map = None
56 + python_unmap_and_release = None
57 + CudaRTLibrary = None
58 + lib_name = None
59 + libcudart = None
60 +
61 +# py_device, py_alignedSize, py_d_mem, py_p_memHandle
62 +HandleType = Tuple[int, int, int, int]
63 +
64 +
65 +@dataclasses.dataclass
66 +class AllocationData:
67 + handle: HandleType
68 + tag: str
69 + cpu_backup_tensor: Optional[torch.Tensor] = None
70 +
71 +
72 +def create_and_map(allocation_handle: HandleType) -> None:
73 + python_create_and_map(*allocation_handle)
74 +
75 +
76 +def unmap_and_release(allocation_handle: HandleType) -> None:
77 + python_unmap_and_release(*allocation_handle)
78 +
79 +
80 +def get_pluggable_allocator(
81 + python_malloc_fn: Callable[[int],
82 + int], python_free_func: Callable[[int, int],
83 + None]
84 +) -> torch.cuda.memory.CUDAPluggableAllocator:
85 + init_module(python_malloc_fn, python_free_func)
86 + new_alloc = torch.cuda.memory.CUDAPluggableAllocator(
87 + lib_name, 'my_malloc', 'my_free')
88 + return new_alloc
89 +
90 +
91 +@contextmanager
92 +def use_memory_pool_with_allocator(
93 + python_malloc_fn: Callable[[int], int],
94 + python_free_func: Callable[[int, int], None]) -> None:
95 + new_alloc = get_pluggable_allocator(python_malloc_fn, python_free_func)
96 + mem_pool = torch.cuda.memory.MemPool(new_alloc._allocator)
97 + with torch.cuda.memory.use_mem_pool(mem_pool):
98 + yield mem_pool
99 +
100 +
101 +class CuMemAllocator:
102 + """
103 + A singleton class that manages a memory pool for CUDA tensors.
104 + The memory in this pool can be offloaded or discarded when the
105 + allocator sleeps.
106 +
107 + Inside the `use_memory_pool(tag)` context, all tensors created will
108 + be allocated in the memory pool, and has the same tag as the
109 + tag passed to the context.
110 +
111 + When we call `sleep`, all tensors with the specified tag will be
112 + offloaded to CPU memory, and the rest of the tensors will be discarded.
113 + When we call `wake_up`, all tensors that are previously offloaded
114 + will be loaded back to GPU memory, and the rest of the tensors will
115 + have empty memory.
116 +
117 + Why it needs to be a singleton?
118 + When allocated tensors are garbage collected, PyTorch will call
119 + the free callback, which will call the `python_free_callback` method.
120 + The C-extension uses a global variable to store the function of an
121 + instance of this class. If we create multiple instances of this class,
122 + the global variable will be overwritten and the free callback will
123 + not work as expected.
124 + """
125 + instance: "CuMemAllocator" = None
126 + default_tag: str = "default"
127 +
128 + @staticmethod
129 + def get_instance() -> "CuMemAllocator":
130 + """
131 + CuMemAllocator is a singleton class.
132 + We cannot call the constructor directly.
133 + Call this method to get the instance.
134 + """
135 + assert cumem_available, "cumem allocator is not available"
136 + if CuMemAllocator.instance is None:
137 + CuMemAllocator.instance = CuMemAllocator()
138 + return CuMemAllocator.instance
139 +
140 + def __init__(self):
141 + self.pointer_to_data: Dict[int, AllocationData] = {}
142 + self.current_tag: str = CuMemAllocator.default_tag
143 +
144 + def python_malloc_callback(self, allocation_handle: HandleType) -> None:
145 + """
146 + Internal method to store the allocation data
147 + when memory is allocated in the memory pool."""
148 + py_d_mem = allocation_handle[2]
149 + self.pointer_to_data[py_d_mem] = AllocationData(
150 + allocation_handle, self.current_tag)
151 + return
152 +
153 + def python_free_callback(self, ptr: int) -> HandleType:
154 + """
155 + Internal method to look up the allocation data
156 + when memory is freed in the memory pool."""
157 + data = self.pointer_to_data.pop(ptr)
158 + if data.cpu_backup_tensor is not None:
159 + data.cpu_backup_tensor = None
160 + return data.handle
161 +
162 + def sleep(
163 + self,
164 + offload_tags: Optional[Union[Tuple[str, ...],
165 + str]] = None) -> None:
166 + """
167 + Put the allocator in sleep mode.
168 + All data in the memory allocation with the specified tag will be
169 + offloaded to CPU memory, and others will be discarded.
170 +
171 + :param offload_tags: The tags of the memory allocation that will be
172 + offloaded. The rest of the memory allocation will be discarded.
173 + """
174 + if offload_tags is None:
175 + # by default, allocated tensors are offloaded
176 + # when the allocator sleeps
177 + offload_tags = (CuMemAllocator.default_tag, )
178 + elif isinstance(offload_tags, str):
179 + offload_tags = (offload_tags, )
180 +
181 + assert isinstance(offload_tags, tuple)
182 +
183 + for ptr, data in self.pointer_to_data.items():
184 + handle = data.handle
185 + if data.tag in offload_tags:
186 + size_in_bytes = handle[1]
187 + cpu_backup_tensor = torch.empty(
188 + size_in_bytes,
189 + dtype=torch.uint8,
190 + device='cpu',
191 + pin_memory=is_pin_memory_available())
192 + cpu_ptr = cpu_backup_tensor.data_ptr()
193 + libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
194 + data.cpu_backup_tensor = cpu_backup_tensor
195 + unmap_and_release(handle)
196 +
197 + def wake_up(self):
198 + """
199 + Wake up the allocator from sleep mode.
200 + All data that is previously offloaded will be loaded back to GPU
201 + memory, and the rest of the data will have empty memory."""
202 + for ptr, data in self.pointer_to_data.items():
203 + handle = data.handle
204 + create_and_map(handle)
205 + if data.cpu_backup_tensor is not None:
206 + cpu_backup_tensor = data.cpu_backup_tensor
207 + if cpu_backup_tensor is not None:
208 + size_in_bytes = cpu_backup_tensor.numel(
209 + ) * cpu_backup_tensor.element_size()
210 + cpu_ptr = cpu_backup_tensor.data_ptr()
211 + libcudart.cudaMemcpy(ptr, cpu_ptr, size_in_bytes)
212 + data.cpu_backup_tensor = None
213 +
214 + @contextmanager
215 + def use_memory_pool(self, tag: Optional[str] = None):
216 + """
217 + A context manager to use the memory pool.
218 + All memory allocation created inside the context will be allocated
219 + in the memory pool, and has the specified tag.
220 +
221 + :param tag: The tag of the memory allocation. If None, the default tag
222 + will be used.
223 + """
224 + if tag is None:
225 + tag = CuMemAllocator.default_tag
226 +
227 + assert isinstance(tag, str)
228 +
229 + old_tag = self.current_tag
230 + self.current_tag = tag
231 + with use_memory_pool_with_allocator(self.python_malloc_callback,
232 + self.python_free_callback):
233 + yield
234 + # PyTorch's bug, calling torch.cuda.empty_cache() will error
235 + # when using pluggable allocator, see
236 + # https://github.com/pytorch/pytorch/issues/145168 .
237 + # if we have some memory allocated and then freed,
238 + # the memory will not be released.
239 + # right now it is fine, because we only use this allocator
240 + # during weight loading and kv cache creation, where we only
241 + # allocate memory.
242 + # TODO: we need to find a way to release the memory,
243 + # i.e. calling torch.cuda.empty_cache()
244 + self.current_tag = old_tag
245 +
246 + def get_current_usage(self) -> int:
247 + """
248 + Get the total number of bytes allocated in the memory pool.
249 + """
250 + sum_bytes: int = 0
251 + for ptr, data in self.pointer_to_data.items():
252 + handle = data.handle
253 + sum_bytes += handle[1]
254 + return sum_bytes
vllm/engine/arg_utils.py
+10 -1
@@ -197,6 +197,7 @@ class EngineArgs:
197 kv_transfer_config: Optional[KVTransferConfig] = None
198
199 generation_config: Optional[str] = None
200 + enable_sleep_mode: bool = False
201
202 def __post_init__(self):
203 if not self.tokenizer:
@@ -955,6 +956,12 @@ class EngineArgs:
956 "loaded from model. If set to a folder path, the generation config "
957 "will be loaded from the specified folder path.")
958
959 + parser.add_argument("--enable-sleep-mode",
960 + action="store_true",
961 + default=False,
962 + help="Enable sleep mode for the engine. "
963 + "(only cuda platform is supported)")
964 +
965 return parser
966
967 @classmethod
@@ -999,7 +1006,9 @@ class EngineArgs:
1006 override_neuron_config=self.override_neuron_config,
1007 override_pooler_config=self.override_pooler_config,
1008 logits_processor_pattern=self.logits_processor_pattern,
1002 - generation_config=self.generation_config)
1009 + generation_config=self.generation_config,
1010 + enable_sleep_mode=self.enable_sleep_mode,
1011 + )
1012
1013 def create_load_config(self) -> LoadConfig:
1014 return LoadConfig(
vllm/engine/llm_engine.py
+10
@@ -1818,6 +1818,16 @@ class LLMEngine:
1818 def stop_profile(self) -> None:
1819 self.model_executor.stop_profile()
1820
1821 + def sleep(self, level: int = 1) -> None:
1822 + assert self.vllm_config.model_config.enable_sleep_mode, (
1823 + "Sleep mode is not enabled in the model config")
1824 + self.model_executor.sleep(level=level)
1825 +
1826 + def wake_up(self) -> None:
1827 + assert self.vllm_config.model_config.enable_sleep_mode, (
1828 + "Sleep mode is not enabled in the model config")
1829 + self.model_executor.wake_up()
1830 +
1831 def check_health(self) -> None:
1832 if self.tokenizer:
1833 self.tokenizer.check_health()
vllm/entrypoints/llm.py
+23
@@ -1132,6 +1132,29 @@ class LLM:
1132 def stop_profile(self) -> None:
1133 self.llm_engine.stop_profile()
1134
1135 + def sleep(self, level: int = 1):
1136 + """
1137 + Put the engine to sleep. The engine should not process any requests.
1138 + The caller should guarantee that no requests are being processed
1139 + during the sleep period, before `wake_up` is called.
1140 +
1141 + :param level: The sleep level. Level 1 sleep will offload the model
1142 + weights and discard the kv cache. The content of kv cache is
1143 + forgotten. Level 1 sleep is good for sleeping and waking up the
1144 + engine to run the same model again. The model weights are backed
1145 + up in CPU memory. Please make sure there's enough CPU memory to
1146 + store the model weights. Level 2 sleep will discard both the model
1147 + weights and the kv cache. The content of both the model weights
1148 + and kv cache is forgotten. Level 2 sleep is good for sleeping and
1149 + waking up the engine to run a different model or update the model,
1150 + where previous model weights are not needed. It reduces CPU memory
1151 + pressure.
1152 + """
1153 + self.llm_engine.sleep(level=level)
1154 +
1155 + def wake_up(self):
1156 + self.llm_engine.wake_up()
1157 +
1158 # LEGACY
1159 def _convert_v1_inputs(
1160 self,
vllm/executor/executor_base.py
+11
@@ -193,6 +193,17 @@ class ExecutorBase(ABC):
193 def stop_profile(self) -> None:
194 self.collective_rpc("stop_profile")
195
196 + def sleep(self, level: int = 1):
197 + if self.cache_config.enable_prefix_caching:
198 + # TODO: support sleep with prefix caching
199 + # by resetting the prefix cache state,
200 + # after https://github.com/vllm-project/vllm/pull/12284
201 + raise ValueError("Cannot sleep when prefix caching is enabled.")
202 + self.collective_rpc("sleep", kwargs=dict(level=level))
203 +
204 + def wake_up(self):
205 + self.collective_rpc("wake_up")
206 +
207 def save_sharded_state(
208 self,
209 path: str,
vllm/v1/worker/gpu_worker.py
+38 -2
@@ -9,12 +9,14 @@ import torch.nn as nn
9
10 import vllm.envs as envs
11 from vllm.config import ParallelConfig, VllmConfig
12 +from vllm.device_allocator.cumem import CuMemAllocator
13 from vllm.distributed import (ensure_model_parallel_initialized,
14 init_distributed_environment,
15 set_custom_all_reduce)
16 from vllm.logger import init_logger
17 from vllm.model_executor import set_random_seed
18 from vllm.platforms import current_platform
19 +from vllm.utils import GiB_bytes
20 from vllm.v1.core.scheduler import SchedulerOutput
21 from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
22 from vllm.v1.outputs import ModelRunnerOutput
@@ -77,6 +79,23 @@ class Worker:
79 else:
80 self.profiler = None
81
82 + def sleep(self, level: int = 1) -> None:
83 + free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
84 + allocator = CuMemAllocator.get_instance()
85 + allocator.sleep(offload_tags=("weights", ) if level == 1 else tuple())
86 + free_bytes_after_sleep, total = torch.cuda.mem_get_info()
87 + freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
88 + used_bytes = total - free_bytes_after_sleep
89 + assert freed_bytes >= 0, "Memory usage increased after sleeping."
90 + logger.info(
91 + "Sleep mode freed %.2f GiB memory, "
92 + "%.2f GiB memory is still in use.", freed_bytes / GiB_bytes,
93 + used_bytes / GiB_bytes)
94 +
95 + def wake_up(self) -> None:
96 + allocator = CuMemAllocator.get_instance()
97 + allocator.wake_up()
98 +
99 def init_device(self):
100 if self.device_config.device.type == "cuda":
101 # torch.distributed.all_reduce does not free the input tensor until
@@ -110,7 +129,17 @@ class Worker:
129 self.model_runner = GPUModelRunner(self.vllm_config, self.device)
130
131 def load_model(self) -> None:
113 - self.model_runner.load_model()
132 + if self.vllm_config.model_config.enable_sleep_mode:
133 + allocator = CuMemAllocator.get_instance()
134 + assert allocator.get_current_usage() == 0, (
135 + "Sleep mode can only be "
136 + "used for one instance per process.")
137 + context = allocator.use_memory_pool(tag="weights")
138 + else:
139 + from contextlib import nullcontext
140 + context = nullcontext()
141 + with context:
142 + self.model_runner.load_model()
143
144 @torch.inference_mode()
145 def determine_available_memory(self) -> int:
@@ -167,7 +196,14 @@ class Worker:
196
197 def initialize_cache(self, kv_cache_config: KVCacheConfig) -> None:
198 """Allocate GPU KV cache with the specified kv_cache_config."""
170 - self.model_runner.initialize_kv_cache(kv_cache_config)
199 + if self.vllm_config.model_config.enable_sleep_mode:
200 + allocator = CuMemAllocator.get_instance()
201 + context = allocator.use_memory_pool(tag="kv_cache")
202 + else:
203 + from contextlib import nullcontext
204 + context = nullcontext()
205 + with context:
206 + self.model_runner.initialize_kv_cache(kv_cache_config)
207
208 def compile_or_warm_up_model(self) -> None:
209 if not self.model_config.enforce_eager:
vllm/worker/worker.py
+37 -2
@@ -8,6 +8,7 @@ import torch.distributed
8
9 import vllm.envs as envs
10 from vllm.config import VllmConfig
11 +from vllm.device_allocator.cumem import CuMemAllocator
12 from vllm.distributed import (ensure_kv_transfer_initialized,
13 ensure_model_parallel_initialized,
14 init_distributed_environment,
@@ -120,6 +121,23 @@ class Worker(LocalOrDistributedWorkerBase):
121 raise RuntimeError("Profiler is not enabled.")
122 self.profiler.stop()
123
124 + def sleep(self, level: int = 1) -> None:
125 + free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
126 + allocator = CuMemAllocator.get_instance()
127 + allocator.sleep(offload_tags=("weights", ) if level == 1 else tuple())
128 + free_bytes_after_sleep, total = torch.cuda.mem_get_info()
129 + freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
130 + used_bytes = total - free_bytes_after_sleep
131 + assert freed_bytes >= 0, "Memory usage increased after sleeping."
132 + logger.info(
133 + "Sleep mode freed %.2f GiB memory, "
134 + "%.2f GiB memory is still in use.", freed_bytes / GiB_bytes,
135 + used_bytes / GiB_bytes)
136 +
137 + def wake_up(self) -> None:
138 + allocator = CuMemAllocator.get_instance()
139 + allocator.wake_up()
140 +
141 def init_device(self) -> None:
142 if self.device_config.device.type == "cuda":
143 # torch.distributed.all_reduce does not free the input tensor until
@@ -151,7 +169,17 @@ class Worker(LocalOrDistributedWorkerBase):
169 set_random_seed(self.model_config.seed)
170
171 def load_model(self):
154 - self.model_runner.load_model()
172 + if self.vllm_config.model_config.enable_sleep_mode:
173 + allocator = CuMemAllocator.get_instance()
174 + assert allocator.get_current_usage() == 0, (
175 + "Sleep mode can only be "
176 + "used for one instance per process.")
177 + context = allocator.use_memory_pool(tag="weights")
178 + else:
179 + from contextlib import nullcontext
180 + context = nullcontext()
181 + with context:
182 + self.model_runner.load_model()
183
184 def save_sharded_state(
185 self,
@@ -270,7 +298,14 @@ class Worker(LocalOrDistributedWorkerBase):
298 self.cache_config.num_gpu_blocks = num_gpu_blocks
299 self.cache_config.num_cpu_blocks = num_cpu_blocks
300
273 - self._init_cache_engine()
301 + if self.vllm_config.model_config.enable_sleep_mode:
302 + allocator = CuMemAllocator.get_instance()
303 + context = allocator.use_memory_pool(tag="kv_cache")
304 + else:
305 + from contextlib import nullcontext
306 + context = nullcontext()
307 + with context:
308 + self._init_cache_engine()
309 self._warm_up_model()
310
311 def _init_cache_engine(self):