A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[core][model] yet another cpu offload implementation (#6496)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
youkaichao committed
Jul 17, 2024 at 20:54 UTC
1c27d25fb57285d2c5bb6d17d70549ab4b8f45a7
7 files changed
+128
-4
.buildkite/test-pipeline.yaml
+1
@@ -140,6 +140,7 @@ steps:
140
# install tensorizer for tensorize_vllm_model.py
141
- pip install awscli tensorizer
142
- python3 offline_inference.py
143
+ - python3 cpu_offload.py
144
- python3 offline_inference_with_prefix.py
145
- python3 llm_engine_example.py
146
- python3 llava_example.py
examples/cpu_offload.py
new
+22
@@ -0,0 +1,22 @@
1
+from vllm import LLM, SamplingParams
2
+
3
+# Sample prompts.
4
+prompts = [
5
+ "Hello, my name is",
6
+ "The president of the United States is",
7
+ "The capital of France is",
8
+ "The future of AI is",
9
+]
10
+# Create a sampling params object.
11
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12
+
13
+# Create an LLM.
14
+llm = LLM(model="meta-llama/Llama-2-13b-chat-hf", cpu_offload_gb=10)
15
+# Generate texts from the prompts. The output is a list of RequestOutput objects
16
+# that contain the prompt, generated text, and other information.
17
+outputs = llm.generate(prompts, sampling_params)
18
+# Print the outputs.
19
+for output in outputs:
20
+ prompt = output.prompt
21
+ generated_text = output.outputs[0].text
22
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/config.py
+2
@@ -433,6 +433,7 @@ class CacheConfig:
433
num_gpu_blocks_override: Optional[int] = None,
434
sliding_window: Optional[int] = None,
435
enable_prefix_caching: bool = False,
436
+ cpu_offload_gb: float = 0,
437
) -> None:
438
self.block_size = block_size
439
self.gpu_memory_utilization = gpu_memory_utilization
@@ -441,6 +442,7 @@ class CacheConfig:
442
self.cache_dtype = cache_dtype
443
self.sliding_window = sliding_window
444
self.enable_prefix_caching = enable_prefix_caching
445
+ self.cpu_offload_gb = cpu_offload_gb
446
self._verify_args()
447
self._verify_cache_dtype()
448
self._verify_prefix_caching()
vllm/engine/arg_utils.py
+23
-1
@@ -45,6 +45,7 @@ class EngineArgs:
45
disable_sliding_window: bool = False
46
use_v2_block_manager: bool = False
47
swap_space: int = 4 # GiB
48
+ cpu_offload_gb: int = 0 # GiB
49
gpu_memory_utilization: float = 0.90
50
max_num_batched_tokens: Optional[int] = None
51
max_num_seqs: int = 256
@@ -303,6 +304,20 @@ class EngineArgs:
304
type=int,
305
default=EngineArgs.swap_space,
306
help='CPU swap space size (GiB) per GPU.')
307
+ parser.add_argument(
308
+ '--cpu-offload-gb',
309
+ type=float,
310
+ default=0,
311
+ help='The space in GiB to offload to CPU, per GPU. '
312
+ 'Default is 0, which means no offloading. Intuitively, '
313
+ 'this argument can be seen as a virtual way to increase '
314
+ 'the GPU memory size. For example, if you have one 24 GB '
315
+ 'GPU and set this to 10, virtually you can think of it as '
316
+ 'a 34 GB GPU. Then you can load a 13B model with BF16 weight,'
317
+ 'which requires at least 26GB GPU memory. Note that this '
318
+ 'requires fast CPU-GPU interconnect, as part of the model is'
319
+ 'loaded from CPU memory to GPU memory on the fly in each '
320
+ 'model forward pass.')
321
parser.add_argument(
322
'--gpu-memory-utilization',
323
type=float,
@@ -633,6 +648,11 @@ class EngineArgs:
648
raise ValueError(
649
"BitsAndBytes load format and QLoRA adapter only support "
650
f"'bitsandbytes' quantization, but got {self.quantization}")
651
+
652
+ assert self.cpu_offload_gb >= 0, (
653
+ "CPU offload space must be non-negative"
654
+ f", but got {self.cpu_offload_gb}")
655
+
656
multimodal_config = MultiModalConfig()
657
658
device_config = DeviceConfig(device=self.device)
@@ -666,7 +686,9 @@ class EngineArgs:
686
cache_dtype=self.kv_cache_dtype,
687
num_gpu_blocks_override=self.num_gpu_blocks_override,
688
sliding_window=model_config.get_sliding_window(),
669
- enable_prefix_caching=self.enable_prefix_caching)
689
+ enable_prefix_caching=self.enable_prefix_caching,
690
+ cpu_offload_gb=self.cpu_offload_gb,
691
+ )
692
parallel_config = ParallelConfig(
693
pipeline_parallel_size=self.pipeline_parallel_size,
694
tensor_parallel_size=self.tensor_parallel_size,
vllm/entrypoints/llm.py
+6
@@ -69,6 +69,10 @@ class LLM:
69
when their `best_of` sampling parameters are larger than 1. If all
70
requests will have `best_of=1`, you can safely set this to 0.
71
Otherwise, too small values may cause out-of-memory (OOM) errors.
72
+ cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
73
+ the model weights. This virtually increases the GPU memory space
74
+ you can use to hold the model weights, at the cost of CPU-GPU data
75
+ transfer for every forward pass.
76
enforce_eager: Whether to enforce eager execution. If True, we will
77
disable CUDA graph and always execute the model in eager mode.
78
If False, we will use CUDA graph and eager execution in hybrid.
@@ -114,6 +118,7 @@ class LLM:
118
seed: int = 0,
119
gpu_memory_utilization: float = 0.9,
120
swap_space: int = 4,
121
+ cpu_offload_gb: float = 0,
122
enforce_eager: bool = False,
123
max_context_len_to_capture: Optional[int] = None,
124
max_seq_len_to_capture: int = 8192,
@@ -141,6 +146,7 @@ class LLM:
146
seed=seed,
147
gpu_memory_utilization=gpu_memory_utilization,
148
swap_space=swap_space,
149
+ cpu_offload_gb=cpu_offload_gb,
150
enforce_eager=enforce_eager,
151
max_context_len_to_capture=max_context_len_to_capture,
152
max_seq_len_to_capture=max_seq_len_to_capture,
vllm/model_executor/models/utils.py
+70
-3
@@ -1,8 +1,10 @@
1
from typing import Callable, Dict, List, Tuple
2
3
import torch
4
+from torch.func import functional_call
5
6
from vllm.multimodal import BatchedTensors
7
+from vllm.utils import is_pin_memory_available
8
9
10
def merge_vision_embeddings(input_ids: torch.Tensor,
@@ -52,6 +54,70 @@ class PPMissingLayer(torch.nn.Identity):
54
super().__init__()
55
56
57
+_CPU_OFFLOAD_BYTES = 0
58
+_CPU_OFFLOAD_MAX_BYTES = 0
59
+
60
+
61
+def set_cpu_offload_max_bytes(max_bytes: int) -> None:
62
+ global _CPU_OFFLOAD_MAX_BYTES, _CPU_OFFLOAD_BYTES
63
+ _CPU_OFFLOAD_BYTES = 0
64
+ _CPU_OFFLOAD_MAX_BYTES = max_bytes
65
+
66
+
67
+def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module:
68
+ device = next(module.parameters()).device
69
+
70
+ if device == torch.device("cpu"):
71
+ return module
72
+
73
+ global _CPU_OFFLOAD_MAX_BYTES, _CPU_OFFLOAD_BYTES
74
+ if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES:
75
+ return module
76
+
77
+ pin_memory = is_pin_memory_available()
78
+
79
+ # offload parameters to CPU
80
+ # use pin_memory if possible, which helps cudagraph capture speed
81
+ for p in module.parameters():
82
+ if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES:
83
+ # we use per-parameter offloading
84
+ # one module might have some parameters offloaded and some not
85
+ break
86
+
87
+ # `torch.empty_like` does not support `pin_memory` argument
88
+ cpu_data = torch.empty(size=p.data.size(),
89
+ dtype=p.data.dtype,
90
+ layout=p.data.layout,
91
+ device='cpu',
92
+ pin_memory=pin_memory)
93
+ cpu_data.copy_(p.data)
94
+ p.data = cpu_data
95
+ _CPU_OFFLOAD_BYTES += p.data.numel() * p.data.element_size()
96
+
97
+ state_dict: Dict[str, torch.Tensor] = module.state_dict()
98
+
99
+ original_forward = module.forward
100
+
101
+ def forward(*args, **kwargs):
102
+ module.forward = original_forward
103
+ device_state = {
104
+ # here we blindly call `to(device)`
105
+ # if the parameter is already on the device, it will be a no-op
106
+ k: v.to(device, non_blocking=True)
107
+ for k, v in state_dict.items()
108
+ }
109
+ output = functional_call(module,
110
+ device_state,
111
+ args=args,
112
+ kwargs=kwargs)
113
+ module.forward = forward
114
+ return output
115
+
116
+ module.forward = forward
117
+
118
+ return module
119
+
120
+
121
def make_layers(
122
num_hidden_layers: int, layer_fn: Callable[[], torch.nn.Module]
123
) -> Tuple[int, int, torch.nn.ModuleList]:
@@ -64,9 +130,10 @@ def make_layers(
130
get_pp_group().rank_in_group,
131
get_pp_group().world_size)
132
modules = torch.nn.ModuleList(
67
- [PPMissingLayer() for _ in range(start_layer)] +
68
- [layer_fn() for _ in range(start_layer, end_layer)] +
69
- [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
133
+ [PPMissingLayer() for _ in range(start_layer)] + [
134
+ maybe_offload_to_cpu(layer_fn())
135
+ for _ in range(start_layer, end_layer)
136
+ ] + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
137
return start_layer, end_layer, modules
138
139
vllm/worker/model_runner.py
+4
@@ -39,6 +39,7 @@ from vllm.model_executor.model_loader import get_model
39
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
40
from vllm.model_executor.models.interfaces import (supports_lora,
41
supports_vision)
42
+from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
43
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
44
MultiModalInputs)
45
from vllm.prompt_adapter.layers import PromptAdapterMapping
@@ -544,6 +545,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
545
self.flashinfer_prefill_workspace_buffer = None
546
self.flashinfer_prefill_wrapper = None
547
548
+ set_cpu_offload_max_bytes(
549
+ int(self.cache_config.cpu_offload_gb * 1024**3))
550
+
551
def load_model(self) -> None:
552
with CudaMemoryProfiler() as m:
553
self.model = get_model(model_config=self.model_config,