@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c27d25fb5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[core][model] yet another cpu offload implementation (#6496)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

youkaichao committed Jul 17, 2024 at 20:54 UTC 1c27d25fb57285d2c5bb6d17d70549ab4b8f45a7
7 files changed +128 -4
.buildkite/test-pipeline.yaml
+1
@@ -140,6 +140,7 @@ steps:
140 # install tensorizer for tensorize_vllm_model.py
141 - pip install awscli tensorizer
142 - python3 offline_inference.py
143 + - python3 cpu_offload.py
144 - python3 offline_inference_with_prefix.py
145 - python3 llm_engine_example.py
146 - python3 llava_example.py
examples/cpu_offload.py new
+22
@@ -0,0 +1,22 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +# Sample prompts.
4 +prompts = [
5 + "Hello, my name is",
6 + "The president of the United States is",
7 + "The capital of France is",
8 + "The future of AI is",
9 +]
10 +# Create a sampling params object.
11 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12 +
13 +# Create an LLM.
14 +llm = LLM(model="meta-llama/Llama-2-13b-chat-hf", cpu_offload_gb=10)
15 +# Generate texts from the prompts. The output is a list of RequestOutput objects
16 +# that contain the prompt, generated text, and other information.
17 +outputs = llm.generate(prompts, sampling_params)
18 +# Print the outputs.
19 +for output in outputs:
20 + prompt = output.prompt
21 + generated_text = output.outputs[0].text
22 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/config.py
+2
@@ -433,6 +433,7 @@ class CacheConfig:
433 num_gpu_blocks_override: Optional[int] = None,
434 sliding_window: Optional[int] = None,
435 enable_prefix_caching: bool = False,
436 + cpu_offload_gb: float = 0,
437 ) -> None:
438 self.block_size = block_size
439 self.gpu_memory_utilization = gpu_memory_utilization
@@ -441,6 +442,7 @@ class CacheConfig:
442 self.cache_dtype = cache_dtype
443 self.sliding_window = sliding_window
444 self.enable_prefix_caching = enable_prefix_caching
445 + self.cpu_offload_gb = cpu_offload_gb
446 self._verify_args()
447 self._verify_cache_dtype()
448 self._verify_prefix_caching()
vllm/engine/arg_utils.py
+23 -1
@@ -45,6 +45,7 @@ class EngineArgs:
45 disable_sliding_window: bool = False
46 use_v2_block_manager: bool = False
47 swap_space: int = 4 # GiB
48 + cpu_offload_gb: int = 0 # GiB
49 gpu_memory_utilization: float = 0.90
50 max_num_batched_tokens: Optional[int] = None
51 max_num_seqs: int = 256
@@ -303,6 +304,20 @@ class EngineArgs:
304 type=int,
305 default=EngineArgs.swap_space,
306 help='CPU swap space size (GiB) per GPU.')
307 + parser.add_argument(
308 + '--cpu-offload-gb',
309 + type=float,
310 + default=0,
311 + help='The space in GiB to offload to CPU, per GPU. '
312 + 'Default is 0, which means no offloading. Intuitively, '
313 + 'this argument can be seen as a virtual way to increase '
314 + 'the GPU memory size. For example, if you have one 24 GB '
315 + 'GPU and set this to 10, virtually you can think of it as '
316 + 'a 34 GB GPU. Then you can load a 13B model with BF16 weight,'
317 + 'which requires at least 26GB GPU memory. Note that this '
318 + 'requires fast CPU-GPU interconnect, as part of the model is'
319 + 'loaded from CPU memory to GPU memory on the fly in each '
320 + 'model forward pass.')
321 parser.add_argument(
322 '--gpu-memory-utilization',
323 type=float,
@@ -633,6 +648,11 @@ class EngineArgs:
648 raise ValueError(
649 "BitsAndBytes load format and QLoRA adapter only support "
650 f"'bitsandbytes' quantization, but got {self.quantization}")
651 +
652 + assert self.cpu_offload_gb >= 0, (
653 + "CPU offload space must be non-negative"
654 + f", but got {self.cpu_offload_gb}")
655 +
656 multimodal_config = MultiModalConfig()
657
658 device_config = DeviceConfig(device=self.device)
@@ -666,7 +686,9 @@ class EngineArgs:
686 cache_dtype=self.kv_cache_dtype,
687 num_gpu_blocks_override=self.num_gpu_blocks_override,
688 sliding_window=model_config.get_sliding_window(),
669 - enable_prefix_caching=self.enable_prefix_caching)
689 + enable_prefix_caching=self.enable_prefix_caching,
690 + cpu_offload_gb=self.cpu_offload_gb,
691 + )
692 parallel_config = ParallelConfig(
693 pipeline_parallel_size=self.pipeline_parallel_size,
694 tensor_parallel_size=self.tensor_parallel_size,
vllm/entrypoints/llm.py
+6
@@ -69,6 +69,10 @@ class LLM:
69 when their `best_of` sampling parameters are larger than 1. If all
70 requests will have `best_of=1`, you can safely set this to 0.
71 Otherwise, too small values may cause out-of-memory (OOM) errors.
72 + cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
73 + the model weights. This virtually increases the GPU memory space
74 + you can use to hold the model weights, at the cost of CPU-GPU data
75 + transfer for every forward pass.
76 enforce_eager: Whether to enforce eager execution. If True, we will
77 disable CUDA graph and always execute the model in eager mode.
78 If False, we will use CUDA graph and eager execution in hybrid.
@@ -114,6 +118,7 @@ class LLM:
118 seed: int = 0,
119 gpu_memory_utilization: float = 0.9,
120 swap_space: int = 4,
121 + cpu_offload_gb: float = 0,
122 enforce_eager: bool = False,
123 max_context_len_to_capture: Optional[int] = None,
124 max_seq_len_to_capture: int = 8192,
@@ -141,6 +146,7 @@ class LLM:
146 seed=seed,
147 gpu_memory_utilization=gpu_memory_utilization,
148 swap_space=swap_space,
149 + cpu_offload_gb=cpu_offload_gb,
150 enforce_eager=enforce_eager,
151 max_context_len_to_capture=max_context_len_to_capture,
152 max_seq_len_to_capture=max_seq_len_to_capture,
vllm/model_executor/models/utils.py
+70 -3
@@ -1,8 +1,10 @@
1 from typing import Callable, Dict, List, Tuple
2
3 import torch
4 +from torch.func import functional_call
5
6 from vllm.multimodal import BatchedTensors
7 +from vllm.utils import is_pin_memory_available
8
9
10 def merge_vision_embeddings(input_ids: torch.Tensor,
@@ -52,6 +54,70 @@ class PPMissingLayer(torch.nn.Identity):
54 super().__init__()
55
56
57 +_CPU_OFFLOAD_BYTES = 0
58 +_CPU_OFFLOAD_MAX_BYTES = 0
59 +
60 +
61 +def set_cpu_offload_max_bytes(max_bytes: int) -> None:
62 + global _CPU_OFFLOAD_MAX_BYTES, _CPU_OFFLOAD_BYTES
63 + _CPU_OFFLOAD_BYTES = 0
64 + _CPU_OFFLOAD_MAX_BYTES = max_bytes
65 +
66 +
67 +def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module:
68 + device = next(module.parameters()).device
69 +
70 + if device == torch.device("cpu"):
71 + return module
72 +
73 + global _CPU_OFFLOAD_MAX_BYTES, _CPU_OFFLOAD_BYTES
74 + if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES:
75 + return module
76 +
77 + pin_memory = is_pin_memory_available()
78 +
79 + # offload parameters to CPU
80 + # use pin_memory if possible, which helps cudagraph capture speed
81 + for p in module.parameters():
82 + if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES:
83 + # we use per-parameter offloading
84 + # one module might have some parameters offloaded and some not
85 + break
86 +
87 + # `torch.empty_like` does not support `pin_memory` argument
88 + cpu_data = torch.empty(size=p.data.size(),
89 + dtype=p.data.dtype,
90 + layout=p.data.layout,
91 + device='cpu',
92 + pin_memory=pin_memory)
93 + cpu_data.copy_(p.data)
94 + p.data = cpu_data
95 + _CPU_OFFLOAD_BYTES += p.data.numel() * p.data.element_size()
96 +
97 + state_dict: Dict[str, torch.Tensor] = module.state_dict()
98 +
99 + original_forward = module.forward
100 +
101 + def forward(*args, **kwargs):
102 + module.forward = original_forward
103 + device_state = {
104 + # here we blindly call `to(device)`
105 + # if the parameter is already on the device, it will be a no-op
106 + k: v.to(device, non_blocking=True)
107 + for k, v in state_dict.items()
108 + }
109 + output = functional_call(module,
110 + device_state,
111 + args=args,
112 + kwargs=kwargs)
113 + module.forward = forward
114 + return output
115 +
116 + module.forward = forward
117 +
118 + return module
119 +
120 +
121 def make_layers(
122 num_hidden_layers: int, layer_fn: Callable[[], torch.nn.Module]
123 ) -> Tuple[int, int, torch.nn.ModuleList]:
@@ -64,9 +130,10 @@ def make_layers(
130 get_pp_group().rank_in_group,
131 get_pp_group().world_size)
132 modules = torch.nn.ModuleList(
67 - [PPMissingLayer() for _ in range(start_layer)] +
68 - [layer_fn() for _ in range(start_layer, end_layer)] +
69 - [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
133 + [PPMissingLayer() for _ in range(start_layer)] + [
134 + maybe_offload_to_cpu(layer_fn())
135 + for _ in range(start_layer, end_layer)
136 + ] + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)])
137 return start_layer, end_layer, modules
138
139
vllm/worker/model_runner.py
+4
@@ -39,6 +39,7 @@ from vllm.model_executor.model_loader import get_model
39 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
40 from vllm.model_executor.models.interfaces import (supports_lora,
41 supports_vision)
42 +from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
43 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
44 MultiModalInputs)
45 from vllm.prompt_adapter.layers import PromptAdapterMapping
@@ -544,6 +545,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
545 self.flashinfer_prefill_workspace_buffer = None
546 self.flashinfer_prefill_wrapper = None
547
548 + set_cpu_offload_max_bytes(
549 + int(self.cache_config.cpu_offload_gb * 1024**3))
550 +
551 def load_model(self) -> None:
552 with CudaMemoryProfiler() as m:
553 self.model = get_model(model_config=self.model_config,