A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Support cpu offloading with fp8 quantization (#6960)
Michael Goin committed
Jul 31, 2024 at 15:47 UTC
460c1884e3cb781730f85cb5591a85d5864bdac8
3 files changed
+116
-33
tests/basic_correctness/test_cpu_offload.py
+37
-6
@@ -1,4 +1,6 @@
1
-from vllm.utils import is_hip
1
+import pytest
2
+
3
+from tests.quantization.utils import is_quant_method_supported
4
5
from ..utils import compare_two_settings
6
@@ -6,8 +8,37 @@ from ..utils import compare_two_settings
8
def test_cpu_offload():
9
compare_two_settings("meta-llama/Llama-2-7b-hf", [],
10
["--cpu-offload-gb", "4"])
9
- if not is_hip():
10
- # compressed-tensors quantization is currently not supported in ROCm.
11
- compare_two_settings(
12
- "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t", [],
13
- ["--cpu-offload-gb", "1"])
11
+
12
+
13
+@pytest.mark.skipif(not is_quant_method_supported("fp8"),
14
+ reason="fp8 is not supported on this GPU type.")
15
+def test_cpu_offload_fp8():
16
+ # Test quantization of an unquantized checkpoint
17
+ compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
18
+ ["--quantization", "fp8"],
19
+ ["--quantization", "fp8", "--cpu-offload-gb", "2"])
20
+ # Test loading a quantized checkpoint
21
+ compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
22
+ ["--cpu-offload-gb", "2"])
23
+
24
+
25
+@pytest.mark.skipif(not is_quant_method_supported("awq"),
26
+ reason="awq is not supported on this GPU type.")
27
+def test_cpu_offload_awq():
28
+ compare_two_settings("casperhansen/llama-3-8b-instruct-awq", [],
29
+ ["--cpu-offload-gb", "2"])
30
+
31
+
32
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
33
+ reason="gptq_marlin is not supported on this GPU type.")
34
+def test_cpu_offload_compressed_tensors():
35
+ # Test wNa16
36
+ compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
37
+ ["--cpu-offload-gb", "1"])
38
+ # Test w4a16_marlin24
39
+ compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
40
+ [], ["--cpu-offload-gb", "1"])
41
+ # Test w8a8
42
+ compare_two_settings(
43
+ "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
44
+ ["--cpu-offload-gb", "1"])
vllm/model_executor/model_loader/loader.py
+53
-3
@@ -7,6 +7,7 @@ import json
7
import math
8
import os
9
from abc import ABC, abstractmethod
10
+from contextlib import contextmanager
11
from typing import Any, Dict, Generator, List, Optional, Tuple, Type
12
13
import huggingface_hub
@@ -37,7 +38,49 @@ from vllm.model_executor.models.interfaces import (has_inner_state,
38
supports_vision)
39
from vllm.model_executor.utils import set_weight_attrs
40
from vllm.platforms import current_platform
40
-from vllm.utils import is_tpu
41
+from vllm.utils import is_pin_memory_available, is_tpu
42
+
43
+
44
+@contextmanager
45
+def device_loading_context(module: torch.nn.Module,
46
+ target_device: torch.device):
47
+ if target_device.type == "cpu":
48
+ # If target is CPU, no need to move anything
49
+ yield module
50
+ return
51
+
52
+ original_device_states: Dict[str, torch.device] = {}
53
+
54
+ # Store original device states and move parameters to GPU if they're on CPU
55
+ for name, p in module.named_parameters():
56
+ if p.device.type == "cpu":
57
+ original_device_states[name] = p.device
58
+ p.data = p.data.to(target_device)
59
+ # Parameters already on target device are not touched
60
+
61
+ try:
62
+ yield module
63
+
64
+ finally:
65
+ # Restore parameters to their original devices, ignoring new parameters
66
+ pin_memory = is_pin_memory_available()
67
+ for name, p in module.named_parameters():
68
+ if name in original_device_states:
69
+ original_device: torch.device = original_device_states[name]
70
+ if original_device.type == "cpu":
71
+ # `torch.empty_like` does not support `pin_memory` argument
72
+ cpu_data = torch.empty_strided(size=p.data.size(),
73
+ stride=p.data.stride(),
74
+ dtype=p.data.dtype,
75
+ layout=p.data.layout,
76
+ device="cpu",
77
+ pin_memory=pin_memory)
78
+ cpu_data.copy_(p.data)
79
+ p.data = cpu_data
80
+ else:
81
+ p.data = p.data.to(original_device)
82
+ # New parameters or parameters already on target device are untouched
83
+
84
85
logger = init_logger(__name__)
86
@@ -275,8 +318,9 @@ class DefaultModelLoader(BaseModelLoader):
318
parallel_config: ParallelConfig,
319
scheduler_config: SchedulerConfig,
320
cache_config: CacheConfig) -> nn.Module:
321
+ target_device = torch.device(device_config.device)
322
with set_default_torch_dtype(model_config.dtype):
279
- with torch.device(device_config.device):
323
+ with target_device:
324
model = _initialize_model(model_config, self.load_config,
325
lora_config, multimodal_config,
326
cache_config, scheduler_config)
@@ -291,7 +335,13 @@ class DefaultModelLoader(BaseModelLoader):
335
for _, module in model.named_modules():
336
quant_method = getattr(module, "quant_method", None)
337
if quant_method is not None:
294
- quant_method.process_weights_after_loading(module)
338
+ # When quant methods need to process weights after loading
339
+ # (for repacking, quantizing, etc), they expect parameters
340
+ # to be on the global target device. This scope is for the
341
+ # case where cpu offloading is used, where we will move the
342
+ # parameters onto device for processing and back off after.
343
+ with device_loading_context(module, target_device):
344
+ quant_method.process_weights_after_loading(module)
345
return model.eval()
346
347
vllm/model_executor/models/utils.py
+26
-24
@@ -87,6 +87,7 @@ def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module:
87
88
# offload parameters to CPU
89
# use pin_memory if possible, which helps cudagraph capture speed
90
+ offloaded_parameters = False
91
for p in module.parameters():
92
if _CPU_OFFLOAD_BYTES >= _CPU_OFFLOAD_MAX_BYTES:
93
# we use per-parameter offloading
@@ -94,35 +95,36 @@ def maybe_offload_to_cpu(module: torch.nn.Module) -> torch.nn.Module:
95
break
96
97
# `torch.empty_like` does not support `pin_memory` argument
97
- cpu_data = torch.empty(size=p.data.size(),
98
- dtype=p.data.dtype,
99
- layout=p.data.layout,
100
- device='cpu',
101
- pin_memory=pin_memory)
98
+ cpu_data = torch.empty_strided(size=p.data.size(),
99
+ stride=p.data.stride(),
100
+ dtype=p.data.dtype,
101
+ layout=p.data.layout,
102
+ device='cpu',
103
+ pin_memory=pin_memory)
104
cpu_data.copy_(p.data)
105
p.data = cpu_data
106
_CPU_OFFLOAD_BYTES += p.data.numel() * p.data.element_size()
107
+ offloaded_parameters = True
108
+
109
+ if offloaded_parameters:
110
+ original_forward = module.forward
111
+
112
+ def forward(*args, **kwargs):
113
+ module.forward = original_forward
114
+ device_state = {
115
+ # here we blindly call `to(device)`
116
+ # if the parameter is already on the device, it will be a no-op
117
+ k: v.to(device, non_blocking=True)
118
+ for k, v in module.state_dict().items()
119
+ }
120
+ output = functional_call(module,
121
+ device_state,
122
+ args=args,
123
+ kwargs=kwargs)
124
+ module.forward = forward
125
+ return output
126
106
- state_dict: Dict[str, torch.Tensor] = module.state_dict()
107
-
108
- original_forward = module.forward
109
-
110
- def forward(*args, **kwargs):
111
- module.forward = original_forward
112
- device_state = {
113
- # here we blindly call `to(device)`
114
- # if the parameter is already on the device, it will be a no-op
115
- k: v.to(device, non_blocking=True)
116
- for k, v in state_dict.items()
117
- }
118
- output = functional_call(module,
119
- device_state,
120
- args=args,
121
- kwargs=kwargs)
127
module.forward = forward
123
- return output
124
-
125
- module.forward = forward
128
129
return module
130