@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f9a5600649
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix GPTQ and GPTQ Marlin CPU Offloading (#7225)

Michael Goin committed Aug 6, 2024 at 21:34 UTC f9a56006497999a0ab8e32289f4fddabf2ed69bd
4 files changed +33 -14
tests/basic_correctness/test_cpu_offload.py
+21 -4
@@ -22,11 +22,28 @@ def test_cpu_offload_fp8():
22 ["--cpu-offload-gb", "2"])
23
24
25 -@pytest.mark.skipif(not is_quant_method_supported("awq"),
26 - reason="awq is not supported on this GPU type.")
25 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
26 + reason="gptq_marlin is not supported on this GPU type.")
27 +def test_cpu_offload_gptq():
28 + # Test GPTQ Marlin
29 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
30 + ["--cpu-offload-gb", "1"])
31 + # Test GPTQ
32 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
33 + ["--quantization", "gptq"],
34 + ["--quantization", "gptq", "--cpu-offload-gb", "1"])
35 +
36 +
37 +@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
38 + reason="awq_marlin is not supported on this GPU type.")
39 def test_cpu_offload_awq():
28 - compare_two_settings("casperhansen/llama-3-8b-instruct-awq", [],
29 - ["--cpu-offload-gb", "2"])
40 + # Test AWQ Marlin
41 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
42 + ["--cpu-offload-gb", "1"])
43 + # Test AWQ
44 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
45 + ["--quantization", "awq"],
46 + ["--quantization", "awq", "--cpu-offload-gb", "1"])
47
48
49 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
tests/utils.py
+3 -2
@@ -266,8 +266,9 @@ def compare_two_settings(model: str,
266 arg1_results = results[:n]
267 arg2_results = results[n:]
268 for arg1_result, arg2_result in zip(arg1_results, arg2_results):
269 - assert arg1_result == arg2_result, \
270 - f"Results for {model=} are not the same with {arg1=} and {arg2=}"
269 + assert arg1_result == arg2_result, (
270 + f"Results for {model=} are not the same with {arg1=} and {arg2=}. "
271 + f"{arg1_result=} != {arg2_result=}")
272
273
274 def init_test_distributed_environment(
vllm/model_executor/layers/quantization/gptq.py
+9 -7
@@ -204,13 +204,7 @@ class GPTQLinearMethod(LinearMethodBase):
204
205 layer.exllama_state = exllama_state
206
207 - def apply(self,
208 - layer: torch.nn.Module,
209 - x: torch.Tensor,
210 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
211 - qweight = layer.qweight
212 - out_shape = x.shape[:-1] + (qweight.shape[-1], )
213 - reshaped_x = x.reshape(-1, x.shape[-1])
207 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
208 # exllama needs to shuffle the weight after the weight is loaded
209 # here we do the shuffle on first forward pass
210 if layer.exllama_state == ExllamaState.UNINITIALIZED:
@@ -222,6 +216,14 @@ class GPTQLinearMethod(LinearMethodBase):
216 layer.exllama_state = ExllamaState.READY
217 ops.gptq_shuffle(layer.qweight, layer.g_idx,
218 self.quant_config.weight_bits)
219 +
220 + def apply(self,
221 + layer: torch.nn.Module,
222 + x: torch.Tensor,
223 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
224 + out_shape = x.shape[:-1] + (layer.qweight.shape[-1], )
225 + reshaped_x = x.reshape(-1, x.shape[-1])
226 +
227 output = ops.gptq_gemm(reshaped_x, layer.qweight, layer.qzeros,
228 layer.scales, layer.g_idx,
229 layer.exllama_state == ExllamaState.READY,
vllm/model_executor/layers/quantization/gptq_marlin.py
-1
@@ -251,7 +251,6 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
251 scales_and_zp_size,
252 output_size_per_partition // self.quant_config.pack_factor,
253 dtype=torch.int32,
254 - device="meta",
254 ),
255 requires_grad=False,
256 )