@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e165528778
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[CI] Move quantization cpu offload tests out of fastcheck (#7574)

Michael Goin committed Aug 16, 2024 at 00:16 UTC e165528778d1bfeb8e9bd8a33d6cd64fb6c78e4e
2 files changed +59 -55
tests/basic_correctness/test_cpu_offload.py
-55
@@ -1,61 +1,6 @@
1 -import pytest
2 -
3 -from tests.quantization.utils import is_quant_method_supported
4 -
1 from ..utils import compare_two_settings
2
3
4 def test_cpu_offload():
5 compare_two_settings("meta-llama/Llama-2-7b-hf", [],
6 ["--cpu-offload-gb", "4"])
11 -
12 -
13 -@pytest.mark.skipif(not is_quant_method_supported("fp8"),
14 - reason="fp8 is not supported on this GPU type.")
15 -def test_cpu_offload_fp8():
16 - # Test quantization of an unquantized checkpoint
17 - compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
18 - ["--quantization", "fp8"],
19 - ["--quantization", "fp8", "--cpu-offload-gb", "2"])
20 - # Test loading a quantized checkpoint
21 - compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
22 - ["--cpu-offload-gb", "2"])
23 -
24 -
25 -@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
26 - reason="gptq_marlin is not supported on this GPU type.")
27 -def test_cpu_offload_gptq():
28 - # Test GPTQ Marlin
29 - compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
30 - ["--cpu-offload-gb", "1"])
31 - # Test GPTQ
32 - compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
33 - ["--quantization", "gptq"],
34 - ["--quantization", "gptq", "--cpu-offload-gb", "1"])
35 -
36 -
37 -@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
38 - reason="awq_marlin is not supported on this GPU type.")
39 -def test_cpu_offload_awq():
40 - # Test AWQ Marlin
41 - compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
42 - ["--cpu-offload-gb", "1"])
43 - # Test AWQ
44 - compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
45 - ["--quantization", "awq"],
46 - ["--quantization", "awq", "--cpu-offload-gb", "1"])
47 -
48 -
49 -@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
50 - reason="gptq_marlin is not supported on this GPU type.")
51 -def test_cpu_offload_compressed_tensors():
52 - # Test wNa16
53 - compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
54 - ["--cpu-offload-gb", "1"])
55 - # Test w4a16_marlin24
56 - compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
57 - [], ["--cpu-offload-gb", "1"])
58 - # Test w8a8
59 - compare_two_settings(
60 - "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
61 - ["--cpu-offload-gb", "1"])
tests/quantization/test_cpu_offload.py new
+59
@@ -0,0 +1,59 @@
1 +# Expanded quantized model tests for CPU offloading
2 +# Base tests: tests/basic_correctness/test_cpu_offload.py
3 +
4 +import pytest
5 +
6 +from tests.quantization.utils import is_quant_method_supported
7 +
8 +from ..utils import compare_two_settings
9 +
10 +
11 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
12 + reason="fp8 is not supported on this GPU type.")
13 +def test_cpu_offload_fp8():
14 + # Test quantization of an unquantized checkpoint
15 + compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
16 + ["--quantization", "fp8"],
17 + ["--quantization", "fp8", "--cpu-offload-gb", "2"])
18 + # Test loading a quantized checkpoint
19 + compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
20 + ["--cpu-offload-gb", "2"])
21 +
22 +
23 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
24 + reason="gptq_marlin is not supported on this GPU type.")
25 +def test_cpu_offload_gptq():
26 + # Test GPTQ Marlin
27 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
28 + ["--cpu-offload-gb", "1"])
29 + # Test GPTQ
30 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
31 + ["--quantization", "gptq"],
32 + ["--quantization", "gptq", "--cpu-offload-gb", "1"])
33 +
34 +
35 +@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
36 + reason="awq_marlin is not supported on this GPU type.")
37 +def test_cpu_offload_awq():
38 + # Test AWQ Marlin
39 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
40 + ["--cpu-offload-gb", "1"])
41 + # Test AWQ
42 + compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
43 + ["--quantization", "awq"],
44 + ["--quantization", "awq", "--cpu-offload-gb", "1"])
45 +
46 +
47 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
48 + reason="gptq_marlin is not supported on this GPU type.")
49 +def test_cpu_offload_compressed_tensors():
50 + # Test wNa16
51 + compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
52 + ["--cpu-offload-gb", "1"])
53 + # Test w4a16_marlin24
54 + compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
55 + [], ["--cpu-offload-gb", "1"])
56 + # Test w8a8
57 + compare_two_settings(
58 + "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
59 + ["--cpu-offload-gb", "1"])