A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[CI] Move quantization cpu offload tests out of fastcheck (#7574)
Michael Goin committed
Aug 16, 2024 at 00:16 UTC
e165528778d1bfeb8e9bd8a33d6cd64fb6c78e4e
2 files changed
+59
-55
tests/basic_correctness/test_cpu_offload.py
-55
@@ -1,61 +1,6 @@
1
-import pytest
2
-
3
-from tests.quantization.utils import is_quant_method_supported
4
-
1
from ..utils import compare_two_settings
2
3
4
def test_cpu_offload():
5
compare_two_settings("meta-llama/Llama-2-7b-hf", [],
6
["--cpu-offload-gb", "4"])
11
-
12
-
13
-@pytest.mark.skipif(not is_quant_method_supported("fp8"),
14
- reason="fp8 is not supported on this GPU type.")
15
-def test_cpu_offload_fp8():
16
- # Test quantization of an unquantized checkpoint
17
- compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
18
- ["--quantization", "fp8"],
19
- ["--quantization", "fp8", "--cpu-offload-gb", "2"])
20
- # Test loading a quantized checkpoint
21
- compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
22
- ["--cpu-offload-gb", "2"])
23
-
24
-
25
-@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
26
- reason="gptq_marlin is not supported on this GPU type.")
27
-def test_cpu_offload_gptq():
28
- # Test GPTQ Marlin
29
- compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
30
- ["--cpu-offload-gb", "1"])
31
- # Test GPTQ
32
- compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
33
- ["--quantization", "gptq"],
34
- ["--quantization", "gptq", "--cpu-offload-gb", "1"])
35
-
36
-
37
-@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
38
- reason="awq_marlin is not supported on this GPU type.")
39
-def test_cpu_offload_awq():
40
- # Test AWQ Marlin
41
- compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
42
- ["--cpu-offload-gb", "1"])
43
- # Test AWQ
44
- compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
45
- ["--quantization", "awq"],
46
- ["--quantization", "awq", "--cpu-offload-gb", "1"])
47
-
48
-
49
-@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
50
- reason="gptq_marlin is not supported on this GPU type.")
51
-def test_cpu_offload_compressed_tensors():
52
- # Test wNa16
53
- compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
54
- ["--cpu-offload-gb", "1"])
55
- # Test w4a16_marlin24
56
- compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
57
- [], ["--cpu-offload-gb", "1"])
58
- # Test w8a8
59
- compare_two_settings(
60
- "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
61
- ["--cpu-offload-gb", "1"])
tests/quantization/test_cpu_offload.py
new
+59
@@ -0,0 +1,59 @@
1
+# Expanded quantized model tests for CPU offloading
2
+# Base tests: tests/basic_correctness/test_cpu_offload.py
3
+
4
+import pytest
5
+
6
+from tests.quantization.utils import is_quant_method_supported
7
+
8
+from ..utils import compare_two_settings
9
+
10
+
11
+@pytest.mark.skipif(not is_quant_method_supported("fp8"),
12
+ reason="fp8 is not supported on this GPU type.")
13
+def test_cpu_offload_fp8():
14
+ # Test quantization of an unquantized checkpoint
15
+ compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
16
+ ["--quantization", "fp8"],
17
+ ["--quantization", "fp8", "--cpu-offload-gb", "2"])
18
+ # Test loading a quantized checkpoint
19
+ compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
20
+ ["--cpu-offload-gb", "2"])
21
+
22
+
23
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
24
+ reason="gptq_marlin is not supported on this GPU type.")
25
+def test_cpu_offload_gptq():
26
+ # Test GPTQ Marlin
27
+ compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
28
+ ["--cpu-offload-gb", "1"])
29
+ # Test GPTQ
30
+ compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
31
+ ["--quantization", "gptq"],
32
+ ["--quantization", "gptq", "--cpu-offload-gb", "1"])
33
+
34
+
35
+@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
36
+ reason="awq_marlin is not supported on this GPU type.")
37
+def test_cpu_offload_awq():
38
+ # Test AWQ Marlin
39
+ compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
40
+ ["--cpu-offload-gb", "1"])
41
+ # Test AWQ
42
+ compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
43
+ ["--quantization", "awq"],
44
+ ["--quantization", "awq", "--cpu-offload-gb", "1"])
45
+
46
+
47
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
48
+ reason="gptq_marlin is not supported on this GPU type.")
49
+def test_cpu_offload_compressed_tensors():
50
+ # Test wNa16
51
+ compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
52
+ ["--cpu-offload-gb", "1"])
53
+ # Test w4a16_marlin24
54
+ compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
55
+ [], ["--cpu-offload-gb", "1"])
56
+ # Test w8a8
57
+ compare_two_settings(
58
+ "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
59
+ ["--cpu-offload-gb", "1"])