A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[CI] Fix failing FP8 cpu offload test (#13170)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Feb 12, 2025 at 14:16 UTC
14b7899d10217b31d98ba78ade7768fbd735ca4e
1 file changed
+6
-6
tests/quantization/test_cpu_offload.py
+6
-6
@@ -1,5 +1,5 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
# Expanded quantized model tests for CPU offloading
4
# Base tests: tests/basic_correctness/test_cpu_offload.py
5
@@ -14,13 +14,13 @@ from ..utils import compare_two_settings
14
reason="fp8 is not supported on this GPU type.")
15
def test_cpu_offload_fp8():
16
# Test quantization of an unquantized checkpoint
17
- compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
17
+ compare_two_settings("meta-llama/Llama-3.2-1B-Instruct",
18
["--quantization", "fp8"],
19
- ["--quantization", "fp8", "--cpu-offload-gb", "2"],
19
+ ["--quantization", "fp8", "--cpu-offload-gb", "1"],
20
max_wait_seconds=480)
21
# Test loading a quantized checkpoint
22
- compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
23
- ["--cpu-offload-gb", "2"],
22
+ compare_two_settings("neuralmagic/Qwen2-1.5B-Instruct-FP8", [],
23
+ ["--cpu-offload-gb", "1"],
24
max_wait_seconds=480)
25
26