@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14b7899d10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[CI] Fix failing FP8 cpu offload test (#13170)

Signed-off-by: mgoin <mgoin64@gmail.com>

Michael Goin committed Feb 12, 2025 at 14:16 UTC 14b7899d10217b31d98ba78ade7768fbd735ca4e
1 file changed +6 -6
tests/quantization/test_cpu_offload.py
+6 -6
@@ -1,5 +1,5 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 # Expanded quantized model tests for CPU offloading
4 # Base tests: tests/basic_correctness/test_cpu_offload.py
5
@@ -14,13 +14,13 @@ from ..utils import compare_two_settings
14 reason="fp8 is not supported on this GPU type.")
15 def test_cpu_offload_fp8():
16 # Test quantization of an unquantized checkpoint
17 - compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
17 + compare_two_settings("meta-llama/Llama-3.2-1B-Instruct",
18 ["--quantization", "fp8"],
19 - ["--quantization", "fp8", "--cpu-offload-gb", "2"],
19 + ["--quantization", "fp8", "--cpu-offload-gb", "1"],
20 max_wait_seconds=480)
21 # Test loading a quantized checkpoint
22 - compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
23 - ["--cpu-offload-gb", "2"],
22 + compare_two_settings("neuralmagic/Qwen2-1.5B-Instruct-FP8", [],
23 + ["--cpu-offload-gb", "1"],
24 max_wait_seconds=480)
25
26