@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e51b6a626
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[ci][test] adjust max wait time for cpu offloading test (#7709)

youkaichao committed Aug 20, 2024 at 17:12 UTC 9e51b6a626e6ddd16abbe6d198960d5d1cfbd063
2 files changed +34 -22
tests/quantization/test_cpu_offload.py
+18 -9
@@ -14,10 +14,12 @@ def test_cpu_offload_fp8():
14 # Test quantization of an unquantized checkpoint
15 compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
16 ["--quantization", "fp8"],
17 - ["--quantization", "fp8", "--cpu-offload-gb", "2"])
17 + ["--quantization", "fp8", "--cpu-offload-gb", "2"],
18 + max_wait_seconds=480)
19 # Test loading a quantized checkpoint
20 compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
20 - ["--cpu-offload-gb", "2"])
21 + ["--cpu-offload-gb", "2"],
22 + max_wait_seconds=480)
23
24
25 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
@@ -25,11 +27,13 @@ def test_cpu_offload_fp8():
27 def test_cpu_offload_gptq():
28 # Test GPTQ Marlin
29 compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
28 - ["--cpu-offload-gb", "1"])
30 + ["--cpu-offload-gb", "1"],
31 + max_wait_seconds=480)
32 # Test GPTQ
33 compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
34 ["--quantization", "gptq"],
32 - ["--quantization", "gptq", "--cpu-offload-gb", "1"])
35 + ["--quantization", "gptq", "--cpu-offload-gb", "1"],
36 + max_wait_seconds=480)
37
38
39 @pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
@@ -37,11 +41,13 @@ def test_cpu_offload_gptq():
41 def test_cpu_offload_awq():
42 # Test AWQ Marlin
43 compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
40 - ["--cpu-offload-gb", "1"])
44 + ["--cpu-offload-gb", "1"],
45 + max_wait_seconds=480)
46 # Test AWQ
47 compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
48 ["--quantization", "awq"],
44 - ["--quantization", "awq", "--cpu-offload-gb", "1"])
49 + ["--quantization", "awq", "--cpu-offload-gb", "1"],
50 + max_wait_seconds=480)
51
52
53 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
@@ -49,11 +55,14 @@ def test_cpu_offload_awq():
55 def test_cpu_offload_compressed_tensors():
56 # Test wNa16
57 compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
52 - ["--cpu-offload-gb", "1"])
58 + ["--cpu-offload-gb", "1"],
59 + max_wait_seconds=480)
60 # Test w4a16_marlin24
61 compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
55 - [], ["--cpu-offload-gb", "1"])
62 + [], ["--cpu-offload-gb", "1"],
63 + max_wait_seconds=480)
64 # Test w8a8
65 compare_two_settings(
66 "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
59 - ["--cpu-offload-gb", "1"])
67 + ["--cpu-offload-gb", "1"],
68 + max_wait_seconds=480)
tests/utils.py
+16 -13
@@ -56,16 +56,14 @@ VLLM_PATH = Path(__file__).parent.parent
56
57 class RemoteOpenAIServer:
58 DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
59 - MAX_START_WAIT_S = 240 # wait for server to start for 240 seconds
60 -
61 - def __init__(
62 - self,
63 - model: str,
64 - cli_args: List[str],
65 - *,
66 - env_dict: Optional[Dict[str, str]] = None,
67 - auto_port: bool = True,
68 - ) -> None:
59 +
60 + def __init__(self,
61 + model: str,
62 + cli_args: List[str],
63 + *,
64 + env_dict: Optional[Dict[str, str]] = None,
65 + auto_port: bool = True,
66 + max_wait_seconds: Optional[float] = None) -> None:
67 if auto_port:
68 if "-p" in cli_args or "--port" in cli_args:
69 raise ValueError("You have manually specified the port"
@@ -90,8 +88,9 @@ class RemoteOpenAIServer:
88 env=env,
89 stdout=sys.stdout,
90 stderr=sys.stderr)
91 + max_wait_seconds = max_wait_seconds or 240
92 self._wait_for_server(url=self.url_for("health"),
94 - timeout=self.MAX_START_WAIT_S)
93 + timeout=max_wait_seconds)
94
95 def __enter__(self):
96 return self
@@ -145,7 +144,8 @@ def compare_two_settings(model: str,
144 arg1: List[str],
145 arg2: List[str],
146 env1: Optional[Dict[str, str]] = None,
148 - env2: Optional[Dict[str, str]] = None):
147 + env2: Optional[Dict[str, str]] = None,
148 + max_wait_seconds: Optional[float] = None) -> None:
149 """
150 Launch API server with two different sets of arguments/environments
151 and compare the results of the API calls.
@@ -164,7 +164,10 @@ def compare_two_settings(model: str,
164 token_ids = tokenizer(prompt)["input_ids"]
165 results = []
166 for args, env in ((arg1, env1), (arg2, env2)):
167 - with RemoteOpenAIServer(model, args, env_dict=env) as server:
167 + with RemoteOpenAIServer(model,
168 + args,
169 + env_dict=env,
170 + max_wait_seconds=max_wait_seconds) as server:
171 client = server.get_client()
172
173 # test models list