A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[ci][test] adjust max wait time for cpu offloading test (#7709)
youkaichao committed
Aug 20, 2024 at 17:12 UTC
9e51b6a626e6ddd16abbe6d198960d5d1cfbd063
2 files changed
+34
-22
tests/quantization/test_cpu_offload.py
+18
-9
@@ -14,10 +14,12 @@ def test_cpu_offload_fp8():
14
# Test quantization of an unquantized checkpoint
15
compare_two_settings("meta-llama/Meta-Llama-3-8B-Instruct",
16
["--quantization", "fp8"],
17
- ["--quantization", "fp8", "--cpu-offload-gb", "2"])
17
+ ["--quantization", "fp8", "--cpu-offload-gb", "2"],
18
+ max_wait_seconds=480)
19
# Test loading a quantized checkpoint
20
compare_two_settings("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", [],
20
- ["--cpu-offload-gb", "2"])
21
+ ["--cpu-offload-gb", "2"],
22
+ max_wait_seconds=480)
23
24
25
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
@@ -25,11 +27,13 @@ def test_cpu_offload_fp8():
27
def test_cpu_offload_gptq():
28
# Test GPTQ Marlin
29
compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4", [],
28
- ["--cpu-offload-gb", "1"])
30
+ ["--cpu-offload-gb", "1"],
31
+ max_wait_seconds=480)
32
# Test GPTQ
33
compare_two_settings("Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
34
["--quantization", "gptq"],
32
- ["--quantization", "gptq", "--cpu-offload-gb", "1"])
35
+ ["--quantization", "gptq", "--cpu-offload-gb", "1"],
36
+ max_wait_seconds=480)
37
38
39
@pytest.mark.skipif(not is_quant_method_supported("awq_marlin"),
@@ -37,11 +41,13 @@ def test_cpu_offload_gptq():
41
def test_cpu_offload_awq():
42
# Test AWQ Marlin
43
compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ", [],
40
- ["--cpu-offload-gb", "1"])
44
+ ["--cpu-offload-gb", "1"],
45
+ max_wait_seconds=480)
46
# Test AWQ
47
compare_two_settings("Qwen/Qwen2-1.5B-Instruct-AWQ",
48
["--quantization", "awq"],
44
- ["--quantization", "awq", "--cpu-offload-gb", "1"])
49
+ ["--quantization", "awq", "--cpu-offload-gb", "1"],
50
+ max_wait_seconds=480)
51
52
53
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
@@ -49,11 +55,14 @@ def test_cpu_offload_awq():
55
def test_cpu_offload_compressed_tensors():
56
# Test wNa16
57
compare_two_settings("nm-testing/tinyllama-oneshot-w4a16-channel-v2", [],
52
- ["--cpu-offload-gb", "1"])
58
+ ["--cpu-offload-gb", "1"],
59
+ max_wait_seconds=480)
60
# Test w4a16_marlin24
61
compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
55
- [], ["--cpu-offload-gb", "1"])
62
+ [], ["--cpu-offload-gb", "1"],
63
+ max_wait_seconds=480)
64
# Test w8a8
65
compare_two_settings(
66
"nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", [],
59
- ["--cpu-offload-gb", "1"])
67
+ ["--cpu-offload-gb", "1"],
68
+ max_wait_seconds=480)
tests/utils.py
+16
-13
@@ -56,16 +56,14 @@ VLLM_PATH = Path(__file__).parent.parent
56
57
class RemoteOpenAIServer:
58
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
59
- MAX_START_WAIT_S = 240 # wait for server to start for 240 seconds
60
-
61
- def __init__(
62
- self,
63
- model: str,
64
- cli_args: List[str],
65
- *,
66
- env_dict: Optional[Dict[str, str]] = None,
67
- auto_port: bool = True,
68
- ) -> None:
59
+
60
+ def __init__(self,
61
+ model: str,
62
+ cli_args: List[str],
63
+ *,
64
+ env_dict: Optional[Dict[str, str]] = None,
65
+ auto_port: bool = True,
66
+ max_wait_seconds: Optional[float] = None) -> None:
67
if auto_port:
68
if "-p" in cli_args or "--port" in cli_args:
69
raise ValueError("You have manually specified the port"
@@ -90,8 +88,9 @@ class RemoteOpenAIServer:
88
env=env,
89
stdout=sys.stdout,
90
stderr=sys.stderr)
91
+ max_wait_seconds = max_wait_seconds or 240
92
self._wait_for_server(url=self.url_for("health"),
94
- timeout=self.MAX_START_WAIT_S)
93
+ timeout=max_wait_seconds)
94
95
def __enter__(self):
96
return self
@@ -145,7 +144,8 @@ def compare_two_settings(model: str,
144
arg1: List[str],
145
arg2: List[str],
146
env1: Optional[Dict[str, str]] = None,
148
- env2: Optional[Dict[str, str]] = None):
147
+ env2: Optional[Dict[str, str]] = None,
148
+ max_wait_seconds: Optional[float] = None) -> None:
149
"""
150
Launch API server with two different sets of arguments/environments
151
and compare the results of the API calls.
@@ -164,7 +164,10 @@ def compare_two_settings(model: str,
164
token_ids = tokenizer(prompt)["input_ids"]
165
results = []
166
for args, env in ((arg1, env1), (arg2, env2)):
167
- with RemoteOpenAIServer(model, args, env_dict=env) as server:
167
+ with RemoteOpenAIServer(model,
168
+ args,
169
+ env_dict=env,
170
+ max_wait_seconds=max_wait_seconds) as server:
171
client = server.get_client()
172
173
# test models list