A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[ci]try to fix flaky multi-step tests (#11894)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 9, 2025 at 22:47 UTC
bd8287221187279c668ac10c3edd5242b8d8b429
2 files changed
+8
-4
tests/multi_step/test_correctness_async_llm.py
+1
-2
@@ -16,7 +16,6 @@ NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
16
NUM_PROMPTS = [10]
17
18
DEFAULT_SERVER_ARGS: List[str] = [
19
- "--disable-log-requests",
19
"--worker-use-ray",
20
"--gpu-memory-utilization",
21
"0.85",
@@ -110,7 +109,7 @@ async def test_multi_step(
109
110
# Spin up client/server & issue completion API requests.
111
# Default `max_wait_seconds` is 240 but was empirically
113
- # was raised 3x to 720 *just for this test* due to
112
+ # was raised 5x to 1200 *just for this test* due to
113
# observed timeouts in GHA CI
114
ref_completions = await completions_with_server_args(
115
prompts,
tests/utils.py
+7
-2
@@ -157,13 +157,19 @@ class RemoteOpenAIServer:
157
def url_for(self, *parts: str) -> str:
158
return self.url_root + "/" + "/".join(parts)
159
160
- def get_client(self):
160
+ def get_client(self, **kwargs):
161
+ if "timeout" not in kwargs:
162
+ kwargs["timeout"] = 600
163
return openai.OpenAI(
164
base_url=self.url_for("v1"),
165
api_key=self.DUMMY_API_KEY,
166
+ max_retries=0,
167
+ **kwargs,
168
)
169
170
def get_async_client(self, **kwargs):
171
+ if "timeout" not in kwargs:
172
+ kwargs["timeout"] = 600
173
return openai.AsyncOpenAI(base_url=self.url_for("v1"),
174
api_key=self.DUMMY_API_KEY,
175
max_retries=0,
@@ -780,7 +786,6 @@ async def completions_with_server_args(
786
assert len(max_tokens) == len(prompts)
787
788
outputs = None
783
- max_wait_seconds = 240 * 3 # 240 is default
789
with RemoteOpenAIServer(model_name,
790
server_cli_args,
791
max_wait_seconds=max_wait_seconds) as server: