A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 22:30 UTC
[ROCm][CI] Specifying time outs for the lm eval models (#44255)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jun 4, 2026 at 09:35 UTC
3e7703676829a2698d8f33027c64296e5135832b
4 files changed
+19
-4
tests/evals/gsm8k/configs/DeepSeek-V2-Lite-Instruct-FP8.yaml
+1
@@ -2,4 +2,5 @@ model_name: "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8"
2
accuracy_threshold: 0.72
3
num_questions: 1319
4
num_fewshot: 5
5
+rocm_request_timeout_seconds: 1800
6
server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen1.5-MoE-W4A16-CT.yaml
+1
@@ -2,4 +2,5 @@ model_name: "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16"
2
accuracy_threshold: 0.45
3
num_questions: 1319
4
num_fewshot: 5
5
+rocm_request_timeout_seconds: 1800
6
server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/gsm8k_eval.py
+4
-4
@@ -106,7 +106,7 @@ async def call_vllm_api(
106
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
107
return text, completion_tokens
108
except Exception as e:
109
- print(f"Error calling vLLM API: {e}")
109
+ print(f"Error calling vLLM API ({type(e).__name__}): {e}")
110
return "", 0
111
112
@@ -177,6 +177,7 @@ def evaluate_gsm8k(
177
port: int = 8000,
178
temperature: float = 0.0,
179
seed: int | None = 42,
180
+ request_timeout_seconds: float = 600,
181
) -> dict[str, float | int]:
182
"""
183
Evaluate GSM8K accuracy using vLLM serve endpoint.
@@ -205,9 +206,8 @@ def evaluate_gsm8k(
206
output_tokens[i] = tokens
207
return answer, tokens
208
208
- async with aiohttp.ClientSession(
209
- timeout=aiohttp.ClientTimeout(total=600)
210
- ) as session:
209
+ timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
210
+ async with aiohttp.ClientSession(timeout=timeout) as session:
211
tasks = [get_answer(session, i) for i in range(num_questions)]
212
await tqdm.gather(*tasks, desc="Evaluating")
213
tests/evals/gsm8k/test_gsm8k_correctness.py
+13
@@ -39,11 +39,18 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
39
host = f"http://{host}"
40
41
# Run GSM8K evaluation
42
+ request_timeout_seconds = eval_config.get("request_timeout_seconds", 600)
43
+ if current_platform.is_rocm():
44
+ request_timeout_seconds = eval_config.get(
45
+ "rocm_request_timeout_seconds", request_timeout_seconds
46
+ )
47
+
48
results = evaluate_gsm8k(
49
num_questions=eval_config["num_questions"],
50
num_shots=eval_config["num_fewshot"],
51
host=host,
52
port=port,
53
+ request_timeout_seconds=request_timeout_seconds,
54
)
55
56
return results
@@ -90,6 +97,12 @@ def test_gsm8k_correctness(config_filename):
97
print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
98
print(f"Number of questions: {eval_config['num_questions']}")
99
print(f"Number of few-shot examples: {eval_config['num_fewshot']}")
100
+ request_timeout_seconds = eval_config.get("request_timeout_seconds", 600)
101
+ if current_platform.is_rocm():
102
+ request_timeout_seconds = eval_config.get(
103
+ "rocm_request_timeout_seconds", request_timeout_seconds
104
+ )
105
+ print(f"Request timeout: {request_timeout_seconds}s")
106
print(f"Server args: {' '.join(server_args)}")
107
print(f"Environment variables: {env_dict}")
108