A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
[CI][Metrics] Stabilize tests with polling and subprocess guards (#34566)
test_abort_metrics_reset is flaky due to hardware-dependent fixed sleeps: replace fixed sleeps with polling. test_metrics_exist_run_batch passes even when the engine crashes on startup (false positive): add subprocess lifecycle guards. Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 16, 2026 at 04:52 UTC
1e828573b4a788971220c17a41350c2068b4c810
1 file changed
+66
-24
tests/entrypoints/instrumentator/test_metrics.py
+66
-24
@@ -17,6 +17,7 @@ from transformers import AutoTokenizer
17
from tests.conftest import LocalAssetServer
18
from tests.utils import RemoteOpenAIServer
19
from vllm import version
20
+from vllm.utils.network_utils import get_open_port
21
22
MODELS = {
23
"text": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
@@ -315,14 +316,26 @@ async def test_abort_metrics_reset(
316
client.completions.create(
317
model=model_name,
318
prompt=prompt_ids,
318
- max_tokens=100, # Long generation to give time to abort
319
+ max_tokens=500, # Long generation to give time to abort
320
temperature=0.0,
321
)
322
)
323
tasks.append(task)
324
324
- # Wait a bit for requests to start processing
325
- await asyncio.sleep(0.5)
325
+ # Poll until we see running requests rather than using a fixed sleep,
326
+ # since generation speed varies across hardware.
327
+ try:
328
+ await _poll_until(
329
+ lambda: _get_running_metrics_from_api(server)[0] > 0,
330
+ timeout=10.0,
331
+ interval=0.1,
332
+ description="running_requests > 0",
333
+ )
334
+ except TimeoutError:
335
+ for task in tasks:
336
+ task.cancel()
337
+ await asyncio.gather(*tasks, return_exceptions=True)
338
+ pytest.fail("Requests never appeared as running in metrics")
339
340
# Check that we have running requests
341
running_requests, waiting_requests, kv_cache_usage = _get_running_metrics_from_api(
@@ -336,13 +349,15 @@ async def test_abort_metrics_reset(
349
# Cancel all tasks to abort the requests
350
for task in tasks:
351
task.cancel()
339
-
340
- # Wait for cancellations to be processed
341
- await asyncio.sleep(1.0)
342
-
343
- # Check that metrics have reset to zero
344
- response = requests.get(server.url_for("metrics"))
345
- assert response.status_code == HTTPStatus.OK
352
+ await asyncio.gather(*tasks, return_exceptions=True)
353
+
354
+ # Poll until metrics reset rather than using a fixed sleep
355
+ await _poll_until(
356
+ lambda: _get_running_metrics_from_api(server) == (0, 0, 0),
357
+ timeout=10.0,
358
+ interval=0.2,
359
+ description="gauge metrics back to zero",
360
+ )
361
362
# Verify running and waiting requests counts and KV cache usage are zero
363
running_requests_after, waiting_requests_after, kv_cache_usage_after = (
@@ -360,6 +375,18 @@ async def test_abort_metrics_reset(
375
)
376
377
378
+async def _poll_until(
379
+ predicate, *, timeout: float, interval: float = 0.5, description: str = "condition"
380
+):
381
+ """Poll until predicate() returns True, or raise TimeoutError."""
382
+ start = time.time()
383
+ while time.time() - start < timeout:
384
+ if predicate():
385
+ return
386
+ await asyncio.sleep(interval)
387
+ raise TimeoutError(f"Timed out after {timeout}s waiting for: {description}")
388
+
389
+
390
def _get_running_metrics_from_api(server: RemoteOpenAIServer):
391
"""Return (running_count, waiting_count, kv_cache_usage)"""
392
@@ -399,7 +426,7 @@ def test_metrics_exist_run_batch():
426
input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}""" # noqa: E501
427
428
base_url = "0.0.0.0"
402
- port = "8001"
429
+ port = str(get_open_port())
430
server_url = f"http://{base_url}:{port}"
431
432
with (
@@ -427,17 +454,32 @@ def test_metrics_exist_run_batch():
454
],
455
)
456
430
- def is_server_up(url):
457
+ try:
458
+
459
+ def is_server_up(url):
460
+ try:
461
+ response = requests.get(url)
462
+ return response.status_code == 200
463
+ except requests.ConnectionError:
464
+ return False
465
+
466
+ start = time.time()
467
+ timeout = 120
468
+ while not is_server_up(server_url):
469
+ if proc.poll() is not None:
470
+ pytest.fail(
471
+ f"Batch process exited early with returncode={proc.returncode}"
472
+ )
473
+ if time.time() - start > timeout:
474
+ pytest.fail("Batch server did not start within timeout")
475
+ time.sleep(1)
476
+
477
+ response = requests.get(server_url + "/metrics")
478
+ assert response.status_code == HTTPStatus.OK
479
+ finally:
480
+ proc.terminate()
481
try:
432
- response = requests.get(url)
433
- return response.status_code == 200
434
- except requests.ConnectionError:
435
- return False
436
-
437
- while not is_server_up(server_url):
438
- time.sleep(1)
439
-
440
- response = requests.get(server_url + "/metrics")
441
- assert response.status_code == HTTPStatus.OK
442
-
443
- proc.wait()
482
+ proc.wait(timeout=15)
483
+ except subprocess.TimeoutExpired:
484
+ proc.kill()
485
+ proc.wait(timeout=5)