@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 234860399b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 23:00 UTC

[Frontend][Core] Revert "Add shutdown timeout" (#34730 and #36270) (#36628)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Mar 10, 2026 at 13:20 UTC 234860399b9d390bf59bfe1f19c2e2304ac5c806

14 files changed +95 -761

tests/entrypoints/openai/test_shutdown.py

-459

@@ -1,20 +1,14 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Integration tests for shutdown behavior, timeout, and signal handling."""

5 -import asyncio

4 import signal

5 import subprocess

6 import sys

7 import time

10 -from dataclasses import dataclass, field

12 -import httpx

9 import openai

14 -import psutil

10 import pytest

17 -from tests.utils import RemoteOpenAIServer

12 from vllm.platforms import current_platform

13 from vllm.utils.network_utils import get_open_port

@@ -24,101 +18,6 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"

18 _IS_ROCM = current_platform.is_rocm()

19 _SERVER_STARTUP_TIMEOUT = 120

20 _PROCESS_EXIT_TIMEOUT = 15

27 -_SHUTDOWN_DETECTION_TIMEOUT = 10

28 -_CHILD_CLEANUP_TIMEOUT = 10

29 -

30 -

31 -def _get_child_pids(parent_pid: int) -> list[int]:

32 - try:

33 - parent = psutil.Process(parent_pid)

34 - return [c.pid for c in parent.children(recursive=True)]

35 - except psutil.NoSuchProcess:

36 - return []

37 -

38 -

39 -async def _assert_children_cleaned_up(

40 - child_pids: list[int],

41 - timeout: float = _CHILD_CLEANUP_TIMEOUT,

42 -):

43 - """Wait for child processes to exit and fail if any remain."""

44 - if not child_pids:

45 - return

46 -

47 - deadline = time.time() + timeout

48 - while time.time() < deadline:

49 - still_alive = []

50 - for pid in child_pids:

51 - try:

52 - p = psutil.Process(pid)

53 - if p.is_running() and p.status() != psutil.STATUS_ZOMBIE:

54 - still_alive.append(pid)

55 - except psutil.NoSuchProcess:

56 - pass

57 - if not still_alive:

58 - return

59 - await asyncio.sleep(0.5)

60 -

61 - pytest.fail(

62 - f"Child processes {still_alive} still alive after {timeout}s. "

63 - f"Process cleanup may not be working correctly."

64 - )

65 -

66 -

67 -@dataclass

68 -class ShutdownState:

69 - got_503: bool = False

70 - got_500: bool = False

71 - requests_after_sigterm: int = 0

72 - aborted_requests: int = 0

73 - connection_errors: int = 0

74 - stop_requesting: bool = False

75 - errors: list[str] = field(default_factory=list)

76 -

77 -

78 -async def _concurrent_request_loop(

79 - client: openai.AsyncOpenAI,

80 - state: ShutdownState,

81 - sigterm_sent: asyncio.Event | None = None,

82 - concurrency: int = 10,

83 -):

84 - """Run multiple concurrent requests to keep the server busy."""

85 -

86 - async def single_request():

87 - while not state.stop_requesting:

88 - try:

89 - response = await client.completions.create(

90 - model=MODEL_NAME,

91 - prompt="Write a story: ",

92 - max_tokens=200,

93 - )

94 - if sigterm_sent is not None and sigterm_sent.is_set():

95 - state.requests_after_sigterm += 1

96 - # Check if any choice has finish_reason='abort'

97 - if any(choice.finish_reason == "abort" for choice in response.choices):

98 - state.aborted_requests += 1

99 - except openai.APIStatusError as e:

100 - if e.status_code == 503:

101 - state.got_503 = True

102 - elif e.status_code == 500:

103 - state.got_500 = True

104 - else:

105 - state.errors.append(f"API error: {e}")

106 - except (openai.APIConnectionError, httpx.RemoteProtocolError):

107 - state.connection_errors += 1

108 - if sigterm_sent is not None and sigterm_sent.is_set():

109 - break

110 - except Exception as e:

111 - state.errors.append(f"Unexpected error: {e}")

112 - break

113 - await asyncio.sleep(0.01)

114 -

115 - tasks = [asyncio.create_task(single_request()) for _ in range(concurrency)]

116 - try:

117 - await asyncio.gather(*tasks, return_exceptions=True)

118 - finally:

119 - for t in tasks:

120 - if not t.done():

121 - t.cancel()

23 @pytest.mark.asyncio

@@ -204,361 +103,3 @@ async def test_shutdown_on_engine_failure():

103

104 return_code = proc.wait(timeout=_PROCESS_EXIT_TIMEOUT)

105 assert return_code is not None

207 -

208 -

209 -@pytest.mark.asyncio

210 -async def test_wait_timeout_completes_requests():

211 - """Verify wait timeout: new requests rejected, in-flight requests complete."""

212 - server_args = [

213 - "--dtype",

214 - "bfloat16",

215 - "--max-model-len",

216 - "256",

217 - "--enforce-eager",

218 - "--gpu-memory-utilization",

219 - "0.05",

220 - "--max-num-seqs",

221 - "4",

222 - "--shutdown-timeout",

223 - "30",

224 - ]

225 -

226 - with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:

227 - client = remote_server.get_async_client()

228 - proc = remote_server.proc

229 - child_pids = _get_child_pids(proc.pid)

230 -

231 - state = ShutdownState()

232 - sigterm_sent = asyncio.Event()

233 -

234 - request_task = asyncio.create_task(

235 - _concurrent_request_loop(client, state, sigterm_sent, concurrency=10)

236 - )

237 -

238 - await asyncio.sleep(0.5)

239 - proc.send_signal(signal.SIGTERM)

240 - sigterm_sent.set()

241 -

242 - try:

243 - await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)

244 - except asyncio.TimeoutError:

245 - pass

246 - finally:

247 - state.stop_requesting = True

248 - if not request_task.done():

249 - request_task.cancel()

250 - await asyncio.gather(request_task, return_exceptions=True)

251 -

252 - # wait timeout should complete in-flight requests

253 - assert state.requests_after_sigterm > 0, (

254 - f"Wait timeout should complete in-flight requests. "

255 - f"503: {state.got_503}, 500: {state.got_500}, "

256 - f"conn_errors: {state.connection_errors}, errors: {state.errors}"

257 - )

258 - # server must stop accepting new requests (503, 500, or connection close)

259 - assert state.got_503 or state.got_500 or state.connection_errors > 0, (

260 - f"Server should stop accepting requests. "

261 - f"completed: {state.requests_after_sigterm}, errors: {state.errors}"

262 - )

263 -

264 - await _assert_children_cleaned_up(child_pids)

265 -

266 -

267 -@pytest.mark.asyncio

268 -@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])

269 -async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):

270 - server_args = [

271 - "--dtype",

272 - "bfloat16",

273 - "--max-model-len",

274 - "256",

275 - "--enforce-eager",

276 - "--gpu-memory-utilization",

277 - "0.05",

278 - "--max-num-seqs",

279 - "4",

280 - "--shutdown-timeout",

281 - "0",

282 - ]

283 -

284 - with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:

285 - proc = remote_server.proc

286 - child_pids = _get_child_pids(proc.pid)

287 -

288 - if wait_for_engine_idle > 0:

289 - client = remote_server.get_async_client()

290 - # Send requests to ensure engine is fully initialized

291 - for _ in range(2):

292 - await client.completions.create(

293 - model=MODEL_NAME,

294 - prompt="Test request: ",

295 - max_tokens=10,

296 - )

297 - # Wait for engine to become idle

298 - await asyncio.sleep(wait_for_engine_idle)

299 -

300 - start_time = time.time()

301 - proc.send_signal(signal.SIGTERM)

302 -

303 - # abort timeout (0) should exit promptly

304 - for _ in range(20):

305 - if proc.poll() is not None:

306 - break

307 - time.sleep(0.1)

308 -

309 - if proc.poll() is None:

310 - proc.kill()

311 - proc.wait(timeout=5)

312 - pytest.fail("Process did not exit after SIGTERM with abort timeout")

313 -

314 - exit_time = time.time() - start_time

315 - assert exit_time < 2, f"Default shutdown took too long: {exit_time:.1f}s"

316 - assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"

317 -

318 - await _assert_children_cleaned_up(child_pids)

319 -

320 -

321 -@pytest.mark.asyncio

322 -async def test_wait_timeout_with_short_duration():

323 - """Verify server exits cleanly with a short wait timeout."""

324 - wait_timeout = 3

325 - server_args = [

326 - "--dtype",

327 - "bfloat16",

328 - "--max-model-len",

329 - "256",

330 - "--enforce-eager",

331 - "--gpu-memory-utilization",

332 - "0.05",

333 - "--max-num-seqs",

334 - "4",

335 - "--shutdown-timeout",

336 - str(wait_timeout),

337 - ]

338 -

339 - with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:

340 - client = remote_server.get_async_client()

341 - proc = remote_server.proc

342 - child_pids = _get_child_pids(proc.pid)

343 -

344 - state = ShutdownState()

345 - request_task = asyncio.create_task(

346 - _concurrent_request_loop(client, state, concurrency=3)

347 - )

348 -

349 - await asyncio.sleep(0.5)

350 -

351 - start_time = time.time()

352 - proc.send_signal(signal.SIGTERM)

353 -

354 - # server should exit within wait_timeout + buffer

355 - max_wait = wait_timeout + 15

356 - for _ in range(int(max_wait * 10)):

357 - if proc.poll() is not None:

358 - break

359 - time.sleep(0.1)

360 -

361 - exit_time = time.time() - start_time

362 -

363 - state.stop_requesting = True

364 - if not request_task.done():

365 - request_task.cancel()

366 - await asyncio.gather(request_task, return_exceptions=True)

367 -

368 - if proc.poll() is None:

369 - proc.kill()

370 - proc.wait(timeout=5)

371 - pytest.fail(f"Process did not exit within {max_wait}s after SIGTERM")

372 -

373 - assert exit_time < wait_timeout + 10, (

374 - f"Took too long to exit ({exit_time:.1f}s), expected <{wait_timeout + 10}s"

375 - )

376 - assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"

377 -

378 - await _assert_children_cleaned_up(child_pids)

379 -

380 -

381 -@pytest.mark.asyncio

382 -async def test_abort_timeout_fails_inflight_requests():

383 - """Verify abort timeout (0) immediately aborts in-flight requests."""

384 - server_args = [

385 - "--dtype",

386 - "bfloat16",

387 - "--max-model-len",

388 - "256",

389 - "--enforce-eager",

390 - "--gpu-memory-utilization",

391 - "0.05",

392 - "--max-num-seqs",

393 - "4",

394 - "--shutdown-timeout",

395 - "0",

396 - ]

397 -

398 - with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:

399 - client = remote_server.get_async_client()

400 - proc = remote_server.proc

401 - child_pids = _get_child_pids(proc.pid)

402 -

403 - state = ShutdownState()

404 - sigterm_sent = asyncio.Event()

405 -

406 - request_task = asyncio.create_task(

407 - _concurrent_request_loop(client, state, sigterm_sent, concurrency=10)

408 - )

409 -

410 - await asyncio.sleep(0.5)

411 -

412 - proc.send_signal(signal.SIGTERM)

413 - sigterm_sent.set()

414 -

415 - try:

416 - await asyncio.wait_for(request_task, timeout=5)

417 - except asyncio.TimeoutError:

418 - pass

419 - finally:

420 - state.stop_requesting = True

421 - if not request_task.done():

422 - request_task.cancel()

423 - await asyncio.gather(request_task, return_exceptions=True)

424 -

425 - # With abort timeout (0), requests should be aborted (finish_reason='abort')

426 - # or rejected (connection errors or API errors)

427 - assert (

428 - state.aborted_requests > 0

429 - or state.connection_errors > 0

430 - or state.got_500

431 - or state.got_503

432 - ), (

433 - f"Abort timeout should cause request aborts or failures. "

434 - f"aborted: {state.aborted_requests}, "

435 - f"503: {state.got_503}, 500: {state.got_500}, "

436 - f"conn_errors: {state.connection_errors}, "

437 - f"completed: {state.requests_after_sigterm}"

438 - )

439 -

440 - # Verify fast shutdown

441 - start_time = time.time()

442 - for _ in range(100):

443 - if proc.poll() is not None:

444 - break

445 - time.sleep(0.1)

446 -

447 - exit_time = time.time() - start_time

448 - assert exit_time < 10, f"Abort timeout shutdown took too long: {exit_time:.1f}s"

449 -

450 - await _assert_children_cleaned_up(child_pids)

451 -

452 -

453 -@pytest.mark.asyncio

454 -async def test_request_rejection_during_shutdown():

455 - """Verify new requests are rejected with error during shutdown."""

456 - server_args = [

457 - "--dtype",

458 - "bfloat16",

459 - "--max-model-len",

460 - "256",

461 - "--enforce-eager",

462 - "--gpu-memory-utilization",

463 - "0.05",

464 - "--max-num-seqs",

465 - "4",

466 - "--shutdown-timeout",

467 - "30",

468 - ]

469 -

470 - with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:

471 - client = remote_server.get_async_client()

472 - proc = remote_server.proc

473 - child_pids = _get_child_pids(proc.pid)

474 -

475 - proc.send_signal(signal.SIGTERM)

476 -

477 - await asyncio.sleep(1.0)

478 -

479 - # Try to send new requests - they should be rejected

480 - rejected_count = 0

481 - for _ in range(10):

482 - try:

483 - await client.completions.create(

484 - model=MODEL_NAME, prompt="Hello", max_tokens=10

485 - )

486 - except (

487 - openai.APIStatusError,

488 - openai.APIConnectionError,

489 - httpx.RemoteProtocolError,

490 - ):

491 - rejected_count += 1

492 - await asyncio.sleep(0.1)

493 -

494 - assert rejected_count > 0, (

495 - f"Expected requests to be rejected during shutdown, "

496 - f"but {rejected_count} were rejected out of 10"

497 - )

498 -

499 - await _assert_children_cleaned_up(child_pids)

500 -

501 -

502 -@pytest.mark.asyncio

503 -async def test_multi_api_server_shutdown():

504 - """Verify shutdown works with multiple API servers."""

505 - server_args = [

506 - "--dtype",

507 - "bfloat16",

508 - "--max-model-len",

509 - "256",

510 - "--enforce-eager",

511 - "--gpu-memory-utilization",

512 - "0.05",

513 - "--max-num-seqs",

514 - "4",

515 - "--shutdown-timeout",

516 - "30",

517 - "--api-server-count",

518 - "2",

519 - ]

520 -

521 - with RemoteOpenAIServer(MODEL_NAME, server_args, auto_port=True) as remote_server:

522 - client = remote_server.get_async_client()

523 - proc = remote_server.proc

524 - child_pids = _get_child_pids(proc.pid)

525 -

526 - assert len(child_pids) >= 2, (

527 - f"Expected at least 2 child processes, got {len(child_pids)}"

528 - )

529 -

530 - state = ShutdownState()

531 - sigterm_sent = asyncio.Event()

532 -

533 - # Start concurrent requests across both API servers

534 - request_task = asyncio.create_task(

535 - _concurrent_request_loop(client, state, sigterm_sent, concurrency=8)

536 - )

537 -

538 - await asyncio.sleep(0.5)

539 -

540 - # Send SIGTERM to parent - should propagate to all children

541 - proc.send_signal(signal.SIGTERM)

542 - sigterm_sent.set()

543 -

544 - try:

545 - await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)

546 - except asyncio.TimeoutError:

547 - pass

548 - finally:

549 - state.stop_requesting = True

550 - if not request_task.done():

551 - request_task.cancel()

552 - await asyncio.gather(request_task, return_exceptions=True)

553 -

554 - for _ in range(300): # up to 30 seconds

555 - if proc.poll() is not None:

556 - break

557 - time.sleep(0.1)

558 -

559 - if proc.poll() is None:

560 - proc.kill()

561 - proc.wait(timeout=5)

562 - pytest.fail("Process did not exit after SIGTERM")

563 -

564 - await _assert_children_cleaned_up(child_pids)

tests/entrypoints/test_api_server_process_manager.py

+7 -15

@@ -79,7 +79,7 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):

79 finally:

80 # Always clean up the processes

81 print("Cleaning up processes...")

82 - manager.shutdown()

82 + manager.close()

84 # Give processes time to terminate

85 time.sleep(0.2)

@@ -111,8 +111,6 @@ def test_wait_for_completion_or_failure(api_server_args):

111 wait_for_completion_or_failure(api_server_manager=manager)

112 except Exception as e:

113 result["exception"] = e

114 - finally:

115 - manager.shutdown()

114

115 # Start a thread to run wait_for_completion_or_failure

116 wait_thread = threading.Thread(target=run_with_exception_capture, daemon=True)

@@ -145,7 +143,7 @@ def test_wait_for_completion_or_failure(api_server_args):

143 assert not proc.is_alive(), f"Process {i} should not be alive"

144

145 finally:

148 - manager.shutdown()

146 + manager.close()

147 time.sleep(0.2)

148

149

@@ -176,14 +174,11 @@ def test_normal_completion(api_server_args):

174 # since all processes have already

175 # terminated, it should return immediately

176 # with no error

179 - try:

180 - wait_for_completion_or_failure(api_server_manager=manager)

181 - finally:

182 - manager.shutdown()

177 + wait_for_completion_or_failure(api_server_manager=manager)

178

179 finally:

180 # Clean up just in case

186 - manager.shutdown()

181 + manager.close()

182 time.sleep(0.2)

183

184

@@ -206,7 +201,7 @@ def test_external_process_monitoring(api_server_args):

201 def __init__(self, proc):

202 self.proc = proc

203

209 - def shutdown(self):

204 + def close(self):

205 if self.proc.is_alive():

206 self.proc.terminate()

207 self.proc.join(timeout=0.5)

@@ -231,9 +226,6 @@ def test_external_process_monitoring(api_server_args):

226 )

227 except Exception as e:

228 result["exception"] = e

234 - finally:

235 - manager.shutdown()

236 - mock_coordinator.shutdown()

229

230 # Start a thread to run wait_for_completion_or_failure

231 wait_thread = threading.Thread(target=run_with_exception_capture, daemon=True)

@@ -267,6 +259,6 @@ def test_external_process_monitoring(api_server_args):

259

260 finally:

261 # Clean up

270 - manager.shutdown()

271 - mock_coordinator.shutdown()

262 + manager.close()

263 + mock_coordinator.close()

264 time.sleep(0.2)

vllm/config/vllm.py

-6

@@ -327,12 +327,6 @@ class VllmConfig:

327 weight_transfer_config: WeightTransferConfig | None = None

328 """The configurations for weight transfer during RL training."""

329

330 - shutdown_timeout: int = Field(default=0, ge=0)

331 - """Shutdown grace period for in-flight requests. Shutdown will be delayed for

332 - up to this amount of time to allow already-running requests to complete. Any

333 - remaining requests are aborted once the timeout is reached.

334 - """

335 -

330 def compute_hash(self) -> str:

331 """

332 WARNING: Whenever a new field is added to this config,

vllm/engine/arg_utils.py

-11

@@ -606,8 +606,6 @@ class EngineArgs:

606 kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend

607 tokens_only: bool = False

608

609 - shutdown_timeout: int = 0

610 -

609 weight_transfer_config: WeightTransferConfig | None = get_field(

610 VllmConfig,

611 "weight_transfer_config",

@@ -1310,14 +1308,6 @@ class EngineArgs:

1308 default=False,

1309 action=argparse.BooleanOptionalAction,

1310 )

1313 -

1314 - parser.add_argument(

1315 - "--shutdown-timeout",

1316 - type=int,

1317 - default=0,

1318 - help="Shutdown timeout in seconds. 0 = abort, >0 = wait.",

1319 - )

1320 -

1311 return parser

1312

1313 @classmethod

@@ -1926,7 +1916,6 @@ class EngineArgs:

1916 optimization_level=self.optimization_level,

1917 performance_mode=self.performance_mode,

1918 weight_transfer_config=self.weight_transfer_config,

1929 - shutdown_timeout=self.shutdown_timeout,

1919 )

1920

1921 return config

vllm/engine/protocol.py

-5

@@ -200,11 +200,6 @@ class EngineClient(ABC):

200 """Return whether the engine is currently paused."""

201 ...

202

203 - @abstractmethod

204 - def shutdown(self, timeout: float | None = None) -> None:

205 - """Shutdown the engine with optional timeout."""

206 - ...

207 -

203 async def scale_elastic_ep(

204 self, new_data_parallel_size: int, drain_timeout: int = 300

205 ) -> None:

vllm/entrypoints/cli/serve.py

+6 -42

@@ -3,7 +3,6 @@

4 import argparse

5 import signal

6 -import time

7 import uvloop

@@ -225,12 +224,8 @@ def run_headless(args: argparse.Namespace):

224 try:

225 engine_manager.join_first()

226 finally:

228 - timeout = None

229 - if shutdown_requested:

230 - timeout = vllm_config.shutdown_timeout

231 - logger.info("Waiting up to %d seconds for processes to exit", timeout)

232 - engine_manager.shutdown(timeout=timeout)

227 logger.info("Shutting down.")

228 + engine_manager.close()

229

230

231 def run_multi_api_server(args: argparse.Namespace):

@@ -241,19 +236,6 @@ def run_multi_api_server(args: argparse.Namespace):

236 if num_api_servers > 1:

237 setup_multiprocess_prometheus()

238

244 - shutdown_requested = False

245 -

246 - # Catch SIGTERM and SIGINT to allow graceful shutdown.

247 - def signal_handler(signum, frame):

248 - nonlocal shutdown_requested

249 - logger.debug("Received %d signal.", signum)

250 - if not shutdown_requested:

251 - shutdown_requested = True

252 - raise SystemExit

253 -

254 - signal.signal(signal.SIGTERM, signal_handler)

255 - signal.signal(signal.SIGINT, signal_handler)

256 -

239 listen_address, sock = setup_server(args)

240

241 engine_args = vllm.AsyncEngineArgs.from_cli_args(args)

@@ -315,29 +297,11 @@ def run_multi_api_server(args: argparse.Namespace):

297 api_server_manager = APIServerProcessManager(**api_server_manager_kwargs)

298

299 # Wait for API servers

318 - try:

319 - wait_for_completion_or_failure(

320 - api_server_manager=api_server_manager,

321 - engine_manager=local_engine_manager,

322 - coordinator=coordinator,

323 - )

324 - finally:

325 - timeout = shutdown_by = None

326 - if shutdown_requested:

327 - timeout = vllm_config.shutdown_timeout

328 - shutdown_by = time.monotonic() + timeout

329 - logger.info("Waiting up to %d seconds for processes to exit", timeout)

330 -

331 - def to_timeout(deadline: float | None) -> float | None:

332 - return (

333 - deadline if deadline is None else max(deadline - time.monotonic(), 0.0)

334 - )

335 -

336 - api_server_manager.shutdown(timeout=timeout)

337 - if local_engine_manager:

338 - local_engine_manager.shutdown(timeout=to_timeout(shutdown_by))

339 - if coordinator:

340 - coordinator.shutdown(timeout=to_timeout(shutdown_by))

300 + wait_for_completion_or_failure(

301 + api_server_manager=api_server_manager,

302 + engine_manager=local_engine_manager,

303 + coordinator=coordinator,

304 + )

305

306

307 def run_api_server_worker_proc(

vllm/entrypoints/launcher.py

+5 -23

@@ -4,7 +4,6 @@

4 import asyncio

5 import signal

6 import socket

7 -from functools import partial

7 from typing import Any

9 import uvicorn

@@ -92,10 +91,12 @@ async def serve_http(

91 )

92 )

95 - shutdown_event = asyncio.Event()

96 -

94 def signal_handler() -> None:

98 - shutdown_event.set()

95 + # prevents the uvicorn signal handler to exit early

96 + server_task.cancel()

97 + watchdog_task.cancel()

98 + if ssl_cert_refresher:

99 + ssl_cert_refresher.stop()

100

101 async def dummy_shutdown() -> None:

102 pass

@@ -103,24 +104,6 @@ async def serve_http(

104 loop.add_signal_handler(signal.SIGINT, signal_handler)

105 loop.add_signal_handler(signal.SIGTERM, signal_handler)

106

106 - async def handle_shutdown() -> None:

107 - await shutdown_event.wait()

108 -

109 - engine_client = app.state.engine_client

110 - timeout = engine_client.vllm_config.shutdown_timeout

111 -

112 - await loop.run_in_executor(

113 - None, partial(engine_client.shutdown, timeout=timeout)

114 - )

115 -

116 - server.should_exit = True

117 - server_task.cancel()

118 - watchdog_task.cancel()

119 - if ssl_cert_refresher:

120 - ssl_cert_refresher.stop()

121 -

122 - shutdown_task = loop.create_task(handle_shutdown())

123 -

107 try:

108 await server_task

109 return dummy_shutdown()

@@ -137,7 +120,6 @@ async def serve_http(

120 logger.info("Shutting down FastAPI HTTP server.")

121 return server.shutdown()

122 finally:

140 - shutdown_task.cancel()

123 watchdog_task.cancel()

124

125

vllm/v1/engine/__init__.py

-2

@@ -226,8 +226,6 @@ class EngineCoreRequestType(enum.Enum):

226 UTILITY = b"\x03"

227 # Sentinel used within EngineCoreProc.

228 EXECUTOR_FAILED = b"\x04"

229 - # Sentinel to wake up input_queue.get() during shutdown.

230 - WAKEUP = b"\x05"

229

230

231 class ReconfigureDistributedRequest(msgspec.Struct):

vllm/v1/engine/async_llm.py

+3 -2

@@ -264,15 +264,16 @@ class AsyncLLM(EngineClient):

264 def __del__(self):

265 self.shutdown()

266

267 - def shutdown(self, timeout: float | None = None) -> None:

267 + def shutdown(self):

268 """Shutdown, cleaning up the background proc and IPC."""

269 +

270 shutdown_prometheus()

271

272 if renderer := getattr(self, "renderer", None):

273 renderer.shutdown()

274

275 if engine_core := getattr(self, "engine_core", None):

275 - engine_core.shutdown(timeout=timeout)

276 + engine_core.shutdown()

277

278 handler = getattr(self, "output_handler", None)

279 if handler is not None:

vllm/v1/engine/coordinator.py

+2 -4

@@ -104,10 +104,8 @@ class DPCoordinator:

104 """Returns tuple of ZMQ input address, output address."""

105 return self.coord_in_address, self.coord_out_address

106

107 - def shutdown(self, timeout: float | None = None) -> None:

108 - """Shutdown coordinator process with configurable timeout."""

109 - if self._finalizer.detach() is not None:

110 - shutdown([self.proc], timeout=timeout)

107 + def close(self):

108 + self._finalizer()

109

110

111 class EngineState:

vllm/v1/engine/core.py

+43 -127

@@ -9,7 +9,6 @@ from collections import defaultdict, deque

9 from collections.abc import Callable, Generator

10 from concurrent.futures import Future

11 from contextlib import ExitStack, contextmanager

12 -from enum import IntEnum

12 from functools import partial

13 from inspect import isclass, signature

14 from logging import DEBUG

@@ -62,7 +61,6 @@ from vllm.v1.engine import (

61 from vllm.v1.engine.utils import (

62 EngineHandshakeMetadata,

63 EngineZmqAddresses,

65 - SignalCallback,

64 get_device_indices,

65 )

66 from vllm.v1.executor import Executor

@@ -773,12 +771,6 @@ class EngineCore:

771 raise NotImplementedError

772

773

776 -class EngineShutdownState(IntEnum):

777 - RUNNING = 0

778 - REQUESTED = 1

779 - SHUTTING_DOWN = 2

780 -

781 -

774 class EngineCoreProc(EngineCore):

775 """ZMQ-wrapper for running EngineCore in background process."""

776

@@ -806,7 +798,6 @@ class EngineCoreProc(EngineCore):

798 self.engine_index = engine_index

799 identity = self.engine_index.to_bytes(length=2, byteorder="little")

800 self.engines_running = False

809 - self.shutdown_state = EngineShutdownState.RUNNING

801

802 with self._perform_handshakes(

803 handshake_address,

@@ -1037,11 +1028,25 @@ class EngineCoreProc(EngineCore):

1028 def run_engine_core(*args, dp_rank: int = 0, local_dp_rank: int = 0, **kwargs):

1029 """Launch EngineCore busy loop in background process."""

1030

1031 + # Signal handler used for graceful termination.

1032 + # SystemExit exception is only raised once to allow this and worker

1033 + # processes to terminate without error

1034 + shutdown_requested = False

1035 +

1036 # Ensure we can serialize transformer config after spawning

1037 maybe_register_config_serialize_by_value()

1038

1039 + def signal_handler(signum, frame):

1040 + nonlocal shutdown_requested

1041 + if not shutdown_requested:

1042 + shutdown_requested = True

1043 + raise SystemExit()

1044 +

1045 + # Either SIGTERM or SIGINT will terminate the engine_core

1046 + signal.signal(signal.SIGTERM, signal_handler)

1047 + signal.signal(signal.SIGINT, signal_handler)

1048 +

1049 engine_core: EngineCoreProc | None = None

1044 - signal_callback: SignalCallback | None = None

1050 try:

1051 vllm_config: VllmConfig = kwargs["vllm_config"]

1052 parallel_config: ParallelConfig = vllm_config.parallel_config

@@ -1089,22 +1094,6 @@ class EngineCoreProc(EngineCore):

1094 engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)

1095

1096 assert engine_core is not None

1092 -

1093 - def wakeup_engine():

1094 - # Wakes up idle engine via input_queue when shutdown is requested

1095 - # Not safe in a signal handler - we may interrupt the main thread

1096 - # while it is holding the non-reentrant input_queue.mutex

1097 - engine_core.input_queue.put_nowait((EngineCoreRequestType.WAKEUP, None))

1098 -

1099 - signal_callback = SignalCallback(wakeup_engine)

1100 -

1101 - def signal_handler(signum, frame):

1102 - engine_core.shutdown_state = EngineShutdownState.REQUESTED

1103 - signal_callback.trigger()

1104 -

1105 - signal.signal(signal.SIGTERM, signal_handler)

1106 - signal.signal(signal.SIGINT, signal_handler)

1107 -

1097 engine_core.run_busy_loop()

1098

1099 except SystemExit:

@@ -1118,10 +1107,6 @@ class EngineCoreProc(EngineCore):

1107 engine_core._send_engine_dead()

1108 raise e

1109 finally:

1121 - signal.signal(signal.SIGTERM, signal.SIG_DFL)

1122 - signal.signal(signal.SIGINT, signal.SIG_DFL)

1123 - if signal_callback is not None:

1124 - signal_callback.stop()

1110 if engine_core is not None:

1111 engine_core.shutdown()

1112

@@ -1136,25 +1121,21 @@ class EngineCoreProc(EngineCore):

1121 or bool(self.batch_queue)

1122 )

1123

1139 - def is_running(self) -> bool:

1140 - """Returns true if shutdown has not been requested."""

1141 - return self.shutdown_state == EngineShutdownState.RUNNING

1142 -

1124 def run_busy_loop(self):

1125 """Core busy loop of the EngineCore."""

1145 - while self._handle_shutdown():

1126 +

1127 + # Loop until process is sent a SIGINT or SIGTERM

1128 + while True:

1129 # 1) Poll the input queue until there is work to do.

1130 self._process_input_queue()

1131 # 2) Step the engine core and return the outputs.

1132 self._process_engine_step()

1133

1151 - raise SystemExit

1152 -

1134 def _process_input_queue(self):

1135 """Exits when an engine step needs to be performed."""

1136

1137 waited = False

1157 - while not self.has_work() and self.is_running():

1138 + while not self.has_work():

1139 # Notify callbacks waiting for engine to become idle.

1140 self._notify_idle_state_callbacks()

1141 if self.input_queue.empty():

@@ -1206,60 +1187,18 @@ class EngineCoreProc(EngineCore):

1187 callback = self._idle_state_callbacks.pop()

1188 callback(self)

1189

1209 - def _handle_shutdown(self) -> bool:

1210 - # Check if shutdown was requested and handle it

1211 - if self.shutdown_state == EngineShutdownState.RUNNING:

1212 - return True

1213 -

1214 - if self.shutdown_state == EngineShutdownState.REQUESTED:

1215 - shutdown_timeout = self.vllm_config.shutdown_timeout

1216 -

1217 - logger.info("Shutdown initiated (timeout=%d)", shutdown_timeout)

1218 -

1219 - if shutdown_timeout == 0:

1220 - num_requests = self.scheduler.get_num_unfinished_requests()

1221 - if num_requests > 0:

1222 - logger.info("Aborting %d requests", num_requests)

1223 - aborted_reqs = self.scheduler.finish_requests(

1224 - None, RequestStatus.FINISHED_ABORTED

1225 - )

1226 - self._send_abort_outputs(aborted_reqs)

1227 - else:

1228 - num_requests = self.scheduler.get_num_unfinished_requests()

1229 - if num_requests > 0:

1230 - logger.info(

1231 - "Draining %d in-flight requests (timeout=%ds)",

1232 - num_requests,

1233 - shutdown_timeout,

1234 - )

1235 -

1236 - self.shutdown_state = EngineShutdownState.SHUTTING_DOWN

1237 -

1238 - # Exit when no work remaining

1239 - if not self.has_work():

1240 - logger.info("Shutdown complete")

1241 - return False

1242 -

1243 - return True

1244 -

1190 def _handle_client_request(

1191 self, request_type: EngineCoreRequestType, request: Any

1192 ) -> None:

1193 """Dispatch request from client."""

1194

1250 - if request_type == EngineCoreRequestType.WAKEUP:

1251 - return

1252 - elif request_type == EngineCoreRequestType.ADD:

1195 + if request_type == EngineCoreRequestType.ADD:

1196 req, request_wave = request

1254 - if self._reject_add_in_shutdown(req):

1255 - return

1197 self.add_request(req, request_wave)

1198 elif request_type == EngineCoreRequestType.ABORT:

1199 self.abort_requests(request)

1200 elif request_type == EngineCoreRequestType.UTILITY:

1201 client_idx, call_id, method_name, args = request

1261 - if self._reject_utility_in_shutdown(client_idx, call_id, method_name):

1262 - return

1202 output = UtilityOutput(call_id)

1203 # Lazily look-up utility method so that failure will be handled/returned.

1204 get_result = lambda: (method := getattr(self, method_name)) and method(

@@ -1276,27 +1215,6 @@ class EngineCoreProc(EngineCore):

1215 "Unrecognized input request type encountered: %s", request_type

1216 )

1217

1279 - def _reject_add_in_shutdown(self, request: Request) -> bool:

1280 - if self.shutdown_state == EngineShutdownState.RUNNING:

1281 - return False

1282 -

1283 - logger.info("Rejecting request %s (server shutting down)", request.request_id)

1284 - self._send_abort_outputs_to_client([request.request_id], request.client_index)

1285 - return True

1286 -

1287 - def _reject_utility_in_shutdown(

1288 - self, client_idx: int, call_id: int, method_name: str

1289 - ) -> bool:

1290 - if self.shutdown_state == EngineShutdownState.RUNNING:

1291 - return False

1292 -

1293 - logger.warning("Rejecting utility call %s (server shutting down)", method_name)

1294 - output = UtilityOutput(call_id, failure_message="Server shutting down")

1295 - self.output_queue.put_nowait(

1296 - (client_idx, EngineCoreOutputs(utility_output=output))

1297 - )

1298 - return True

1299 -

1218 @staticmethod

1219 def _invoke_utility_method(

1220 name: str, get_result: Callable, output: UtilityOutput, enqueue_output: Callable

@@ -1510,7 +1428,22 @@ class EngineCoreProc(EngineCore):

1428 logger.exception(

1429 "Unexpected error pre-processing request %s", request.request_id

1430 )

1513 - self._send_error_outputs_to_client([request.request_id], request.client_index)

1431 + self.output_queue.put_nowait(

1432 + (

1433 + request.client_index,

1434 + EngineCoreOutputs(

1435 + engine_index=self.engine_index,

1436 + finished_requests={request.request_id},

1437 + outputs=[

1438 + EngineCoreOutput(

1439 + request_id=request.request_id,

1440 + new_token_ids=[],

1441 + finish_reason=FinishReason.ERROR,

1442 + )

1443 + ],

1444 + ),

1445 + )

1446 + )

1447

1448 def pause_scheduler(

1449 self, mode: PauseMode = "abort", clear_cache: bool = True

@@ -1553,26 +1486,6 @@ class EngineCoreProc(EngineCore):

1486 self._idle_state_callbacks.append(partial(engine_idle_callback, future=future))

1487 return future

1488

1556 - def _send_finish_outputs_to_client(

1557 - self, req_ids: list[str], client_index: int, finish_reason: FinishReason

1558 - ) -> None:

1559 - outputs = [

1560 - EngineCoreOutput(req_id, [], finish_reason=finish_reason)

1561 - for req_id in req_ids

1562 - ]

1563 - eco = EngineCoreOutputs(finished_requests=req_ids, outputs=outputs)

1564 - self.output_queue.put_nowait((client_index, eco))

1565 -

1566 - def _send_abort_outputs_to_client(

1567 - self, req_ids: list[str], client_index: int

1568 - ) -> None:

1569 - self._send_finish_outputs_to_client(req_ids, client_index, FinishReason.ABORT)

1570 -

1571 - def _send_error_outputs_to_client(

1572 - self, req_ids: list[str], client_index: int

1573 - ) -> None:

1574 - self._send_finish_outputs_to_client(req_ids, client_index, FinishReason.ERROR)

1575 -

1489 def _send_abort_outputs(self, aborted_reqs: list[tuple[str, int]]) -> None:

1490 # TODO(nick) this will be moved inside the scheduler

1491 if aborted_reqs:

@@ -1581,7 +1494,12 @@ class EngineCoreProc(EngineCore):

1494 for req_id, client_index in aborted_reqs:

1495 by_client[client_index].add(req_id)

1496 for client_index, req_ids in by_client.items():

1584 - self._send_abort_outputs_to_client(list(req_ids), client_index)

1497 + outputs = [

1498 + EngineCoreOutput(req_id, [], finish_reason=FinishReason.ABORT)

1499 + for req_id in req_ids

1500 + ]

1501 + eco = EngineCoreOutputs(finished_requests=req_ids, outputs=outputs)

1502 + self.output_queue.put_nowait((client_index, eco))

1503

1504

1505 class DPEngineCoreProc(EngineCoreProc):

@@ -1699,7 +1617,7 @@ class DPEngineCoreProc(EngineCoreProc):

1617 """Core busy loop of the EngineCore for data parallel case."""

1618

1619 # Loop until process is sent a SIGINT or SIGTERM

1702 - while self._handle_shutdown():

1620 + while True:

1621 # 1) Poll the input queue until there is work to do.

1622 self._process_input_queue()

1623

@@ -1747,8 +1665,6 @@ class DPEngineCoreProc(EngineCoreProc):

1665 self.current_wave += 1

1666 self.step_counter = 0

1667

1750 - raise SystemExit

1751 -

1668 def _has_global_unfinished_reqs(self, local_unfinished: bool) -> bool:

1669 # Optimization - only perform finish-sync all-reduce every 32 steps.

1670 self.step_counter += 1

vllm/v1/engine/core_client.py

+12 -12

@@ -128,7 +128,7 @@ class EngineCoreClient(ABC):

128 return AsyncMPClient(*client_args)

129

130 @abstractmethod

131 - def shutdown(self, timeout: float | None = None) -> None: ...

131 + def shutdown(self): ...

132

133 def get_output(self) -> EngineCoreOutputs:

134 raise NotImplementedError

@@ -298,7 +298,7 @@ class InprocClient(EngineCoreClient):

298 if len(request_ids) > 0:

299 self.engine_core.abort_requests(request_ids)

300

301 - def shutdown(self, timeout: float | None = None) -> None:

301 + def shutdown(self) -> None:

302 self.engine_core.shutdown()

303

304 def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None:

@@ -390,9 +390,9 @@ class BackgroundResources:

390

391 self.engine_dead = True

392 if self.engine_manager is not None:

393 - self.engine_manager.shutdown()

393 + self.engine_manager.close()

394 if self.coordinator is not None:

395 - self.coordinator.shutdown()

395 + self.coordinator.close()

396

397 if isinstance(self.output_socket, zmq.asyncio.Socket):

398 # Async case.

@@ -568,7 +568,10 @@ class MPClient(EngineCoreClient):

568 )

569

570 with launch_core_engines(

571 - vllm_config, executor_class, log_stats, addresses

571 + vllm_config,

572 + executor_class,

573 + log_stats,

574 + addresses,

575 ) as (engine_manager, coordinator, addresses):

576 self.resources.coordinator = coordinator

577 self.resources.engine_manager = engine_manager

@@ -634,12 +637,9 @@ class MPClient(EngineCoreClient):

637 if not success:

638 self._finalizer()

639

637 - def shutdown(self, timeout: float | None = None) -> None:

638 - """Shutdown engine manager under timeout and clean up resources."""

639 - if self._finalizer.detach() is not None:

640 - if self.resources.engine_manager is not None:

641 - self.resources.engine_manager.shutdown(timeout=timeout)

642 - self.resources()

640 + def shutdown(self):

641 + # Terminate background resources.

642 + self._finalizer()

643

644 def _format_exception(self, e: Exception) -> Exception:

645 """If errored, use EngineDeadError so root cause is clear."""

@@ -683,7 +683,7 @@ class MPClient(EngineCoreClient):

683 sentinels = [proc.sentinel for proc in engine_processes]

684 died = multiprocessing.connection.wait(sentinels)

685 _self = self_ref()

686 - if not _self or not _self._finalizer.alive or _self.resources.engine_dead:

686 + if not _self or _self.resources.engine_dead:

687 return

688 _self.resources.engine_dead = True

689 proc_name = next(

vllm/v1/engine/utils.py

+5 -34

@@ -3,7 +3,6 @@

4 import contextlib

5 import os

6 -import threading

6 import weakref

7 from collections.abc import Callable, Iterator

8 from dataclasses import dataclass

@@ -152,12 +151,11 @@ class CoreEngineProcManager:

151 finally:

152 # Kill other procs if not all are running.

153 if self.finished_procs():

155 - self.shutdown()

154 + self.close()

155

157 - def shutdown(self, timeout: float | None = None) -> None:

158 - """Shutdown engine core processes with configurable timeout."""

159 - if self._finalizer.detach() is not None:

160 - shutdown(self.processes, timeout=timeout)

156 + def close(self):

157 + """Shutdown all procs."""

158 + self._finalizer()

159

160 def join_first(self):

161 """Wait for any process to exit."""

@@ -175,33 +173,6 @@ class CoreEngineProcManager:

173 }

174

175

178 -class SignalCallback:

179 - """Safely trigger a callback from signal handler context via a dedicated thread."""

180 -

181 - def __init__(self, callback: Callable[[], None]):

182 - self._callback = callback

183 - self._event = threading.Event()

184 - self._stopped = False

185 - self._thread = threading.Thread(

186 - target=self._run,

187 - daemon=True,

188 - name="signal-callback",

189 - )

190 - self._thread.start()

191 -

192 - def _run(self):

193 - self._event.wait()

194 - if not self._stopped:

195 - self._callback()

196 -

197 - def trigger(self):

198 - self._event.set()

199 -

200 - def stop(self):

201 - self._stopped = True

202 - self._event.set()

203 -

204 -

176 @contextlib.contextmanager

177 def set_device_control_env_var(

178 vllm_config: VllmConfig, local_dp_rank: int

@@ -797,7 +768,7 @@ class CoreEngineActorManager:

768 def get_run_refs(self):

769 return self.run_refs

770

800 - def shutdown(self, timeout: float | None = None) -> None:

771 + def close(self):

772 import ray

773

774 for actor in self.local_engine_actors + self.remote_engine_actors:

vllm/v1/utils.py

+12 -19

@@ -220,10 +220,8 @@ class APIServerProcessManager:

220 # The extra processes are managed by their owners

221 self._finalizer = weakref.finalize(self, shutdown, self.processes)

222

223 - def shutdown(self, timeout: float | None = None) -> None:

224 - """Shutdown API server processes with configurable timeout"""

225 - if self._finalizer.detach() is not None:

226 - shutdown(self.processes, timeout=timeout)

223 + def close(self) -> None:

224 + self._finalizer()

225

226

227 def wait_for_completion_or_failure(

@@ -290,30 +288,25 @@ def wait_for_completion_or_failure(

288 except Exception as e:

289 logger.exception("Exception occurred while running API servers: %s", str(e))

290 raise

291 + finally:

292 + logger.info("Terminating remaining processes ...")

293 + api_server_manager.close()

294 + if coordinator:

295 + coordinator.close()

296 + if engine_manager:

297 + engine_manager.close()

298

299

300 # Note(rob): shutdown function cannot be a bound method,

301 # else the gc cannot collect the object.

297 -def shutdown(procs: list[BaseProcess], timeout: float | None = None) -> None:

298 - """Shutdown processes with timeout.

299 -

300 - Args:

301 - procs: List of processes to shutdown

302 - timeout: Maximum time in seconds to wait for graceful shutdown

303 - """

304 - if timeout is None:

305 - timeout = 0.0

306 -

307 - # Allow at least 5 seconds for remaining procs to terminate.

308 - timeout = max(timeout, 5.0)

309 -

302 +def shutdown(procs: list[BaseProcess]):

303 # Shutdown the process.

304 for proc in procs:

305 if proc.is_alive():

306 proc.terminate()

307

315 - # Allow time for remaining procs to terminate.

316 - deadline = time.monotonic() + timeout

308 + # Allow 5 seconds for remaining procs to terminate.

309 + deadline = time.monotonic() + 5

310 for proc in procs:

311 remaining = deadline - time.monotonic()

312 if remaining <= 0: