@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 812e7e7364

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 19:30 UTC

[Bugfix][V1] Fix TOCTOU race causing intermittent `EADDRINUSE` on multi-API-server DP startup (#42585)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Signed-off-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Vadim Gimpelson committed May 27, 2026 at 01:06 UTC 812e7e73648573142808fdeea0ffc30adb247ba3

7 files changed +316 -50

tests/entrypoints/test_api_server_process_manager.py

+129 -1

@@ -8,8 +8,14 @@ import time

8 from unittest.mock import patch

10 import pytest

11 +import zmq

12 -from vllm.v1.utils import APIServerProcessManager, wait_for_completion_or_failure

13 +from vllm.utils.network_utils import make_zmq_socket, split_zmq_path

14 +from vllm.v1.utils import (

15 + APIServerProcessManager,

16 + get_engine_client_zmq_addr,

17 + wait_for_completion_or_failure,

18 +)

20 # Global variables to control worker behavior

21 WORKER_RUNTIME_SECONDS = 0.5

@@ -23,6 +29,39 @@ def mock_run_api_server_worker(listen_address, sock, args, client_config=None):

29 print("Mock worker completed successfully")

32 +# Module-level stub for the gather_actual_addresses test. Must be

33 +# importable by `multiprocessing.spawn` (no closures, no nesting).

34 +def defer_addresses_stub_worker(listen_address, sock, args, client_config):

35 + """Bind ROUTER/PULL with a kernel-assigned port, report the actual

36 + endpoints back via the pipe, then exit."""

37 + ctx = zmq.Context()

38 + try:

39 + in_sock = make_zmq_socket(

40 + ctx, client_config["input_address"], zmq.ROUTER, bind=True

41 + )

42 + out_sock = make_zmq_socket(

43 + ctx, client_config["output_address"], zmq.PULL, bind=True

44 + )

45 + try:

46 + pipe = client_config["actual_address_pipe"]

47 + try:

48 + pipe.send(

49 + {

50 + "input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(),

51 + "output_address": out_sock.getsockopt(

52 + zmq.LAST_ENDPOINT

53 + ).decode(),

54 + }

55 + )

56 + finally:

57 + pipe.close()

58 + finally:

59 + in_sock.close(linger=0)

60 + out_sock.close(linger=0)

61 + finally:

62 + ctx.term()

63 +

64 +

65 @pytest.fixture

66 def api_server_args():

67 """Fixture to provide arguments for APIServerProcessManager."""

@@ -268,3 +307,92 @@ def test_external_process_monitoring(api_server_args):

307 manager.shutdown()

308 mock_coordinator.shutdown()

309 time.sleep(0.2)

310 +

311 +

312 +@pytest.mark.timeout(60)

313 +def test_gather_actual_addresses_end_to_end():

314 + """Each child binds ROUTER/PULL with a kernel-picked port and reports

315 + the bound endpoints back via its per-child pipe; the manager surfaces

316 + them via :py:meth:`gather_actual_addresses`."""

317 + host = "127.0.0.1"

318 + num_servers = 4

319 +

320 + placeholder_inputs = [

321 + get_engine_client_zmq_addr(local_only=False, host=host)

322 + for _ in range(num_servers)

323 + ]

324 + placeholder_outputs = [

325 + get_engine_client_zmq_addr(local_only=False, host=host)

326 + for _ in range(num_servers)

327 + ]

328 + for addr in placeholder_inputs + placeholder_outputs:

329 + assert addr == f"tcp://{host}:0", addr

330 +

331 + sock = socket.socket()

332 + manager = APIServerProcessManager(

333 + listen_address=f"tcp://{host}:0",

334 + sock=sock,

335 + args="test_args",

336 + num_servers=num_servers,

337 + input_addresses=placeholder_inputs,

338 + output_addresses=placeholder_outputs,

339 + target_server_fn=defer_addresses_stub_worker,

340 + )

341 +

342 + try:

343 + assert len(manager.processes) == num_servers

344 + actual_inputs, actual_outputs = manager.gather_actual_addresses(timeout=15.0)

345 + finally:

346 + manager.shutdown()

347 + time.sleep(0.2)

348 + sock.close()

349 +

350 + assert len(actual_inputs) == num_servers

351 + assert len(actual_outputs) == num_servers

352 +

353 + for addr in actual_inputs + actual_outputs:

354 + scheme, parsed_host, port = split_zmq_path(addr)

355 + assert scheme == "tcp", addr

356 + assert parsed_host == host, addr

357 + assert port and int(port) > 0, addr

358 +

359 + all_addrs = actual_inputs + actual_outputs

360 + assert len(set(all_addrs)) == len(all_addrs), all_addrs

361 +

362 +

363 +@pytest.mark.timeout(30)

364 +def test_gather_actual_addresses_child_crash_before_report():

365 + """A child that exits before sending its endpoints must surface a

366 + clear ``RuntimeError`` rather than hang or return ``None`` slots."""

367 + host = "127.0.0.1"

368 + num_servers = 2

369 + placeholder_inputs = [

370 + get_engine_client_zmq_addr(local_only=False, host=host)

371 + for _ in range(num_servers)

372 + ]

373 + placeholder_outputs = [

374 + get_engine_client_zmq_addr(local_only=False, host=host)

375 + for _ in range(num_servers)

376 + ]

377 +

378 + sock = socket.socket()

379 + manager = APIServerProcessManager(

380 + listen_address=f"tcp://{host}:0",

381 + sock=sock,

382 + args="test_args",

383 + num_servers=num_servers,

384 + input_addresses=placeholder_inputs,

385 + output_addresses=placeholder_outputs,

386 + # mock_run_api_server_worker exits without touching

387 + # ``actual_address_pipe`` — simulates a child that dies before

388 + # reporting its bound addresses.

389 + target_server_fn=mock_run_api_server_worker,

390 + )

391 + try:

392 + # Sentinel-first vs pipe-EOF-first both produce "reporting".

393 + with pytest.raises(RuntimeError, match="reporting"):

394 + manager.gather_actual_addresses(timeout=10.0)

395 + finally:

396 + manager.shutdown()

397 + time.sleep(0.2)

398 + sock.close()

vllm/entrypoints/cli/serve.py

+14 -1

@@ -308,7 +308,14 @@ def run_multi_api_server(args: argparse.Namespace):

308

309 from vllm.v1.engine.utils import get_engine_zmq_addresses

310

311 - addresses = get_engine_zmq_addresses(vllm_config, num_api_servers)

311 + # Per-API-server ports are picked by the kernel at each child's bind()

312 + # to avoid parent-probe vs child-bind TOCTOU; Rust front-end opts out

313 + # because it has no port-report-back channel.

314 + addresses = get_engine_zmq_addresses(

315 + vllm_config,

316 + num_api_servers,

317 + defer_api_server_ports=not rust_frontend_path,

318 + )

319

320 with launch_core_engines(

321 vllm_config, executor_class, log_stats, addresses, num_api_servers

@@ -341,6 +348,12 @@ def run_multi_api_server(args: argparse.Namespace):

348 tensor_queue=tensor_queue,

349 )

350

351 + # Forward each child's bound endpoints to the engine handshake

352 + # (runs on ``with`` exit).

353 + actual_inputs, actual_outputs = api_server_manager.gather_actual_addresses()

354 + addresses.inputs = actual_inputs

355 + addresses.outputs = actual_outputs

356 +

357 # Wait for API servers.

358 try:

359 wait_for_completion_or_failure(

vllm/v1/engine/async_llm.py

+2 -2

@@ -81,7 +81,7 @@ class AsyncLLM(EngineClient):

81 start_engine_loop: bool = True,

82 stat_loggers: list[StatLoggerFactory] | None = None,

83 aggregate_engine_logging: bool = False,

84 - client_addresses: dict[str, str] | None = None,

84 + client_addresses: dict[str, Any] | None = None,

85 client_count: int = 1,

86 client_index: int = 0,

87 ) -> None:

@@ -209,7 +209,7 @@ class AsyncLLM(EngineClient):

209 enable_log_requests: bool = False,

210 aggregate_engine_logging: bool = False,

211 disable_log_stats: bool = False,

212 - client_addresses: dict[str, str] | None = None,

212 + client_addresses: dict[str, Any] | None = None,

213 client_count: int = 1,

214 client_index: int = 0,

215 ) -> "AsyncLLM":

vllm/v1/engine/coordinator.py

+4 -11

@@ -11,7 +11,7 @@ import zmq

12 from vllm.config import ParallelConfig

13 from vllm.logger import init_logger

14 -from vllm.utils.network_utils import get_tcp_uri, make_zmq_socket

14 +from vllm.utils.network_utils import make_zmq_socket

15 from vllm.utils.system_utils import get_mp_context, set_process_title

16 from vllm.v1.engine import EngineCoreOutputs, EngineCoreRequestType

17 from vllm.v1.serial_utils import MsgpackDecoder

@@ -91,16 +91,9 @@ class DPCoordinator:

91 if parallel_config.enable_elastic_ep:

92 local_only_eng = False

94 - def bind_address(local_only: bool) -> str:

95 - return (

96 - get_engine_client_zmq_addr(local_only=True, host=host)

97 - if local_only

98 - else get_tcp_uri(host, 0)

99 - )

100 -

101 - front_publish_address = bind_address(local_only)

102 - back_publish_address = bind_address(local_only_eng)

103 - back_output_address = bind_address(local_only_eng)

94 + front_publish_address = get_engine_client_zmq_addr(local_only, host=host)

95 + back_publish_address = get_engine_client_zmq_addr(local_only_eng, host=host)

96 + back_output_address = get_engine_client_zmq_addr(local_only_eng, host=host)

98 context = get_mp_context()

99 parent_zmq_addr_pipe, child_zmq_addr_pipe = context.Pipe(duplex=False)

vllm/v1/engine/core_client.py

+38 -6

@@ -11,6 +11,7 @@ from collections import defaultdict, deque

11 from collections.abc import Awaitable, Callable, Sequence

12 from concurrent.futures import Future

13 from dataclasses import dataclass

14 +from multiprocessing.connection import Connection

15 from multiprocessing.queues import Queue

16 from threading import Thread

17 from typing import Any, TypeAlias, TypeVar

@@ -108,7 +109,7 @@ class EngineCoreClient(ABC):

109 vllm_config: VllmConfig,

110 executor_class: type[Executor],

111 log_stats: bool,

111 - client_addresses: dict[str, str] | None = None,

112 + client_addresses: dict[str, Any] | None = None,

113 client_count: int = 1,

114 client_index: int = 0,

115 ) -> "AsyncMPClient":

@@ -476,7 +477,7 @@ class MPClient(EngineCoreClient):

477 vllm_config: VllmConfig,

478 executor_class: type[Executor],

479 log_stats: bool,

479 - client_addresses: dict[str, str] | None = None,

480 + client_addresses: dict[str, Any] | None = None,

481 ):

482 self.vllm_config = vllm_config

483

@@ -507,7 +508,7 @@ class MPClient(EngineCoreClient):

508 output_address = client_addresses["output_address"]

509 self.stats_update_address = client_addresses.get("stats_update_address")

510 # Tensor queues passed via client_addresses for multi-API-server case

510 - tensor_queue = client_addresses.get("tensor_queue") # type: ignore[assignment]

511 + tensor_queue = client_addresses.get("tensor_queue")

512 self.input_socket = self.resources.input_socket = make_zmq_socket(

513 self.ctx,

514 input_address,

@@ -518,6 +519,28 @@ class MPClient(EngineCoreClient):

519 self.resources.output_socket = make_zmq_socket(

520 self.ctx, output_address, zmq.PULL

521 )

522 +

523 + # Report bound endpoints back so the parent can forward

524 + # them to engines (mirrors the DPCoordinator pattern).

525 + actual_address_pipe: Connection | None = client_addresses.get(

526 + "actual_address_pipe"

527 + )

528 + if actual_address_pipe is not None:

529 + try:

530 + actual_input = self.input_socket.getsockopt(

531 + zmq.LAST_ENDPOINT

532 + ).decode()

533 + actual_output = self.resources.output_socket.getsockopt(

534 + zmq.LAST_ENDPOINT

535 + ).decode()

536 + actual_address_pipe.send(

537 + {

538 + "input_address": actual_input,

539 + "output_address": actual_output,

540 + }

541 + )

542 + finally:

543 + actual_address_pipe.close()

544 else:

545 # Engines are managed by this client.

546 addresses = get_engine_zmq_addresses(vllm_config)

@@ -532,6 +555,15 @@ class MPClient(EngineCoreClient):

555 self.ctx, addresses.outputs[0], zmq.PULL

556 )

557

558 + # Resolve ``tcp://host:0`` placeholders to bound endpoints

559 + # before engines DEALER-connect. No-op for IPC.

560 + addresses.inputs[0] = self.input_socket.getsockopt(

561 + zmq.LAST_ENDPOINT

562 + ).decode()

563 + addresses.outputs[0] = self.resources.output_socket.getsockopt(

564 + zmq.LAST_ENDPOINT

565 + ).decode()

566 +

567 with launch_core_engines(

568 vllm_config, executor_class, log_stats, addresses

569 ) as (engine_manager, coordinator, addresses, tensor_queue):

@@ -893,7 +925,7 @@ class AsyncMPClient(MPClient):

925 vllm_config: VllmConfig,

926 executor_class: type[Executor],

927 log_stats: bool,

896 - client_addresses: dict[str, str] | None = None,

928 + client_addresses: dict[str, Any] | None = None,

929 client_count: int = 1,

930 client_index: int = 0,

931 ):

@@ -1143,7 +1175,7 @@ class DPAsyncMPClient(AsyncMPClient):

1175 vllm_config: VllmConfig,

1176 executor_class: type[Executor],

1177 log_stats: bool,

1146 - client_addresses: dict[str, str] | None = None,

1178 + client_addresses: dict[str, Any] | None = None,

1179 client_count: int = 1,

1180 client_index: int = 0,

1181 ):

@@ -1323,7 +1355,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):

1355 vllm_config: VllmConfig,

1356 executor_class: type[Executor],

1357 log_stats: bool,

1326 - client_addresses: dict[str, str] | None = None,

1358 + client_addresses: dict[str, Any] | None = None,

1359 client_count: int = 1,

1360 client_index: int = 0,

1361 ):

vllm/v1/engine/utils.py

+30 -13

@@ -23,7 +23,12 @@ from vllm.logger import init_logger

23 from vllm.platforms import current_platform

24 from vllm.ray.ray_env import get_env_vars_to_copy

25 from vllm.utils import numa_utils

26 -from vllm.utils.network_utils import get_open_zmq_ipc_path, zmq_socket_ctx

26 +from vllm.utils.network_utils import (

27 + get_open_port,

28 + get_open_zmq_ipc_path,

29 + get_tcp_uri,

30 + zmq_socket_ctx,

31 +)

32 from vllm.utils.system_utils import get_mp_context

33 from vllm.v1.engine.coordinator import DPCoordinator

34 from vllm.v1.executor import Executor

@@ -955,8 +960,19 @@ class CoreEngineActorManager:

960 def get_engine_zmq_addresses(

961 vllm_config: VllmConfig,

962 num_api_servers: int = 1,

963 + *,

964 + defer_api_server_ports: bool = True,

965 ) -> EngineZmqAddresses:

959 - """Allocate ZMQ addresses for engine-client communication."""

966 + """Allocate ZMQ addresses for engine-client communication.

967 +

968 + By default each TCP address is a ``tcp://host:0`` placeholder; the

969 + consumer (API-server child or single-process ``MPClient``) binds, then

970 + recovers the kernel-assigned port via ``getsockopt(zmq.LAST_ENDPOINT)``

971 + and writes it back into ``addresses`` before the engine handshake.

972 +

973 + Set ``defer_api_server_ports=False`` only when the consumer cannot

974 + report a bound port back (e.g. the Rust front-end). IPC paths are

975 + unaffected."""

976 parallel_config = vllm_config.parallel_config

977 local_engine_count = parallel_config.data_parallel_size_local

978 local_start_index = parallel_config.data_parallel_rank_local

@@ -978,15 +994,14 @@ def get_engine_zmq_addresses(

994 if parallel_config.enable_elastic_ep:

995 client_local_only = False

996

997 + def _addr() -> str:

998 + if client_local_only:

999 + return get_open_zmq_ipc_path()

1000 + return get_tcp_uri(host, 0 if defer_api_server_ports else get_open_port())

1001 +

1002 return EngineZmqAddresses(

982 - inputs=[

983 - get_engine_client_zmq_addr(client_local_only, host)

984 - for _ in range(num_api_servers)

985 - ],

986 - outputs=[

987 - get_engine_client_zmq_addr(client_local_only, host)

988 - for _ in range(num_api_servers)

989 - ],

1003 + inputs=[_addr() for _ in range(num_api_servers)],

1004 + outputs=[_addr() for _ in range(num_api_servers)],

1005 )

1006

1007

@@ -1095,9 +1110,11 @@ def launch_core_engines(

1110 if parallel_config.enable_elastic_ep:

1111 handshake_local_only = False

1112

1098 - handshake_address = get_engine_client_zmq_addr(

1099 - handshake_local_only, host, parallel_config.data_parallel_rpc_port

1100 - )

1113 + # Preserve "port=0 means auto-pick" for the handshake address, which

1114 + # is consumed by engines spawned in this process and so cannot defer

1115 + # port resolution to bind time.

1116 + rpc_port = parallel_config.data_parallel_rpc_port or get_open_port()

1117 + handshake_address = get_engine_client_zmq_addr(handshake_local_only, host, rpc_port)

1118

1119 if local_engines_only and dp_rank > 0:

1120 assert not handshake_local_only

vllm/v1/utils.py

+99 -16

@@ -29,7 +29,7 @@ from torch.autograd.profiler import record_function

29 import vllm.envs as envs

30 from vllm.logger import init_logger

31 from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message

32 -from vllm.utils.network_utils import get_open_port, get_open_zmq_ipc_path, get_tcp_uri

32 +from vllm.utils.network_utils import get_open_zmq_ipc_path, get_tcp_uri

33 from vllm.utils.system_utils import decorate_logs, kill_process_tree, set_process_title

34 from vllm.v1.core.sched.output import SchedulerOutput

@@ -144,20 +144,18 @@ class CpuGpuBuffer:

144 return self.cpu[:n].copy_(self.gpu[:n], non_blocking=True)

145

146

147 -def get_engine_client_zmq_addr(local_only: bool, host: str, port: int = 0) -> str:

148 - """Assign a new ZMQ socket address.

147 +def get_engine_client_zmq_addr(

148 + local_only: bool,

149 + host: str,

150 + port: int = 0,

151 +) -> str:

152 + """Return an IPC path (``local_only=True``) or ``tcp://host:port``.

153

150 - If local_only is True, participants are colocated and so a unique IPC

151 - address will be returned.

152 -

153 - Otherwise, the provided host and port will be used to construct a TCP

154 - address (port == 0 means assign an available port)."""

155 -

156 - return (

157 - get_open_zmq_ipc_path()

158 - if local_only

159 - else (get_tcp_uri(host, port or get_open_port()))

160 - )

154 + ``port=0`` lets the kernel assign the port at ``bind()`` time; the

155 + caller must recover it via ``getsockopt(zmq.LAST_ENDPOINT)``."""

156 + if local_only:

157 + return get_open_zmq_ipc_path()

158 + return get_tcp_uri(host, port)

159

160

161 class APIServerProcessManager:

@@ -181,6 +179,12 @@ class APIServerProcessManager:

179 ):

180 """Initialize and start API server worker processes.

181

182 + ``input_addresses``/``output_addresses`` may contain

183 + ``tcp://host:0`` placeholders; each child must report the actual

184 + bound endpoint over its ``actual_address_pipe`` in ``client_config``

185 + and the parent collects them via

186 + :py:meth:`gather_actual_addresses`.

187 +

188 Args:

189 target_server_fn: Override function to call for each API server process

190 listen_address: Address to listen for client connections

@@ -196,14 +200,14 @@ class APIServerProcessManager:

200 self.sock = sock

201 self.args = args

202

199 - # Start API servers

203 spawn_context = multiprocessing.get_context("spawn")

204 self.processes: list[BaseProcess] = []

205 + self._address_pipes: list[connection.Connection] = []

206

207 for i, in_addr, out_addr in zip(

208 range(num_servers), input_addresses, output_addresses

209 ):

206 - client_config = {

210 + client_config: dict[str, Any] = {

211 "input_address": in_addr,

212 "output_address": out_addr,

213 "client_count": num_servers,

@@ -214,6 +218,10 @@ class APIServerProcessManager:

218 if tensor_queue is not None:

219 client_config["tensor_queue"] = tensor_queue

220

221 + parent_recv, child_send = spawn_context.Pipe(duplex=False)

222 + self._address_pipes.append(parent_recv)

223 + client_config["actual_address_pipe"] = child_send

224 +

225 proc = spawn_context.Process(

226 target=target_server_fn or run_api_server_worker_proc,

227 name=f"ApiServer_{i}",

@@ -222,14 +230,89 @@ class APIServerProcessManager:

230 self.processes.append(proc)

231 proc.start()

232

233 + # Drop parent's write end so reader sees EOF on child death.

234 + child_send.close()

235 +

236 logger.info("Started %d API server processes", len(self.processes))

237

238 # Shutdown only the API server processes on garbage collection

239 # The extra processes are managed by their owners

240 self._finalizer = weakref.finalize(self, shutdown, self.processes)

241

242 + def gather_actual_addresses(

243 + self,

244 + timeout: float = 60.0,

245 + ) -> tuple[list[str], list[str]]:

246 + """Return (inputs, outputs) reported by each child, indexed by

247 + ``client_index``. Raises ``RuntimeError`` on timeout or premature

248 + child exit."""

249 + n = len(self._address_pipes)

250 + inputs: list[str | None] = [None] * n

251 + outputs: list[str | None] = [None] * n

252 + pending: dict[connection.Connection, int] = {

253 + pipe: i for i, pipe in enumerate(self._address_pipes)

254 + }

255 + sentinel_to_idx: dict[Any, int] = {

256 + proc.sentinel: i for i, proc in enumerate(self.processes)

257 + }

258 +

259 + deadline = time.monotonic() + timeout

260 + try:

261 + while pending:

262 + remaining = deadline - time.monotonic()

263 + if remaining <= 0:

264 + missing = [self.processes[i].name for i in pending.values()]

265 + raise RuntimeError(

266 + f"Timed out after {timeout:.1f}s waiting for "

267 + f"API server(s) to report bound ZMQ addresses: "

268 + f"{missing}"

269 + )

270 + waitables: list[Any] = list(pending.keys()) + list(

271 + sentinel_to_idx.keys()

272 + )

273 + ready = connection.wait(waitables, timeout=remaining)

274 + # Drain pipes before checking sentinels: a child that sent

275 + # its message and then exited can surface both events in

276 + # the same poll, and we must record the success first.

277 + for item in ready:

278 + if isinstance(item, connection.Connection) and item in pending:

279 + idx = pending.pop(item)

280 + try:

281 + msg: dict[str, str] = item.recv()

282 + except EOFError as e:

283 + raise RuntimeError(

284 + f"API server {self.processes[idx].name} "

285 + f"closed its address pipe without "

286 + f"reporting its bound ZMQ addresses"

287 + ) from e

288 + inputs[idx] = msg["input_address"]

289 + outputs[idx] = msg["output_address"]

290 + item.close()

291 + for item in ready:

292 + if item in sentinel_to_idx:

293 + idx = sentinel_to_idx.pop(item)

294 + pipe = self._address_pipes[idx]

295 + if pipe in pending:

296 + proc = self.processes[idx]

297 + raise RuntimeError(

298 + f"API server process {proc.name} exited "

299 + f"(code={proc.exitcode}) before reporting "

300 + f"its bound ZMQ addresses"

301 + )

302 + finally:

303 + for pipe in pending:

304 + with contextlib.suppress(Exception):

305 + pipe.close()

306 +

307 + return inputs, outputs # type: ignore[return-value]

308 +

309 def shutdown(self, timeout: float | None = None) -> None:

310 """Shutdown API server processes with configurable timeout"""

311 + for pipe in self._address_pipes:

312 + with contextlib.suppress(Exception):

313 + pipe.close()

314 + self._address_pipes = []

315 +

316 if self._finalizer.detach() is not None:

317 shutdown(self.processes, timeout=timeout)

318