A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 22:45 UTC
fix nixl side-channel host selection (#41806)
Signed-off-by: Shahar Mor <smor@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
shaharmor98 committed
May 11, 2026 at 10:40 UTC
f9f770ca0b02f8b83ae92997a68efa797611ce36
8 files changed
+179
-2
tests/test_envs.py
+8
@@ -28,6 +28,14 @@ def test_getattr_without_cache(monkeypatch: pytest.MonkeyPatch):
28
assert not hasattr(envs.__getattr__, "cache_info")
29
30
31
+def test_nixl_side_channel_host_is_not_compile_factor(
32
+ monkeypatch: pytest.MonkeyPatch,
33
+):
34
+ monkeypatch.setenv("VLLM_NIXL_SIDE_CHANNEL_HOST", "10.0.0.15")
35
+
36
+ assert "VLLM_NIXL_SIDE_CHANNEL_HOST" not in envs.compile_factors()
37
+
38
+
39
def test_getattr_with_cache(monkeypatch: pytest.MonkeyPatch):
40
monkeypatch.setenv("VLLM_HOST_IP", "1.1.1.1")
41
monkeypatch.setenv("VLLM_PORT", "1234")
tests/test_ray_env.py
+14
@@ -6,6 +6,7 @@ import os
6
from unittest.mock import patch
7
8
from vllm.ray.ray_env import get_env_vars_to_copy
9
+from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS
10
11
# ---------------------------------------------------------------------------
12
# Default prefix matching
@@ -106,6 +107,19 @@ class TestExclusion:
107
result = get_env_vars_to_copy(exclude_vars={"CUDA_VISIBLE_DEVICES"})
108
assert "CUDA_VISIBLE_DEVICES" not in result
109
110
+ @patch.dict(
111
+ os.environ,
112
+ {
113
+ "VLLM_HOST_IP": "10.0.0.1",
114
+ "VLLM_NIXL_SIDE_CHANNEL_HOST": "10.0.0.1",
115
+ },
116
+ clear=False,
117
+ )
118
+ def test_worker_specific_host_vars_are_excluded(self):
119
+ result = get_env_vars_to_copy(exclude_vars=WORKER_SPECIFIC_ENV_VARS)
120
+ assert "VLLM_HOST_IP" not in result
121
+ assert "VLLM_NIXL_SIDE_CHANNEL_HOST" not in result
122
+
123
@patch.dict(os.environ, {"LMCACHE_LOCAL_CPU": "True"}, clear=False)
124
@patch(
125
"vllm.ray.ray_env.RAY_NON_CARRY_OVER_ENV_VARS",
tests/v1/engine/test_core_engine_actor_manager.py
new
+136
@@ -0,0 +1,136 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import os
5
+import uuid
6
+from pathlib import Path
7
+from types import SimpleNamespace
8
+from typing import Any
9
+
10
+import pytest
11
+import ray
12
+
13
+from vllm.v1.engine.core import EngineCoreActorMixin
14
+from vllm.v1.engine.utils import CoreEngineActorManager, EngineZmqAddresses
15
+
16
+
17
+class _StubEngineCoreActor(EngineCoreActorMixin):
18
+ def __init__(
19
+ self,
20
+ vllm_config: Any,
21
+ local_client: bool,
22
+ addresses: EngineZmqAddresses,
23
+ executor_class: type[Any],
24
+ log_stats: bool,
25
+ dp_rank: int = 0,
26
+ local_dp_rank: int = 0,
27
+ ):
28
+ # Exercise the production Ray actor mixin without loading a model.
29
+ EngineCoreActorMixin.__init__(
30
+ self, vllm_config, addresses, dp_rank, local_dp_rank
31
+ )
32
+
33
+ def _set_visible_devices(self, vllm_config: Any, local_dp_rank: int) -> None:
34
+ pass
35
+
36
+ def wait_for_init(self) -> None:
37
+ pass
38
+
39
+ def run(self) -> None:
40
+ pass
41
+
42
+ def get_nixl_side_channel_host(self) -> str | None:
43
+ return os.environ.get("VLLM_NIXL_SIDE_CHANNEL_HOST")
44
+
45
+
46
+class _DummyExecutor:
47
+ pass
48
+
49
+
50
+def _make_vllm_config() -> SimpleNamespace:
51
+ return SimpleNamespace(
52
+ parallel_config=SimpleNamespace(
53
+ data_parallel_size=1,
54
+ data_parallel_size_local=1,
55
+ enable_elastic_ep=False,
56
+ world_size=1,
57
+ ),
58
+ model_config=SimpleNamespace(is_moe=False),
59
+ kv_transfer_config=None,
60
+ )
61
+
62
+
63
+def _make_addresses() -> EngineZmqAddresses:
64
+ return EngineZmqAddresses(
65
+ inputs=["tcp://127.0.0.1:12345"],
66
+ outputs=["tcp://127.0.0.1:12346"],
67
+ )
68
+
69
+
70
+def _make_cpu_placement_group():
71
+ pg = ray.util.placement_group(
72
+ [{"CPU": 0.001}, {"CPU": 1.0}],
73
+ strategy="PACK",
74
+ )
75
+ ray.get(pg.ready())
76
+ return pg
77
+
78
+
79
+@pytest.fixture
80
+def ray_context():
81
+ started_ray = False
82
+ if not ray.is_initialized():
83
+ project_root = str(Path(__file__).resolve().parents[3])
84
+ ray.init(
85
+ num_cpus=2,
86
+ runtime_env={"env_vars": {"PYTHONPATH": project_root}},
87
+ log_to_driver=False,
88
+ )
89
+ started_ray = True
90
+
91
+ yield
92
+
93
+ if started_ray:
94
+ ray.shutdown()
95
+
96
+
97
+@pytest.mark.usefixtures("ray_context")
98
+def test_driver_nixl_side_channel_host_does_not_leak_to_engine_core_actor(
99
+ monkeypatch: pytest.MonkeyPatch,
100
+) -> None:
101
+ driver_marker = f"driver-only-nixl-host-{uuid.uuid4()}"
102
+ created_placement_groups: list[Any] = []
103
+ manager: CoreEngineActorManager | None = None
104
+
105
+ def create_dp_placement_groups(vllm_config: Any):
106
+ pg = _make_cpu_placement_group()
107
+ created_placement_groups.append(pg)
108
+ return [pg], [0]
109
+
110
+ monkeypatch.setenv("VLLM_NIXL_SIDE_CHANNEL_HOST", driver_marker)
111
+ monkeypatch.setattr("vllm.v1.engine.core.EngineCoreActor", _StubEngineCoreActor)
112
+ monkeypatch.setattr(
113
+ CoreEngineActorManager,
114
+ "create_dp_placement_groups",
115
+ staticmethod(create_dp_placement_groups),
116
+ )
117
+
118
+ try:
119
+ manager = CoreEngineActorManager(
120
+ vllm_config=_make_vllm_config(),
121
+ addresses=_make_addresses(),
122
+ executor_class=_DummyExecutor,
123
+ log_stats=False,
124
+ )
125
+ actor = manager.local_engine_actors[0]
126
+ actor_host = ray.get(actor.get_nixl_side_channel_host.remote())
127
+ node_host = ray.util.get_node_ip_address()
128
+
129
+ assert actor_host != driver_marker
130
+ assert actor_host == node_host
131
+ finally:
132
+ if manager is not None:
133
+ manager.shutdown()
134
+ else:
135
+ for pg in created_placement_groups:
136
+ ray.util.remove_placement_group(pg)
vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py
+2
-1
@@ -209,6 +209,7 @@ class NixlConnectorScheduler:
209
encoded_data,
210
ready_event,
211
self._stop_event,
212
+ self.side_channel_host,
213
self.side_channel_port,
214
),
215
daemon=True,
@@ -222,6 +223,7 @@ class NixlConnectorScheduler:
223
encoded_data: dict[int, Any],
224
ready_event: threading.Event,
225
stop_event: threading.Event,
226
+ host: str,
227
port: int,
228
):
229
"""Background thread for getting new NIXL handshakes."""
@@ -229,7 +231,6 @@ class NixlConnectorScheduler:
231
# to a better approach via HTTP endpoint soon.
232
233
# Listen for new requests for metadata.
232
- host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST
234
path = make_zmq_path("tcp", host, port)
235
logger.debug("Starting listening on path: %s", path)
236
with zmq_ctx(zmq.ROUTER, path) as sock:
vllm/envs.py
+1
@@ -1884,6 +1884,7 @@ def compile_factors() -> dict[str, object]:
1884
"VLLM_SERVER_DEV_MODE",
1885
"VLLM_DP_MASTER_IP",
1886
"VLLM_DP_MASTER_PORT",
1887
+ "VLLM_NIXL_SIDE_CHANNEL_HOST",
1888
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
1889
"VLLM_CI_USE_S3",
1890
"VLLM_MODEL_REDIRECT_PATH",
vllm/v1/engine/core.py
+12
@@ -2005,6 +2005,8 @@ class EngineCoreActorMixin:
2005
vllm_config.parallel_config.data_parallel_index = dp_rank
2006
vllm_config.parallel_config.data_parallel_rank_local = local_dp_rank
2007
2008
+ self._set_nixl_side_channel_host()
2009
+
2010
# Set CUDA_VISIBLE_DEVICES as early as possible in actor life cycle
2011
# NOTE: in MP we set CUDA_VISIBLE_DEVICES at process creation time,
2012
# and this cannot be done in the same way for Ray because:
@@ -2024,6 +2026,16 @@ class EngineCoreActorMixin:
2026
# of ray.
2027
self._set_visible_devices(vllm_config, local_dp_rank)
2028
2029
+ @staticmethod
2030
+ def _set_nixl_side_channel_host():
2031
+ import ray
2032
+
2033
+ # The driver-side value is excluded from Ray actor env propagation.
2034
+ # Fill in an actor-local default while preserving explicit overrides.
2035
+ os.environ.setdefault(
2036
+ "VLLM_NIXL_SIDE_CHANNEL_HOST", ray.util.get_node_ip_address()
2037
+ )
2038
+
2039
def _set_visible_devices(self, vllm_config: VllmConfig, local_dp_rank: int):
2040
from vllm.platforms import current_platform
2041
vllm/v1/engine/utils.py
+5
-1
@@ -27,6 +27,7 @@ from vllm.utils.network_utils import get_open_zmq_ipc_path, zmq_socket_ctx
27
from vllm.utils.system_utils import get_mp_context
28
from vllm.v1.engine.coordinator import DPCoordinator
29
from vllm.v1.executor import Executor
30
+from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS
31
from vllm.v1.utils import get_engine_client_zmq_addr, shutdown
32
33
if TYPE_CHECKING:
@@ -356,7 +357,10 @@ class CoreEngineActorManager:
357
self.local_engine_actors: list[ray.ActorHandle] = []
358
self.remote_engine_actors: list[ray.ActorHandle] = []
359
359
- env_vars_list = get_env_vars_to_copy(destination=actor_class.__name__)
360
+ env_vars_list = get_env_vars_to_copy(
361
+ destination=actor_class.__name__,
362
+ exclude_vars=WORKER_SPECIFIC_ENV_VARS,
363
+ )
364
self.env_vars_dict = {
365
name: os.environ[name] for name in env_vars_list if name in os.environ
366
}
vllm/v1/executor/ray_utils.py
+1
@@ -33,6 +33,7 @@ PG_WAIT_TIMEOUT = 1800
33
WORKER_SPECIFIC_ENV_VARS: set[str] = {
34
"VLLM_HOST_IP",
35
"VLLM_HOST_PORT",
36
+ "VLLM_NIXL_SIDE_CHANNEL_HOST",
37
"LOCAL_RANK",
38
"CUDA_VISIBLE_DEVICES",
39
"HIP_VISIBLE_DEVICES",