A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
[Ray] Propagate third-party env vars to Ray workers via prefix matching (#34383)
Signed-off-by: Kourosh Hakhamaneshi <kourosh@anyscale.com> Co-authored-by: Cursor <cursoragent@cursor.com>
kourosh hakhamaneshi committed
Feb 17, 2026 at 01:08 UTC
c464b573749d956a1e7a5721c44853677b5c9659
5 files changed
+279
-36
.buildkite/test_areas/misc.yaml
+2
@@ -123,6 +123,7 @@ steps:
123
- tests/test_inputs.py
124
- tests/test_outputs.py
125
- tests/test_pooling_params.py
126
+ - tests/test_ray_env.py
127
- tests/multimodal
128
- tests/renderers
129
- tests/standalone_tests/lazy_imports.py
@@ -136,6 +137,7 @@ steps:
137
- pytest -v -s test_inputs.py
138
- pytest -v -s test_outputs.py
139
- pytest -v -s test_pooling_params.py
140
+ - pytest -v -s test_ray_env.py
141
- pytest -v -s -m 'cpu_test' multimodal
142
- pytest -v -s renderers
143
- pytest -v -s tokenizers_
tests/test_ray_env.py
new
+194
@@ -0,0 +1,194 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Tests for vllm.ray.ray_env — env var propagation to Ray workers."""
4
+
5
+import os
6
+from unittest.mock import patch
7
+
8
+from vllm.ray.ray_env import get_env_vars_to_copy
9
+
10
+# ---------------------------------------------------------------------------
11
+# Default prefix matching
12
+# ---------------------------------------------------------------------------
13
+
14
+
15
+class TestDefaultPrefixes:
16
+ """Built-in prefixes (VLLM_, LMCACHE_, NCCL_, UCX_, HF_, HUGGING_FACE_)
17
+ should be forwarded without any extra configuration."""
18
+
19
+ @patch.dict(os.environ, {"LMCACHE_LOCAL_CPU": "True"}, clear=False)
20
+ def test_lmcache_prefix(self):
21
+ result = get_env_vars_to_copy()
22
+ assert "LMCACHE_LOCAL_CPU" in result
23
+
24
+ @patch.dict(os.environ, {"NCCL_DEBUG": "INFO"}, clear=False)
25
+ def test_nccl_prefix(self):
26
+ result = get_env_vars_to_copy()
27
+ assert "NCCL_DEBUG" in result
28
+
29
+ @patch.dict(os.environ, {"UCX_TLS": "rc"}, clear=False)
30
+ def test_ucx_prefix(self):
31
+ result = get_env_vars_to_copy()
32
+ assert "UCX_TLS" in result
33
+
34
+ @patch.dict(os.environ, {"HF_TOKEN": "secret"}, clear=False)
35
+ def test_hf_token_via_prefix(self):
36
+ result = get_env_vars_to_copy()
37
+ assert "HF_TOKEN" in result
38
+
39
+ @patch.dict(os.environ, {"HUGGING_FACE_HUB_TOKEN": "secret"}, clear=False)
40
+ def test_hugging_face_prefix(self):
41
+ result = get_env_vars_to_copy()
42
+ assert "HUGGING_FACE_HUB_TOKEN" in result
43
+
44
+
45
+# ---------------------------------------------------------------------------
46
+# Default extra vars
47
+# ---------------------------------------------------------------------------
48
+
49
+
50
+class TestDefaultExtraVars:
51
+ """Individual vars listed in VLLM_RAY_EXTRA_ENV_VARS_TO_COPY's default."""
52
+
53
+ def test_pythonhashseed_in_result(self):
54
+ """PYTHONHASHSEED should always be in the result set (as a name to
55
+ copy) regardless of whether it is actually set in os.environ."""
56
+ result = get_env_vars_to_copy()
57
+ assert "PYTHONHASHSEED" in result
58
+
59
+
60
+# ---------------------------------------------------------------------------
61
+# User-supplied extensions
62
+# ---------------------------------------------------------------------------
63
+
64
+
65
+class TestUserExtensions:
66
+ """Users can add prefixes and extra vars at deploy time."""
67
+
68
+ @patch.dict(
69
+ os.environ,
70
+ {
71
+ "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY": "MYLIB_",
72
+ "MYLIB_FOO": "bar",
73
+ },
74
+ clear=False,
75
+ )
76
+ def test_user_prefix(self):
77
+ """User-supplied prefixes are additive — built-in defaults are kept."""
78
+ result = get_env_vars_to_copy()
79
+ assert "MYLIB_FOO" in result
80
+
81
+ @patch.dict(
82
+ os.environ,
83
+ {
84
+ "VLLM_RAY_EXTRA_ENV_VARS_TO_COPY": "MY_SECRET",
85
+ "MY_SECRET": "val",
86
+ },
87
+ clear=False,
88
+ )
89
+ def test_user_extra_var(self):
90
+ """User-supplied extras are additive — PYTHONHASHSEED still included."""
91
+ result = get_env_vars_to_copy()
92
+ assert "MY_SECRET" in result
93
+ assert "PYTHONHASHSEED" in result
94
+
95
+
96
+# ---------------------------------------------------------------------------
97
+# Exclusion
98
+# ---------------------------------------------------------------------------
99
+
100
+
101
+class TestExclusion:
102
+ """exclude_vars and RAY_NON_CARRY_OVER_ENV_VARS take precedence."""
103
+
104
+ @patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0,1"}, clear=False)
105
+ def test_exclude_vars(self):
106
+ result = get_env_vars_to_copy(exclude_vars={"CUDA_VISIBLE_DEVICES"})
107
+ assert "CUDA_VISIBLE_DEVICES" not in result
108
+
109
+ @patch.dict(os.environ, {"LMCACHE_LOCAL_CPU": "True"}, clear=False)
110
+ @patch(
111
+ "vllm.ray.ray_env.RAY_NON_CARRY_OVER_ENV_VARS",
112
+ {"LMCACHE_LOCAL_CPU"},
113
+ )
114
+ def test_non_carry_over_blacklist(self):
115
+ result = get_env_vars_to_copy()
116
+ assert "LMCACHE_LOCAL_CPU" not in result
117
+
118
+
119
+# ---------------------------------------------------------------------------
120
+# additional_vars (platform extension point)
121
+# ---------------------------------------------------------------------------
122
+
123
+
124
+class TestAdditionalVars:
125
+ """The additional_vars parameter supports platform-specific vars."""
126
+
127
+ @patch.dict(os.environ, {"CUSTOM_PLATFORM_VAR": "1"}, clear=False)
128
+ def test_additional_vars_passthrough(self):
129
+ result = get_env_vars_to_copy(additional_vars={"CUSTOM_PLATFORM_VAR"})
130
+ assert "CUSTOM_PLATFORM_VAR" in result
131
+
132
+
133
+# ---------------------------------------------------------------------------
134
+# Edge cases
135
+# ---------------------------------------------------------------------------
136
+
137
+
138
+class TestEdgeCases:
139
+ """Prefix matching should be strict (startswith, not contains)."""
140
+
141
+ @patch.dict(os.environ, {"LMCACH_TYPO": "1"}, clear=False)
142
+ def test_prefix_no_partial_match(self):
143
+ """'LMCACH_' does not match the 'LMCACHE_' prefix."""
144
+ result = get_env_vars_to_copy()
145
+ assert "LMCACH_TYPO" not in result
146
+
147
+ @patch.dict(
148
+ os.environ,
149
+ {
150
+ "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY": " MYLIB_ , OTHER_ ",
151
+ },
152
+ clear=False,
153
+ )
154
+ def test_csv_whitespace_handling(self):
155
+ """Whitespace around commas and tokens should be stripped."""
156
+ result = get_env_vars_to_copy()
157
+ # MYLIB_ and OTHER_ should be parsed as valid prefixes — no crash
158
+ assert isinstance(result, set)
159
+
160
+ @patch.dict(
161
+ os.environ,
162
+ {
163
+ "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY": "MYLIB_",
164
+ "LMCACHE_BACKEND": "cpu",
165
+ "NCCL_DEBUG": "INFO",
166
+ "MYLIB_FOO": "bar",
167
+ },
168
+ clear=False,
169
+ )
170
+ def test_user_prefix_additive(self):
171
+ """Setting VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY does NOT drop defaults."""
172
+ result = get_env_vars_to_copy()
173
+ # Built-in defaults still present
174
+ assert "LMCACHE_BACKEND" in result
175
+ assert "NCCL_DEBUG" in result
176
+ # User addition also present
177
+ assert "MYLIB_FOO" in result
178
+
179
+ @patch.dict(
180
+ os.environ,
181
+ {
182
+ "VLLM_RAY_EXTRA_ENV_VARS_TO_COPY": "MY_FLAG",
183
+ "PYTHONHASHSEED": "42",
184
+ "MY_FLAG": "1",
185
+ },
186
+ clear=False,
187
+ )
188
+ def test_user_extra_additive(self):
189
+ """Setting VLLM_RAY_EXTRA_ENV_VARS_TO_COPY does NOT drop defaults."""
190
+ result = get_env_vars_to_copy()
191
+ # Built-in default still present
192
+ assert "PYTHONHASHSEED" in result
193
+ # User addition also present
194
+ assert "MY_FLAG" in result
vllm/envs.py
+15
@@ -139,6 +139,8 @@ if TYPE_CHECKING:
139
VLLM_ENABLE_MOE_DP_CHUNK: bool = True
140
VLLM_RANDOMIZE_DP_DUMMY_INPUTS: bool = False
141
VLLM_RAY_DP_PACK_STRATEGY: Literal["strict", "fill", "span"] = "strict"
142
+ VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY: str = ""
143
+ VLLM_RAY_EXTRA_ENV_VARS_TO_COPY: str = ""
144
VLLM_MARLIN_USE_ATOMIC_ADD: bool = False
145
VLLM_MARLIN_INPUT_DTYPE: Literal["int8", "fp8"] | None = None
146
VLLM_MXFP4_USE_MARLIN: bool | None = None
@@ -1090,6 +1092,19 @@ environment_variables: dict[str, Callable[[], Any]] = {
1092
"VLLM_RAY_DP_PACK_STRATEGY": lambda: os.getenv(
1093
"VLLM_RAY_DP_PACK_STRATEGY", "strict"
1094
),
1095
+ # Comma-separated *additional* prefixes of env vars to copy from the
1096
+ # driver to Ray workers. These are merged with the built-in defaults
1097
+ # defined in ``vllm.ray.ray_env`` (VLLM_, etc.). Example: "MYLIB_,OTHER_"
1098
+ "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY": lambda: os.getenv(
1099
+ "VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY", ""
1100
+ ),
1101
+ # Comma-separated *additional* individual env var names to copy from
1102
+ # the driver to Ray workers. Merged with the built-in defaults
1103
+ # defined in ``vllm.ray.ray_env`` (PYTHONHASHSEED).
1104
+ # Example: "MY_SECRET,MY_FLAG"
1105
+ "VLLM_RAY_EXTRA_ENV_VARS_TO_COPY": lambda: os.getenv(
1106
+ "VLLM_RAY_EXTRA_ENV_VARS_TO_COPY", ""
1107
+ ),
1108
# Whether to use S3 path for model loading in CI via RunAI Streamer
1109
"VLLM_CI_USE_S3": lambda: os.environ.get("VLLM_CI_USE_S3", "0") == "1",
1110
# Use model_redirect to redirect the model name to a local folder.
vllm/ray/ray_env.py
+67
-30
@@ -10,8 +10,7 @@ logger = init_logger(__name__)
10
11
CONFIG_HOME = envs.VLLM_CONFIG_ROOT
12
13
-# This file contains a list of env vars that should not be copied
14
-# from the driver to the Ray workers.
13
+# Env vars that should NOT be copied from the driver to Ray workers.
14
RAY_NON_CARRY_OVER_ENV_VARS_FILE = os.path.join(
15
CONFIG_HOME, "ray_non_carry_over_env_vars.json"
16
)
@@ -29,51 +28,89 @@ except json.JSONDecodeError:
28
)
29
RAY_NON_CARRY_OVER_ENV_VARS = set()
30
31
+# ---------------------------------------------------------------------------
32
+# Built-in defaults for env var propagation.
33
+# Users can add more via VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY and
34
+# VLLM_RAY_EXTRA_ENV_VARS_TO_COPY (additive, not replacing).
35
+# ---------------------------------------------------------------------------
36
+DEFAULT_ENV_VAR_PREFIXES: set[str] = {
37
+ "VLLM_",
38
+ "LMCACHE_",
39
+ "NCCL_",
40
+ "UCX_",
41
+ "HF_",
42
+ "HUGGING_FACE_",
43
+}
44
+
45
+DEFAULT_EXTRA_ENV_VARS: set[str] = {
46
+ "PYTHONHASHSEED",
47
+}
48
+
49
+
50
+def _parse_csv(value: str) -> set[str]:
51
+ """Split a comma-separated string into a set of stripped, non-empty tokens."""
52
+ return {tok.strip() for tok in value.split(",") if tok.strip()}
53
+
54
55
def get_env_vars_to_copy(
56
exclude_vars: set[str] | None = None,
57
additional_vars: set[str] | None = None,
58
destination: str | None = None,
59
) -> set[str]:
38
- """
39
- Get the environment variables to copy to downstream Ray actors.
60
+ """Return the env var names to copy from the driver to Ray actors.
61
41
- Example use cases:
42
- - Copy environment variables from RayDistributedExecutor to Ray workers.
43
- - Copy environment variables from RayDPClient to Ray DPEngineCoreActor.
62
+ The result is the union of:
63
+
64
+ 1. Env vars registered in ``vllm.envs.environment_variables``.
65
+ 2. Env vars in ``os.environ`` matching a prefix in
66
+ ``DEFAULT_ENV_VAR_PREFIXES`` + ``VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY``.
67
+ 3. Individual names in ``DEFAULT_EXTRA_ENV_VARS`` +
68
+ ``VLLM_RAY_EXTRA_ENV_VARS_TO_COPY``.
69
+ 4. Caller-supplied *additional_vars* (e.g. platform-specific).
70
+
71
+ Minus any names in *exclude_vars* or ``RAY_NON_CARRY_OVER_ENV_VARS``.
72
73
Args:
46
- exclude_vars: A set of vllm defined environment variables to exclude
47
- from copying.
48
- additional_vars: A set of additional environment variables to copy.
49
- If a variable is in both exclude_vars and additional_vars, it will
50
- be excluded.
51
- destination: The destination of the environment variables.
52
- Returns:
53
- A set of environment variables to copy.
74
+ exclude_vars: Env vars to exclude (e.g. worker-specific ones).
75
+ additional_vars: Extra individual env var names to copy. Useful
76
+ for caller-specific vars (e.g. platform env vars).
77
+ destination: Label used in log messages only.
78
"""
55
- exclude_vars = exclude_vars or set()
56
- additional_vars = additional_vars or set()
79
+ exclude = (exclude_vars or set()) | RAY_NON_CARRY_OVER_ENV_VARS
80
58
- env_vars_to_copy = {
59
- v
60
- for v in set(envs.environment_variables).union(additional_vars)
61
- if v not in exclude_vars and v not in RAY_NON_CARRY_OVER_ENV_VARS
62
- }
81
+ # -- prefixes (built-in + user-supplied, additive) ----------------------
82
+ prefixes = DEFAULT_ENV_VAR_PREFIXES | _parse_csv(
83
+ envs.VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY
84
+ )
85
64
- to_destination = " to " + destination if destination is not None else ""
86
+ # -- collect env var names ----------------------------------------------
87
+ # 1. vLLM's registered env vars
88
+ result = set(envs.environment_variables)
89
+ # 2. Prefix-matched vars present in the current environment
90
+ result |= {name for name in os.environ if any(name.startswith(p) for p in prefixes)}
91
+ # 3. Individual extra vars (built-in + user-supplied, additive)
92
+ result |= DEFAULT_EXTRA_ENV_VARS | _parse_csv(envs.VLLM_RAY_EXTRA_ENV_VARS_TO_COPY)
93
+ # 4. Caller-supplied extra vars (e.g. platform-specific)
94
+ result |= additional_vars or set()
95
+ # 5. Exclude worker-specific and user-blacklisted vars
96
+ result -= exclude
97
66
- logger.info(
67
- "RAY_NON_CARRY_OVER_ENV_VARS from config: %s", RAY_NON_CARRY_OVER_ENV_VARS
68
- )
98
+ # -- logging ------------------------------------------------------------
99
+ dest = f" to {destination}" if destination else ""
100
+ logger.info("Env var prefixes to copy: %s", sorted(prefixes))
101
logger.info(
102
"Copying the following environment variables%s: %s",
71
- to_destination,
72
- [v for v in env_vars_to_copy if v in os.environ],
103
+ dest,
104
+ sorted(v for v in result if v in os.environ),
105
)
106
+ if RAY_NON_CARRY_OVER_ENV_VARS:
107
+ logger.info(
108
+ "RAY_NON_CARRY_OVER_ENV_VARS from config: %s",
109
+ RAY_NON_CARRY_OVER_ENV_VARS,
110
+ )
111
logger.info(
75
- "If certain env vars should NOT be copied, add them to %s file",
112
+ "To exclude env vars from copying, add them to %s",
113
RAY_NON_CARRY_OVER_ENV_VARS_FILE,
114
)
115
79
- return env_vars_to_copy
116
+ return result
vllm/v1/executor/ray_executor.py
+1
-6
@@ -73,9 +73,6 @@ class RayDistributedExecutor(Executor):
73
"ROCR_VISIBLE_DEVICES",
74
}
75
76
- # These non-vLLM env vars are copied from the driver to workers
77
- ADDITIONAL_ENV_VARS = {"HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"}
78
-
76
uses_ray: bool = True
77
supports_pp: bool = True
78
@@ -339,9 +336,7 @@ class RayDistributedExecutor(Executor):
336
# Environment variables to copy from driver to workers
337
env_vars_to_copy = get_env_vars_to_copy(
338
exclude_vars=self.WORKER_SPECIFIC_ENV_VARS,
342
- additional_vars=set(current_platform.additional_env_vars).union(
343
- self.ADDITIONAL_ENV_VARS
344
- ),
339
+ additional_vars=set(current_platform.additional_env_vars),
340
destination="workers",
341
)
342