A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[core] Pass all driver env vars to ray workers unless excluded (#14099)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Mar 3, 2025 at 19:44 UTC
bf13d40972357e41779bc7dbfe729246b1c247c1
1 file changed
+27
-3
vllm/executor/ray_distributed_executor.py
+27
-3
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import asyncio
4
+import json
5
import os
6
from collections import defaultdict
7
from dataclasses import dataclass
@@ -48,6 +49,24 @@ class RayWorkerMetaData:
49
50
51
class RayDistributedExecutor(DistributedExecutorBase):
52
+ """Ray-based distributed executor"""
53
+
54
+ # These env vars are worker-specific, therefore are NOT copied
55
+ # from the driver to the workers
56
+ WORKER_SPECIFIC_ENV_VARS = {
57
+ "VLLM_HOST_IP", "VLLM_HOST_PORT", "LOCAL_RANK", "CUDA_VISIBLE_DEVICES"
58
+ }
59
+
60
+ config_home = envs.VLLM_CONFIG_ROOT
61
+ # This file contains a list of env vars that should not be copied
62
+ # from the driver to the Ray workers.
63
+ non_carry_over_env_vars_file = os.path.join(
64
+ config_home, "ray_non_carry_over_env_vars.json")
65
+ if os.path.exists(non_carry_over_env_vars_file):
66
+ with open(non_carry_over_env_vars_file) as f:
67
+ non_carry_over_env_vars = set(json.load(f))
68
+ else:
69
+ non_carry_over_env_vars = set()
70
71
uses_ray: bool = True
72
@@ -311,9 +330,9 @@ class RayDistributedExecutor(DistributedExecutorBase):
330
331
# Environment variables to copy from driver to workers
332
env_vars_to_copy = [
314
- "VLLM_ATTENTION_BACKEND", "TPU_CHIPS_PER_HOST_BOUNDS",
315
- "TPU_HOST_BOUNDS", "VLLM_USE_V1", "VLLM_TRACE_FUNCTION",
316
- "VLLM_TORCH_PROFILER_DIR", "VLLM_TEST_ENABLE_EP"
333
+ v for v in envs.environment_variables
334
+ if v not in self.WORKER_SPECIFIC_ENV_VARS
335
+ and v not in self.non_carry_over_env_vars
336
]
337
338
# Copy existing env vars to each worker's args
@@ -323,9 +342,14 @@ class RayDistributedExecutor(DistributedExecutorBase):
342
if name in os.environ:
343
args[name] = os.environ[name]
344
345
+ logger.info("non_carry_over_env_vars from config: %s",
346
+ self.non_carry_over_env_vars)
347
logger.info(
348
"Copying the following environment variables to workers: %s",
349
[v for v in env_vars_to_copy if v in os.environ])
350
+ logger.info(
351
+ "If certain env vars should NOT be copied to workers, add them to "
352
+ "%s file", self.non_carry_over_env_vars_file)
353
354
self._env_vars_for_all_workers = (
355
all_args_to_update_environment_variables)