@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b8a7c1cb0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] centralize all usage of environment variables (#4548)

youkaichao committed May 2, 2024 at 11:13 UTC 5b8a7c1cb0f1bb81266bae98944c055a8abb1a68
18 files changed +220 -64
vllm/attention/backends/rocm_flash_attn.py
+2 -3
@@ -1,10 +1,10 @@
1 """Attention layer ROCm GPUs."""
2 -import os
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple, Type
4
5 import torch
6
7 +import vllm.envs as envs
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 AttentionMetadata,
10 AttentionMetadataPerStage)
@@ -156,8 +156,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
156
157 self.use_naive_attn = False
158 # NOTE: Allow for switching between Triton and CK. Defaulting to triton.
159 - self.use_triton_flash_attn = (os.environ.get(
160 - "VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1"))
159 + self.use_triton_flash_attn = envs.VLLM_USE_TRITON_FLASH_ATTN
160 if self.use_triton_flash_attn:
161 from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
162 triton_attention)
vllm/attention/selector.py
+2 -4
@@ -1,18 +1,16 @@
1 import enum
2 -import os
2 from functools import lru_cache
3 from typing import Type
4
5 import torch
6
7 +import vllm.envs as envs
8 from vllm.attention.backends.abstract import AttentionBackend
9 from vllm.logger import init_logger
10 from vllm.utils import is_cpu, is_hip
11
12 logger = init_logger(__name__)
13
14 -VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
15 -
14
15 class _Backend(enum.Enum):
16 FLASH_ATTN = enum.auto()
@@ -79,7 +77,7 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
77 "package is not found. Please install it for better performance.")
78 return _Backend.XFORMERS
79
82 - backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
80 + backend_by_env_var = envs.VLLM_ATTENTION_BACKEND
81 if backend_by_env_var is not None:
82 return _Backend[backend_by_env_var]
83
vllm/config.py
-5
@@ -1,6 +1,5 @@
1 import enum
2 import json
3 -import os
3 from dataclasses import dataclass, field, fields
4 from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
5
@@ -24,10 +23,6 @@ if TYPE_CHECKING:
23
24 logger = init_logger(__name__)
25
27 -# If true, will load models from ModelScope instead of Hugging Face Hub.
28 -VLLM_USE_MODELSCOPE = os.environ.get("VLLM_USE_MODELSCOPE",
29 - "False").lower() == "true"
30 -
26 _GB = 1 << 30
27
28
vllm/distributed/device_communicators/custom_all_reduce.py
+4 -4
@@ -1,10 +1,10 @@
1 -import os
1 from contextlib import contextmanager
2 from typing import Any, List, Optional
3
4 import torch
5 import torch.distributed as dist
6
7 +import vllm.envs as envs
8 from vllm.logger import init_logger
9
10 try:
@@ -54,9 +54,9 @@ def init_custom_ar() -> None:
54 return
55 # test nvlink first, this will filter out most of the cases
56 # where custom allreduce is not supported
57 - if "CUDA_VISIBLE_DEVICES" in os.environ:
58 - device_ids = list(
59 - map(int, os.environ["CUDA_VISIBLE_DEVICES"].split(",")))
57 + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
58 + if cuda_visible_devices:
59 + device_ids = list(map(int, cuda_visible_devices.split(",")))
60 else:
61 device_ids = list(range(num_dev))
62 # this checks hardware and driver support for NVLink
vllm/distributed/parallel_state.py
+2 -2
@@ -4,11 +4,11 @@
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 """Tensor and pipeline parallel groups."""
6 import contextlib
7 -import os
7 from typing import Optional
8
9 import torch
10
11 +import vllm.envs as envs
12 from vllm.logger import init_logger
13
14 logger = init_logger(__name__)
@@ -80,7 +80,7 @@ def init_distributed_environment(
80 # local_rank is not available in torch ProcessGroup,
81 # see https://github.com/pytorch/pytorch/issues/122816
82 if local_rank == -1 and distributed_init_method == "env://":
83 - local_rank = int(os.environ['LOCAL_RANK'])
83 + local_rank = envs.LOCAL_RANK
84 global _LOCAL_RANK
85 _LOCAL_RANK = local_rank
86
vllm/distributed/utils.py
+5 -2
@@ -9,6 +9,7 @@ from typing import Dict, Optional, Sequence
9 import torch
10 import torch.distributed as dist
11
12 +import vllm.envs as envs
13 from vllm.logger import init_logger
14
15 from .parallel_state import get_cpu_world_group, get_local_rank
@@ -102,11 +103,13 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
103 is_distributed = dist.is_initialized()
104
105 num_dev = torch.cuda.device_count()
105 - cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None)
106 + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
107 if cuda_visible_devices is None:
108 cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
109 + VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
110 path = os.path.expanduser(
109 - f"~/.config/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
111 + f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
112 + )
113 os.makedirs(os.path.dirname(path), exist_ok=True)
114 if (not is_distributed or get_local_rank() == 0) \
115 and (not os.path.exists(path)):
vllm/engine/async_llm_engine.py
+2 -3
@@ -1,5 +1,4 @@
1 import asyncio
2 -import os
2 import time
3 from functools import partial
4 from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
@@ -7,6 +6,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
6
7 from transformers import PreTrainedTokenizer
8
9 +import vllm.envs as envs
10 from vllm.config import DecodingConfig, ModelConfig
11 from vllm.core.scheduler import SchedulerOutputs
12 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -20,8 +20,7 @@ from vllm.sequence import MultiModalData, SamplerOutput
20 from vllm.usage.usage_lib import UsageContext
21
22 logger = init_logger(__name__)
23 -ENGINE_ITERATION_TIMEOUT_S = int(
24 - os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60"))
23 +ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
24
25
26 class AsyncEngineDeadError(RuntimeError):
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -1,7 +1,6 @@
1 import asyncio
2 import importlib
3 import inspect
4 -import os
4 import re
5 from contextlib import asynccontextmanager
6 from http import HTTPStatus
@@ -16,6 +15,7 @@ from prometheus_client import make_asgi_app
15 from starlette.routing import Mount
16
17 import vllm
18 +import vllm.envs as envs
19 from vllm.engine.arg_utils import AsyncEngineArgs
20 from vllm.engine.async_llm_engine import AsyncLLMEngine
21 from vllm.entrypoints.openai.cli_args import make_arg_parser
@@ -129,7 +129,7 @@ if __name__ == "__main__":
129 allow_headers=args.allowed_headers,
130 )
131
132 - if token := os.environ.get("VLLM_API_KEY") or args.api_key:
132 + if token := envs.VLLM_API_KEY or args.api_key:
133
134 @app.middleware("http")
135 async def authentication(request: Request, call_next):
vllm/envs.py new
+160
@@ -0,0 +1,160 @@
1 +import os
2 +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
3 +
4 +if TYPE_CHECKING:
5 + VLLM_HOST_IP: str = ""
6 + VLLM_USE_MODELSCOPE: bool = False
7 + VLLM_INSTANCE_ID: Optional[str] = None
8 + VLLM_NCCL_SO_PATH: Optional[str] = None
9 + LD_LIBRARY_PATH: Optional[str] = None
10 + VLLM_USE_TRITON_FLASH_ATTN: bool = False
11 + LOCAL_RANK: int = 0
12 + CUDA_VISIBLE_DEVICES: Optional[str] = None
13 + VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
14 + VLLM_API_KEY: Optional[str] = None
15 + S3_ACCESS_KEY_ID: Optional[str] = None
16 + S3_SECRET_ACCESS_KEY: Optional[str] = None
17 + S3_ENDPOINT_URL: Optional[str] = None
18 + VLLM_CONFIG_ROOT: str = ""
19 + VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
20 + VLLM_NO_USAGE_STATS: bool = False
21 + VLLM_DO_NOT_TRACK: bool = False
22 + VLLM_USAGE_SOURCE: str = ""
23 + VLLM_CONFIGURE_LOGGING: int = 1
24 + VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
25 + VLLM_TRACE_FUNCTION: int = 0
26 + VLLM_ATTENTION_BACKEND: Optional[str] = None
27 + VLLM_CPU_KVCACHE_SPACE: int = 0
28 + VLLM_USE_RAY_COMPILED_DAG: bool = False
29 + VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
30 +
31 +environment_variables: Dict[str, Callable[[], Any]] = {
32 + # used in distributed environment to determine the master address
33 + 'VLLM_HOST_IP':
34 + lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
35 +
36 + # If true, will load models from ModelScope instead of Hugging Face Hub.
37 + # note that the value is true or false, not numbers
38 + "VLLM_USE_MODELSCOPE":
39 + lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true",
40 +
41 + # Instance id represents an instance of the VLLM. All processes in the same
42 + # instance should have the same instance id.
43 + "VLLM_INSTANCE_ID":
44 + lambda: os.environ.get("VLLM_INSTANCE_ID", None),
45 +
46 + # path to cudatoolkit home directory, under which should be bin, include,
47 + # and lib directories.
48 + "CUDA_HOME":
49 + lambda: os.environ.get("CUDA_HOME", None),
50 +
51 + # Path to the NCCL library file. It is needed because nccl>=2.19 brought
52 + # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
53 + "VLLM_NCCL_SO_PATH":
54 + lambda: os.environ.get("VLLM_NCCL_SO_PATH", None),
55 +
56 + # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl
57 + # library file in the locations specified by `LD_LIBRARY_PATH`
58 + "LD_LIBRARY_PATH":
59 + lambda: os.environ.get("LD_LIBRARY_PATH", None),
60 +
61 + # flag to control if vllm should use triton flash attention
62 + "VLLM_USE_TRITON_FLASH_ATTN":
63 + lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
64 + ("true", "1")),
65 +
66 + # local rank of the process in the distributed setting, used to determine
67 + # the GPU device id
68 + "LOCAL_RANK":
69 + lambda: int(os.environ.get("LOCAL_RANK", "0")),
70 +
71 + # used to control the visible devices in the distributed setting
72 + "CUDA_VISIBLE_DEVICES":
73 + lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
74 +
75 + # timeout for each iteration in the engine
76 + "VLLM_ENGINE_ITERATION_TIMEOUT_S":
77 + lambda: int(os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")),
78 +
79 + # API key for VLLM API server
80 + "VLLM_API_KEY":
81 + lambda: os.environ.get("VLLM_API_KEY", None),
82 +
83 + # S3 access information, used for tensorizer to load model from S3
84 + "S3_ACCESS_KEY_ID":
85 + lambda: os.environ.get("S3_ACCESS_KEY", None),
86 + "S3_SECRET_ACCESS_KEY":
87 + lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
88 + "S3_ENDPOINT_URL":
89 + lambda: os.environ.get("S3_ENDPOINT_URL", None),
90 +
91 + # Root directory for VLLM configuration files
92 + # Note that this not only affects how vllm finds its configuration files
93 + # during runtime, but also affects how vllm installs its configuration
94 + # files during **installation**.
95 + "VLLM_CONFIG_ROOT":
96 + lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
97 + "XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
98 +
99 + # Usage stats collection
100 + "VLLM_USAGE_STATS_SERVER":
101 + lambda: os.environ.get("VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"),
102 + "VLLM_NO_USAGE_STATS":
103 + lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1",
104 + "VLLM_DO_NOT_TRACK":
105 + lambda: (os.environ.get("VLLM_DO_NOT_TRACK", None) or os.environ.get(
106 + "DO_NOT_TRACK", None) or "0") == "1",
107 + "VLLM_USAGE_SOURCE":
108 + lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"),
109 +
110 + # Logging configuration
111 + # If set to 0, vllm will not configure logging
112 + # If set to 1, vllm will configure logging using the default configuration
113 + # or the configuration file specified by VLLM_LOGGING_CONFIG_PATH
114 + "VLLM_CONFIGURE_LOGGING":
115 + lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")),
116 + "VLLM_LOGGING_CONFIG_PATH":
117 + lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"),
118 +
119 + # Trace function calls
120 + # If set to 1, vllm will trace function calls
121 + # Useful for debugging
122 + "VLLM_TRACE_FUNCTION":
123 + lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
124 +
125 + # Backend for attention computation
126 + # Available options:
127 + # - "TORCH_SDPA": use torch.nn.MultiheadAttention
128 + # - "FLASH_ATTN": use FlashAttention
129 + # - "XFORMERS": use XFormers
130 + # - "ROCM_FLASH": use ROCmFlashAttention
131 + "VLLM_ATTENTION_BACKEND":
132 + lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
133 +
134 + # CPU key-value cache space
135 + # default is 4GB
136 + "VLLM_CPU_KVCACHE_SPACE":
137 + lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")),
138 +
139 + # If the env var is set, it uses the Ray's compiled DAG API
140 + # which optimizes the control plane overhead.
141 + # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
142 + "VLLM_USE_RAY_COMPILED_DAG":
143 + lambda: bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0)),
144 +
145 + # Use dedicated multiprocess context for workers.
146 + # Both spawn and fork work
147 + "VLLM_WORKER_MULTIPROC_METHOD":
148 + lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn"),
149 +}
150 +
151 +
152 +def __getattr__(name):
153 + # lazy evaluation of environment variables
154 + if name in environment_variables:
155 + return environment_variables[name]()
156 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
157 +
158 +
159 +def __dir__():
160 + return list(environment_variables.keys())
vllm/executor/cpu_executor.py
+2 -3
@@ -1,8 +1,8 @@
1 -import os
1 from typing import Dict, List, Set, Tuple
2
3 import torch
4
5 +import vllm.envs as envs
6 from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8 from vllm.logger import init_logger
@@ -152,8 +152,7 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
152 logger.warning("Prefix caching is not supported on CPU, disable it.")
153 config.enable_prefix_caching = False
154
155 - kv_cache_space_str = os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")
156 - kv_cache_space = int(kv_cache_space_str)
155 + kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
156
157 if kv_cache_space >= 0:
158 if kv_cache_space == 0:
vllm/executor/multiproc_worker_utils.py
+2 -3
@@ -12,6 +12,7 @@ from multiprocessing.process import BaseProcess
12 from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
13 TypeVar, Union)
14
15 +import vllm.envs as envs
16 from vllm.logger import init_logger
17
18 logger = init_logger(__name__)
@@ -26,9 +27,7 @@ RESET = '\033[0;0m'
27
28 JOIN_TIMEOUT_S = 2
29
29 -# Use dedicated multiprocess context for workers.
30 -# Both spawn and fork work
31 -mp_method = os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
30 +mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
31 mp = multiprocessing.get_context(mp_method)
32
33
vllm/executor/ray_gpu_executor.py
+3 -5
@@ -5,6 +5,7 @@ from collections import defaultdict
5 from itertools import islice, repeat
6 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8 +import vllm.envs as envs
9 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10 DistributedGPUExecutor, DistributedGPUExecutorAsync)
11 from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -21,10 +22,7 @@ if TYPE_CHECKING:
22
23 logger = init_logger(__name__)
24
24 -# If the env var is set, it uses the Ray's compiled DAG API
25 -# which optimizes the control plane overhead.
26 -# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
27 -USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
25 +USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
26
27
28 class RayGPUExecutor(DistributedGPUExecutor):
@@ -145,7 +143,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
143 "VLLM_INSTANCE_ID":
144 VLLM_INSTANCE_ID,
145 "VLLM_TRACE_FUNCTION":
148 - os.getenv("VLLM_TRACE_FUNCTION", "0"),
146 + str(envs.VLLM_TRACE_FUNCTION),
147 }, ) for (node_id, _) in worker_node_and_gpu_ids]
148 self._run_workers("update_environment_variables",
149 all_args=all_args_to_update_environment_variables)
vllm/logger.py
+4 -2
@@ -10,8 +10,10 @@ from logging.config import dictConfig
10 from os import path
11 from typing import Dict, Optional
12
13 -VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
14 -VLLM_LOGGING_CONFIG_PATH = os.getenv("VLLM_LOGGING_CONFIG_PATH")
13 +import vllm.envs as envs
14 +
15 +VLLM_CONFIGURE_LOGGING = envs.VLLM_CONFIGURE_LOGGING
16 +VLLM_LOGGING_CONFIG_PATH = envs.VLLM_LOGGING_CONFIG_PATH
17
18 _FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s"
19 _DATE_FORMAT = "%m-%d %H:%M:%S"
vllm/model_executor/model_loader/loader.py
+4 -3
@@ -9,9 +9,10 @@ import huggingface_hub
9 import torch
10 from torch import nn
11
12 -from vllm.config import (VLLM_USE_MODELSCOPE, DeviceConfig, LoadConfig,
13 - LoadFormat, LoRAConfig, ModelConfig, ParallelConfig,
14 - SchedulerConfig, VisionLanguageConfig)
12 +from vllm.config import (DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13 + ModelConfig, ParallelConfig, SchedulerConfig,
14 + VisionLanguageConfig)
15 +from vllm.envs import VLLM_USE_MODELSCOPE
16 from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization.base_config import (
18 QuantizationConfig)
vllm/model_executor/model_loader/tensorizer.py
+5 -7
@@ -11,6 +11,7 @@ import torch
11 from torch import nn
12 from transformers import PretrainedConfig
13
14 +import vllm.envs as envs
15 from vllm.config import ModelConfig, ParallelConfig
16 from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization.base_config import (
@@ -142,13 +143,10 @@ class TensorizerArgs:
143
144 def __post_init__(self):
145 self.file_obj = self.tensorizer_uri
145 - self.s3_access_key_id = (self.s3_access_key_id
146 - or os.environ.get("S3_ACCESS_KEY_ID")) or None
147 - self.s3_secret_access_key = (
148 - self.s3_secret_access_key
149 - or os.environ.get("S3_SECRET_ACCESS_KEY")) or None
150 - self.s3_endpoint = (self.s3_endpoint
151 - or os.environ.get("S3_ENDPOINT_URL")) or None
146 + self.s3_access_key_id = self.s3_access_key_id or envs.S3_ACCESS_KEY_ID
147 + self.s3_secret_access_key = (self.s3_secret_access_key
148 + or envs.S3_SECRET_ACCESS_KEY)
149 + self.s3_endpoint = self.s3_endpoint or envs.S3_ENDPOINT_URL
150 self.stream_params = {
151 "s3_access_key_id": self.s3_access_key_id,
152 "s3_secret_access_key": self.s3_secret_access_key,
vllm/transformers_utils/tokenizer.py
+1 -1
@@ -5,7 +5,7 @@ import huggingface_hub
5 from transformers import (AutoTokenizer, PreTrainedTokenizer,
6 PreTrainedTokenizerFast)
7
8 -from vllm.config import VLLM_USE_MODELSCOPE
8 +from vllm.envs import VLLM_USE_MODELSCOPE
9 from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
11 from vllm.transformers_utils.tokenizers import BaichuanTokenizer
vllm/usage/usage_lib.py
+9 -7
@@ -15,20 +15,22 @@ import psutil
15 import requests
16 import torch
17
18 -_config_home = os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
18 +import vllm.envs as envs
19 +
20 +_config_home = envs.VLLM_CONFIG_ROOT
21 _USAGE_STATS_JSON_PATH = os.path.join(_config_home, "vllm/usage_stats.json")
22 _USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
23 "vllm/do_not_track")
24 _USAGE_STATS_ENABLED = None
23 -_USAGE_STATS_SERVER = os.environ.get("VLLM_USAGE_STATS_SERVER",
24 - "https://stats.vllm.ai")
25 +_USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
26
27
28 def is_usage_stats_enabled():
29 """Determine whether or not we can send usage stats to the server.
30 The logic is as follows:
31 - By default, it should be enabled.
31 - - Two environment variables can disable it:
32 + - Three environment variables can disable it:
33 + - VLLM_DO_NOT_TRACK=1
34 - DO_NOT_TRACK=1
35 - VLLM_NO_USAGE_STATS=1
36 - A file in the home directory can disable it if it exists:
@@ -36,8 +38,8 @@ def is_usage_stats_enabled():
38 """
39 global _USAGE_STATS_ENABLED
40 if _USAGE_STATS_ENABLED is None:
39 - do_not_track = os.environ.get("DO_NOT_TRACK", "0") == "1"
40 - no_usage_stats = os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1"
41 + do_not_track = envs.VLLM_DO_NOT_TRACK
42 + no_usage_stats = envs.VLLM_NO_USAGE_STATS
43 do_not_track_file = os.path.exists(_USAGE_STATS_DO_NOT_TRACK_PATH)
44
45 _USAGE_STATS_ENABLED = not (do_not_track or no_usage_stats
@@ -167,7 +169,7 @@ class UsageMessage:
169
170 # Metadata
171 self.log_time = _get_current_timestamp_ns()
170 - self.source = os.environ.get("VLLM_USAGE_SOURCE", "production")
172 + self.source = envs.VLLM_USAGE_SOURCE
173
174 data = vars(self)
175 if extra_kvs:
vllm/utils.py
+11 -8
@@ -21,6 +21,7 @@ import psutil
21 import torch
22 from packaging.version import Version, parse
23
24 +import vllm.envs as envs
25 from vllm.logger import enable_trace_function_call, init_logger
26
27 T = TypeVar("T")
@@ -174,7 +175,7 @@ def get_vllm_instance_id():
175 Instance id represents an instance of the VLLM. All processes in the same
176 instance should have the same instance id.
177 """
177 - return os.environ.get("VLLM_INSTANCE_ID", f"vllm-instance-{random_uuid()}")
178 + return envs.VLLM_INSTANCE_ID or f"vllm-instance-{random_uuid()}"
179
180
181 @lru_cache(maxsize=None)
@@ -243,7 +244,7 @@ def merge_async_iterators(
244
245
246 def get_ip() -> str:
246 - host_ip = os.environ.get("HOST_IP")
247 + host_ip = envs.VLLM_HOST_IP
248 if host_ip:
249 return host_ip
250
@@ -269,7 +270,8 @@ def get_ip() -> str:
270
271 warnings.warn(
272 "Failed to get the IP address, using 0.0.0.0 by default."
272 - "The value can be set by the environment variable HOST_IP.",
273 + "The value can be set by the environment variable"
274 + " VLLM_HOST_IP or HOST_IP.",
275 stacklevel=2)
276 return "0.0.0.0"
277
@@ -314,7 +316,7 @@ def cdiv(a: int, b: int) -> int:
316
317 @lru_cache(maxsize=None)
318 def get_nvcc_cuda_version() -> Optional[Version]:
317 - cuda_home = os.environ.get('CUDA_HOME')
319 + cuda_home = envs.CUDA_HOME
320 if not cuda_home:
321 cuda_home = '/usr/local/cuda'
322 if os.path.isfile(cuda_home + '/bin/nvcc'):
@@ -581,7 +583,7 @@ def find_library(lib_name: str) -> str:
583 # libcuda.so.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libcuda.so.1
584 locs = [line.split()[-1] for line in libs.splitlines() if lib_name in line]
585 # `LD_LIBRARY_PATH` searches the library in the user-defined paths
584 - env_ld_library_path = os.getenv("LD_LIBRARY_PATH")
586 + env_ld_library_path = envs.LD_LIBRARY_PATH
587 if not locs and env_ld_library_path:
588 locs = [
589 os.path.join(dir, lib_name)
@@ -594,14 +596,15 @@ def find_library(lib_name: str) -> str:
596
597
598 def find_nccl_library():
597 - so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
599 + so_file = envs.VLLM_NCCL_SO_PATH
600 + VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
601
602 # check if we have vllm-managed nccl
603 vllm_nccl_path = None
604 if torch.version.cuda is not None:
605 cuda_major = torch.version.cuda.split(".")[0]
606 path = os.path.expanduser(
604 - f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
607 + f"{VLLM_CONFIG_ROOT}/vllm/nccl/cu{cuda_major}/libnccl.so.*")
608 files = glob.glob(path)
609 vllm_nccl_path = files[0] if files else None
610
@@ -626,7 +629,7 @@ def enable_trace_function_call_for_thread() -> None:
629 if enabled via the VLLM_TRACE_FUNCTION environment variable
630 """
631
629 - if int(os.getenv("VLLM_TRACE_FUNCTION", "0")):
632 + if envs.VLLM_TRACE_FUNCTION:
633 tmp_dir = tempfile.gettempdir()
634 filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
635 f"_thread_{threading.get_ident()}_"