A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] centralize all usage of environment variables (#4548)
youkaichao committed
May 2, 2024 at 11:13 UTC
5b8a7c1cb0f1bb81266bae98944c055a8abb1a68
18 files changed
+220
-64
vllm/attention/backends/rocm_flash_attn.py
+2
-3
@@ -1,10 +1,10 @@
1
"""Attention layer ROCm GPUs."""
2
-import os
2
from dataclasses import dataclass
3
from typing import Dict, List, Optional, Tuple, Type
4
5
import torch
6
7
+import vllm.envs as envs
8
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
AttentionMetadata,
10
AttentionMetadataPerStage)
@@ -156,8 +156,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
156
157
self.use_naive_attn = False
158
# NOTE: Allow for switching between Triton and CK. Defaulting to triton.
159
- self.use_triton_flash_attn = (os.environ.get(
160
- "VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1"))
159
+ self.use_triton_flash_attn = envs.VLLM_USE_TRITON_FLASH_ATTN
160
if self.use_triton_flash_attn:
161
from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
162
triton_attention)
vllm/attention/selector.py
+2
-4
@@ -1,18 +1,16 @@
1
import enum
2
-import os
2
from functools import lru_cache
3
from typing import Type
4
5
import torch
6
7
+import vllm.envs as envs
8
from vllm.attention.backends.abstract import AttentionBackend
9
from vllm.logger import init_logger
10
from vllm.utils import is_cpu, is_hip
11
12
logger = init_logger(__name__)
13
14
-VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
15
-
14
15
class _Backend(enum.Enum):
16
FLASH_ATTN = enum.auto()
@@ -79,7 +77,7 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
77
"package is not found. Please install it for better performance.")
78
return _Backend.XFORMERS
79
82
- backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
80
+ backend_by_env_var = envs.VLLM_ATTENTION_BACKEND
81
if backend_by_env_var is not None:
82
return _Backend[backend_by_env_var]
83
vllm/config.py
-5
@@ -1,6 +1,5 @@
1
import enum
2
import json
3
-import os
3
from dataclasses import dataclass, field, fields
4
from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
5
@@ -24,10 +23,6 @@ if TYPE_CHECKING:
23
24
logger = init_logger(__name__)
25
27
-# If true, will load models from ModelScope instead of Hugging Face Hub.
28
-VLLM_USE_MODELSCOPE = os.environ.get("VLLM_USE_MODELSCOPE",
29
- "False").lower() == "true"
30
-
26
_GB = 1 << 30
27
28
vllm/distributed/device_communicators/custom_all_reduce.py
+4
-4
@@ -1,10 +1,10 @@
1
-import os
1
from contextlib import contextmanager
2
from typing import Any, List, Optional
3
4
import torch
5
import torch.distributed as dist
6
7
+import vllm.envs as envs
8
from vllm.logger import init_logger
9
10
try:
@@ -54,9 +54,9 @@ def init_custom_ar() -> None:
54
return
55
# test nvlink first, this will filter out most of the cases
56
# where custom allreduce is not supported
57
- if "CUDA_VISIBLE_DEVICES" in os.environ:
58
- device_ids = list(
59
- map(int, os.environ["CUDA_VISIBLE_DEVICES"].split(",")))
57
+ cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
58
+ if cuda_visible_devices:
59
+ device_ids = list(map(int, cuda_visible_devices.split(",")))
60
else:
61
device_ids = list(range(num_dev))
62
# this checks hardware and driver support for NVLink
vllm/distributed/parallel_state.py
+2
-2
@@ -4,11 +4,11 @@
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
import contextlib
7
-import os
7
from typing import Optional
8
9
import torch
10
11
+import vllm.envs as envs
12
from vllm.logger import init_logger
13
14
logger = init_logger(__name__)
@@ -80,7 +80,7 @@ def init_distributed_environment(
80
# local_rank is not available in torch ProcessGroup,
81
# see https://github.com/pytorch/pytorch/issues/122816
82
if local_rank == -1 and distributed_init_method == "env://":
83
- local_rank = int(os.environ['LOCAL_RANK'])
83
+ local_rank = envs.LOCAL_RANK
84
global _LOCAL_RANK
85
_LOCAL_RANK = local_rank
86
vllm/distributed/utils.py
+5
-2
@@ -9,6 +9,7 @@ from typing import Dict, Optional, Sequence
9
import torch
10
import torch.distributed as dist
11
12
+import vllm.envs as envs
13
from vllm.logger import init_logger
14
15
from .parallel_state import get_cpu_world_group, get_local_rank
@@ -102,11 +103,13 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
103
is_distributed = dist.is_initialized()
104
105
num_dev = torch.cuda.device_count()
105
- cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None)
106
+ cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
107
if cuda_visible_devices is None:
108
cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
109
+ VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
110
path = os.path.expanduser(
109
- f"~/.config/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
111
+ f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
112
+ )
113
os.makedirs(os.path.dirname(path), exist_ok=True)
114
if (not is_distributed or get_local_rank() == 0) \
115
and (not os.path.exists(path)):
vllm/engine/async_llm_engine.py
+2
-3
@@ -1,5 +1,4 @@
1
import asyncio
2
-import os
2
import time
3
from functools import partial
4
from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
@@ -7,6 +6,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
6
7
from transformers import PreTrainedTokenizer
8
9
+import vllm.envs as envs
10
from vllm.config import DecodingConfig, ModelConfig
11
from vllm.core.scheduler import SchedulerOutputs
12
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -20,8 +20,7 @@ from vllm.sequence import MultiModalData, SamplerOutput
20
from vllm.usage.usage_lib import UsageContext
21
22
logger = init_logger(__name__)
23
-ENGINE_ITERATION_TIMEOUT_S = int(
24
- os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60"))
23
+ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
24
25
26
class AsyncEngineDeadError(RuntimeError):
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -1,7 +1,6 @@
1
import asyncio
2
import importlib
3
import inspect
4
-import os
4
import re
5
from contextlib import asynccontextmanager
6
from http import HTTPStatus
@@ -16,6 +15,7 @@ from prometheus_client import make_asgi_app
15
from starlette.routing import Mount
16
17
import vllm
18
+import vllm.envs as envs
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.engine.async_llm_engine import AsyncLLMEngine
21
from vllm.entrypoints.openai.cli_args import make_arg_parser
@@ -129,7 +129,7 @@ if __name__ == "__main__":
129
allow_headers=args.allowed_headers,
130
)
131
132
- if token := os.environ.get("VLLM_API_KEY") or args.api_key:
132
+ if token := envs.VLLM_API_KEY or args.api_key:
133
134
@app.middleware("http")
135
async def authentication(request: Request, call_next):
vllm/envs.py
new
+160
@@ -0,0 +1,160 @@
1
+import os
2
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
3
+
4
+if TYPE_CHECKING:
5
+ VLLM_HOST_IP: str = ""
6
+ VLLM_USE_MODELSCOPE: bool = False
7
+ VLLM_INSTANCE_ID: Optional[str] = None
8
+ VLLM_NCCL_SO_PATH: Optional[str] = None
9
+ LD_LIBRARY_PATH: Optional[str] = None
10
+ VLLM_USE_TRITON_FLASH_ATTN: bool = False
11
+ LOCAL_RANK: int = 0
12
+ CUDA_VISIBLE_DEVICES: Optional[str] = None
13
+ VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
14
+ VLLM_API_KEY: Optional[str] = None
15
+ S3_ACCESS_KEY_ID: Optional[str] = None
16
+ S3_SECRET_ACCESS_KEY: Optional[str] = None
17
+ S3_ENDPOINT_URL: Optional[str] = None
18
+ VLLM_CONFIG_ROOT: str = ""
19
+ VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
20
+ VLLM_NO_USAGE_STATS: bool = False
21
+ VLLM_DO_NOT_TRACK: bool = False
22
+ VLLM_USAGE_SOURCE: str = ""
23
+ VLLM_CONFIGURE_LOGGING: int = 1
24
+ VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
25
+ VLLM_TRACE_FUNCTION: int = 0
26
+ VLLM_ATTENTION_BACKEND: Optional[str] = None
27
+ VLLM_CPU_KVCACHE_SPACE: int = 0
28
+ VLLM_USE_RAY_COMPILED_DAG: bool = False
29
+ VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
30
+
31
+environment_variables: Dict[str, Callable[[], Any]] = {
32
+ # used in distributed environment to determine the master address
33
+ 'VLLM_HOST_IP':
34
+ lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
35
+
36
+ # If true, will load models from ModelScope instead of Hugging Face Hub.
37
+ # note that the value is true or false, not numbers
38
+ "VLLM_USE_MODELSCOPE":
39
+ lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true",
40
+
41
+ # Instance id represents an instance of the VLLM. All processes in the same
42
+ # instance should have the same instance id.
43
+ "VLLM_INSTANCE_ID":
44
+ lambda: os.environ.get("VLLM_INSTANCE_ID", None),
45
+
46
+ # path to cudatoolkit home directory, under which should be bin, include,
47
+ # and lib directories.
48
+ "CUDA_HOME":
49
+ lambda: os.environ.get("CUDA_HOME", None),
50
+
51
+ # Path to the NCCL library file. It is needed because nccl>=2.19 brought
52
+ # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
53
+ "VLLM_NCCL_SO_PATH":
54
+ lambda: os.environ.get("VLLM_NCCL_SO_PATH", None),
55
+
56
+ # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl
57
+ # library file in the locations specified by `LD_LIBRARY_PATH`
58
+ "LD_LIBRARY_PATH":
59
+ lambda: os.environ.get("LD_LIBRARY_PATH", None),
60
+
61
+ # flag to control if vllm should use triton flash attention
62
+ "VLLM_USE_TRITON_FLASH_ATTN":
63
+ lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
64
+ ("true", "1")),
65
+
66
+ # local rank of the process in the distributed setting, used to determine
67
+ # the GPU device id
68
+ "LOCAL_RANK":
69
+ lambda: int(os.environ.get("LOCAL_RANK", "0")),
70
+
71
+ # used to control the visible devices in the distributed setting
72
+ "CUDA_VISIBLE_DEVICES":
73
+ lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
74
+
75
+ # timeout for each iteration in the engine
76
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S":
77
+ lambda: int(os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")),
78
+
79
+ # API key for VLLM API server
80
+ "VLLM_API_KEY":
81
+ lambda: os.environ.get("VLLM_API_KEY", None),
82
+
83
+ # S3 access information, used for tensorizer to load model from S3
84
+ "S3_ACCESS_KEY_ID":
85
+ lambda: os.environ.get("S3_ACCESS_KEY", None),
86
+ "S3_SECRET_ACCESS_KEY":
87
+ lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
88
+ "S3_ENDPOINT_URL":
89
+ lambda: os.environ.get("S3_ENDPOINT_URL", None),
90
+
91
+ # Root directory for VLLM configuration files
92
+ # Note that this not only affects how vllm finds its configuration files
93
+ # during runtime, but also affects how vllm installs its configuration
94
+ # files during **installation**.
95
+ "VLLM_CONFIG_ROOT":
96
+ lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
97
+ "XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
98
+
99
+ # Usage stats collection
100
+ "VLLM_USAGE_STATS_SERVER":
101
+ lambda: os.environ.get("VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"),
102
+ "VLLM_NO_USAGE_STATS":
103
+ lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1",
104
+ "VLLM_DO_NOT_TRACK":
105
+ lambda: (os.environ.get("VLLM_DO_NOT_TRACK", None) or os.environ.get(
106
+ "DO_NOT_TRACK", None) or "0") == "1",
107
+ "VLLM_USAGE_SOURCE":
108
+ lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"),
109
+
110
+ # Logging configuration
111
+ # If set to 0, vllm will not configure logging
112
+ # If set to 1, vllm will configure logging using the default configuration
113
+ # or the configuration file specified by VLLM_LOGGING_CONFIG_PATH
114
+ "VLLM_CONFIGURE_LOGGING":
115
+ lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")),
116
+ "VLLM_LOGGING_CONFIG_PATH":
117
+ lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"),
118
+
119
+ # Trace function calls
120
+ # If set to 1, vllm will trace function calls
121
+ # Useful for debugging
122
+ "VLLM_TRACE_FUNCTION":
123
+ lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
124
+
125
+ # Backend for attention computation
126
+ # Available options:
127
+ # - "TORCH_SDPA": use torch.nn.MultiheadAttention
128
+ # - "FLASH_ATTN": use FlashAttention
129
+ # - "XFORMERS": use XFormers
130
+ # - "ROCM_FLASH": use ROCmFlashAttention
131
+ "VLLM_ATTENTION_BACKEND":
132
+ lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
133
+
134
+ # CPU key-value cache space
135
+ # default is 4GB
136
+ "VLLM_CPU_KVCACHE_SPACE":
137
+ lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")),
138
+
139
+ # If the env var is set, it uses the Ray's compiled DAG API
140
+ # which optimizes the control plane overhead.
141
+ # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
142
+ "VLLM_USE_RAY_COMPILED_DAG":
143
+ lambda: bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0)),
144
+
145
+ # Use dedicated multiprocess context for workers.
146
+ # Both spawn and fork work
147
+ "VLLM_WORKER_MULTIPROC_METHOD":
148
+ lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn"),
149
+}
150
+
151
+
152
+def __getattr__(name):
153
+ # lazy evaluation of environment variables
154
+ if name in environment_variables:
155
+ return environment_variables[name]()
156
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
157
+
158
+
159
+def __dir__():
160
+ return list(environment_variables.keys())
vllm/executor/cpu_executor.py
+2
-3
@@ -1,8 +1,8 @@
1
-import os
1
from typing import Dict, List, Set, Tuple
2
3
import torch
4
5
+import vllm.envs as envs
6
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8
from vllm.logger import init_logger
@@ -152,8 +152,7 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
152
logger.warning("Prefix caching is not supported on CPU, disable it.")
153
config.enable_prefix_caching = False
154
155
- kv_cache_space_str = os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")
156
- kv_cache_space = int(kv_cache_space_str)
155
+ kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
156
157
if kv_cache_space >= 0:
158
if kv_cache_space == 0:
vllm/executor/multiproc_worker_utils.py
+2
-3
@@ -12,6 +12,7 @@ from multiprocessing.process import BaseProcess
12
from typing import (Any, Callable, Dict, Generic, List, Optional, TextIO,
13
TypeVar, Union)
14
15
+import vllm.envs as envs
16
from vllm.logger import init_logger
17
18
logger = init_logger(__name__)
@@ -26,9 +27,7 @@ RESET = '\033[0;0m'
27
28
JOIN_TIMEOUT_S = 2
29
29
-# Use dedicated multiprocess context for workers.
30
-# Both spawn and fork work
31
-mp_method = os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
30
+mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
31
mp = multiprocessing.get_context(mp_method)
32
33
vllm/executor/ray_gpu_executor.py
+3
-5
@@ -5,6 +5,7 @@ from collections import defaultdict
5
from itertools import islice, repeat
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8
+import vllm.envs as envs
9
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10
DistributedGPUExecutor, DistributedGPUExecutorAsync)
11
from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -21,10 +22,7 @@ if TYPE_CHECKING:
22
23
logger = init_logger(__name__)
24
24
-# If the env var is set, it uses the Ray's compiled DAG API
25
-# which optimizes the control plane overhead.
26
-# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
27
-USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
25
+USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
26
27
28
class RayGPUExecutor(DistributedGPUExecutor):
@@ -145,7 +143,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
143
"VLLM_INSTANCE_ID":
144
VLLM_INSTANCE_ID,
145
"VLLM_TRACE_FUNCTION":
148
- os.getenv("VLLM_TRACE_FUNCTION", "0"),
146
+ str(envs.VLLM_TRACE_FUNCTION),
147
}, ) for (node_id, _) in worker_node_and_gpu_ids]
148
self._run_workers("update_environment_variables",
149
all_args=all_args_to_update_environment_variables)
vllm/logger.py
+4
-2
@@ -10,8 +10,10 @@ from logging.config import dictConfig
10
from os import path
11
from typing import Dict, Optional
12
13
-VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
14
-VLLM_LOGGING_CONFIG_PATH = os.getenv("VLLM_LOGGING_CONFIG_PATH")
13
+import vllm.envs as envs
14
+
15
+VLLM_CONFIGURE_LOGGING = envs.VLLM_CONFIGURE_LOGGING
16
+VLLM_LOGGING_CONFIG_PATH = envs.VLLM_LOGGING_CONFIG_PATH
17
18
_FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s"
19
_DATE_FORMAT = "%m-%d %H:%M:%S"
vllm/model_executor/model_loader/loader.py
+4
-3
@@ -9,9 +9,10 @@ import huggingface_hub
9
import torch
10
from torch import nn
11
12
-from vllm.config import (VLLM_USE_MODELSCOPE, DeviceConfig, LoadConfig,
13
- LoadFormat, LoRAConfig, ModelConfig, ParallelConfig,
14
- SchedulerConfig, VisionLanguageConfig)
12
+from vllm.config import (DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13
+ ModelConfig, ParallelConfig, SchedulerConfig,
14
+ VisionLanguageConfig)
15
+from vllm.envs import VLLM_USE_MODELSCOPE
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization.base_config import (
18
QuantizationConfig)
vllm/model_executor/model_loader/tensorizer.py
+5
-7
@@ -11,6 +11,7 @@ import torch
11
from torch import nn
12
from transformers import PretrainedConfig
13
14
+import vllm.envs as envs
15
from vllm.config import ModelConfig, ParallelConfig
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization.base_config import (
@@ -142,13 +143,10 @@ class TensorizerArgs:
143
144
def __post_init__(self):
145
self.file_obj = self.tensorizer_uri
145
- self.s3_access_key_id = (self.s3_access_key_id
146
- or os.environ.get("S3_ACCESS_KEY_ID")) or None
147
- self.s3_secret_access_key = (
148
- self.s3_secret_access_key
149
- or os.environ.get("S3_SECRET_ACCESS_KEY")) or None
150
- self.s3_endpoint = (self.s3_endpoint
151
- or os.environ.get("S3_ENDPOINT_URL")) or None
146
+ self.s3_access_key_id = self.s3_access_key_id or envs.S3_ACCESS_KEY_ID
147
+ self.s3_secret_access_key = (self.s3_secret_access_key
148
+ or envs.S3_SECRET_ACCESS_KEY)
149
+ self.s3_endpoint = self.s3_endpoint or envs.S3_ENDPOINT_URL
150
self.stream_params = {
151
"s3_access_key_id": self.s3_access_key_id,
152
"s3_secret_access_key": self.s3_secret_access_key,
vllm/transformers_utils/tokenizer.py
+1
-1
@@ -5,7 +5,7 @@ import huggingface_hub
5
from transformers import (AutoTokenizer, PreTrainedTokenizer,
6
PreTrainedTokenizerFast)
7
8
-from vllm.config import VLLM_USE_MODELSCOPE
8
+from vllm.envs import VLLM_USE_MODELSCOPE
9
from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
11
from vllm.transformers_utils.tokenizers import BaichuanTokenizer
vllm/usage/usage_lib.py
+9
-7
@@ -15,20 +15,22 @@ import psutil
15
import requests
16
import torch
17
18
-_config_home = os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
18
+import vllm.envs as envs
19
+
20
+_config_home = envs.VLLM_CONFIG_ROOT
21
_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "vllm/usage_stats.json")
22
_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
23
"vllm/do_not_track")
24
_USAGE_STATS_ENABLED = None
23
-_USAGE_STATS_SERVER = os.environ.get("VLLM_USAGE_STATS_SERVER",
24
- "https://stats.vllm.ai")
25
+_USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
26
27
28
def is_usage_stats_enabled():
29
"""Determine whether or not we can send usage stats to the server.
30
The logic is as follows:
31
- By default, it should be enabled.
31
- - Two environment variables can disable it:
32
+ - Three environment variables can disable it:
33
+ - VLLM_DO_NOT_TRACK=1
34
- DO_NOT_TRACK=1
35
- VLLM_NO_USAGE_STATS=1
36
- A file in the home directory can disable it if it exists:
@@ -36,8 +38,8 @@ def is_usage_stats_enabled():
38
"""
39
global _USAGE_STATS_ENABLED
40
if _USAGE_STATS_ENABLED is None:
39
- do_not_track = os.environ.get("DO_NOT_TRACK", "0") == "1"
40
- no_usage_stats = os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1"
41
+ do_not_track = envs.VLLM_DO_NOT_TRACK
42
+ no_usage_stats = envs.VLLM_NO_USAGE_STATS
43
do_not_track_file = os.path.exists(_USAGE_STATS_DO_NOT_TRACK_PATH)
44
45
_USAGE_STATS_ENABLED = not (do_not_track or no_usage_stats
@@ -167,7 +169,7 @@ class UsageMessage:
169
170
# Metadata
171
self.log_time = _get_current_timestamp_ns()
170
- self.source = os.environ.get("VLLM_USAGE_SOURCE", "production")
172
+ self.source = envs.VLLM_USAGE_SOURCE
173
174
data = vars(self)
175
if extra_kvs:
vllm/utils.py
+11
-8
@@ -21,6 +21,7 @@ import psutil
21
import torch
22
from packaging.version import Version, parse
23
24
+import vllm.envs as envs
25
from vllm.logger import enable_trace_function_call, init_logger
26
27
T = TypeVar("T")
@@ -174,7 +175,7 @@ def get_vllm_instance_id():
175
Instance id represents an instance of the VLLM. All processes in the same
176
instance should have the same instance id.
177
"""
177
- return os.environ.get("VLLM_INSTANCE_ID", f"vllm-instance-{random_uuid()}")
178
+ return envs.VLLM_INSTANCE_ID or f"vllm-instance-{random_uuid()}"
179
180
181
@lru_cache(maxsize=None)
@@ -243,7 +244,7 @@ def merge_async_iterators(
244
245
246
def get_ip() -> str:
246
- host_ip = os.environ.get("HOST_IP")
247
+ host_ip = envs.VLLM_HOST_IP
248
if host_ip:
249
return host_ip
250
@@ -269,7 +270,8 @@ def get_ip() -> str:
270
271
warnings.warn(
272
"Failed to get the IP address, using 0.0.0.0 by default."
272
- "The value can be set by the environment variable HOST_IP.",
273
+ "The value can be set by the environment variable"
274
+ " VLLM_HOST_IP or HOST_IP.",
275
stacklevel=2)
276
return "0.0.0.0"
277
@@ -314,7 +316,7 @@ def cdiv(a: int, b: int) -> int:
316
317
@lru_cache(maxsize=None)
318
def get_nvcc_cuda_version() -> Optional[Version]:
317
- cuda_home = os.environ.get('CUDA_HOME')
319
+ cuda_home = envs.CUDA_HOME
320
if not cuda_home:
321
cuda_home = '/usr/local/cuda'
322
if os.path.isfile(cuda_home + '/bin/nvcc'):
@@ -581,7 +583,7 @@ def find_library(lib_name: str) -> str:
583
# libcuda.so.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libcuda.so.1
584
locs = [line.split()[-1] for line in libs.splitlines() if lib_name in line]
585
# `LD_LIBRARY_PATH` searches the library in the user-defined paths
584
- env_ld_library_path = os.getenv("LD_LIBRARY_PATH")
586
+ env_ld_library_path = envs.LD_LIBRARY_PATH
587
if not locs and env_ld_library_path:
588
locs = [
589
os.path.join(dir, lib_name)
@@ -594,14 +596,15 @@ def find_library(lib_name: str) -> str:
596
597
598
def find_nccl_library():
597
- so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
599
+ so_file = envs.VLLM_NCCL_SO_PATH
600
+ VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
601
602
# check if we have vllm-managed nccl
603
vllm_nccl_path = None
604
if torch.version.cuda is not None:
605
cuda_major = torch.version.cuda.split(".")[0]
606
path = os.path.expanduser(
604
- f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
607
+ f"{VLLM_CONFIG_ROOT}/vllm/nccl/cu{cuda_major}/libnccl.so.*")
608
files = glob.glob(path)
609
vllm_nccl_path = files[0] if files else None
610
@@ -626,7 +629,7 @@ def enable_trace_function_call_for_thread() -> None:
629
if enabled via the VLLM_TRACE_FUNCTION environment variable
630
"""
631
629
- if int(os.getenv("VLLM_TRACE_FUNCTION", "0")):
632
+ if envs.VLLM_TRACE_FUNCTION:
633
tmp_dir = tempfile.gettempdir()
634
filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
635
f"_thread_{threading.get_ident()}_"