A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf] Cache vllm.env.__getattr__ result to avoid recomputation (#26146)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com>
Jialin Ouyang committed
Oct 14, 2025 at 14:03 UTC
380f17527c273c04372bb8a7a6acbb5efb132e51
4 files changed
+84
-2
tests/test_envs.py
+48
-1
@@ -6,7 +6,54 @@ from unittest.mock import patch
6
7
import pytest
8
9
-from vllm.envs import env_list_with_choices, env_with_choices
9
+import vllm.envs as envs
10
+from vllm.envs import (
11
+ enable_envs_cache,
12
+ env_list_with_choices,
13
+ env_with_choices,
14
+ environment_variables,
15
+)
16
+
17
+
18
+def test_getattr_without_cache(monkeypatch: pytest.MonkeyPatch):
19
+ assert envs.VLLM_HOST_IP == ""
20
+ assert envs.VLLM_PORT is None
21
+ monkeypatch.setenv("VLLM_HOST_IP", "1.1.1.1")
22
+ monkeypatch.setenv("VLLM_PORT", "1234")
23
+ assert envs.VLLM_HOST_IP == "1.1.1.1"
24
+ assert envs.VLLM_PORT == 1234
25
+ # __getattr__ is not decorated with functools.cache
26
+ assert not hasattr(envs.__getattr__, "cache_info")
27
+
28
+
29
+def test_getattr_with_cache(monkeypatch: pytest.MonkeyPatch):
30
+ monkeypatch.setenv("VLLM_HOST_IP", "1.1.1.1")
31
+ monkeypatch.setenv("VLLM_PORT", "1234")
32
+ # __getattr__ is not decorated with functools.cache
33
+ assert not hasattr(envs.__getattr__, "cache_info")
34
+
35
+ # Enable envs cache and ignore ongoing environment changes
36
+ enable_envs_cache()
37
+
38
+ # __getattr__ is not decorated with functools.cache
39
+ assert hasattr(envs.__getattr__, "cache_info")
40
+ start_hits = envs.__getattr__.cache_info().hits
41
+
42
+ # 2 more hits due to VLLM_HOST_IP and VLLM_PORT accesses
43
+ assert envs.VLLM_HOST_IP == "1.1.1.1"
44
+ assert envs.VLLM_PORT == 1234
45
+ assert envs.__getattr__.cache_info().hits == start_hits + 2
46
+
47
+ # All environment variables are cached
48
+ for environment_variable in environment_variables:
49
+ envs.__getattr__(environment_variable)
50
+ assert envs.__getattr__.cache_info().hits == start_hits + 2 + len(
51
+ environment_variables
52
+ )
53
+
54
+ # Reset envs.__getattr__ back to none-cached version to
55
+ # avoid affecting other tests
56
+ envs.__getattr__ = envs.__getattr__.__wrapped__
57
58
59
class TestEnvWithChoices:
vllm/envs.py
+26
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import functools
5
import hashlib
6
import json
7
import os
@@ -1408,12 +1409,36 @@ environment_variables: dict[str, Callable[[], Any]] = {
1409
1410
1411
def __getattr__(name: str):
1411
- # lazy evaluation of environment variables
1412
+ """
1413
+ Gets environment variables lazily.
1414
+
1415
+ NOTE: After enable_envs_cache() invocation (which triggered after service
1416
+ initialization), all environment variables will be cached.
1417
+ """
1418
if name in environment_variables:
1419
return environment_variables[name]()
1420
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
1421
1422
1423
+def enable_envs_cache() -> None:
1424
+ """
1425
+ Enables caching of environment variables. This is useful for performance
1426
+ reasons, as it avoids the need to re-evaluate environment variables on
1427
+ every call.
1428
+
1429
+ NOTE: Currently, it's invoked after service initialization to reduce
1430
+ runtime overhead. This also means that environment variables should NOT
1431
+ be updated after the service is initialized.
1432
+ """
1433
+ # Tag __getattr__ with functools.cache
1434
+ global __getattr__
1435
+ __getattr__ = functools.cache(__getattr__)
1436
+
1437
+ # Cache all environment variables
1438
+ for key in environment_variables:
1439
+ __getattr__(key)
1440
+
1441
+
1442
def __dir__():
1443
return list(environment_variables.keys())
1444
vllm/v1/engine/core.py
+5
@@ -20,6 +20,7 @@ import zmq
20
from vllm.config import ParallelConfig, VllmConfig
21
from vllm.distributed import stateless_destroy_torch_distributed_process_group
22
from vllm.distributed.parallel_state import is_global_first_rank
23
+from vllm.envs import enable_envs_cache
24
from vllm.logger import init_logger
25
from vllm.logging_utils.dump_input import dump_engine_exception
26
from vllm.lora.request import LoRARequest
@@ -601,6 +602,10 @@ class EngineCoreProc(EngineCore):
602
# If enable, attach GC debugger after static variable freeze.
603
maybe_attach_gc_debug_callback()
604
605
+ # Enable environment variable cache (e.g. assume no more
606
+ # environment variable overrides after this point)
607
+ enable_envs_cache()
608
+
609
@contextmanager
610
def _perform_handshakes(
611
self,
vllm/v1/executor/multiproc_executor.py
+5
@@ -33,6 +33,7 @@ from vllm.distributed.parallel_state import (
33
get_pp_group,
34
get_tp_group,
35
)
36
+from vllm.envs import enable_envs_cache
37
from vllm.logger import init_logger
38
from vllm.utils import (
39
_maybe_force_spawn,
@@ -455,6 +456,10 @@ class WorkerProc:
456
# Load model
457
self.worker.load_model()
458
459
+ # Enable environment variable cache (e.g. assume no more
460
+ # environment variable overrides after this point)
461
+ enable_envs_cache()
462
+
463
@staticmethod
464
def make_worker_process(
465
vllm_config: VllmConfig,