A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] consider related env vars for torch.compiled cache hash (#14953)
Signed-off-by: DefTruth <31974251+DefTruth@users.noreply.github.com>
DefTruth committed
Mar 23, 2025 at 23:53 UTC
6ebaf9ac71387228951fd1642662a020080eb037
2 files changed
+46
vllm/compilation/backends.py
+5
@@ -357,6 +357,11 @@ class VllmBackend:
357
# graph.
358
359
factors = []
360
+ # 0. factors come from the env, for example, The values of
361
+ # VLLM_PP_LAYER_PARTITION will affects the computation graph.
362
+ env_hash = envs.compute_hash()
363
+ factors.append(env_hash)
364
+
365
# 1. factors come from the vllm_config (it mainly summarizes how the
366
# model is created)
367
config_hash = vllm_config.compute_hash()
vllm/envs.py
+41
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import hashlib
4
import os
5
import tempfile
6
from typing import TYPE_CHECKING, Any, Callable, Optional
@@ -651,3 +652,43 @@ def set_vllm_use_v1(use_v1: bool):
652
"explicitly by the user. Please raise this as a Github "
653
"Issue and explicitly set VLLM_USE_V1=0 or 1.")
654
os.environ["VLLM_USE_V1"] = "1" if use_v1 else "0"
655
+
656
+
657
+def compute_hash() -> str:
658
+ """
659
+ WARNING: Whenever a new key is added to this environment
660
+ variables, ensure that it is included in the factors list if
661
+ it affects the computation graph. For example, different values
662
+ of VLLM_PP_LAYER_PARTITION will generate different computation
663
+ graphs, so it is included in the factors list. The env vars that
664
+ affect the choice of different kernels or attention backends should
665
+ also be included in the factors list.
666
+ """
667
+ factors: list[Any] = []
668
+
669
+ # summarize environment variables
670
+ def factorize(name: str):
671
+ if __getattr__(name):
672
+ factors.append(__getattr__(name))
673
+ else:
674
+ factors.append("None")
675
+
676
+ # The values of envs may affects the computation graph.
677
+ # TODO(DefTruth): hash all environment variables?
678
+ # for key in environment_variables:
679
+ # factorize(key)
680
+ environment_variables_to_hash = [
681
+ "VLLM_PP_LAYER_PARTITION",
682
+ "VLLM_MLA_DISABLE",
683
+ "VLLM_USE_TRITON_FLASH_ATTN",
684
+ "VLLM_USE_TRITON_AWQ",
685
+ "VLLM_DP_RANK",
686
+ "VLLM_DP_SIZE",
687
+ ]
688
+ for key in environment_variables_to_hash:
689
+ if key in environment_variables:
690
+ factorize(key)
691
+
692
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
693
+
694
+ return hash_str