@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2298e69b5f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[ci][bugfix] fix kernel tests (#10431)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 18, 2024 at 15:29 UTC 2298e69b5f1dc77f00aee687a3843a4dae12cb91
1 file changed +13 -10
vllm/plugins/__init__.py
+13 -10
@@ -6,9 +6,6 @@ import vllm.envs as envs
6
7 if TYPE_CHECKING:
8 from vllm.config import CompilationConfig, VllmConfig
9 -else:
10 - CompilationConfig = None
11 - VllmConfig = None
9
10 logger = logging.getLogger(__name__)
11
@@ -50,23 +47,23 @@ def load_general_plugins():
47 logger.exception("Failed to load plugin %s", plugin.name)
48
49
53 -_compilation_config: Optional[CompilationConfig] = None
50 +_compilation_config: Optional["CompilationConfig"] = None
51
52
56 -def set_compilation_config(config: Optional[CompilationConfig]):
53 +def set_compilation_config(config: Optional["CompilationConfig"]):
54 global _compilation_config
55 _compilation_config = config
56
57
61 -def get_compilation_config() -> Optional[CompilationConfig]:
58 +def get_compilation_config() -> Optional["CompilationConfig"]:
59 return _compilation_config
60
61
65 -_current_vllm_config: Optional[VllmConfig] = None
62 +_current_vllm_config: Optional["VllmConfig"] = None
63
64
65 @contextmanager
69 -def set_current_vllm_config(vllm_config: VllmConfig):
66 +def set_current_vllm_config(vllm_config: "VllmConfig"):
67 """
68 Temporarily set the current VLLM config.
69 Used during model initialization.
@@ -87,6 +84,12 @@ def set_current_vllm_config(vllm_config: VllmConfig):
84 _current_vllm_config = old_vllm_config
85
86
90 -def get_current_vllm_config() -> VllmConfig:
91 - assert _current_vllm_config is not None, "Current VLLM config is not set."
87 +def get_current_vllm_config() -> "VllmConfig":
88 + if _current_vllm_config is None:
89 + # in ci, usually when we test custom ops/modules directly,
90 + # we don't set the vllm config. In that case, we set a default
91 + # config.
92 + logger.warning("Current VLLM config is not set.")
93 + from vllm.config import VllmConfig
94 + return VllmConfig()
95 return _current_vllm_config