A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[ci][bugfix] fix kernel tests (#10431)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 18, 2024 at 15:29 UTC
2298e69b5f1dc77f00aee687a3843a4dae12cb91
1 file changed
+13
-10
vllm/plugins/__init__.py
+13
-10
@@ -6,9 +6,6 @@ import vllm.envs as envs
6
7
if TYPE_CHECKING:
8
from vllm.config import CompilationConfig, VllmConfig
9
-else:
10
- CompilationConfig = None
11
- VllmConfig = None
9
10
logger = logging.getLogger(__name__)
11
@@ -50,23 +47,23 @@ def load_general_plugins():
47
logger.exception("Failed to load plugin %s", plugin.name)
48
49
53
-_compilation_config: Optional[CompilationConfig] = None
50
+_compilation_config: Optional["CompilationConfig"] = None
51
52
56
-def set_compilation_config(config: Optional[CompilationConfig]):
53
+def set_compilation_config(config: Optional["CompilationConfig"]):
54
global _compilation_config
55
_compilation_config = config
56
57
61
-def get_compilation_config() -> Optional[CompilationConfig]:
58
+def get_compilation_config() -> Optional["CompilationConfig"]:
59
return _compilation_config
60
61
65
-_current_vllm_config: Optional[VllmConfig] = None
62
+_current_vllm_config: Optional["VllmConfig"] = None
63
64
65
@contextmanager
69
-def set_current_vllm_config(vllm_config: VllmConfig):
66
+def set_current_vllm_config(vllm_config: "VllmConfig"):
67
"""
68
Temporarily set the current VLLM config.
69
Used during model initialization.
@@ -87,6 +84,12 @@ def set_current_vllm_config(vllm_config: VllmConfig):
84
_current_vllm_config = old_vllm_config
85
86
90
-def get_current_vllm_config() -> VllmConfig:
91
- assert _current_vllm_config is not None, "Current VLLM config is not set."
87
+def get_current_vllm_config() -> "VllmConfig":
88
+ if _current_vllm_config is None:
89
+ # in ci, usually when we test custom ops/modules directly,
90
+ # we don't set the vllm config. In that case, we set a default
91
+ # config.
92
+ logger.warning("Current VLLM config is not set.")
93
+ from vllm.config import VllmConfig
94
+ return VllmConfig()
95
return _current_vllm_config