A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[plugin][torch.compile] allow to add custom compile backend (#8445)
youkaichao committed
Sep 13, 2024 at 09:32 UTC
0a4806f0a99880df1f74b10a6dceaf638cd3981c
2 files changed
+16
-1
vllm/plugins/__init__.py
+13
@@ -1,4 +1,5 @@
1
import logging
2
+from typing import Callable, Optional, Union
3
4
import vllm.envs as envs
5
@@ -29,3 +30,15 @@ def load_general_plugins():
30
except Exception:
31
logger.exception("Failed to load general plugin: %s",
32
plugin.name)
33
+
34
+
35
+_torch_compile_backend: Optional[Union[Callable, str]] = None
36
+
37
+
38
+def set_torch_compile_backend(backend: Union[Callable, str]):
39
+ global _torch_compile_backend
40
+ _torch_compile_backend = backend
41
+
42
+
43
+def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
44
+ return _torch_compile_backend
vllm/worker/model_runner.py
+3
-1
@@ -1064,10 +1064,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1064
"This may lead to less accurate results!")
1065
1066
if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1067
+ from vllm.plugins import get_torch_compile_backend
1068
+ backend = get_torch_compile_backend() or "eager"
1069
self.model = torch.compile(
1070
self.model,
1071
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
1070
- backend="eager")
1072
+ backend=backend)
1073
1074
def save_sharded_state(
1075
self,