@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a4806f0a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[plugin][torch.compile] allow to add custom compile backend (#8445)

youkaichao committed Sep 13, 2024 at 09:32 UTC 0a4806f0a99880df1f74b10a6dceaf638cd3981c
2 files changed +16 -1
vllm/plugins/__init__.py
+13
@@ -1,4 +1,5 @@
1 import logging
2 +from typing import Callable, Optional, Union
3
4 import vllm.envs as envs
5
@@ -29,3 +30,15 @@ def load_general_plugins():
30 except Exception:
31 logger.exception("Failed to load general plugin: %s",
32 plugin.name)
33 +
34 +
35 +_torch_compile_backend: Optional[Union[Callable, str]] = None
36 +
37 +
38 +def set_torch_compile_backend(backend: Union[Callable, str]):
39 + global _torch_compile_backend
40 + _torch_compile_backend = backend
41 +
42 +
43 +def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
44 + return _torch_compile_backend
vllm/worker/model_runner.py
+3 -1
@@ -1064,10 +1064,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1064 "This may lead to less accurate results!")
1065
1066 if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1067 + from vllm.plugins import get_torch_compile_backend
1068 + backend = get_torch_compile_backend() or "eager"
1069 self.model = torch.compile(
1070 self.model,
1071 fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
1070 - backend="eager")
1072 + backend=backend)
1073
1074 def save_sharded_state(
1075 self,