A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[4/N][torch.compile] clean up set_torch_compile_backend (#10401)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 17, 2024 at 23:57 UTC
51bb12d17b374d5c4521cd01e5b066fd2419a8fa
7 files changed
+49
-42
vllm/compilation/backends.py
+2
-14
@@ -2,15 +2,14 @@ import copy
2
import dataclasses
3
import operator
4
from contextlib import ExitStack
5
-from typing import (Any, Callable, Dict, List, Optional, Sequence, Set, Tuple,
6
- Union)
5
+from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
6
from unittest.mock import patch
7
8
import torch
9
import torch.fx as fx
10
11
import vllm.envs as envs
13
-from vllm.config import CompilationConfig, CompilationLevel
12
+from vllm.config import CompilationConfig
13
from vllm.logger import init_logger
14
from vllm.utils import combine_fx_passes, weak_ref_tensors
15
@@ -684,14 +683,3 @@ class PiecewiseBackend:
683
684
entry.cudagraph.replay()
685
return entry.output
687
-
688
-
689
-def select_default_backend(level: int) -> Union[str, Callable]:
690
- if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
691
- backend_str = "eager"
692
- return backend_str
693
- assert level == CompilationLevel.PIECEWISE
694
-
695
- from vllm.plugins import get_current_vllm_config
696
- compilation_config = get_current_vllm_config().compilation_config
697
- return VllmBackend(compilation_config)
vllm/compilation/wrapper.py
+3
-8
@@ -32,14 +32,9 @@ class TorchCompileWrapperWithCustomDispatcher:
32
# default compilation settings
33
# compiling the forward method
34
35
- # choose the compile backend
36
-
37
- # if the user has set the backend, use it
38
- from vllm.plugins import get_torch_compile_backend
39
- backend = get_torch_compile_backend()
40
- if backend is None:
41
- from vllm.compilation.backends import select_default_backend
42
- backend = select_default_backend(compilation_level)
35
+ from vllm.plugins import get_current_vllm_config
36
+ backend = get_current_vllm_config(
37
+ ).compilation_config.init_backend()
38
39
compiled_callable = torch.compile(
40
self.forward,
vllm/config.py
+30
-1
@@ -22,7 +22,7 @@ from vllm.transformers_utils.config import (
22
get_hf_text_config, get_pooling_config,
23
get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
24
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
25
- identity, print_warning_once)
25
+ identity, print_warning_once, resolve_obj_by_qualname)
26
27
if TYPE_CHECKING:
28
from ray.util.placement_group import PlacementGroup
@@ -2072,6 +2072,13 @@ class CompilationConfig(BaseModel):
2072
- 1: dynamo as is.
2073
- 2: dynamo once.
2074
- 3: piecewise compilation.
2075
+ - backend: the backend for compilation. It needs to be a string.
2076
+ - "" (empty string): use the default backend.
2077
+ - "eager"/"openxla"/...: use the specified backend registered in PyTorch.
2078
+ - "full.module.name": a qualified name which can be used to import the backend function.
2079
+ We use string to avoid serialization issues when using compilation in a distributed setting.
2080
+ When the compilation level is 1 or 2, the backend is used for the compilation directly (it sees the whole graph).
2081
+ When the compilation level is 3, the backend is used for the piecewise compilation (it sees a part of the graph).
2082
- custom_ops: fine-grained control over which custom ops to enable/disable.
2083
Use 'all' to enable all, 'none' to disable all.
2084
Also specify a list of custom op names to enable (prefixed with a '+'),
@@ -2139,6 +2146,7 @@ class CompilationConfig(BaseModel):
2146
certain small batchsizes, where inductor is good at optimizing.
2147
""" # noqa
2148
level: int = 0
2149
+ backend: str = ""
2150
custom_ops: List[str] = Field(default_factory=list)
2151
2152
use_inductor: bool = True
@@ -2182,6 +2190,27 @@ class CompilationConfig(BaseModel):
2190
func = __import__(module).__dict__[func_name]
2191
self.inductor_compile_config[k] = func
2192
2193
+ def init_backend(self) -> Union[str, Callable]:
2194
+ if self.level == CompilationLevel.NO_COMPILATION:
2195
+ raise ValueError("No compilation level is set.")
2196
+
2197
+ from torch._dynamo.backends.registry import list_backends
2198
+ torch_backends = list_backends(exclude_tags=tuple())
2199
+ if self.level in [
2200
+ CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE
2201
+ ]:
2202
+ if self.backend == "":
2203
+ return "eager"
2204
+ if self.backend in torch_backends:
2205
+ return self.backend
2206
+ return resolve_obj_by_qualname(self.backend)
2207
+
2208
+ # TODO: pass user-specified backend to piecewise compilation
2209
+ # merge with the config use_inductor
2210
+ assert self.level == CompilationLevel.PIECEWISE
2211
+ from vllm.compilation.backends import VllmBackend
2212
+ return VllmBackend(self)
2213
+
2214
def init_during_runtime(self):
2215
"""To complete the initialization of config,
2216
we need to know the compile context, which is only available
vllm/platforms/tpu.py
+3
-4
@@ -3,8 +3,6 @@ from typing import TYPE_CHECKING
3
4
import torch
5
6
-from vllm.plugins import set_torch_compile_backend
7
-
6
from .interface import Platform, PlatformEnum
7
8
if TYPE_CHECKING:
@@ -12,8 +10,6 @@ if TYPE_CHECKING:
10
else:
11
VllmConfig = None
12
15
-set_torch_compile_backend("openxla")
16
-
13
14
class TpuPlatform(Platform):
15
_enum = PlatformEnum.TPU
@@ -38,3 +34,6 @@ class TpuPlatform(Platform):
34
compilation_config.level = CompilationLevel.DYNAMO_ONCE
35
assert compilation_config.level < CompilationLevel.PIECEWISE,\
36
"TPU does not support Inductor."
37
+
38
+ if compilation_config.backend == "":
39
+ compilation_config.backend = "openxla"
vllm/plugins/__init__.py
+1
-13
@@ -1,6 +1,6 @@
1
import logging
2
from contextlib import contextmanager
3
-from typing import TYPE_CHECKING, Callable, Optional, Union
3
+from typing import TYPE_CHECKING, Optional
4
5
import vllm.envs as envs
6
@@ -50,18 +50,6 @@ def load_general_plugins():
50
logger.exception("Failed to load plugin %s", plugin.name)
51
52
53
-_torch_compile_backend: Optional[Union[Callable, str]] = None
54
-
55
-
56
-def set_torch_compile_backend(backend: Union[Callable, str]):
57
- global _torch_compile_backend
58
- _torch_compile_backend = backend
59
-
60
-
61
-def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
62
- return _torch_compile_backend
63
-
64
-
53
_compilation_config: Optional[CompilationConfig] = None
54
55
vllm/utils.py
+9
@@ -1600,3 +1600,12 @@ def direct_register_custom_op(
1600
my_lib.impl(op_name, op_func, "CUDA")
1601
if fake_impl is not None:
1602
my_lib._register_fake(op_name, fake_impl)
1603
+
1604
+
1605
+def resolve_obj_by_qualname(qualname: str) -> Any:
1606
+ """
1607
+ Resolve an object by its fully qualified name.
1608
+ """
1609
+ module_name, obj_name = qualname.rsplit(".", 1)
1610
+ module = importlib.import_module(module_name)
1611
+ return getattr(module, obj_name)
vllm/worker/model_runner.py
+1
-2
@@ -1143,8 +1143,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1143
1144
if self.vllm_config.compilation_config.level ==\
1145
CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1146
- from vllm.plugins import get_torch_compile_backend
1147
- backend = get_torch_compile_backend() or "eager"
1146
+ backend = self.vllm_config.compilation_config.init_backend()
1147
self.model = torch.compile(
1148
self.model,
1149
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,