@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51bb12d17b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[4/N][torch.compile] clean up set_torch_compile_backend (#10401)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 17, 2024 at 23:57 UTC 51bb12d17b374d5c4521cd01e5b066fd2419a8fa
7 files changed +49 -42
vllm/compilation/backends.py
+2 -14
@@ -2,15 +2,14 @@ import copy
2 import dataclasses
3 import operator
4 from contextlib import ExitStack
5 -from typing import (Any, Callable, Dict, List, Optional, Sequence, Set, Tuple,
6 - Union)
5 +from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
6 from unittest.mock import patch
7
8 import torch
9 import torch.fx as fx
10
11 import vllm.envs as envs
13 -from vllm.config import CompilationConfig, CompilationLevel
12 +from vllm.config import CompilationConfig
13 from vllm.logger import init_logger
14 from vllm.utils import combine_fx_passes, weak_ref_tensors
15
@@ -684,14 +683,3 @@ class PiecewiseBackend:
683
684 entry.cudagraph.replay()
685 return entry.output
687 -
688 -
689 -def select_default_backend(level: int) -> Union[str, Callable]:
690 - if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
691 - backend_str = "eager"
692 - return backend_str
693 - assert level == CompilationLevel.PIECEWISE
694 -
695 - from vllm.plugins import get_current_vllm_config
696 - compilation_config = get_current_vllm_config().compilation_config
697 - return VllmBackend(compilation_config)
vllm/compilation/wrapper.py
+3 -8
@@ -32,14 +32,9 @@ class TorchCompileWrapperWithCustomDispatcher:
32 # default compilation settings
33 # compiling the forward method
34
35 - # choose the compile backend
36 -
37 - # if the user has set the backend, use it
38 - from vllm.plugins import get_torch_compile_backend
39 - backend = get_torch_compile_backend()
40 - if backend is None:
41 - from vllm.compilation.backends import select_default_backend
42 - backend = select_default_backend(compilation_level)
35 + from vllm.plugins import get_current_vllm_config
36 + backend = get_current_vllm_config(
37 + ).compilation_config.init_backend()
38
39 compiled_callable = torch.compile(
40 self.forward,
vllm/config.py
+30 -1
@@ -22,7 +22,7 @@ from vllm.transformers_utils.config import (
22 get_hf_text_config, get_pooling_config,
23 get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
24 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
25 - identity, print_warning_once)
25 + identity, print_warning_once, resolve_obj_by_qualname)
26
27 if TYPE_CHECKING:
28 from ray.util.placement_group import PlacementGroup
@@ -2072,6 +2072,13 @@ class CompilationConfig(BaseModel):
2072 - 1: dynamo as is.
2073 - 2: dynamo once.
2074 - 3: piecewise compilation.
2075 + - backend: the backend for compilation. It needs to be a string.
2076 + - "" (empty string): use the default backend.
2077 + - "eager"/"openxla"/...: use the specified backend registered in PyTorch.
2078 + - "full.module.name": a qualified name which can be used to import the backend function.
2079 + We use string to avoid serialization issues when using compilation in a distributed setting.
2080 + When the compilation level is 1 or 2, the backend is used for the compilation directly (it sees the whole graph).
2081 + When the compilation level is 3, the backend is used for the piecewise compilation (it sees a part of the graph).
2082 - custom_ops: fine-grained control over which custom ops to enable/disable.
2083 Use 'all' to enable all, 'none' to disable all.
2084 Also specify a list of custom op names to enable (prefixed with a '+'),
@@ -2139,6 +2146,7 @@ class CompilationConfig(BaseModel):
2146 certain small batchsizes, where inductor is good at optimizing.
2147 """ # noqa
2148 level: int = 0
2149 + backend: str = ""
2150 custom_ops: List[str] = Field(default_factory=list)
2151
2152 use_inductor: bool = True
@@ -2182,6 +2190,27 @@ class CompilationConfig(BaseModel):
2190 func = __import__(module).__dict__[func_name]
2191 self.inductor_compile_config[k] = func
2192
2193 + def init_backend(self) -> Union[str, Callable]:
2194 + if self.level == CompilationLevel.NO_COMPILATION:
2195 + raise ValueError("No compilation level is set.")
2196 +
2197 + from torch._dynamo.backends.registry import list_backends
2198 + torch_backends = list_backends(exclude_tags=tuple())
2199 + if self.level in [
2200 + CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE
2201 + ]:
2202 + if self.backend == "":
2203 + return "eager"
2204 + if self.backend in torch_backends:
2205 + return self.backend
2206 + return resolve_obj_by_qualname(self.backend)
2207 +
2208 + # TODO: pass user-specified backend to piecewise compilation
2209 + # merge with the config use_inductor
2210 + assert self.level == CompilationLevel.PIECEWISE
2211 + from vllm.compilation.backends import VllmBackend
2212 + return VllmBackend(self)
2213 +
2214 def init_during_runtime(self):
2215 """To complete the initialization of config,
2216 we need to know the compile context, which is only available
vllm/platforms/tpu.py
+3 -4
@@ -3,8 +3,6 @@ from typing import TYPE_CHECKING
3
4 import torch
5
6 -from vllm.plugins import set_torch_compile_backend
7 -
6 from .interface import Platform, PlatformEnum
7
8 if TYPE_CHECKING:
@@ -12,8 +10,6 @@ if TYPE_CHECKING:
10 else:
11 VllmConfig = None
12
15 -set_torch_compile_backend("openxla")
16 -
13
14 class TpuPlatform(Platform):
15 _enum = PlatformEnum.TPU
@@ -38,3 +34,6 @@ class TpuPlatform(Platform):
34 compilation_config.level = CompilationLevel.DYNAMO_ONCE
35 assert compilation_config.level < CompilationLevel.PIECEWISE,\
36 "TPU does not support Inductor."
37 +
38 + if compilation_config.backend == "":
39 + compilation_config.backend = "openxla"
vllm/plugins/__init__.py
+1 -13
@@ -1,6 +1,6 @@
1 import logging
2 from contextlib import contextmanager
3 -from typing import TYPE_CHECKING, Callable, Optional, Union
3 +from typing import TYPE_CHECKING, Optional
4
5 import vllm.envs as envs
6
@@ -50,18 +50,6 @@ def load_general_plugins():
50 logger.exception("Failed to load plugin %s", plugin.name)
51
52
53 -_torch_compile_backend: Optional[Union[Callable, str]] = None
54 -
55 -
56 -def set_torch_compile_backend(backend: Union[Callable, str]):
57 - global _torch_compile_backend
58 - _torch_compile_backend = backend
59 -
60 -
61 -def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
62 - return _torch_compile_backend
63 -
64 -
53 _compilation_config: Optional[CompilationConfig] = None
54
55
vllm/utils.py
+9
@@ -1600,3 +1600,12 @@ def direct_register_custom_op(
1600 my_lib.impl(op_name, op_func, "CUDA")
1601 if fake_impl is not None:
1602 my_lib._register_fake(op_name, fake_impl)
1603 +
1604 +
1605 +def resolve_obj_by_qualname(qualname: str) -> Any:
1606 + """
1607 + Resolve an object by its fully qualified name.
1608 + """
1609 + module_name, obj_name = qualname.rsplit(".", 1)
1610 + module = importlib.import_module(module_name)
1611 + return getattr(module, obj_name)
vllm/worker/model_runner.py
+1 -2
@@ -1143,8 +1143,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1143
1144 if self.vllm_config.compilation_config.level ==\
1145 CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1146 - from vllm.plugins import get_torch_compile_backend
1147 - backend = get_torch_compile_backend() or "eager"
1146 + backend = self.vllm_config.compilation_config.init_backend()
1147 self.model = torch.compile(
1148 self.model,
1149 fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,