A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[torch.compile] avoid Dynamo guard evaluation overhead (#7898)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
youkaichao committed
Aug 28, 2024 at 16:10 UTC
ce6bf3a2cff4860c5661cac2280e0a28bedb6440
9 files changed
+190
-11
.buildkite/run-tpu-test.sh
+1
-1
@@ -12,4 +12,4 @@ remove_docker_container
12
# For HF_TOKEN.
13
source /etc/environment
14
# Run a simple end-to-end example.
15
-docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
15
+docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
.buildkite/test-pipeline.yaml
+1
@@ -173,6 +173,7 @@ steps:
173
- vllm/
174
commands:
175
- pytest -v -s ./compile/test_full_graph.py
176
+ - pytest -v -s ./compile/test_wrapper.py
177
178
179
- label: Vision Language Models Test # 42min
tests/compile/test_wrapper.py
new
+59
@@ -0,0 +1,59 @@
1
+from typing import Optional
2
+
3
+import torch
4
+
5
+from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
6
+
7
+
8
+class MyMod(torch.nn.Module):
9
+
10
+ def forward(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
11
+ if cache is not None:
12
+ return x + cache
13
+ return x * 2
14
+
15
+
16
+class MyWrapper(TorchCompileWrapperWithCustomDispacther):
17
+
18
+ def __init__(self, model):
19
+ self.model = model
20
+ compiled_callable = torch.compile(self.forward, backend="eager")
21
+ super().__init__(compiled_callable)
22
+
23
+ def forward(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
24
+ # this is the function to be compiled
25
+ return self.model(x, cache)
26
+
27
+ def __call__(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
28
+ # let torch.compile compile twice
29
+ if len(self.compiled_codes) == 2:
30
+ dispatch_id = 0 if cache is None else 1
31
+ with self.dispatch_to_code(dispatch_id):
32
+ return self.forward(x, cache)
33
+ else:
34
+ return self.compiled_callable(x, cache)
35
+
36
+
37
+def test_torch_compile_wrapper():
38
+ mod = MyMod()
39
+ wrappers = []
40
+ for i in range(3):
41
+ torch._dynamo.reset()
42
+ wrapper = MyWrapper(mod)
43
+ wrappers.append(wrapper)
44
+ x = torch.tensor([1])
45
+ wrapper(x, None) # profile run, compile
46
+ # create a cache tensor
47
+ cache = torch.tensor([2])
48
+ wrapper(x, cache) # warm up with cache, recompile
49
+
50
+ # for new input, dispatch to the compiled code directly
51
+ new_x = torch.tensor([3])
52
+ assert wrapper(new_x,
53
+ None).item() == 6 # dispatch to the first compiled code
54
+ assert wrapper(
55
+ new_x, cache).item() == 5 # dispatch to the second compiled code
56
+
57
+ for wrapper in wrappers:
58
+ # make sure they have independent compiled codes
59
+ assert len(wrapper.compiled_codes) == 2
tests/tpu/__init__.py
tests/tpu/test_custom_dispatcher.py
new
+9
@@ -0,0 +1,9 @@
1
+from ..utils import compare_two_settings
2
+
3
+
4
+def test_custom_dispatcher():
5
+ compare_two_settings("google/gemma-2b",
6
+ arg1=["--enforce-eager"],
7
+ arg2=["--enforce-eager"],
8
+ env1={"VLLM_DYNAMO_USE_CUSTOM_DISPATCHER": "0"},
9
+ env2={})
vllm/compilation/__init__.py
vllm/compilation/wrapper.py
new
+81
@@ -0,0 +1,81 @@
1
+import os
2
+import sys
3
+from abc import abstractmethod
4
+from contextlib import contextmanager
5
+from types import CodeType
6
+from typing import Callable, List
7
+
8
+import torch
9
+
10
+import vllm.envs as envs
11
+
12
+
13
+class TorchCompileWrapperWithCustomDispacther:
14
+ """
15
+ A wrapper class for torch.compile, with a custom dispatch logic.
16
+ Subclasses should:
17
+ 1. Implement the forward method
18
+ 2. Implement the dispatch logic in the __call__ method
19
+ It can use `self.compiled_codes` to access the compiled bytecode,
20
+ and `with self.dispatch_to_code(index):` to dispatch to
21
+ the compiled code.
22
+ 3. Implement the `__init__` method to determine how to call
23
+ `torch.compile` over the forward method.
24
+ """
25
+
26
+ def __init__(self, compiled_callable: Callable):
27
+ self.compiled_callable = compiled_callable
28
+ self.original_code_object = self.__class__.forward.__code__
29
+ self.compiled_codes: List[CodeType] = []
30
+ torch._dynamo.convert_frame.register_bytecode_hook(self.bytecode_hook)
31
+
32
+ # read the env var to determine whether to use the custom dispatcher
33
+ # subclasses can use this to switch between the custom dispatcher
34
+ # and the default Dynamo guard mechanism.
35
+ self.use_custom_dispatcher: bool = \
36
+ envs.VLLM_DYNAMO_USE_CUSTOM_DISPATCHER
37
+
38
+ def __call__(self, *args, **kwargs):
39
+ """Implement the dispatch logic here, beyond the torch.compile level.
40
+ NOTE: this function can have additional arguments beyond the forward
41
+ method, for directly dispatching to the compiled code.
42
+ """
43
+ return self.compiled_callable(*args, **kwargs)
44
+
45
+ @abstractmethod
46
+ def forward(self, *args, **kwargs):
47
+ ...
48
+
49
+ def bytecode_hook(self, old_code: CodeType, new_code: CodeType):
50
+ """Hook to save the compiled bytecode for direct execution."""
51
+ if old_code is not self.original_code_object:
52
+ return
53
+ # code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25
54
+ frame = sys._getframe()
55
+ while True:
56
+ frame = frame.f_back
57
+ code_name = frame.f_code.co_name
58
+ file_name = frame.f_code.co_filename.split(os.path.sep)[-1]
59
+ if code_name == "_compile" and file_name == "convert_frame.py":
60
+ break
61
+ frame = frame.f_locals["frame"]
62
+ assert frame.f_code == old_code
63
+
64
+ if frame.f_locals["self"] is not self:
65
+ return
66
+
67
+ self.compiled_codes.append(new_code)
68
+
69
+ @contextmanager
70
+ def dispatch_to_code(self, index: int):
71
+ """Context manager to dispatch to the compiled code.
72
+ Why does this work? Because Dynamo guarantees that the compiled
73
+ bytecode has exactly the same arguments, cell variables, and free
74
+ variables as the original code. Therefore we can directly switch
75
+ the code object in the function and call it.
76
+
77
+ See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7 for more details.
78
+ """ # noqa
79
+ self.__class__.forward.__code__ = self.compiled_codes[index]
80
+ yield
81
+ self.__class__.forward.__code__ = self.original_code_object
vllm/envs.py
+4
@@ -196,6 +196,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
196
# Internal flag to enable Dynamo graph capture
197
"VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
198
lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
199
+ "VLLM_DYNAMO_USE_CUSTOM_DISPATCHER":
200
+ lambda:
201
+ (os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
202
+ ("true", "1")),
203
204
# local rank of the process in the distributed setting, used to determine
205
# the GPU device id
vllm/worker/tpu_model_runner.py
+35
-10
@@ -10,6 +10,7 @@ import torch_xla.core.xla_model as xm
10
import torch_xla.runtime as xr
11
12
from vllm.attention import AttentionMetadata, get_attn_backend
13
+from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
14
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
15
ParallelConfig, SchedulerConfig)
16
from vllm.logger import init_logger
@@ -144,11 +145,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
145
)
146
model = model.eval()
147
xm.wait_device_ops()
147
- model = ModelWrapper(model)
148
- self.model = torch.compile(model,
149
- backend="openxla",
150
- fullgraph=True,
151
- dynamic=False)
148
+ self.model = ModelWrapper(model)
149
150
def _dummy_run(
151
self,
@@ -235,8 +232,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
232
torch._dynamo.mark_dynamic(t, 0)
233
torch._dynamo.mark_dynamic(p, 0)
234
# Dummy run.
238
- self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
239
- num_samples, kv_caches)
235
+ self.model(token_ids,
236
+ position_ids,
237
+ attn_metadata,
238
+ input_lens,
239
+ t,
240
+ p,
241
+ num_samples,
242
+ kv_caches,
243
+ is_prompt=is_prompt)
244
245
def warmup_model(
246
self,
@@ -530,7 +534,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
534
if getattr(arg, "context_lens", None) is not None:
535
arg.context_lens = arg.context_lens.to(self.device)
536
new_args.append(arg)
533
- return self.model(*new_args)
537
+ return self.model(*new_args, is_prompt=is_prompt)
538
539
num_prefills = model_input.attn_metadata.num_prefills
540
is_prompt = num_prefills > 0
@@ -601,11 +605,32 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
605
return [SamplerOutput(sampler_outputs)]
606
607
604
-class ModelWrapper(nn.Module):
608
+class ModelWrapper(TorchCompileWrapperWithCustomDispacther):
609
610
def __init__(self, model: nn.Module):
607
- super().__init__()
611
self.model = model
612
+ compiled_callable = torch.compile(self.forward,
613
+ backend="openxla",
614
+ fullgraph=True,
615
+ dynamic=False)
616
+ super().__init__(compiled_callable)
617
+
618
+ def __call__(self, *args, is_prompt: bool, **kwargs):
619
+ if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher:
620
+ # not fully compiled yet, or not using the custom dispatcher,
621
+ # let PyTorch handle it
622
+ return self.compiled_callable(*args, **kwargs)
623
+ # the 3 compiled codes are:
624
+ # 0: for profiling
625
+ # 1: for prompt
626
+ # 2: for decode
627
+ # dispatch to the compiled code directly, skip PyTorch
628
+ if is_prompt:
629
+ with self.dispatch_to_code(1):
630
+ return self.forward(*args, **kwargs)
631
+ else:
632
+ with self.dispatch_to_code(2):
633
+ return self.forward(*args, **kwargs)
634
635
def forward(
636
self,