@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce6bf3a2cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[torch.compile] avoid Dynamo guard evaluation overhead (#7898)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

youkaichao committed Aug 28, 2024 at 16:10 UTC ce6bf3a2cff4860c5661cac2280e0a28bedb6440
9 files changed +190 -11
.buildkite/run-tpu-test.sh
+1 -1
@@ -12,4 +12,4 @@ remove_docker_container
12 # For HF_TOKEN.
13 source /etc/environment
14 # Run a simple end-to-end example.
15 -docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
15 +docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
.buildkite/test-pipeline.yaml
+1
@@ -173,6 +173,7 @@ steps:
173 - vllm/
174 commands:
175 - pytest -v -s ./compile/test_full_graph.py
176 + - pytest -v -s ./compile/test_wrapper.py
177
178
179 - label: Vision Language Models Test # 42min
tests/compile/test_wrapper.py new
+59
@@ -0,0 +1,59 @@
1 +from typing import Optional
2 +
3 +import torch
4 +
5 +from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
6 +
7 +
8 +class MyMod(torch.nn.Module):
9 +
10 + def forward(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
11 + if cache is not None:
12 + return x + cache
13 + return x * 2
14 +
15 +
16 +class MyWrapper(TorchCompileWrapperWithCustomDispacther):
17 +
18 + def __init__(self, model):
19 + self.model = model
20 + compiled_callable = torch.compile(self.forward, backend="eager")
21 + super().__init__(compiled_callable)
22 +
23 + def forward(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
24 + # this is the function to be compiled
25 + return self.model(x, cache)
26 +
27 + def __call__(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
28 + # let torch.compile compile twice
29 + if len(self.compiled_codes) == 2:
30 + dispatch_id = 0 if cache is None else 1
31 + with self.dispatch_to_code(dispatch_id):
32 + return self.forward(x, cache)
33 + else:
34 + return self.compiled_callable(x, cache)
35 +
36 +
37 +def test_torch_compile_wrapper():
38 + mod = MyMod()
39 + wrappers = []
40 + for i in range(3):
41 + torch._dynamo.reset()
42 + wrapper = MyWrapper(mod)
43 + wrappers.append(wrapper)
44 + x = torch.tensor([1])
45 + wrapper(x, None) # profile run, compile
46 + # create a cache tensor
47 + cache = torch.tensor([2])
48 + wrapper(x, cache) # warm up with cache, recompile
49 +
50 + # for new input, dispatch to the compiled code directly
51 + new_x = torch.tensor([3])
52 + assert wrapper(new_x,
53 + None).item() == 6 # dispatch to the first compiled code
54 + assert wrapper(
55 + new_x, cache).item() == 5 # dispatch to the second compiled code
56 +
57 + for wrapper in wrappers:
58 + # make sure they have independent compiled codes
59 + assert len(wrapper.compiled_codes) == 2
tests/tpu/__init__.py
tests/tpu/test_custom_dispatcher.py new
+9
@@ -0,0 +1,9 @@
1 +from ..utils import compare_two_settings
2 +
3 +
4 +def test_custom_dispatcher():
5 + compare_two_settings("google/gemma-2b",
6 + arg1=["--enforce-eager"],
7 + arg2=["--enforce-eager"],
8 + env1={"VLLM_DYNAMO_USE_CUSTOM_DISPATCHER": "0"},
9 + env2={})
vllm/compilation/__init__.py
vllm/compilation/wrapper.py new
+81
@@ -0,0 +1,81 @@
1 +import os
2 +import sys
3 +from abc import abstractmethod
4 +from contextlib import contextmanager
5 +from types import CodeType
6 +from typing import Callable, List
7 +
8 +import torch
9 +
10 +import vllm.envs as envs
11 +
12 +
13 +class TorchCompileWrapperWithCustomDispacther:
14 + """
15 + A wrapper class for torch.compile, with a custom dispatch logic.
16 + Subclasses should:
17 + 1. Implement the forward method
18 + 2. Implement the dispatch logic in the __call__ method
19 + It can use `self.compiled_codes` to access the compiled bytecode,
20 + and `with self.dispatch_to_code(index):` to dispatch to
21 + the compiled code.
22 + 3. Implement the `__init__` method to determine how to call
23 + `torch.compile` over the forward method.
24 + """
25 +
26 + def __init__(self, compiled_callable: Callable):
27 + self.compiled_callable = compiled_callable
28 + self.original_code_object = self.__class__.forward.__code__
29 + self.compiled_codes: List[CodeType] = []
30 + torch._dynamo.convert_frame.register_bytecode_hook(self.bytecode_hook)
31 +
32 + # read the env var to determine whether to use the custom dispatcher
33 + # subclasses can use this to switch between the custom dispatcher
34 + # and the default Dynamo guard mechanism.
35 + self.use_custom_dispatcher: bool = \
36 + envs.VLLM_DYNAMO_USE_CUSTOM_DISPATCHER
37 +
38 + def __call__(self, *args, **kwargs):
39 + """Implement the dispatch logic here, beyond the torch.compile level.
40 + NOTE: this function can have additional arguments beyond the forward
41 + method, for directly dispatching to the compiled code.
42 + """
43 + return self.compiled_callable(*args, **kwargs)
44 +
45 + @abstractmethod
46 + def forward(self, *args, **kwargs):
47 + ...
48 +
49 + def bytecode_hook(self, old_code: CodeType, new_code: CodeType):
50 + """Hook to save the compiled bytecode for direct execution."""
51 + if old_code is not self.original_code_object:
52 + return
53 + # code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25
54 + frame = sys._getframe()
55 + while True:
56 + frame = frame.f_back
57 + code_name = frame.f_code.co_name
58 + file_name = frame.f_code.co_filename.split(os.path.sep)[-1]
59 + if code_name == "_compile" and file_name == "convert_frame.py":
60 + break
61 + frame = frame.f_locals["frame"]
62 + assert frame.f_code == old_code
63 +
64 + if frame.f_locals["self"] is not self:
65 + return
66 +
67 + self.compiled_codes.append(new_code)
68 +
69 + @contextmanager
70 + def dispatch_to_code(self, index: int):
71 + """Context manager to dispatch to the compiled code.
72 + Why does this work? Because Dynamo guarantees that the compiled
73 + bytecode has exactly the same arguments, cell variables, and free
74 + variables as the original code. Therefore we can directly switch
75 + the code object in the function and call it.
76 +
77 + See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7 for more details.
78 + """ # noqa
79 + self.__class__.forward.__code__ = self.compiled_codes[index]
80 + yield
81 + self.__class__.forward.__code__ = self.original_code_object
vllm/envs.py
+4
@@ -196,6 +196,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
196 # Internal flag to enable Dynamo graph capture
197 "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
198 lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
199 + "VLLM_DYNAMO_USE_CUSTOM_DISPATCHER":
200 + lambda:
201 + (os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
202 + ("true", "1")),
203
204 # local rank of the process in the distributed setting, used to determine
205 # the GPU device id
vllm/worker/tpu_model_runner.py
+35 -10
@@ -10,6 +10,7 @@ import torch_xla.core.xla_model as xm
10 import torch_xla.runtime as xr
11
12 from vllm.attention import AttentionMetadata, get_attn_backend
13 +from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
14 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
15 ParallelConfig, SchedulerConfig)
16 from vllm.logger import init_logger
@@ -144,11 +145,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
145 )
146 model = model.eval()
147 xm.wait_device_ops()
147 - model = ModelWrapper(model)
148 - self.model = torch.compile(model,
149 - backend="openxla",
150 - fullgraph=True,
151 - dynamic=False)
148 + self.model = ModelWrapper(model)
149
150 def _dummy_run(
151 self,
@@ -235,8 +232,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
232 torch._dynamo.mark_dynamic(t, 0)
233 torch._dynamo.mark_dynamic(p, 0)
234 # Dummy run.
238 - self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
239 - num_samples, kv_caches)
235 + self.model(token_ids,
236 + position_ids,
237 + attn_metadata,
238 + input_lens,
239 + t,
240 + p,
241 + num_samples,
242 + kv_caches,
243 + is_prompt=is_prompt)
244
245 def warmup_model(
246 self,
@@ -530,7 +534,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
534 if getattr(arg, "context_lens", None) is not None:
535 arg.context_lens = arg.context_lens.to(self.device)
536 new_args.append(arg)
533 - return self.model(*new_args)
537 + return self.model(*new_args, is_prompt=is_prompt)
538
539 num_prefills = model_input.attn_metadata.num_prefills
540 is_prompt = num_prefills > 0
@@ -601,11 +605,32 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
605 return [SamplerOutput(sampler_outputs)]
606
607
604 -class ModelWrapper(nn.Module):
608 +class ModelWrapper(TorchCompileWrapperWithCustomDispacther):
609
610 def __init__(self, model: nn.Module):
607 - super().__init__()
611 self.model = model
612 + compiled_callable = torch.compile(self.forward,
613 + backend="openxla",
614 + fullgraph=True,
615 + dynamic=False)
616 + super().__init__(compiled_callable)
617 +
618 + def __call__(self, *args, is_prompt: bool, **kwargs):
619 + if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher:
620 + # not fully compiled yet, or not using the custom dispatcher,
621 + # let PyTorch handle it
622 + return self.compiled_callable(*args, **kwargs)
623 + # the 3 compiled codes are:
624 + # 0: for profiling
625 + # 1: for prompt
626 + # 2: for decode
627 + # dispatch to the compiled code directly, skip PyTorch
628 + if is_prompt:
629 + with self.dispatch_to_code(1):
630 + return self.forward(*args, **kwargs)
631 + else:
632 + with self.dispatch_to_code(2):
633 + return self.forward(*args, **kwargs)
634
635 def forward(
636 self,