@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e00c094f15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[torch.compile] generic decorators (#9258)

youkaichao committed Oct 10, 2024 at 15:54 UTC e00c094f15e79c5a113fdf975df1ee9018cb65b3
3 files changed +74 -34
vllm/compilation/decorators.py
+58 -30
@@ -1,20 +1,54 @@
1 -from typing import List, Optional, Union
1 +import inspect
2 +from typing import Dict, List, Union
3
4 import torch
5
6 import vllm.envs as envs
6 -from vllm.attention import AttentionMetadata
7 from vllm.compilation.levels import CompilationLevel
8 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9 from vllm.sequence import IntermediateTensors
10 from vllm.utils import supports_dynamo
11
12
13 -def support_compile_llama_style(cls: type):
13 +def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
14 + """
15 + A decorator to add support for compiling the forward method of a class.
16 +
17 + `dynamic_arg_dims` is a dictionary that maps argument names to the dynamic
18 + dimensions of the argument. The dynamic dimensions can be either a single
19 + integer or a list of integers.
20 +
21 + Depending on the value of arguments:
22 +
23 + - if it is a single integer, the corresponding dimension of the argument
24 + will be marked as dynamic.
25 + - if it is `None`, ignored.
26 + - if it is `IntermediateTensors`, all the tensors in the intermediate
27 + tensors will be marked as dynamic.
28 + - otherwise, it will raise an error.
29 +
30 + NOTE: if an argument is `None`, it should always be passed as `None` during
31 + the lifetime of the model, otherwise, it cannot be captured as a single
32 + computation graph.
33 + """
34 +
35 + def cls_decorator_helper(cls: type):
36 + # helper to pass `dynamic_arg_dims`` to `_support_torch_compile``
37 + # to avoid too much indentation for `_support_torch_compile``
38 + sig = inspect.signature(cls.forward)
39 + for k in dynamic_arg_dims:
40 + if k not in sig.parameters:
41 + raise ValueError(
42 + f"Argument {k} not found in the forward method of {cls}")
43 + return _support_torch_compile(cls, dynamic_arg_dims)
44 +
45 + return cls_decorator_helper
46 +
47 +
48 +def _support_torch_compile(cls: type,
49 + dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
50 """
51 A decorator to add support for compiling the forward method of a class.
16 - If a module's **forward signature** is compatible with llama, this
17 - decorator can be used to enable the compilation of the forward method.
52 """
53
54 # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
@@ -37,48 +71,42 @@ def support_compile_llama_style(cls: type):
71
72 cls.__init__ = __init__
73
40 - def __call__(
41 - self,
42 - input_ids: Optional[torch.Tensor],
43 - positions: torch.Tensor,
44 - kv_caches: List[torch.Tensor],
45 - attn_metadata: AttentionMetadata,
46 - intermediate_tensors: Optional[IntermediateTensors],
47 - inputs_embeds: Optional[torch.Tensor] = None,
48 - ) -> Union[torch.Tensor, IntermediateTensors]:
74 + def __call__(self, *args, **kwargs):
75 # torch.compiler.is_compiling() means we are inside the compilation
76 # e.g. TPU has the compilation logic in model runner, so we don't
77 # need to compile the model inside.
78 if torch.compiler.is_compiling():
53 - return self.forward(input_ids, positions, kv_caches, attn_metadata,
54 - intermediate_tensors, inputs_embeds)
79 + return self.forward(*args, **kwargs)
80
81 # the first compilation needs to have dynamic shapes marked
82 if len(self.compiled_codes) < 1:
58 - if input_ids is not None:
59 - torch._dynamo.mark_dynamic(input_ids, 0)
60 - torch._dynamo.mark_dynamic(positions, 0)
61 - if inputs_embeds is not None:
62 - torch._dynamo.mark_dynamic(inputs_embeds, 0)
63 - if intermediate_tensors is not None:
64 - for tensors in intermediate_tensors.tensors.values():
65 - torch._dynamo.mark_dynamic(tensors, 0)
83 + sig = inspect.signature(self.__class__.forward)
84 + bound_args = sig.bind(self, *args, **kwargs)
85 + bound_args.apply_defaults()
86 + for k, dims in dynamic_arg_dims.items():
87 + arg = bound_args.arguments.get(k)
88 + if arg is not None:
89 + if isinstance(arg, torch.Tensor):
90 + torch._dynamo.mark_dynamic(arg, dims)
91 + elif isinstance(arg, IntermediateTensors):
92 + for tensor in arg.tensors.values():
93 + torch._dynamo.mark_dynamic(tensor, dims)
94 + else:
95 + raise ValueError(
96 + "Unsupported dynamic dimensions"
97 + f" {dims} for argument {k} with type {type(arg)}.")
98
99 # if we don't use custom dispatcher, we can directly call the
100 # compiled function and let torch.compile handle the dispatching,
101 # with the overhead of guard evaluation and recompilation.
102 if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
71 - return self.compiled_callable(input_ids, positions, kv_caches,
72 - attn_metadata, intermediate_tensors,
73 - inputs_embeds)
103 + return self.compiled_callable(*args, **kwargs)
104
105 # usually, capturing the model once is enough, and then we can
106 # dispatch to the compiled code directly, without going through
107 # the Dynamo guard mechanism.
108 with self.dispatch_to_code(0):
79 - model_output = self.forward(input_ids, positions, kv_caches,
80 - attn_metadata, intermediate_tensors,
81 - inputs_embeds)
109 + model_output = self.forward(*args, **kwargs)
110 return model_output
111
112 cls.__call__ = __call__
vllm/model_executor/models/gemma2.py
+8 -2
@@ -21,7 +21,7 @@ from torch import nn
21 from transformers import Gemma2Config
22
23 from vllm.attention import Attention, AttentionMetadata
24 -from vllm.compilation.decorators import support_compile_llama_style
24 +from vllm.compilation.decorators import support_torch_compile
25 from vllm.config import CacheConfig, LoRAConfig
26 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
@@ -239,7 +239,13 @@ class Gemma2DecoderLayer(nn.Module):
239 return hidden_states, residual
240
241
242 -@support_compile_llama_style
242 +@support_torch_compile(
243 + dynamic_arg_dims={
244 + "input_ids": 0,
245 + "positions": 0,
246 + "inputs_embeds": 0,
247 + "intermediate_tensors": 0,
248 + })
249 class Gemma2Model(nn.Module):
250
251 def __init__(
vllm/model_executor/models/llama.py
+8 -2
@@ -28,7 +28,7 @@ from torch import nn
28 from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 -from vllm.compilation.decorators import support_compile_llama_style
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
@@ -266,7 +266,13 @@ class LlamaDecoderLayer(nn.Module):
266 return hidden_states, residual
267
268
269 -@support_compile_llama_style
269 +@support_torch_compile(
270 + dynamic_arg_dims={
271 + "input_ids": 0,
272 + "positions": 0,
273 + "inputs_embeds": 0,
274 + "intermediate_tensors": 0,
275 + })
276 class LlamaModel(nn.Module):
277
278 def __init__(