@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 17c79f3c36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] auto infer dynamic_arg_dims from type annotation (#9589)

youkaichao committed Oct 22, 2024 at 13:43 UTC 17c79f3c364be166b68923bced94f902c00bd8bb
3 files changed +65 -19
vllm/compilation/decorators.py
+63 -5
@@ -1,24 +1,58 @@
1 import inspect
2 -from typing import Dict, List, Union
2 +from typing import Dict, List, Optional, Union
3
4 import torch
5
6 import vllm.envs as envs
7 from vllm.compilation.levels import CompilationLevel
8 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9 +from vllm.logger import init_logger
10 from vllm.sequence import IntermediateTensors
11 from vllm.utils import supports_dynamo
12
13 +logger = init_logger(__name__)
14
13 -def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
15 +
16 +def support_torch_compile(
17 + cls: Optional[type] = None,
18 + dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]] = None):
19 """
20 A decorator to add support for compiling the forward method of a class.
21
22 + Usage 1: use directly as a decorator without arguments:
23 +
24 + ```python
25 + @support_torch_compile
26 + class MyModel(nn.Module):
27 + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor]):
28 + ...
29 + ```
30 +
31 + Usage 2: use as a decorator with arguments:
32 +
33 + ```python
34 + @support_torch_compile(dynamic_arg_dims={"x": 0, "y": 0})
35 + class MyModel(nn.Module):
36 + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor]):
37 + ...
38 + ```
39 +
40 `dynamic_arg_dims` is a dictionary that maps argument names to the dynamic
41 dimensions of the argument. The dynamic dimensions can be either a single
42 integer or a list of integers.
43
21 - Depending on the value of arguments:
44 + if `dynamic_arg_dims` is `None`, it is inferred from the type annotation
45 + of the `forward` method, based on the following default rules:
46 +
47 + - if the argument is annotated as `torch.Tensor` or
48 + `Optional[torch.Tensor]`, the first dimension will be
49 + marked as dynamic.
50 + - if the argument is annotated as `IntermediateTensors`, the first
51 + dimension of all the tensors in the intermediate tensors
52 + will be marked as dynamic.
53 +
54 + During runtime, when we actually mark dimensions of tensors,
55 + it depends on the value of arguments:
56
57 - if it is a single integer, the corresponding dimension of the argument
58 will be marked as dynamic.
@@ -38,11 +72,35 @@ def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
72 if not hasattr(cls, 'forward'):
73 raise TypeError("decorated class should have a forward method.")
74 sig = inspect.signature(cls.forward)
41 - for k in dynamic_arg_dims:
75 + inferred_dynamic_arg_dims = dynamic_arg_dims
76 + if inferred_dynamic_arg_dims is None:
77 + inferred_dynamic_arg_dims = {}
78 + for k, v in sig.parameters.items():
79 + if v.annotation in [
80 + torch.Tensor, Optional[torch.Tensor],
81 + IntermediateTensors, Optional[IntermediateTensors]
82 + ]:
83 + inferred_dynamic_arg_dims[k] = 0
84 +
85 + logger.debug(("Inferred dynamic dimensions for "
86 + "forward method of %s: %s"), cls,
87 + list(inferred_dynamic_arg_dims.keys()))
88 +
89 + if len(inferred_dynamic_arg_dims) == 0:
90 + raise ValueError(
91 + "No dynamic dimensions found in the forward method of "
92 + f"{cls}. Please provide dynamic_arg_dims explicitly.")
93 +
94 + for k in inferred_dynamic_arg_dims:
95 if k not in sig.parameters:
96 raise ValueError(
97 f"Argument {k} not found in the forward method of {cls}")
45 - return _support_torch_compile(cls, dynamic_arg_dims)
98 + return _support_torch_compile(cls, inferred_dynamic_arg_dims)
99 +
100 + if cls is not None:
101 + # use `support_torch_compile` as a decorator without arguments
102 + assert isinstance(cls, type)
103 + return cls_decorator_helper(cls)
104
105 return cls_decorator_helper
106
vllm/model_executor/models/gemma2.py
+1 -7
@@ -241,13 +241,7 @@ class Gemma2DecoderLayer(nn.Module):
241 return hidden_states, residual
242
243
244 -@support_torch_compile(
245 - dynamic_arg_dims={
246 - "input_ids": 0,
247 - "positions": 0,
248 - "inputs_embeds": 0,
249 - "intermediate_tensors": 0,
250 - })
244 +@support_torch_compile
245 class Gemma2Model(nn.Module):
246
247 def __init__(
vllm/model_executor/models/llama.py
+1 -7
@@ -268,13 +268,7 @@ class LlamaDecoderLayer(nn.Module):
268 return hidden_states, residual
269
270
271 -@support_torch_compile(
272 - dynamic_arg_dims={
273 - "input_ids": 0,
274 - "positions": 0,
275 - "inputs_embeds": 0,
276 - "intermediate_tensors": 0,
277 - })
271 +@support_torch_compile
272 class LlamaModel(nn.Module):
273
274 def __init__(