A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[torch.compile] integration with compilation control (#9058)
youkaichao committed
Oct 10, 2024 at 12:39 UTC
e4d652ea3ed9b2a60c1582cb2e2605695e61280f
22 files changed
+404
-98
.buildkite/test-pipeline.yaml
+12
-8
@@ -121,7 +121,9 @@ steps:
121
- vllm/core/
122
- tests/distributed
123
- tests/spec_decode/e2e/test_integration_dist_tp4
124
+ - tests/compile
125
commands:
126
+ - pytest -v -s compile/test_basic_correctness.py
127
- pytest -v -s distributed/test_pynccl.py
128
- pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
129
@@ -231,14 +233,16 @@ steps:
233
- vllm/
234
- tests/compile
235
commands:
234
- - pytest -v -s compile/test_full_graph_smoke.py
236
+ - pytest -v -s compile/test_basic_correctness.py
237
236
-- label: "PyTorch Fullgraph Test" # 18min
237
- source_file_dependencies:
238
- - vllm/
239
- - tests/compile
240
- commands:
241
- - pytest -v -s compile/test_full_graph.py
238
+# TODO: re-write in comparison tests, and fix symbolic shape
239
+# for quantization ops.
240
+# - label: "PyTorch Fullgraph Test" # 18min
241
+# source_file_dependencies:
242
+# - vllm/
243
+# - tests/compile
244
+# commands:
245
+# - pytest -v -s compile/test_full_graph.py
246
247
- label: Kernels Test %N # 1h each
248
mirror_hardwares: [amd]
@@ -394,7 +398,7 @@ steps:
398
- tests/distributed/
399
- vllm/compilation
400
commands:
397
- - pytest -v -s ./compile/test_full_graph_multi_gpu.py
401
+ - pytest -v -s ./compile/test_basic_correctness.py
402
- pytest -v -s ./compile/test_wrapper.py
403
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
404
- TARGET_TEST_SUITE=L4 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest basic_correctness/ -v -s -m distributed_2_gpus
tests/compile/test_basic_correctness.py
new
+48
@@ -0,0 +1,48 @@
1
+from typing import Dict, List, Optional
2
+
3
+import pytest
4
+
5
+from vllm.compilation.levels import CompilationLevel
6
+from vllm.utils import cuda_device_count_stateless
7
+
8
+from ..utils import compare_all_settings
9
+
10
+
11
+# we cannot afford testing the full Catesian product
12
+# of all models and all levels
13
+@pytest.mark.parametrize(
14
+ "model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15
+ [
16
+ ("meta-llama/Meta-Llama-3-8B", [], 2, 2, "FLASH_ATTN", "generate",
17
+ True),
18
+ ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
19
+ ["--quantization", "compressed-tensors"
20
+ ], 1, 1, "FLASH_ATTN", "generate", True),
21
+ ("google/gemma-2-2b-it", [], 1, 2, "FLASHINFER", "generate", True),
22
+ # TODO: add multi-modality test for llava
23
+ ("llava-hf/llava-1.5-7b-hf", [], 2, 1, "FLASHINFER", "generate", False)
24
+ ])
25
+def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
26
+ method, fullgraph):
27
+ # this test is run under multiple suits, with different GPUs.
28
+ # make sure we only run the test with correct CUDA devices.
29
+ # don't use "<", as it will duplicate the tests.
30
+ if cuda_device_count_stateless() != pp_size * tp_size:
31
+ pytest.skip("Not correct CUDA devices for the test.")
32
+ import os
33
+ os.environ["VLLM_ATTENTION_BACKEND"] = attn_backend
34
+ if not fullgraph:
35
+ os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0"
36
+ all_args = [["--enforce-eager"] + model_args + ["--max_model_len", "1024"]
37
+ + ["-pp", str(pp_size)] + ["-tp", str(tp_size)]] * 3
38
+ # don't test VLLM_TORCH_COMPILE_LEVEL == 3 case
39
+ # inductor will change the output, so we cannot compare them.
40
+ all_envs: List[Optional[Dict[str, str]]] = [{
41
+ "VLLM_TORCH_COMPILE_LEVEL":
42
+ str(level)
43
+ } for level in [
44
+ CompilationLevel.NO_COMPILATION,
45
+ CompilationLevel.DYNAMO_AS_IS,
46
+ CompilationLevel.DYNAMO_ONCE,
47
+ ]]
48
+ compare_all_settings(model, all_args, all_envs, method=method)
tests/compile/test_full_graph.py
+11
-4
@@ -1,13 +1,20 @@
1
import pytest
2
3
-from vllm.compilation.backends import vllm_backend
3
+from vllm.compilation.levels import CompilationLevel
4
5
+from ..utils import fork_new_process_for_each_test
6
from .utils import TEST_MODELS, check_full_graph_support
7
8
9
@pytest.mark.parametrize("model_info", TEST_MODELS)
9
-@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10
-def test_full_graph(model_info, backend):
10
+@pytest.mark.parametrize(
11
+ "optimization_level",
12
+ [CompilationLevel.DYNAMO_ONCE, CompilationLevel.INDUCTOR])
13
+@fork_new_process_for_each_test
14
+def test_full_graph(model_info, optimization_level):
15
model = model_info[0]
16
model_kwargs = model_info[1]
13
- check_full_graph_support(model, model_kwargs, backend, tp_size=1)
17
+ check_full_graph_support(model,
18
+ model_kwargs,
19
+ optimization_level,
20
+ tp_size=1)
tests/compile/test_full_graph_multi_gpu.py
deleted
-22
@@ -1,22 +0,0 @@
1
-import pytest
2
-
3
-from vllm.compilation.backends import vllm_backend
4
-from vllm.utils import cuda_device_count_stateless
5
-
6
-from ..utils import fork_new_process_for_each_test
7
-from .utils import TEST_MODELS_SMOKE, check_full_graph_support
8
-
9
-
10
-@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
11
-@pytest.mark.parametrize("tp_size", [2])
12
-@pytest.mark.parametrize("backend", ["eager", vllm_backend])
13
-@fork_new_process_for_each_test
14
-def test_full_graph_multi_gpu(model_info, tp_size, backend):
15
- model = model_info[0]
16
- model_kwargs = model_info[1]
17
-
18
- # Skip the test if there are not enough CUDA devices.
19
- if cuda_device_count_stateless() < tp_size:
20
- pytest.skip("Not enough CUDA devices for the test.")
21
-
22
- check_full_graph_support(model, model_kwargs, backend, tp_size=tp_size)
tests/compile/test_full_graph_smoke.py
deleted
-13
@@ -1,13 +0,0 @@
1
-import pytest
2
-
3
-from vllm.compilation.backends import vllm_backend
4
-
5
-from .utils import TEST_MODELS_SMOKE, check_full_graph_support
6
-
7
-
8
-@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
9
-@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10
-def test_full_graph(model_info, backend):
11
- model = model_info[0]
12
- model_kwargs = model_info[1]
13
- check_full_graph_support(model, model_kwargs, backend, tp_size=1)
tests/compile/utils.py
+9
-15
@@ -4,16 +4,9 @@ import torch
4
5
from tests.quantization.utils import is_quant_method_supported
6
from vllm import LLM, SamplingParams
7
-from vllm.plugins import set_torch_compile_backend
7
+from vllm.compilation.levels import CompilationLevel
8
from vllm.utils import is_hip
9
10
-TEST_MODELS_SMOKE = [
11
- ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
12
- "quantization": "compressed-tensors"
13
- }),
14
- ("meta-llama/Meta-Llama-3-8B", {}),
15
-]
16
-
10
TEST_MODELS = [
11
("facebook/opt-125m", {}),
12
("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
@@ -68,20 +61,21 @@ if not is_hip() and is_quant_method_supported("awq"):
61
}))
62
63
71
-def check_full_graph_support(model, model_kwargs, backend, tp_size=1):
64
+def check_full_graph_support(model,
65
+ model_kwargs,
66
+ optimization_level,
67
+ tp_size=1):
68
# make sure these models can be captured in full graph mode
73
- if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
74
- os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
75
- os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
69
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
70
+ os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
71
72
# Inductor doesn't support fp8/gptq_marlin_24 yet.
73
quantization = model_kwargs.get("quantization")
74
if (quantization == "fp8" or quantization == "gptq_marlin"
80
- or quantization == "gptq_marlin_24") and backend != "eager":
75
+ or quantization == "gptq_marlin_24"
76
+ ) and optimization_level >= CompilationLevel.INDUCTOR:
77
return
78
83
- set_torch_compile_backend(backend)
84
-
79
prompts = [
80
"Hello, my name is",
81
"The president of the United States is",
tests/tpu/test_compilation.py
+3
-1
@@ -5,9 +5,11 @@ import tempfile
5
6
import depyf
7
8
+from vllm.compilation.levels import CompilationLevel
9
+
10
# disable custom dispatcher, let Dynamo takes over
11
# all the control
10
-os.environ['VLLM_DYNAMO_USE_CUSTOM_DISPATCHER'] = "0"
12
+os.environ['VLLM_TORCH_COMPILE_LEVEL'] = str(CompilationLevel.DYNAMO_AS_IS)
13
14
temp_dir = tempfile.mkdtemp()
15
with depyf.prepare_debug(temp_dir):
tests/tpu/test_custom_dispatcher.py
+8
-5
@@ -1,5 +1,7 @@
1
import os
2
3
+from vllm.compilation.levels import CompilationLevel
4
+
5
from ..utils import compare_two_settings
6
7
# --enforce-eager on TPU causes graph compilation
@@ -9,8 +11,9 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
11
12
13
def test_custom_dispatcher():
12
- compare_two_settings("google/gemma-2b",
13
- arg1=["--enforce-eager"],
14
- arg2=["--enforce-eager"],
15
- env1={"VLLM_DYNAMO_USE_CUSTOM_DISPATCHER": "0"},
16
- env2={})
14
+ compare_two_settings(
15
+ "google/gemma-2b",
16
+ arg1=["--enforce-eager"],
17
+ arg2=["--enforce-eager"],
18
+ env1={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_ONCE)},
19
+ env2={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_AS_IS)})
vllm/compilation/backends.py
+114
-1
@@ -1,8 +1,17 @@
1
+import copy
2
import operator
3
+from typing import Callable, Dict, List, Optional, Tuple, Union
4
5
import torch
6
import torch.fx as fx
7
8
+from vllm.logger import init_logger
9
+
10
+from .compile_context import get_compile_context
11
+from .levels import CompilationLevel
12
+
13
+logger = init_logger(__name__)
14
+
15
16
def fix_functionalization(graph: fx.Graph):
17
"""
@@ -148,9 +157,113 @@ def fix_functionalization(graph: fx.Graph):
157
# print(graph.python_code(root_module="self", verbose=True).src, file=f)
158
159
151
-def vllm_backend(graph, example_inputs):
160
+def wrap_inductor(graph, example_inputs, additional_inductor_config):
161
from torch._inductor import config
162
current_config = config.shallow_copy_dict()
163
from torch._inductor.compile_fx import compile_fx
164
+
165
+ if additional_inductor_config is not None:
166
+ current_config.update(additional_inductor_config)
167
+ if current_config['post_grad_custom_post_pass'] is not None:
168
+ logger.warning(
169
+ "post_grad_custom_post_pass is already set in the config. "
170
+ "Overwriting it with the fix_functionalization")
171
current_config['post_grad_custom_post_pass'] = fix_functionalization
172
return compile_fx(graph, example_inputs, config_patches=current_config)
173
+
174
+
175
+def vllm_backend(
176
+ graph,
177
+ example_inputs,
178
+ additional_inductor_config: Optional[Dict] = None) -> Callable:
179
+
180
+ context = get_compile_context()
181
+ context = copy.deepcopy(context) if context is not None else []
182
+ sizes_to_specialize: List[int] = context
183
+
184
+ # flags for all the seen shapes, whether we need to specialize
185
+ runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
186
+
187
+ # if we need to specialize, the compiled graph for that shape
188
+ runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
189
+
190
+ # this is the first compilation, we will compile a graph with
191
+ # dynamic shape, as the caller will mark first dimension as dynamic
192
+ logger.info("Compiling a graph for general shapes")
193
+ graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
194
+ additional_inductor_config)
195
+
196
+ # TODO: Dynamo does not pass all dynamic shapes.
197
+ # Need to investigate why. It works now because all the dynamic
198
+ # shapes have the same value, and either of them can be used.
199
+ sym_shape_indices = [
200
+ i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
201
+ ]
202
+
203
+ first_run = True
204
+
205
+ # this is the function we return to Dynamo to run finally
206
+ def compiled_graph_wrapper(*args):
207
+
208
+ runtime_shapes: Tuple[int,
209
+ ...] = tuple(args[i] for i in sym_shape_indices)
210
+
211
+ nonlocal first_run
212
+ nonlocal runtime_shapes_to_compile_flags
213
+ nonlocal runtime_shapes_to_compiled_graph
214
+
215
+ if first_run:
216
+ # the first compilation is for profiling, we directly run it
217
+ first_run = False
218
+ return graph_for_symbolic_shape(*args)
219
+
220
+ if runtime_shapes not in runtime_shapes_to_compile_flags:
221
+ # we haven't seen this shape before
222
+ # query if we need to specialize for this shape
223
+ # we only specialize for the first dimension.
224
+ # TODO: investigate if any model needs to specialize
225
+ # beyond the first dimension
226
+ runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
227
+ 0] in sizes_to_specialize
228
+
229
+ if not runtime_shapes_to_compile_flags[runtime_shapes]:
230
+ # we don't need to specialize for this shape
231
+ return graph_for_symbolic_shape(*args)
232
+
233
+ if runtime_shapes not in runtime_shapes_to_compiled_graph:
234
+ # we need to specialize for this shape, and we haven't compiled
235
+ # compile the graph for this shape
236
+ logger.info("Compiling a graph for shapes %s", runtime_shapes)
237
+ runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
238
+ graph, args, additional_inductor_config)
239
+
240
+ return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
241
+
242
+ return compiled_graph_wrapper
243
+
244
+
245
+def select_default_backend(level: int) -> Union[str, Callable]:
246
+ if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
247
+ backend = "eager"
248
+ return backend
249
+ assert level in [
250
+ CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
251
+ ], f"Invalid level {level}"
252
+
253
+ from vllm.compilation.backends import vllm_backend
254
+ from vllm.plugins import get_inductor_additional_configs
255
+ additional_configs = get_inductor_additional_configs()
256
+
257
+ if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
258
+ if "max_autotune" in additional_configs and not additional_configs[
259
+ "max_autotune"]:
260
+ logger.warning(
261
+ "max_autotune is disabled, but is overridden by level %s",
262
+ CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
263
+ additional_configs['max_autotune'] = True
264
+
265
+ from functools import partial
266
+ backend = partial(vllm_backend,
267
+ additional_inductor_config=additional_configs)
268
+
269
+ return backend
vllm/compilation/compile_context.py
new
+23
@@ -0,0 +1,23 @@
1
+from contextlib import contextmanager
2
+from typing import Any
3
+
4
+_compile_context: Any = None
5
+
6
+
7
+def get_compile_context() -> Any:
8
+ """Get the current compile context."""
9
+ return _compile_context
10
+
11
+
12
+@contextmanager
13
+def set_compile_context(context: Any):
14
+ """A context manager that stores the current compile context,
15
+ usually it is a list of sizes to specialize.
16
+ """
17
+ global _compile_context
18
+ prev_context = _compile_context
19
+ _compile_context = context
20
+ try:
21
+ yield
22
+ finally:
23
+ _compile_context = prev_context
vllm/compilation/decorators.py
new
+85
@@ -0,0 +1,85 @@
1
+from typing import List, Optional, Union
2
+
3
+import torch
4
+
5
+import vllm.envs as envs
6
+from vllm.attention import AttentionMetadata
7
+from vllm.compilation.levels import CompilationLevel
8
+from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9
+from vllm.sequence import IntermediateTensors
10
+from vllm.utils import supports_dynamo
11
+
12
+
13
+def support_compile_llama_style(cls: type):
14
+ """
15
+ A decorator to add support for compiling the forward method of a class.
16
+ If a module's **forward signature** is compatible with llama, this
17
+ decorator can be used to enable the compilation of the forward method.
18
+ """
19
+
20
+ # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
21
+ # will handle the compilation, so we don't need to do anything here.
22
+ if envs.VLLM_TORCH_COMPILE_LEVEL in [
23
+ CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
24
+ ] or not supports_dynamo():
25
+ return cls
26
+
27
+ # take care of method resolution order
28
+ # make sure super().__init__ is called on the base class
29
+ # other than TorchCompileWrapperWithCustomDispatcher
30
+ cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
31
+
32
+ old_init = cls.__init__
33
+
34
+ def __init__(self, *args, **kwargs):
35
+ old_init(self, *args, **kwargs)
36
+ TorchCompileWrapperWithCustomDispatcher.__init__(self)
37
+
38
+ cls.__init__ = __init__
39
+
40
+ def __call__(
41
+ self,
42
+ input_ids: Optional[torch.Tensor],
43
+ positions: torch.Tensor,
44
+ kv_caches: List[torch.Tensor],
45
+ attn_metadata: AttentionMetadata,
46
+ intermediate_tensors: Optional[IntermediateTensors],
47
+ inputs_embeds: Optional[torch.Tensor] = None,
48
+ ) -> Union[torch.Tensor, IntermediateTensors]:
49
+ # torch.compiler.is_compiling() means we are inside the compilation
50
+ # e.g. TPU has the compilation logic in model runner, so we don't
51
+ # need to compile the model inside.
52
+ if torch.compiler.is_compiling():
53
+ return self.forward(input_ids, positions, kv_caches, attn_metadata,
54
+ intermediate_tensors, inputs_embeds)
55
+
56
+ # the first compilation needs to have dynamic shapes marked
57
+ if len(self.compiled_codes) < 1:
58
+ if input_ids is not None:
59
+ torch._dynamo.mark_dynamic(input_ids, 0)
60
+ torch._dynamo.mark_dynamic(positions, 0)
61
+ if inputs_embeds is not None:
62
+ torch._dynamo.mark_dynamic(inputs_embeds, 0)
63
+ if intermediate_tensors is not None:
64
+ for tensors in intermediate_tensors.tensors.values():
65
+ torch._dynamo.mark_dynamic(tensors, 0)
66
+
67
+ # if we don't use custom dispatcher, we can directly call the
68
+ # compiled function and let torch.compile handle the dispatching,
69
+ # with the overhead of guard evaluation and recompilation.
70
+ if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
71
+ return self.compiled_callable(input_ids, positions, kv_caches,
72
+ attn_metadata, intermediate_tensors,
73
+ inputs_embeds)
74
+
75
+ # usually, capturing the model once is enough, and then we can
76
+ # dispatch to the compiled code directly, without going through
77
+ # the Dynamo guard mechanism.
78
+ with self.dispatch_to_code(0):
79
+ model_output = self.forward(input_ids, positions, kv_caches,
80
+ attn_metadata, intermediate_tensors,
81
+ inputs_embeds)
82
+ return model_output
83
+
84
+ cls.__call__ = __call__
85
+ return cls
vllm/compilation/levels.py
new
+9
@@ -0,0 +1,9 @@
1
+# constants for the levels of the compilation process
2
+
3
+
4
+class CompilationLevel:
5
+ NO_COMPILATION = 0
6
+ DYNAMO_AS_IS = 1
7
+ DYNAMO_ONCE = 2
8
+ INDUCTOR = 3
9
+ INDUCTOR_MAX_AUTOTUNE = 4
vllm/compilation/wrapper.py
+24
-3
@@ -3,12 +3,14 @@ import sys
3
from abc import abstractmethod
4
from contextlib import contextmanager
5
from types import CodeType
6
-from typing import Callable, List
6
+from typing import Callable, List, Optional
7
8
import torch
9
10
import vllm.envs as envs
11
12
+from .levels import CompilationLevel
13
+
14
15
class TorchCompileWrapperWithCustomDispatcher:
16
"""
@@ -23,7 +25,26 @@ class TorchCompileWrapperWithCustomDispatcher:
25
`torch.compile` over the forward method.
26
"""
27
26
- def __init__(self, compiled_callable: Callable):
28
+ def __init__(self, compiled_callable: Optional[Callable] = None):
29
+
30
+ if compiled_callable is None:
31
+ # default compilation settings
32
+ # compiling the forward method
33
+
34
+ # choose the compile backend
35
+
36
+ # if the user has set the backend, use it
37
+ from vllm.plugins import get_torch_compile_backend
38
+ backend = get_torch_compile_backend()
39
+ if backend is None:
40
+ from vllm.compilation.backends import select_default_backend
41
+ backend = select_default_backend(envs.VLLM_TORCH_COMPILE_LEVEL)
42
+
43
+ compiled_callable = torch.compile(
44
+ self.forward,
45
+ fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
46
+ backend=backend)
47
+
48
self.compiled_callable = compiled_callable
49
self.original_code_object = self.__class__.forward.__code__
50
self.compiled_codes: List[CodeType] = []
@@ -33,7 +54,7 @@ class TorchCompileWrapperWithCustomDispatcher:
54
# subclasses can use this to switch between the custom dispatcher
55
# and the default Dynamo guard mechanism.
56
self.use_custom_dispatcher: bool = \
36
- envs.VLLM_DYNAMO_USE_CUSTOM_DISPATCHER
57
+ envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.DYNAMO_ONCE
58
59
def __call__(self, *args, **kwargs):
60
"""Implement the dispatch logic here, beyond the torch.compile level.
vllm/envs.py
+3
-13
@@ -65,6 +65,7 @@ if TYPE_CHECKING:
65
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66
VLLM_SKIP_P2P_CHECK: bool = False
67
VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
68
+ VLLM_TORCH_COMPILE_LEVEL: int = 0
69
70
71
def get_default_cache_root():
@@ -198,23 +199,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
199
lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
200
("true", "1")),
201
201
- # Internal flag to enable Dynamo graph capture
202
- "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
203
- lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
204
- "VLLM_DYNAMO_USE_CUSTOM_DISPATCHER":
205
- lambda:
206
- (os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
207
- ("true", "1")),
208
-
209
- # Internal flag to control whether we use custom op,
210
- # or use the native pytorch implementation
211
- "VLLM_TEST_COMPILE_NO_CUSTOM_OPS":
212
- lambda: int(os.environ.get("VLLM_TEST_COMPILE_NO_CUSTOM_OPS", "0")),
213
-
202
# Internal flag to enable Dynamo fullgraph capture
203
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
204
lambda: bool(
205
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
206
+ "VLLM_TORCH_COMPILE_LEVEL":
207
+ lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
208
209
# local rank of the process in the distributed setting, used to determine
210
# the GPU device id
vllm/model_executor/custom_op.py
+2
-1
@@ -1,6 +1,7 @@
1
import torch.nn as nn
2
3
import vllm.envs as envs
4
+from vllm.compilation.levels import CompilationLevel
5
from vllm.platforms import current_platform
6
from vllm.utils import is_cpu, is_hip, is_xpu
7
@@ -55,7 +56,7 @@ class CustomOp(nn.Module):
56
# NOTE(woosuk): Here we assume that vLLM was built for only one
57
# specific backend. Currently, we do not support dynamic dispatching.
58
58
- if envs.VLLM_TEST_COMPILE_NO_CUSTOM_OPS:
59
+ if envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.INDUCTOR:
60
return self.forward_native
61
62
if is_hip():
vllm/model_executor/models/gemma2.py
+2
@@ -21,6 +21,7 @@ from torch import nn
21
from transformers import Gemma2Config
22
23
from vllm.attention import Attention, AttentionMetadata
24
+from vllm.compilation.decorators import support_compile_llama_style
25
from vllm.config import CacheConfig, LoRAConfig
26
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
@@ -238,6 +239,7 @@ class Gemma2DecoderLayer(nn.Module):
239
return hidden_states, residual
240
241
242
+@support_compile_llama_style
243
class Gemma2Model(nn.Module):
244
245
def __init__(
vllm/model_executor/models/llama.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import LlamaConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_compile_llama_style
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
@@ -265,6 +266,7 @@ class LlamaDecoderLayer(nn.Module):
266
return hidden_states, residual
267
268
269
+@support_compile_llama_style
270
class LlamaModel(nn.Module):
271
272
def __init__(
vllm/model_executor/models/llava.py
+5
-3
@@ -365,6 +365,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
365
input_ids = None
366
inputs_embeds = None
367
else:
368
+ # always pass the input via `inputs_embeds`
369
+ # to make sure the computation graph is consistent
370
image_input = self._parse_and_validate_image_input(**kwargs)
371
372
if image_input is not None:
@@ -375,10 +377,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
377
inputs_embeds = merge_multimodal_embeddings(
378
input_ids, inputs_embeds, vision_embeddings,
379
self.config.image_token_index)
378
-
379
- input_ids = None
380
else:
381
- inputs_embeds = None
381
+ inputs_embeds = self.language_model.model.get_input_embeddings(
382
+ input_ids)
383
+ input_ids = None
384
385
hidden_states = self.language_model.model(input_ids,
386
positions,
vllm/platforms/tpu.py
+14
@@ -1,7 +1,21 @@
1
+import os
2
+
3
import torch
4
5
+import vllm.envs as envs
6
+from vllm.compilation.levels import CompilationLevel
7
+from vllm.plugins import set_torch_compile_backend
8
+
9
from .interface import Platform, PlatformEnum
10
11
+if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
12
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE)
13
+
14
+assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR,\
15
+ "TPU does not support Inductor."
16
+
17
+set_torch_compile_backend("openxla")
18
+
19
20
class TpuPlatform(Platform):
21
_enum = PlatformEnum.TPU
vllm/plugins/__init__.py
+13
-1
@@ -1,5 +1,5 @@
1
import logging
2
-from typing import Callable, Optional, Union
2
+from typing import Callable, Dict, Optional, Union
3
4
import vllm.envs as envs
5
@@ -42,3 +42,15 @@ def set_torch_compile_backend(backend: Union[Callable, str]):
42
43
def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
44
return _torch_compile_backend
45
+
46
+
47
+_inductor_additional_configs: Dict = {}
48
+
49
+
50
+def set_inductor_additional_configs(configs: Dict):
51
+ global _inductor_additional_configs
52
+ _inductor_additional_configs = configs
53
+
54
+
55
+def get_inductor_additional_configs() -> Dict:
56
+ return _inductor_additional_configs
vllm/sequence.py
+3
-4
@@ -1137,10 +1137,9 @@ class EmbeddingSequenceGroupOutput(
1137
return self.embeddings == other.embeddings
1138
1139
1140
-class IntermediateTensors(
1141
- msgspec.Struct,
1142
- omit_defaults=True, # type: ignore[call-arg]
1143
- array_like=True): # type: ignore[call-arg]
1140
+# cannot use msgspec.Struct here because Dynamo does not support it
1141
+@dataclass
1142
+class IntermediateTensors:
1143
"""For all pipeline stages except the last, we need to return the hidden
1144
states and residuals to be sent to the next stage. This data structure
1145
contains the hidden states and residuals for a request.
vllm/worker/model_runner.py
+14
-4
@@ -18,6 +18,8 @@ import vllm.envs as envs
18
from vllm.attention import AttentionMetadata, get_attn_backend
19
from vllm.attention.backends.abstract import AttentionState
20
from vllm.attention.backends.utils import CommonAttentionState
21
+from vllm.compilation.compile_context import set_compile_context
22
+from vllm.compilation.levels import CompilationLevel
23
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
24
ModelConfig, ObservabilityConfig, ParallelConfig,
25
PromptAdapterConfig, SchedulerConfig)
@@ -1126,10 +1128,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1128
"provided. Defaulting to scaling factors of 1.0. "
1129
"This may lead to less accurate results!")
1130
1129
- if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1130
- from vllm.compilation.backends import vllm_backend
1131
+ if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS \
1132
+ and supports_dynamo():
1133
from vllm.plugins import get_torch_compile_backend
1132
- backend = get_torch_compile_backend() or vllm_backend
1134
+ backend = get_torch_compile_backend() or "eager"
1135
self.model = torch.compile(
1136
self.model,
1137
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
@@ -1289,7 +1291,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1291
batch_size=batch_size,
1292
dtype=self.model_config.dtype,
1293
device=self.device)
1292
- self.execute_model(model_input, kv_caches, intermediate_tensors)
1294
+
1295
+ graph_batch_size = self.max_batchsize_to_capture
1296
+ batch_size_capture_list = [
1297
+ bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1298
+ ]
1299
+ if self.model_config.enforce_eager:
1300
+ batch_size_capture_list = []
1301
+ with set_compile_context(batch_size_capture_list):
1302
+ self.execute_model(model_input, kv_caches, intermediate_tensors)
1303
torch.cuda.synchronize()
1304
return
1305