@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e4d652ea3e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[torch.compile] integration with compilation control (#9058)

youkaichao committed Oct 10, 2024 at 12:39 UTC e4d652ea3ed9b2a60c1582cb2e2605695e61280f
22 files changed +404 -98
.buildkite/test-pipeline.yaml
+12 -8
@@ -121,7 +121,9 @@ steps:
121 - vllm/core/
122 - tests/distributed
123 - tests/spec_decode/e2e/test_integration_dist_tp4
124 + - tests/compile
125 commands:
126 + - pytest -v -s compile/test_basic_correctness.py
127 - pytest -v -s distributed/test_pynccl.py
128 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
129
@@ -231,14 +233,16 @@ steps:
233 - vllm/
234 - tests/compile
235 commands:
234 - - pytest -v -s compile/test_full_graph_smoke.py
236 + - pytest -v -s compile/test_basic_correctness.py
237
236 -- label: "PyTorch Fullgraph Test" # 18min
237 - source_file_dependencies:
238 - - vllm/
239 - - tests/compile
240 - commands:
241 - - pytest -v -s compile/test_full_graph.py
238 +# TODO: re-write in comparison tests, and fix symbolic shape
239 +# for quantization ops.
240 +# - label: "PyTorch Fullgraph Test" # 18min
241 +# source_file_dependencies:
242 +# - vllm/
243 +# - tests/compile
244 +# commands:
245 +# - pytest -v -s compile/test_full_graph.py
246
247 - label: Kernels Test %N # 1h each
248 mirror_hardwares: [amd]
@@ -394,7 +398,7 @@ steps:
398 - tests/distributed/
399 - vllm/compilation
400 commands:
397 - - pytest -v -s ./compile/test_full_graph_multi_gpu.py
401 + - pytest -v -s ./compile/test_basic_correctness.py
402 - pytest -v -s ./compile/test_wrapper.py
403 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
404 - TARGET_TEST_SUITE=L4 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1=1 pytest basic_correctness/ -v -s -m distributed_2_gpus
tests/compile/test_basic_correctness.py new
+48
@@ -0,0 +1,48 @@
1 +from typing import Dict, List, Optional
2 +
3 +import pytest
4 +
5 +from vllm.compilation.levels import CompilationLevel
6 +from vllm.utils import cuda_device_count_stateless
7 +
8 +from ..utils import compare_all_settings
9 +
10 +
11 +# we cannot afford testing the full Catesian product
12 +# of all models and all levels
13 +@pytest.mark.parametrize(
14 + "model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15 + [
16 + ("meta-llama/Meta-Llama-3-8B", [], 2, 2, "FLASH_ATTN", "generate",
17 + True),
18 + ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
19 + ["--quantization", "compressed-tensors"
20 + ], 1, 1, "FLASH_ATTN", "generate", True),
21 + ("google/gemma-2-2b-it", [], 1, 2, "FLASHINFER", "generate", True),
22 + # TODO: add multi-modality test for llava
23 + ("llava-hf/llava-1.5-7b-hf", [], 2, 1, "FLASHINFER", "generate", False)
24 + ])
25 +def test_compile_correctness(model, model_args, pp_size, tp_size, attn_backend,
26 + method, fullgraph):
27 + # this test is run under multiple suits, with different GPUs.
28 + # make sure we only run the test with correct CUDA devices.
29 + # don't use "<", as it will duplicate the tests.
30 + if cuda_device_count_stateless() != pp_size * tp_size:
31 + pytest.skip("Not correct CUDA devices for the test.")
32 + import os
33 + os.environ["VLLM_ATTENTION_BACKEND"] = attn_backend
34 + if not fullgraph:
35 + os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "0"
36 + all_args = [["--enforce-eager"] + model_args + ["--max_model_len", "1024"]
37 + + ["-pp", str(pp_size)] + ["-tp", str(tp_size)]] * 3
38 + # don't test VLLM_TORCH_COMPILE_LEVEL == 3 case
39 + # inductor will change the output, so we cannot compare them.
40 + all_envs: List[Optional[Dict[str, str]]] = [{
41 + "VLLM_TORCH_COMPILE_LEVEL":
42 + str(level)
43 + } for level in [
44 + CompilationLevel.NO_COMPILATION,
45 + CompilationLevel.DYNAMO_AS_IS,
46 + CompilationLevel.DYNAMO_ONCE,
47 + ]]
48 + compare_all_settings(model, all_args, all_envs, method=method)
tests/compile/test_full_graph.py
+11 -4
@@ -1,13 +1,20 @@
1 import pytest
2
3 -from vllm.compilation.backends import vllm_backend
3 +from vllm.compilation.levels import CompilationLevel
4
5 +from ..utils import fork_new_process_for_each_test
6 from .utils import TEST_MODELS, check_full_graph_support
7
8
9 @pytest.mark.parametrize("model_info", TEST_MODELS)
9 -@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10 -def test_full_graph(model_info, backend):
10 +@pytest.mark.parametrize(
11 + "optimization_level",
12 + [CompilationLevel.DYNAMO_ONCE, CompilationLevel.INDUCTOR])
13 +@fork_new_process_for_each_test
14 +def test_full_graph(model_info, optimization_level):
15 model = model_info[0]
16 model_kwargs = model_info[1]
13 - check_full_graph_support(model, model_kwargs, backend, tp_size=1)
17 + check_full_graph_support(model,
18 + model_kwargs,
19 + optimization_level,
20 + tp_size=1)
tests/compile/test_full_graph_multi_gpu.py deleted
-22
@@ -1,22 +0,0 @@
1 -import pytest
2 -
3 -from vllm.compilation.backends import vllm_backend
4 -from vllm.utils import cuda_device_count_stateless
5 -
6 -from ..utils import fork_new_process_for_each_test
7 -from .utils import TEST_MODELS_SMOKE, check_full_graph_support
8 -
9 -
10 -@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
11 -@pytest.mark.parametrize("tp_size", [2])
12 -@pytest.mark.parametrize("backend", ["eager", vllm_backend])
13 -@fork_new_process_for_each_test
14 -def test_full_graph_multi_gpu(model_info, tp_size, backend):
15 - model = model_info[0]
16 - model_kwargs = model_info[1]
17 -
18 - # Skip the test if there are not enough CUDA devices.
19 - if cuda_device_count_stateless() < tp_size:
20 - pytest.skip("Not enough CUDA devices for the test.")
21 -
22 - check_full_graph_support(model, model_kwargs, backend, tp_size=tp_size)
tests/compile/test_full_graph_smoke.py deleted
-13
@@ -1,13 +0,0 @@
1 -import pytest
2 -
3 -from vllm.compilation.backends import vllm_backend
4 -
5 -from .utils import TEST_MODELS_SMOKE, check_full_graph_support
6 -
7 -
8 -@pytest.mark.parametrize("model_info", TEST_MODELS_SMOKE)
9 -@pytest.mark.parametrize("backend", ["eager", vllm_backend])
10 -def test_full_graph(model_info, backend):
11 - model = model_info[0]
12 - model_kwargs = model_info[1]
13 - check_full_graph_support(model, model_kwargs, backend, tp_size=1)
tests/compile/utils.py
+9 -15
@@ -4,16 +4,9 @@ import torch
4
5 from tests.quantization.utils import is_quant_method_supported
6 from vllm import LLM, SamplingParams
7 -from vllm.plugins import set_torch_compile_backend
7 +from vllm.compilation.levels import CompilationLevel
8 from vllm.utils import is_hip
9
10 -TEST_MODELS_SMOKE = [
11 - ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
12 - "quantization": "compressed-tensors"
13 - }),
14 - ("meta-llama/Meta-Llama-3-8B", {}),
15 -]
16 -
10 TEST_MODELS = [
11 ("facebook/opt-125m", {}),
12 ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
@@ -68,20 +61,21 @@ if not is_hip() and is_quant_method_supported("awq"):
61 }))
62
63
71 -def check_full_graph_support(model, model_kwargs, backend, tp_size=1):
64 +def check_full_graph_support(model,
65 + model_kwargs,
66 + optimization_level,
67 + tp_size=1):
68 # make sure these models can be captured in full graph mode
73 - if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
74 - os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
75 - os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
69 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
70 + os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
71
72 # Inductor doesn't support fp8/gptq_marlin_24 yet.
73 quantization = model_kwargs.get("quantization")
74 if (quantization == "fp8" or quantization == "gptq_marlin"
80 - or quantization == "gptq_marlin_24") and backend != "eager":
75 + or quantization == "gptq_marlin_24"
76 + ) and optimization_level >= CompilationLevel.INDUCTOR:
77 return
78
83 - set_torch_compile_backend(backend)
84 -
79 prompts = [
80 "Hello, my name is",
81 "The president of the United States is",
tests/tpu/test_compilation.py
+3 -1
@@ -5,9 +5,11 @@ import tempfile
5
6 import depyf
7
8 +from vllm.compilation.levels import CompilationLevel
9 +
10 # disable custom dispatcher, let Dynamo takes over
11 # all the control
10 -os.environ['VLLM_DYNAMO_USE_CUSTOM_DISPATCHER'] = "0"
12 +os.environ['VLLM_TORCH_COMPILE_LEVEL'] = str(CompilationLevel.DYNAMO_AS_IS)
13
14 temp_dir = tempfile.mkdtemp()
15 with depyf.prepare_debug(temp_dir):
tests/tpu/test_custom_dispatcher.py
+8 -5
@@ -1,5 +1,7 @@
1 import os
2
3 +from vllm.compilation.levels import CompilationLevel
4 +
5 from ..utils import compare_two_settings
6
7 # --enforce-eager on TPU causes graph compilation
@@ -9,8 +11,9 @@ os.environ["VLLM_RPC_TIMEOUT"] = "30000"
11
12
13 def test_custom_dispatcher():
12 - compare_two_settings("google/gemma-2b",
13 - arg1=["--enforce-eager"],
14 - arg2=["--enforce-eager"],
15 - env1={"VLLM_DYNAMO_USE_CUSTOM_DISPATCHER": "0"},
16 - env2={})
14 + compare_two_settings(
15 + "google/gemma-2b",
16 + arg1=["--enforce-eager"],
17 + arg2=["--enforce-eager"],
18 + env1={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_ONCE)},
19 + env2={"VLLM_TORCH_COMPILE_LEVEL": str(CompilationLevel.DYNAMO_AS_IS)})
vllm/compilation/backends.py
+114 -1
@@ -1,8 +1,17 @@
1 +import copy
2 import operator
3 +from typing import Callable, Dict, List, Optional, Tuple, Union
4
5 import torch
6 import torch.fx as fx
7
8 +from vllm.logger import init_logger
9 +
10 +from .compile_context import get_compile_context
11 +from .levels import CompilationLevel
12 +
13 +logger = init_logger(__name__)
14 +
15
16 def fix_functionalization(graph: fx.Graph):
17 """
@@ -148,9 +157,113 @@ def fix_functionalization(graph: fx.Graph):
157 # print(graph.python_code(root_module="self", verbose=True).src, file=f)
158
159
151 -def vllm_backend(graph, example_inputs):
160 +def wrap_inductor(graph, example_inputs, additional_inductor_config):
161 from torch._inductor import config
162 current_config = config.shallow_copy_dict()
163 from torch._inductor.compile_fx import compile_fx
164 +
165 + if additional_inductor_config is not None:
166 + current_config.update(additional_inductor_config)
167 + if current_config['post_grad_custom_post_pass'] is not None:
168 + logger.warning(
169 + "post_grad_custom_post_pass is already set in the config. "
170 + "Overwriting it with the fix_functionalization")
171 current_config['post_grad_custom_post_pass'] = fix_functionalization
172 return compile_fx(graph, example_inputs, config_patches=current_config)
173 +
174 +
175 +def vllm_backend(
176 + graph,
177 + example_inputs,
178 + additional_inductor_config: Optional[Dict] = None) -> Callable:
179 +
180 + context = get_compile_context()
181 + context = copy.deepcopy(context) if context is not None else []
182 + sizes_to_specialize: List[int] = context
183 +
184 + # flags for all the seen shapes, whether we need to specialize
185 + runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
186 +
187 + # if we need to specialize, the compiled graph for that shape
188 + runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
189 +
190 + # this is the first compilation, we will compile a graph with
191 + # dynamic shape, as the caller will mark first dimension as dynamic
192 + logger.info("Compiling a graph for general shapes")
193 + graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
194 + additional_inductor_config)
195 +
196 + # TODO: Dynamo does not pass all dynamic shapes.
197 + # Need to investigate why. It works now because all the dynamic
198 + # shapes have the same value, and either of them can be used.
199 + sym_shape_indices = [
200 + i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
201 + ]
202 +
203 + first_run = True
204 +
205 + # this is the function we return to Dynamo to run finally
206 + def compiled_graph_wrapper(*args):
207 +
208 + runtime_shapes: Tuple[int,
209 + ...] = tuple(args[i] for i in sym_shape_indices)
210 +
211 + nonlocal first_run
212 + nonlocal runtime_shapes_to_compile_flags
213 + nonlocal runtime_shapes_to_compiled_graph
214 +
215 + if first_run:
216 + # the first compilation is for profiling, we directly run it
217 + first_run = False
218 + return graph_for_symbolic_shape(*args)
219 +
220 + if runtime_shapes not in runtime_shapes_to_compile_flags:
221 + # we haven't seen this shape before
222 + # query if we need to specialize for this shape
223 + # we only specialize for the first dimension.
224 + # TODO: investigate if any model needs to specialize
225 + # beyond the first dimension
226 + runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
227 + 0] in sizes_to_specialize
228 +
229 + if not runtime_shapes_to_compile_flags[runtime_shapes]:
230 + # we don't need to specialize for this shape
231 + return graph_for_symbolic_shape(*args)
232 +
233 + if runtime_shapes not in runtime_shapes_to_compiled_graph:
234 + # we need to specialize for this shape, and we haven't compiled
235 + # compile the graph for this shape
236 + logger.info("Compiling a graph for shapes %s", runtime_shapes)
237 + runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
238 + graph, args, additional_inductor_config)
239 +
240 + return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
241 +
242 + return compiled_graph_wrapper
243 +
244 +
245 +def select_default_backend(level: int) -> Union[str, Callable]:
246 + if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
247 + backend = "eager"
248 + return backend
249 + assert level in [
250 + CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
251 + ], f"Invalid level {level}"
252 +
253 + from vllm.compilation.backends import vllm_backend
254 + from vllm.plugins import get_inductor_additional_configs
255 + additional_configs = get_inductor_additional_configs()
256 +
257 + if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
258 + if "max_autotune" in additional_configs and not additional_configs[
259 + "max_autotune"]:
260 + logger.warning(
261 + "max_autotune is disabled, but is overridden by level %s",
262 + CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
263 + additional_configs['max_autotune'] = True
264 +
265 + from functools import partial
266 + backend = partial(vllm_backend,
267 + additional_inductor_config=additional_configs)
268 +
269 + return backend
vllm/compilation/compile_context.py new
+23
@@ -0,0 +1,23 @@
1 +from contextlib import contextmanager
2 +from typing import Any
3 +
4 +_compile_context: Any = None
5 +
6 +
7 +def get_compile_context() -> Any:
8 + """Get the current compile context."""
9 + return _compile_context
10 +
11 +
12 +@contextmanager
13 +def set_compile_context(context: Any):
14 + """A context manager that stores the current compile context,
15 + usually it is a list of sizes to specialize.
16 + """
17 + global _compile_context
18 + prev_context = _compile_context
19 + _compile_context = context
20 + try:
21 + yield
22 + finally:
23 + _compile_context = prev_context
vllm/compilation/decorators.py new
+85
@@ -0,0 +1,85 @@
1 +from typing import List, Optional, Union
2 +
3 +import torch
4 +
5 +import vllm.envs as envs
6 +from vllm.attention import AttentionMetadata
7 +from vllm.compilation.levels import CompilationLevel
8 +from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9 +from vllm.sequence import IntermediateTensors
10 +from vllm.utils import supports_dynamo
11 +
12 +
13 +def support_compile_llama_style(cls: type):
14 + """
15 + A decorator to add support for compiling the forward method of a class.
16 + If a module's **forward signature** is compatible with llama, this
17 + decorator can be used to enable the compilation of the forward method.
18 + """
19 +
20 + # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
21 + # will handle the compilation, so we don't need to do anything here.
22 + if envs.VLLM_TORCH_COMPILE_LEVEL in [
23 + CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
24 + ] or not supports_dynamo():
25 + return cls
26 +
27 + # take care of method resolution order
28 + # make sure super().__init__ is called on the base class
29 + # other than TorchCompileWrapperWithCustomDispatcher
30 + cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
31 +
32 + old_init = cls.__init__
33 +
34 + def __init__(self, *args, **kwargs):
35 + old_init(self, *args, **kwargs)
36 + TorchCompileWrapperWithCustomDispatcher.__init__(self)
37 +
38 + cls.__init__ = __init__
39 +
40 + def __call__(
41 + self,
42 + input_ids: Optional[torch.Tensor],
43 + positions: torch.Tensor,
44 + kv_caches: List[torch.Tensor],
45 + attn_metadata: AttentionMetadata,
46 + intermediate_tensors: Optional[IntermediateTensors],
47 + inputs_embeds: Optional[torch.Tensor] = None,
48 + ) -> Union[torch.Tensor, IntermediateTensors]:
49 + # torch.compiler.is_compiling() means we are inside the compilation
50 + # e.g. TPU has the compilation logic in model runner, so we don't
51 + # need to compile the model inside.
52 + if torch.compiler.is_compiling():
53 + return self.forward(input_ids, positions, kv_caches, attn_metadata,
54 + intermediate_tensors, inputs_embeds)
55 +
56 + # the first compilation needs to have dynamic shapes marked
57 + if len(self.compiled_codes) < 1:
58 + if input_ids is not None:
59 + torch._dynamo.mark_dynamic(input_ids, 0)
60 + torch._dynamo.mark_dynamic(positions, 0)
61 + if inputs_embeds is not None:
62 + torch._dynamo.mark_dynamic(inputs_embeds, 0)
63 + if intermediate_tensors is not None:
64 + for tensors in intermediate_tensors.tensors.values():
65 + torch._dynamo.mark_dynamic(tensors, 0)
66 +
67 + # if we don't use custom dispatcher, we can directly call the
68 + # compiled function and let torch.compile handle the dispatching,
69 + # with the overhead of guard evaluation and recompilation.
70 + if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
71 + return self.compiled_callable(input_ids, positions, kv_caches,
72 + attn_metadata, intermediate_tensors,
73 + inputs_embeds)
74 +
75 + # usually, capturing the model once is enough, and then we can
76 + # dispatch to the compiled code directly, without going through
77 + # the Dynamo guard mechanism.
78 + with self.dispatch_to_code(0):
79 + model_output = self.forward(input_ids, positions, kv_caches,
80 + attn_metadata, intermediate_tensors,
81 + inputs_embeds)
82 + return model_output
83 +
84 + cls.__call__ = __call__
85 + return cls
vllm/compilation/levels.py new
+9
@@ -0,0 +1,9 @@
1 +# constants for the levels of the compilation process
2 +
3 +
4 +class CompilationLevel:
5 + NO_COMPILATION = 0
6 + DYNAMO_AS_IS = 1
7 + DYNAMO_ONCE = 2
8 + INDUCTOR = 3
9 + INDUCTOR_MAX_AUTOTUNE = 4
vllm/compilation/wrapper.py
+24 -3
@@ -3,12 +3,14 @@ import sys
3 from abc import abstractmethod
4 from contextlib import contextmanager
5 from types import CodeType
6 -from typing import Callable, List
6 +from typing import Callable, List, Optional
7
8 import torch
9
10 import vllm.envs as envs
11
12 +from .levels import CompilationLevel
13 +
14
15 class TorchCompileWrapperWithCustomDispatcher:
16 """
@@ -23,7 +25,26 @@ class TorchCompileWrapperWithCustomDispatcher:
25 `torch.compile` over the forward method.
26 """
27
26 - def __init__(self, compiled_callable: Callable):
28 + def __init__(self, compiled_callable: Optional[Callable] = None):
29 +
30 + if compiled_callable is None:
31 + # default compilation settings
32 + # compiling the forward method
33 +
34 + # choose the compile backend
35 +
36 + # if the user has set the backend, use it
37 + from vllm.plugins import get_torch_compile_backend
38 + backend = get_torch_compile_backend()
39 + if backend is None:
40 + from vllm.compilation.backends import select_default_backend
41 + backend = select_default_backend(envs.VLLM_TORCH_COMPILE_LEVEL)
42 +
43 + compiled_callable = torch.compile(
44 + self.forward,
45 + fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
46 + backend=backend)
47 +
48 self.compiled_callable = compiled_callable
49 self.original_code_object = self.__class__.forward.__code__
50 self.compiled_codes: List[CodeType] = []
@@ -33,7 +54,7 @@ class TorchCompileWrapperWithCustomDispatcher:
54 # subclasses can use this to switch between the custom dispatcher
55 # and the default Dynamo guard mechanism.
56 self.use_custom_dispatcher: bool = \
36 - envs.VLLM_DYNAMO_USE_CUSTOM_DISPATCHER
57 + envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.DYNAMO_ONCE
58
59 def __call__(self, *args, **kwargs):
60 """Implement the dispatch logic here, beyond the torch.compile level.
vllm/envs.py
+3 -13
@@ -65,6 +65,7 @@ if TYPE_CHECKING:
65 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
66 VLLM_SKIP_P2P_CHECK: bool = False
67 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
68 + VLLM_TORCH_COMPILE_LEVEL: int = 0
69
70
71 def get_default_cache_root():
@@ -198,23 +199,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
199 lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
200 ("true", "1")),
201
201 - # Internal flag to enable Dynamo graph capture
202 - "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
203 - lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
204 - "VLLM_DYNAMO_USE_CUSTOM_DISPATCHER":
205 - lambda:
206 - (os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
207 - ("true", "1")),
208 -
209 - # Internal flag to control whether we use custom op,
210 - # or use the native pytorch implementation
211 - "VLLM_TEST_COMPILE_NO_CUSTOM_OPS":
212 - lambda: int(os.environ.get("VLLM_TEST_COMPILE_NO_CUSTOM_OPS", "0")),
213 -
202 # Internal flag to enable Dynamo fullgraph capture
203 "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
204 lambda: bool(
205 os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
206 + "VLLM_TORCH_COMPILE_LEVEL":
207 + lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
208
209 # local rank of the process in the distributed setting, used to determine
210 # the GPU device id
vllm/model_executor/custom_op.py
+2 -1
@@ -1,6 +1,7 @@
1 import torch.nn as nn
2
3 import vllm.envs as envs
4 +from vllm.compilation.levels import CompilationLevel
5 from vllm.platforms import current_platform
6 from vllm.utils import is_cpu, is_hip, is_xpu
7
@@ -55,7 +56,7 @@ class CustomOp(nn.Module):
56 # NOTE(woosuk): Here we assume that vLLM was built for only one
57 # specific backend. Currently, we do not support dynamic dispatching.
58
58 - if envs.VLLM_TEST_COMPILE_NO_CUSTOM_OPS:
59 + if envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.INDUCTOR:
60 return self.forward_native
61
62 if is_hip():
vllm/model_executor/models/gemma2.py
+2
@@ -21,6 +21,7 @@ from torch import nn
21 from transformers import Gemma2Config
22
23 from vllm.attention import Attention, AttentionMetadata
24 +from vllm.compilation.decorators import support_compile_llama_style
25 from vllm.config import CacheConfig, LoRAConfig
26 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
@@ -238,6 +239,7 @@ class Gemma2DecoderLayer(nn.Module):
239 return hidden_states, residual
240
241
242 +@support_compile_llama_style
243 class Gemma2Model(nn.Module):
244
245 def __init__(
vllm/model_executor/models/llama.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_compile_llama_style
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
@@ -265,6 +266,7 @@ class LlamaDecoderLayer(nn.Module):
266 return hidden_states, residual
267
268
269 +@support_compile_llama_style
270 class LlamaModel(nn.Module):
271
272 def __init__(
vllm/model_executor/models/llava.py
+5 -3
@@ -365,6 +365,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
365 input_ids = None
366 inputs_embeds = None
367 else:
368 + # always pass the input via `inputs_embeds`
369 + # to make sure the computation graph is consistent
370 image_input = self._parse_and_validate_image_input(**kwargs)
371
372 if image_input is not None:
@@ -375,10 +377,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
377 inputs_embeds = merge_multimodal_embeddings(
378 input_ids, inputs_embeds, vision_embeddings,
379 self.config.image_token_index)
378 -
379 - input_ids = None
380 else:
381 - inputs_embeds = None
381 + inputs_embeds = self.language_model.model.get_input_embeddings(
382 + input_ids)
383 + input_ids = None
384
385 hidden_states = self.language_model.model(input_ids,
386 positions,
vllm/platforms/tpu.py
+14
@@ -1,7 +1,21 @@
1 +import os
2 +
3 import torch
4
5 +import vllm.envs as envs
6 +from vllm.compilation.levels import CompilationLevel
7 +from vllm.plugins import set_torch_compile_backend
8 +
9 from .interface import Platform, PlatformEnum
10
11 +if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
12 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE)
13 +
14 +assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR,\
15 + "TPU does not support Inductor."
16 +
17 +set_torch_compile_backend("openxla")
18 +
19
20 class TpuPlatform(Platform):
21 _enum = PlatformEnum.TPU
vllm/plugins/__init__.py
+13 -1
@@ -1,5 +1,5 @@
1 import logging
2 -from typing import Callable, Optional, Union
2 +from typing import Callable, Dict, Optional, Union
3
4 import vllm.envs as envs
5
@@ -42,3 +42,15 @@ def set_torch_compile_backend(backend: Union[Callable, str]):
42
43 def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
44 return _torch_compile_backend
45 +
46 +
47 +_inductor_additional_configs: Dict = {}
48 +
49 +
50 +def set_inductor_additional_configs(configs: Dict):
51 + global _inductor_additional_configs
52 + _inductor_additional_configs = configs
53 +
54 +
55 +def get_inductor_additional_configs() -> Dict:
56 + return _inductor_additional_configs
vllm/sequence.py
+3 -4
@@ -1137,10 +1137,9 @@ class EmbeddingSequenceGroupOutput(
1137 return self.embeddings == other.embeddings
1138
1139
1140 -class IntermediateTensors(
1141 - msgspec.Struct,
1142 - omit_defaults=True, # type: ignore[call-arg]
1143 - array_like=True): # type: ignore[call-arg]
1140 +# cannot use msgspec.Struct here because Dynamo does not support it
1141 +@dataclass
1142 +class IntermediateTensors:
1143 """For all pipeline stages except the last, we need to return the hidden
1144 states and residuals to be sent to the next stage. This data structure
1145 contains the hidden states and residuals for a request.
vllm/worker/model_runner.py
+14 -4
@@ -18,6 +18,8 @@ import vllm.envs as envs
18 from vllm.attention import AttentionMetadata, get_attn_backend
19 from vllm.attention.backends.abstract import AttentionState
20 from vllm.attention.backends.utils import CommonAttentionState
21 +from vllm.compilation.compile_context import set_compile_context
22 +from vllm.compilation.levels import CompilationLevel
23 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
24 ModelConfig, ObservabilityConfig, ParallelConfig,
25 PromptAdapterConfig, SchedulerConfig)
@@ -1126,10 +1128,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1128 "provided. Defaulting to scaling factors of 1.0. "
1129 "This may lead to less accurate results!")
1130
1129 - if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1130 - from vllm.compilation.backends import vllm_backend
1131 + if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS \
1132 + and supports_dynamo():
1133 from vllm.plugins import get_torch_compile_backend
1132 - backend = get_torch_compile_backend() or vllm_backend
1134 + backend = get_torch_compile_backend() or "eager"
1135 self.model = torch.compile(
1136 self.model,
1137 fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
@@ -1289,7 +1291,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1291 batch_size=batch_size,
1292 dtype=self.model_config.dtype,
1293 device=self.device)
1292 - self.execute_model(model_input, kv_caches, intermediate_tensors)
1294 +
1295 + graph_batch_size = self.max_batchsize_to_capture
1296 + batch_size_capture_list = [
1297 + bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1298 + ]
1299 + if self.model_config.enforce_eager:
1300 + batch_size_capture_list = []
1301 + with set_compile_context(batch_size_capture_list):
1302 + self.execute_model(model_input, kv_caches, intermediate_tensors)
1303 torch.cuda.synchronize()
1304 return
1305