@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 47790f3e32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[torch.compile] add a flag to disable custom op (#8488)

youkaichao committed Sep 14, 2024 at 13:07 UTC 47790f3e328f1fbf250d8f858b6390496c1e61c0
3 files changed +12 -1
tests/compile/test_full_graph.py
+2 -1
@@ -6,7 +6,8 @@ import pytest
6 @pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
7 def test_full_graph(model):
8 # make sure these models can be captured in full graph mode
9 - os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
9 + if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
10 + os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
11
12 from vllm import LLM, SamplingParams
13 prompts = [
vllm/envs.py
+5
@@ -202,6 +202,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
202 (os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
203 ("true", "1")),
204
205 + # Internal flag to control whether we use custom op,
206 + # or use the native pytorch implementation
207 + "VLLM_TEST_COMPILE_NO_CUSTOM_OPS":
208 + lambda: int(os.environ.get("VLLM_TEST_COMPILE_NO_CUSTOM_OPS", "0")),
209 +
210 # Internal flag to enable Dynamo fullgraph capture
211 "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
212 lambda: bool(
vllm/model_executor/custom_op.py
+5
@@ -1,5 +1,6 @@
1 import torch.nn as nn
2
3 +import vllm.envs as envs
4 from vllm.platforms import current_platform
5 from vllm.utils import is_cpu, is_hip, is_xpu
6
@@ -53,6 +54,10 @@ class CustomOp(nn.Module):
54 def dispatch_forward(self):
55 # NOTE(woosuk): Here we assume that vLLM was built for only one
56 # specific backend. Currently, we do not support dynamic dispatching.
57 +
58 + if envs.VLLM_TEST_COMPILE_NO_CUSTOM_OPS:
59 + return self.forward_native
60 +
61 if is_hip():
62 return self.forward_hip
63 elif is_cpu():