A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[torch.compile] add a flag to disable custom op (#8488)
youkaichao committed
Sep 14, 2024 at 13:07 UTC
47790f3e328f1fbf250d8f858b6390496c1e61c0
3 files changed
+12
-1
tests/compile/test_full_graph.py
+2
-1
@@ -6,7 +6,8 @@ import pytest
6
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
7
def test_full_graph(model):
8
# make sure these models can be captured in full graph mode
9
- os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
9
+ if "VLLM_TEST_DYNAMO_GRAPH_CAPTURE" not in os.environ:
10
+ os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
11
12
from vllm import LLM, SamplingParams
13
prompts = [
vllm/envs.py
+5
@@ -202,6 +202,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
202
(os.environ.get("VLLM_DYNAMO_USE_CUSTOM_DISPATCHER", "True").lower() in
203
("true", "1")),
204
205
+ # Internal flag to control whether we use custom op,
206
+ # or use the native pytorch implementation
207
+ "VLLM_TEST_COMPILE_NO_CUSTOM_OPS":
208
+ lambda: int(os.environ.get("VLLM_TEST_COMPILE_NO_CUSTOM_OPS", "0")),
209
+
210
# Internal flag to enable Dynamo fullgraph capture
211
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
212
lambda: bool(
vllm/model_executor/custom_op.py
+5
@@ -1,5 +1,6 @@
1
import torch.nn as nn
2
3
+import vllm.envs as envs
4
from vllm.platforms import current_platform
5
from vllm.utils import is_cpu, is_hip, is_xpu
6
@@ -53,6 +54,10 @@ class CustomOp(nn.Module):
54
def dispatch_forward(self):
55
# NOTE(woosuk): Here we assume that vLLM was built for only one
56
# specific backend. Currently, we do not support dynamic dispatching.
57
+
58
+ if envs.VLLM_TEST_COMPILE_NO_CUSTOM_OPS:
59
+ return self.forward_native
60
+
61
if is_hip():
62
return self.forward_hip
63
elif is_cpu():