A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] use size tuning for specific sizes (#10933)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 5, 2024 at 20:30 UTC
db87eb6c67271eb61ba9fd8559ce811a1a398a4d
1 file changed
+6
vllm/compilation/backends.py
+6
@@ -43,6 +43,12 @@ def wrap_inductor(graph,
43
if additional_inductor_config is not None:
44
current_config.update(additional_inductor_config)
45
46
+ if isinstance(runtime_shape, int):
47
+ # for a specific batchsize, tuning triton kernel parameters
48
+ # can be beneficial
49
+ current_config["max_autotune"] = True
50
+ current_config["coordinate_descent_tuning"] = True
51
+
52
# inductor can inplace modify the graph, so we need to copy it
53
# see https://github.com/pytorch/pytorch/issues/138980
54
graph = copy.deepcopy(graph)