A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Log how much time loading a compiled artifact takes (#16848)
Signed-off-by: rzou <zou3519@gmail.com>
Richard Zou committed
Apr 19, 2025 at 12:50 UTC
682e0b6d2fbef012abde9b90c73da23f3065e2b8
1 file changed
+8
-4
vllm/compilation/backends.py
+8
-4
@@ -110,10 +110,14 @@ class CompilerManager:
110
compiled_graph = self.load(graph, example_inputs, graph_index,
111
runtime_shape)
112
if compiled_graph is not None:
113
- if graph_index == 0:
114
- # adds some info logging for the first graph
115
- logger.info("Directly load the compiled graph for shape %s "
116
- "from the cache", str(runtime_shape)) # noqa
113
+ if graph_index == num_graphs - 1:
114
+ # after loading the last graph for this shape, record the time.
115
+ # there can be multiple graphs due to piecewise compilation.
116
+ now = time.time()
117
+ elapsed = now - compilation_start_time
118
+ logger.info(
119
+ "Directly load the compiled graph(s) for shape %s "
120
+ "from the cache, took %.3f s", str(runtime_shape), elapsed)
121
return compiled_graph
122
123
# no compiler cached the graph, or the cache is disabled,