A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Misc] Improve logging for dynamic shape cache compilation (#20573)
Signed-off-by: kyolebu <kyu@redhat.com>
Kyle Yu committed
Jul 7, 2025 at 20:48 UTC
d2e841a10a273df06c8e57ba5ca8eb9cd8cce79e
1 file changed
+36
-15
vllm/compilation/backends.py
+36
-15
@@ -120,10 +120,15 @@ class CompilerManager:
120
handle = self.cache[(runtime_shape, graph_index, self.compiler.name)]
121
compiled_graph = self.compiler.load(handle, graph, example_inputs,
122
graph_index, runtime_shape)
123
- logger.debug(
124
- "Directly load the %s-th graph for shape %s from %s via "
125
- "handle %s", graph_index, str(runtime_shape), self.compiler.name,
126
- handle)
123
+ if runtime_shape is None:
124
+ logger.debug(
125
+ "Directly load the %s-th graph for dynamic shape from %s via "
126
+ "handle %s", graph_index, self.compiler.name, handle)
127
+ else:
128
+ logger.debug(
129
+ "Directly load the %s-th graph for shape %s from %s via "
130
+ "handle %s", graph_index, str(runtime_shape),
131
+ self.compiler.name, handle)
132
return compiled_graph
133
134
def compile(self,
@@ -152,9 +157,15 @@ class CompilerManager:
157
# there can be multiple graphs due to piecewise compilation.
158
now = time.time()
159
elapsed = now - compilation_start_time
155
- logger.info(
156
- "Directly load the compiled graph(s) for shape %s "
157
- "from the cache, took %.3f s", str(runtime_shape), elapsed)
160
+ if runtime_shape is None:
161
+ logger.info(
162
+ "Directly load the compiled graph(s) for dynamic shape "
163
+ "from the cache, took %.3f s", elapsed)
164
+ else:
165
+ logger.info(
166
+ "Directly load the compiled graph(s) for shape %s "
167
+ "from the cache, took %.3f s", str(runtime_shape),
168
+ elapsed)
169
return compiled_graph
170
171
# no compiler cached the graph, or the cache is disabled,
@@ -178,11 +189,21 @@ class CompilerManager:
189
self.is_cache_updated = True
190
if graph_index == 0:
191
# adds some info logging for the first graph
181
- logger.info("Cache the graph of shape %s for later use",
182
- str(runtime_shape))
183
- logger.debug(
184
- "store the %s-th graph for shape %s from %s via handle %s",
185
- graph_index, str(runtime_shape), self.compiler.name, handle)
192
+ if runtime_shape is None:
193
+ logger.info(
194
+ "Cache the graph for dynamic shape for later use")
195
+ else:
196
+ logger.info("Cache the graph of shape %s for later use",
197
+ str(runtime_shape))
198
+ if runtime_shape is None:
199
+ logger.debug(
200
+ "Store the %s-th graph for dynamic shape from %s via "
201
+ "handle %s", graph_index, self.compiler.name, handle)
202
+ else:
203
+ logger.debug(
204
+ "Store the %s-th graph for shape %s from %s via handle %s",
205
+ graph_index, str(runtime_shape), self.compiler.name,
206
+ handle)
207
208
# after compiling the last graph, record the end time
209
if graph_index == num_graphs - 1:
@@ -190,7 +211,7 @@ class CompilerManager:
211
elapsed = now - compilation_start_time
212
compilation_config.compilation_time += elapsed
213
if runtime_shape is None:
193
- logger.info("Compiling a graph for general shape takes %.2f s",
214
+ logger.info("Compiling a graph for dynamic shape takes %.2f s",
215
elapsed)
216
else:
217
logger.info("Compiling a graph for shape %s takes %.2f s",
@@ -308,7 +329,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
329
i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
330
]
331
global compilation_start_time
311
- compiled_graph_for_general_shape = self.vllm_backend.\
332
+ compiled_graph_for_dynamic_shape = self.vllm_backend.\
333
compiler_manager.compile(
334
submod,
335
args,
@@ -323,7 +344,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
344
self.module.__dict__[target] = piecewise_backend(
345
submod, self.vllm_config, self.graph_pool, index,
346
len(self.compile_submod_names), sym_shape_indices,
326
- compiled_graph_for_general_shape, self.vllm_backend)
347
+ compiled_graph_for_dynamic_shape, self.vllm_backend)
348
349
compilation_counter.num_piecewise_capturable_graphs_seen += 1
350