A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
chore: disable enable_cpp_symbolic_shape_guards (#23048)
Signed-off-by: Xiao Liu <xiszishu@gmail.com>
Xiao committed
Aug 18, 2025 at 20:08 UTC
a4454e9401c9c76150dc3a4d23d10377926b2fb6
1 file changed
+17
-1
vllm/compilation/decorators.py
+17
-1
@@ -267,8 +267,24 @@ def _support_torch_compile(
267
code.co_filename)
268
return inline_call(parent, func, args, kwargs)
269
270
+ # Disable the C++ compilation of symbolic shape guards. C++-fication
271
+ # of symbolic shape guards can improve guard overhead. But, since
272
+ # vllm skip guards anyways, setting this flag to False can improve
273
+ # compile time.
274
+ dynamo_config_patches = {}
275
+ try:
276
+ _ = torch._dynamo.config.enable_cpp_symbolic_shape_guards
277
+ dynamo_config_patches[
278
+ "enable_cpp_symbolic_shape_guards"] = False
279
+ except AttributeError:
280
+ # Note: this config is not available in torch 2.6, we can skip
281
+ # if the config doesn't exist
282
+ logger.debug(
283
+ "enable_cpp_symbolic_shape_guards config not available")
284
+
285
with patch.object(InliningInstructionTranslator, 'inline_call',
271
- patched_inline_call):
286
+ patched_inline_call), torch._dynamo.config.patch(
287
+ **dynamo_config_patches):
288
output = self.compiled_callable(*args, **kwargs)
289
return output
290