A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[Bugfix][torch2.10] Fix test_qwen2_5_vl_compilation with 2.10 RC (#30822)
Signed-off-by: Lucas Kabela <lucaskabela@meta.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Lucas Kabela committed
Dec 18, 2025 at 08:23 UTC
0db5439ded739731474009217a7515fd52c221c8
3 files changed
+13
-7
vllm/compilation/backends.py
+4
-3
@@ -520,6 +520,7 @@ class VllmBackend:
520
self,
521
vllm_config: VllmConfig,
522
prefix: str = "",
523
+ is_encoder: bool = False,
524
):
525
# if the model is initialized with a non-empty prefix,
526
# then usually it's enough to use that prefix,
@@ -530,7 +531,7 @@ class VllmBackend:
531
self.prefix = prefix or model_tag
532
533
# Mark compilation for encoder.
533
- self.is_encoder = model_is_encoder
534
+ self.is_encoder = is_encoder or model_is_encoder
535
536
# Passes to run on the graph post-grad.
537
self.pass_manager = resolve_obj_by_qualname(
@@ -797,7 +798,7 @@ class VllmBackend:
798
or not self.compilation_config.cudagraph_copy_inputs
799
):
800
return VllmSerializableFunction(
800
- graph, example_inputs, self.prefix, self.split_gm
801
+ graph, example_inputs, self.prefix, self.split_gm, self.is_encoder
802
)
803
804
# index of tensors that have symbolic shapes (batch size)
@@ -835,5 +836,5 @@ class VllmBackend:
836
return self.split_gm(*list_args)
837
838
return VllmSerializableFunction(
838
- graph, example_inputs, self.prefix, copy_and_call
839
+ graph, example_inputs, self.prefix, copy_and_call, self.is_encoder
840
)
vllm/compilation/caching.py
+8
-2
@@ -37,12 +37,15 @@ class VllmSerializableFunction(SerializableCallable):
37
serializing the Dynamo fx graph plus example inputs.
38
"""
39
40
- def __init__(self, graph_module, example_inputs, prefix, optimized_call):
40
+ def __init__(
41
+ self, graph_module, example_inputs, prefix, optimized_call, is_encoder=False
42
+ ):
43
assert isinstance(graph_module, torch.fx.GraphModule)
44
self.graph_module = graph_module
45
self.example_inputs = example_inputs
46
self.prefix = prefix
47
self.optimized_call = optimized_call
48
+ self.is_encoder = is_encoder
49
self.shape_env = None
50
sym_input = next(
51
(i for i in self.example_inputs if isinstance(i, torch.SymInt)), None
@@ -106,7 +109,10 @@ class VllmSerializableFunction(SerializableCallable):
109
state["graph_module"] = GraphPickler.loads(state["graph_module"], fake_mode)
110
state["graph_module"].recompile()
111
state["example_inputs"] = GraphPickler.loads(state["example_inputs"], fake_mode)
109
- vllm_backend = VllmBackend(get_current_vllm_config(), state["prefix"])
112
+ is_encoder = state.get("is_encoder", False)
113
+ vllm_backend = VllmBackend(
114
+ get_current_vllm_config(), state["prefix"], is_encoder
115
+ )
116
117
def optimized_call(*example_inputs):
118
"""
vllm/compilation/piecewise_backend.py
+1
-2
@@ -170,8 +170,7 @@ class PiecewiseBackend:
170
range_entry = self._find_range_for_shape(runtime_shape)
171
172
assert range_entry is not None, (
173
- f"Shape out of considered range: {runtime_shape} "
174
- "[1, max_num_batched_tokens]"
173
+ f"Shape: {runtime_shape} out of considered ranges: {self.compile_ranges}"
174
)
175
176
self._maybe_compile_for_range_entry(range_entry, args)