A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 13:45 UTC
[Bugfix][Granite4Vision] Fix deepstack buffer causing decode slowdown in compiled mode (#40917)
Signed-off-by: artemspector <artems@il.ibm.com> Co-authored-by: artemspector <artems@il.ibm.com>
artem-spector committed
Apr 28, 2026 at 10:43 UTC
bde0efdbb78a57dc10375e8d0686cf862332192c
1 file changed
+2
-1
vllm/model_executor/models/granite4_vision.py
+2
-1
@@ -887,9 +887,10 @@ class Granite4VisionForConditionalGeneration(
887
and get_pp_group().is_first_rank
888
and self._ds_layer_indices
889
):
890
+ n = inputs_embeds.size(0)
891
ds: IntermediateTensors | None = IntermediateTensors(
892
{
892
- f"ds_{llm_layer}": self._ds_buffers[lvl]
893
+ f"ds_{llm_layer}": self._ds_buffers[lvl][:n]
894
for lvl, llm_layer in enumerate(self._ds_layer_indices)
895
}
896
)