A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] Fix missing sequence_lengths in EXAONE-4.5 vision encoder (#45073)
Signed-off-by: Jongsu Liam Kim <jongsukim8@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
Jongsu Liam Kim committed
Jun 10, 2026 at 23:44 UTC
ccc05de03888d2ceaec5d039e07a52c938b2ef00
1 file changed
+7
vllm/model_executor/models/exaone4_5.py
+7
@@ -152,6 +152,8 @@ class EXAONE4_5_VisionAttention(nn.Module):
152
rotary_pos_emb_cos: torch.Tensor,
153
rotary_pos_emb_sin: torch.Tensor,
154
max_seqlen: int | None = None,
155
+ sequence_lengths: torch.Tensor
156
+ | None = None, # Only used for FlashInfer CuDNN backend
157
) -> torch.Tensor:
158
# [s, b, c] --> [s, b, head * 3 * head_dim]
159
x, _ = self.qkv(x)
@@ -176,6 +178,7 @@ class EXAONE4_5_VisionAttention(nn.Module):
178
value=v,
179
cu_seqlens=cu_seqlens,
180
max_seqlen=max_seqlen,
181
+ sequence_lengths=sequence_lengths,
182
)
183
184
context_layer = einops.rearrange(
@@ -190,6 +193,7 @@ class EXAONE4_5_VisionAttention(nn.Module):
193
dynamic_arg_dims={
194
"x": 0,
195
"cu_seqlens": 0,
196
+ "sequence_lengths": 0,
197
"rotary_pos_emb_cos": 0,
198
"rotary_pos_emb_sin": 0,
199
},
@@ -241,6 +245,8 @@ class Exaone4_5_VisionBlock(nn.Module):
245
rotary_pos_emb_sin: torch.Tensor,
246
max_seqlen: int | None = None, # Only used for Flash Attention
247
seqlens: list[int] | None = None, # Only used for xFormers
248
+ # Only used for FlashInfer CuDNN backend
249
+ sequence_lengths: torch.Tensor | None = None,
250
) -> torch.Tensor:
251
x_attn = self.attn(
252
self.norm1(x),
@@ -248,6 +254,7 @@ class Exaone4_5_VisionBlock(nn.Module):
254
rotary_pos_emb_cos=rotary_pos_emb_cos,
255
rotary_pos_emb_sin=rotary_pos_emb_sin,
256
max_seqlen=max_seqlen,
257
+ sequence_lengths=sequence_lengths,
258
)
259
x_fused_norm, residual = self.norm2(x, residual=x_attn)
260
x = residual + self.mlp(x_fused_norm)