A high-throughput and memory-efficient inference and serving engine for LLMs
[ROCm][V2] Fix failed assertion in Llama models when using EAGLE with `ROCM_AITER_FA` (#44936)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Micah Williamson committed
Jun 9, 2026 at 13:30 UTC
c9c1540e61b229def0d16395a3489c68a1581f0f
1 file changed
+4
-3
vllm/v1/attention/backends/utils.py
+4
-3
@@ -458,15 +458,16 @@ def split_decodes_prefills_and_extends(
458
num_reqs = common_attn_metadata.num_reqs
459
num_tokens = common_attn_metadata.num_actual_tokens
460
query_start_loc = common_attn_metadata.query_start_loc_cpu
461
+
462
+ if max_query_len <= decode_threshold:
463
+ return num_reqs, 0, 0, num_tokens, 0, 0
464
+
465
# Upper bound is exact for prefill rows; decode rows still satisfy
466
# seq_len > query_len under the optimistic bound, so `seq_lens ==
467
# query_lens` identifies prefills correctly either way.
468
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
469
seq_lens = common_attn_metadata.seq_lens_cpu_upper_bound
470
467
- if max_query_len <= decode_threshold:
468
- return num_reqs, 0, 0, num_tokens, 0, 0
469
-
471
query_lens = query_start_loc[1:] - query_start_loc[:-1]
472
is_prefill_or_extend = query_lens > decode_threshold
473
is_prefill = (seq_lens == query_lens) & is_prefill_or_extend