A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] add reorder_batch AttentionMetadataBuilder (#23798)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Aug 30, 2025 at 21:41 UTC
fb4983e112a81f4df25b92ab98c9c84a5babfec9
1 file changed
+17
vllm/v1/attention/backends/utils.py
+17
@@ -212,6 +212,23 @@ class AttentionMetadataBuilder(abc.ABC, Generic[M]):
212
"""
213
raise NotImplementedError
214
215
+ def reorder_batch(self, input_batch: "InputBatch",
216
+ scheduler_output: "SchedulerOutput") -> bool:
217
+ """
218
+ Update the order of requests in the batch based on the attention
219
+ backend's needs. For example, some attention backends (namely MLA) may
220
+ want to separate requests based on if the attention computation will be
221
+ compute-bound or memory-bound.
222
+
223
+ Args:
224
+ input_batch: input batch
225
+ scheduler_output: scheduler output.
226
+
227
+ Returns:
228
+ True if the batch was modified, False otherwise.
229
+ """
230
+ raise NotImplementedError
231
+
232
def build_for_cudagraph_capture(
233
self, common_attn_metadata: CommonAttentionMetadata) -> M:
234
"""