[Perf][MRV2] Share token-to-request mappings across KV cache groups (#57102)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Juntian Liu committed
Sep 17, 2026 at 20:14 UTC
bf13ecc2f7c4d69b59854f7cef37547be58805e5
1 file changed
+3
vllm/v1/worker/gpu/attn_utils.py
+3
@@ -426,6 +426,7 @@ def build_attn_metadata(
426
seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound[:num_reqs]
427
428
attn_metadata: dict[str, Any] = {}
429
+ token_to_req_indices: torch.Tensor | None = None
430
num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
431
for i in range(num_kv_cache_groups):
432
if not attn_groups[i]:
@@ -472,6 +473,7 @@ def build_attn_metadata(
473
mm_req_doc_ranges=mm_req_doc_ranges,
474
rswa_prefix_lens=rswa_prefix_lens,
475
req_idx=req_idx,
476
+ _token_to_req_indices_cache=token_to_req_indices,
477
**common_attn_metadata_extra_kwargs,
478
)
479
@@ -497,6 +499,7 @@ def build_attn_metadata(
499
)
500
for layer_name in attn_group.layer_names:
501
attn_metadata[layer_name] = metadata
502
+ token_to_req_indices = common_attn_metadata._token_to_req_indices_cache
503
return attn_metadata
504
505