A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 09:30 UTC
[Core] Cache `vllm_is_batch_invariant` (#28304)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Nov 12, 2025 at 05:03 UTC
ac0bb2c3075f87b62afde8ea7dab10207bb71df1
1 file changed
+2
vllm/model_executor/layers/batch_invariant.py
+2
@@ -4,6 +4,7 @@ import contextlib
4
import os
5
from collections import namedtuple
6
from collections.abc import Callable
7
+from functools import cache
8
from typing import Any
9
10
import torch
@@ -857,6 +858,7 @@ def get_batch_invariant_attention_block_size() -> AttentionBlockSize:
858
return AttentionBlockSize(block_m=16, block_n=16)
859
860
861
+@cache
862
def vllm_is_batch_invariant():
863
env_key = "VLLM_BATCH_INVARIANT"
864
is_overridden = False