A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Minor] Avoid frequently creating ConstantList (#12653)
A small optimization to avoid creating a new `ConstantList` every time `request.kv_block_hashes` is used. Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Feb 1, 2025 at 23:43 UTC
abfcdcdf27eb54d2a2104b4bf5091a24ea4ff928
1 file changed
+2
-5
vllm/v1/request.py
+2
-5
@@ -64,6 +64,7 @@ class Request:
64
# Cache the computed kv block hashes of the request to avoid
65
# recomputing.
66
self._kv_block_hashes: List[BlockHashType] = []
67
+ self.kv_block_hashes = ConstantList(self._kv_block_hashes)
68
69
# Read-only views
70
# Prevent directly appending to the these lists since
@@ -121,13 +122,9 @@ class Request:
122
num_tokens = self.mm_positions[input_id]["length"]
123
return num_tokens
124
124
- @property
125
- def kv_block_hashes(self) -> ConstantList["BlockHashType"]:
126
- # Prevent directly appending to the kv_block_hashes.
127
- return ConstantList(self._kv_block_hashes)
128
-
125
def set_kv_block_hashes(self, value: List["BlockHashType"]) -> None:
126
self._kv_block_hashes = value
127
+ self.kv_block_hashes = ConstantList(self._kv_block_hashes)
128
129
def append_kv_block_hashes(self, block_hash: "BlockHashType") -> None:
130
self._kv_block_hashes.append(block_hash)