@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / abfcdcdf27
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Minor] Avoid frequently creating ConstantList (#12653)

A small optimization to avoid creating a new `ConstantList` every time `request.kv_block_hashes` is used. Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Feb 1, 2025 at 23:43 UTC abfcdcdf27eb54d2a2104b4bf5091a24ea4ff928
1 file changed +2 -5
vllm/v1/request.py
+2 -5
@@ -64,6 +64,7 @@ class Request:
64 # Cache the computed kv block hashes of the request to avoid
65 # recomputing.
66 self._kv_block_hashes: List[BlockHashType] = []
67 + self.kv_block_hashes = ConstantList(self._kv_block_hashes)
68
69 # Read-only views
70 # Prevent directly appending to the these lists since
@@ -121,13 +122,9 @@ class Request:
122 num_tokens = self.mm_positions[input_id]["length"]
123 return num_tokens
124
124 - @property
125 - def kv_block_hashes(self) -> ConstantList["BlockHashType"]:
126 - # Prevent directly appending to the kv_block_hashes.
127 - return ConstantList(self._kv_block_hashes)
128 -
125 def set_kv_block_hashes(self, value: List["BlockHashType"]) -> None:
126 self._kv_block_hashes = value
127 + self.kv_block_hashes = ConstantList(self._kv_block_hashes)
128
129 def append_kv_block_hashes(self, block_hash: "BlockHashType") -> None:
130 self._kv_block_hashes.append(block_hash)