A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
avoid python list copy in sequence initialization (#401)
Lily Liu committed
Jul 8, 2023 at 12:42 UTC
2179e4f4c5c40d852044fddbcae80996c2feb092
1 file changed
+5
-3
vllm/sequence.py
+5
-3
@@ -127,7 +127,8 @@ class Sequence:
127
self.logical_token_blocks.append(block)
128
129
def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
130
- while token_ids:
130
+ cursor = 0
131
+ while cursor < len(token_ids):
132
if not self.logical_token_blocks:
133
self._append_logical_block()
134
@@ -137,8 +138,9 @@ class Sequence:
138
last_block = self.logical_token_blocks[-1]
139
140
num_empty_slots = last_block.get_num_empty_slots()
140
- last_block.append_tokens(token_ids[:num_empty_slots])
141
- token_ids = token_ids[num_empty_slots:]
141
+ last_block.append_tokens(token_ids[cursor:cursor +
142
+ num_empty_slots])
143
+ cursor += num_empty_slots
144
145
def append_token_id(
146
self,