@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2179e4f4c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

avoid python list copy in sequence initialization (#401)

Lily Liu committed Jul 8, 2023 at 12:42 UTC 2179e4f4c5c40d852044fddbcae80996c2feb092
1 file changed +5 -3
vllm/sequence.py
+5 -3
@@ -127,7 +127,8 @@ class Sequence:
127 self.logical_token_blocks.append(block)
128
129 def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
130 - while token_ids:
130 + cursor = 0
131 + while cursor < len(token_ids):
132 if not self.logical_token_blocks:
133 self._append_logical_block()
134
@@ -137,8 +138,9 @@ class Sequence:
138 last_block = self.logical_token_blocks[-1]
139
140 num_empty_slots = last_block.get_num_empty_slots()
140 - last_block.append_tokens(token_ids[:num_empty_slots])
141 - token_ids = token_ids[num_empty_slots:]
141 + last_block.append_tokens(token_ids[cursor:cursor +
142 + num_empty_slots])
143 + cursor += num_empty_slots
144
145 def append_token_id(
146 self,