A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[core][misc] remove logical block (#5882)
youkaichao committed
Jun 27, 2024 at 13:34 UTC
64e8d2a783ac976f1b8e84a795f6a607820d6485
3 files changed
+16
-120
vllm/block.py
+1
-81
@@ -1,90 +1,10 @@
1
"""Token blocks."""
2
-import weakref
3
-from collections import defaultdict
4
-from typing import Dict, List
2
+from typing import List
3
4
from vllm.utils import Device
5
8
-_BLANK_TOKEN_ID = -1
9
-
6
DEFAULT_LAST_ACCESSED_TIME = -1
7
12
-TokensBlock = List[int]
13
-
14
-
15
-class BlockPool:
16
- """A pool of logical blocks.
17
- When requests come, we create a lot of logical blocks;
18
- when requests are done, we destroy a lot of logical blocks.
19
- It turns out that creating and destroying logical blocks can be expensive,
20
- especially for the `token_ids` field, which is a list of integers.
21
- To avoid this overhead, we use a pool to manage the logical blocks.
22
- When an old request is done and a new request comes, we can reuse the
23
- logical blocks from the old request to feed the new request.
24
- """
25
-
26
- def __init__(self) -> None:
27
- # block size to list of token blocks
28
- self.pool: Dict[int, List[TokensBlock]] = defaultdict(list)
29
-
30
- def alloc_block(self, block_size: int) -> TokensBlock:
31
- if block_size in self.pool and self.pool[block_size]:
32
- return self.pool[block_size].pop()
33
- return [_BLANK_TOKEN_ID] * block_size
34
-
35
- def del_block(self, block: TokensBlock) -> None:
36
- self.pool[len(block)].append(block)
37
-
38
-
39
-_BLOCK_POOL = BlockPool()
40
-
41
-
42
-class LogicalTokenBlock:
43
- """A block that stores a contiguous chunk of tokens from left to right.
44
-
45
- Logical blocks are used to represent the states of the corresponding
46
- physical blocks in the KV cache.
47
- """
48
-
49
- def __init__(
50
- self,
51
- block_number: int,
52
- block_size: int,
53
- ) -> None:
54
- self.block_number = block_number
55
- self.block_size = block_size
56
-
57
- self.token_ids = _BLOCK_POOL.alloc_block(block_size)
58
- # this finalizer is used to return the block to the pool when the object is deleted # noqa
59
- # NOTE: don't use __del__ because it cannot guarantee the order of finalization, # noqa
60
- # i.e. `self.token_ids` may be deleted before `self`, and we lose
61
- # the opportunity to return the block to the pool
62
- self._finalizer = weakref.finalize(self, _BLOCK_POOL.del_block,
63
- self.token_ids)
64
- self.num_tokens = 0
65
-
66
- def is_empty(self) -> bool:
67
- return self.num_tokens == 0
68
-
69
- def get_num_empty_slots(self) -> int:
70
- return self.block_size - self.num_tokens
71
-
72
- def is_full(self) -> bool:
73
- return self.num_tokens == self.block_size
74
-
75
- def append_tokens(self, token_ids: List[int]) -> None:
76
- assert len(token_ids) <= self.get_num_empty_slots()
77
- curr_idx = self.num_tokens
78
- self.token_ids[curr_idx:curr_idx + len(token_ids)] = token_ids
79
- self.num_tokens += len(token_ids)
80
-
81
- def get_token_ids(self) -> List[int]:
82
- return self.token_ids[:self.num_tokens]
83
-
84
- def get_last_token_id(self) -> int:
85
- assert self.num_tokens > 0
86
- return self.token_ids[self.num_tokens - 1]
87
-
8
9
class PhysicalTokenBlock:
10
"""Represents the state of a block in the KV cache."""
vllm/core/block_manager_v1.py
+9
-10
@@ -262,8 +262,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
262
self.cross_block_tables: Dict[str, BlockTable] = {}
263
264
def _get_seq_num_required_blocks(self, seq: Sequence) -> int:
265
- return 0 if seq is None \
266
- else len(seq.logical_token_blocks)
265
+ return 0 if seq is None else seq.n_blocks
266
267
def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
268
# FIXME(woosuk): Here we assume that all sequences in the group share
@@ -298,7 +297,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
297
ref_count: int, \
298
is_encoder_decoder: bool = True) -> BlockTable:
299
# Allocate new physical token blocks that will store the prompt tokens.
301
- num_prompt_blocks = len(seq.logical_token_blocks)
300
+ num_prompt_blocks = seq.n_blocks
301
302
block_table: BlockTable = []
303
for logical_idx in range(num_prompt_blocks):
@@ -367,7 +366,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
366
367
# Compute a new hash for the block so that it can be shared by other
368
# Sequences
370
- new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
369
+ new_hash = seq.hash_of_block(seq.n_blocks - 1)
370
371
# if new_hash is already in the cached table, then free last_block
372
# and return the cached version
@@ -407,10 +406,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
406
if not self.enable_caching:
407
return self.gpu_allocator.allocate()
408
block_hash: Optional[int] = None
409
+ n_blocks = seq.n_blocks
410
if (self._is_last_block_full(seq)):
411
- block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
412
- num_hashed_tokens = seq.num_hashed_tokens_of_block(
413
- len(seq.logical_token_blocks) - 1)
411
+ block_hash = seq.hash_of_block(n_blocks - 1)
412
+ num_hashed_tokens = seq.num_hashed_tokens_of_block(n_blocks - 1)
413
414
# num_hashed_tokens is used to compute future hashes
415
# (e.g. in the hashing function, it is used to ask the sequence for
@@ -429,12 +428,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
428
num_lookahead_slots: int = 0,
429
) -> List[Tuple[int, int]]:
430
"""Allocate a physical slot for a new token."""
432
- logical_blocks = seq.logical_token_blocks
431
+ n_blocks = seq.n_blocks
432
block_table = self.block_tables[seq.seq_id]
433
# If we need to allocate a new physical block
435
- if len(block_table) < len(logical_blocks):
434
+ if len(block_table) < n_blocks:
435
# Currently this code only supports adding one physical block
437
- assert len(block_table) == len(logical_blocks) - 1
436
+ assert len(block_table) == n_blocks - 1
437
438
if (self.block_sliding_window
439
and len(block_table) >= self.block_sliding_window):
vllm/sequence.py
+6
-29
@@ -1,13 +1,13 @@
1
"""Sequence and its related classes."""
2
import copy
3
import enum
4
+import math
5
from abc import ABC, abstractmethod
6
from dataclasses import dataclass, field
7
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
8
9
import torch
10
10
-from vllm.block import LogicalTokenBlock
11
from vllm.inputs import LLMInputs
12
from vllm.lora.request import LoRARequest
13
from vllm.pooling_params import PoolingParams
@@ -236,9 +236,6 @@ class Sequence:
236
self.output_logprobs: SampleLogprobs = []
237
self.output_text = ""
238
239
- self.logical_token_blocks: List[LogicalTokenBlock] = []
240
- # Initialize the logical token blocks with the prompt token ids.
241
- self._append_tokens_to_blocks(self.prompt_token_ids)
239
self.status = SequenceStatus.WAITING
240
self.stop_reason: Union[int, str, None] = None
241
@@ -248,6 +245,10 @@ class Sequence:
245
# Input + output tokens
246
self.tokens: Optional[List[str]] = None
247
248
+ @property
249
+ def n_blocks(self) -> int:
250
+ return math.ceil(self.get_len() / self.block_size)
251
+
252
@property
253
def prompt(self) -> Optional[str]:
254
return self.inputs.get("prompt")
@@ -287,36 +288,12 @@ class Sequence:
288
"""Reset the sequence states for recomputation."""
289
self.data.reset_state_for_recompute()
290
290
- def _append_logical_block(self) -> None:
291
- block = LogicalTokenBlock(
292
- block_number=len(self.logical_token_blocks),
293
- block_size=self.block_size,
294
- )
295
- self.logical_token_blocks.append(block)
296
-
297
- def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
298
- cursor = 0
299
- while cursor < len(token_ids):
300
- if not self.logical_token_blocks:
301
- self._append_logical_block()
302
-
303
- last_block = self.logical_token_blocks[-1]
304
- if last_block.is_full():
305
- self._append_logical_block()
306
- last_block = self.logical_token_blocks[-1]
307
-
308
- num_empty_slots = last_block.get_num_empty_slots()
309
- last_block.append_tokens(token_ids[cursor:cursor +
310
- num_empty_slots])
311
- cursor += num_empty_slots
312
-
291
def append_token_id(
292
self,
293
token_id: int,
294
logprobs: Dict[int, Logprob],
295
) -> None:
296
assert token_id in logprobs
319
- self._append_tokens_to_blocks([token_id])
297
self.output_logprobs.append(logprobs)
298
self.data.append_token_id(token_id, logprobs[token_id].logprob)
299
@@ -388,7 +365,7 @@ class Sequence:
365
def __repr__(self) -> str:
366
return (f"Sequence(seq_id={self.seq_id}, "
367
f"status={self.status.name}, "
391
- f"num_blocks={len(self.logical_token_blocks)})")
368
+ f"num_blocks={self.n_blocks}, ")
369
370
371
@dataclass