@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06bfb51963
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Add BlockTable class (#11693)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 6, 2025 at 14:24 UTC 06bfb51963953d6ae31b87965bfb91b6eca4fd24
3 files changed +94 -25
vllm/v1/worker/block_table.py new
+78
@@ -0,0 +1,78 @@
1 +from typing import List
2 +
3 +import numpy as np
4 +import torch
5 +
6 +from vllm.logger import init_logger
7 +
8 +logger = init_logger(__name__)
9 +
10 +
11 +class BlockTable:
12 +
13 + def __init__(
14 + self,
15 + max_num_reqs: int,
16 + max_model_len: int,
17 + max_num_blocks_per_req: int,
18 + pin_memory: bool,
19 + device: torch.device,
20 + ):
21 + self.max_num_reqs = max_num_reqs
22 + self.max_model_len = max_model_len
23 + self.max_num_blocks_per_req = max_num_blocks_per_req
24 + self.pin_memory = pin_memory
25 + self.device = device
26 +
27 + self.block_table = torch.zeros(
28 + (max_num_reqs, max_num_blocks_per_req),
29 + device=self.device,
30 + dtype=torch.int32,
31 + )
32 + self.block_table_cpu = torch.zeros(
33 + (max_num_reqs, max_num_blocks_per_req),
34 + device="cpu",
35 + dtype=torch.int32,
36 + pin_memory=pin_memory,
37 + )
38 + self.block_table_np = self.block_table_cpu.numpy()
39 + self.num_blocks_per_row = np.zeros(max_num_reqs, dtype=np.int32)
40 +
41 + def append_row(
42 + self,
43 + row_idx: int,
44 + start: int,
45 + block_ids: List[int],
46 + ) -> None:
47 + num_blocks = len(block_ids)
48 + self.block_table_np[row_idx, start:start + num_blocks] = block_ids
49 + self.num_blocks_per_row[row_idx] = start + num_blocks
50 +
51 + def add_row(self, row_idx: int, block_ids: List[int]) -> None:
52 + self.append_row(row_idx, 0, block_ids)
53 +
54 + def move_row(self, src: int, tgt: int) -> None:
55 + num_blocks = self.num_blocks_per_row[src]
56 + self.block_table_np[tgt, :num_blocks] = self.block_table_np[
57 + src, :num_blocks]
58 + self.num_blocks_per_row[tgt] = num_blocks
59 +
60 + def commit(self, num_reqs: int) -> None:
61 + self.block_table[:num_reqs].copy_(self.block_table_cpu[:num_reqs],
62 + non_blocking=True)
63 +
64 + def clear(self) -> None:
65 + self.block_table.fill_(0)
66 + self.block_table_cpu.fill_(0)
67 +
68 + def get_device_tensor(self) -> torch.Tensor:
69 + """Ruturns the device tensor of the block table."""
70 + return self.block_table
71 +
72 + def get_cpu_tensor(self) -> torch.Tensor:
73 + """Returns the CPU tensor of the block table."""
74 + return self.block_table_cpu
75 +
76 + def get_numpy_array(self) -> np.ndarray:
77 + """Returns the numpy array of the block table."""
78 + return self.block_table_np
vllm/v1/worker/gpu_input_batch.py
+9 -16
@@ -9,6 +9,7 @@ import torch
9 from vllm.multimodal import MultiModalKwargs
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.v1.sample.metadata import SamplingMetadata
12 +from vllm.v1.worker.block_table import BlockTable
13
14 if TYPE_CHECKING:
15 from vllm.multimodal.inputs import PlaceholderRange
@@ -70,19 +71,14 @@ class InputBatch:
71 self.num_prompt_tokens = np.zeros(max_num_reqs, dtype=np.int32)
72 self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
73
73 - # Attention-related.
74 - self.block_table = torch.zeros(
75 - (max_num_reqs, max_num_blocks_per_req),
76 - device=self.device,
77 - dtype=torch.int32,
78 - )
79 - self.block_table_cpu_tensor = torch.zeros(
80 - (max_num_reqs, max_num_blocks_per_req),
81 - device="cpu",
82 - dtype=torch.int32,
74 + # Block table.
75 + self.block_table = BlockTable(
76 + max_num_reqs=max_num_reqs,
77 + max_model_len=max_model_len,
78 + max_num_blocks_per_req=max_num_blocks_per_req,
79 pin_memory=pin_memory,
80 + device=device,
81 )
85 - self.block_table_cpu = self.block_table_cpu_tensor.numpy()
82
83 # Sampling-related.
84 self.temperature = torch.empty((max_num_reqs, ),
@@ -193,8 +189,7 @@ class InputBatch:
189 self.num_tokens[req_index] = request.num_tokens
190
191 self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
196 - num_blocks = len(request.block_ids)
197 - self.block_table_cpu[req_index, :num_blocks] = request.block_ids
192 + self.block_table.add_row(req_index, request.block_ids)
193
194 sampling_params = request.sampling_params
195 self.temperature_cpu[req_index] = sampling_params.temperature
@@ -300,9 +295,7 @@ class InputBatch:
295 self.num_prompt_tokens[last_req_index]
296 self.num_computed_tokens_cpu[
297 empty_index] = self.num_computed_tokens_cpu[last_req_index]
303 - # TODO(woosuk): Optimize the copy of block_table_cpu.
304 - self.block_table_cpu[empty_index] = self.block_table_cpu[
305 - last_req_index]
298 + self.block_table.move_row(last_req_index, empty_index)
299 self.temperature_cpu[empty_index] = self.temperature_cpu[
300 last_req_index]
301 self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
vllm/v1/worker/gpu_model_runner.py
+7 -9
@@ -211,10 +211,9 @@ class GPUModelRunner:
211 if num_new_blocks == 0:
212 continue
213 start_index = len(req_state.block_ids)
214 - end_index = start_index + num_new_blocks
214 req_state.block_ids.extend(req_data.new_block_ids)
216 - self.input_batch.block_table_cpu[
217 - req_index, start_index:end_index] = req_data.new_block_ids
215 + self.input_batch.block_table.append_row(req_index, start_index,
216 + req_data.new_block_ids)
217
218 req_ids_to_add: List[str] = []
219 # Add new requests to the cached states.
@@ -275,9 +274,7 @@ class GPUModelRunner:
274
275 # OPTIMIZATION: Start copying the block table first.
276 # This way, we can overlap the copy with the following CPU operations.
278 - self.input_batch.block_table[:num_reqs].copy_(
279 - self.input_batch.block_table_cpu_tensor[:num_reqs],
280 - non_blocking=True)
277 + self.input_batch.block_table.commit(num_reqs)
278
279 # Get the number of scheduled tokens for each request.
280 # TODO: The Python loop can be slow. Optimize.
@@ -333,8 +330,8 @@ class GPUModelRunner:
330 # NOTE(woosuk): We use torch.index_select instead of np.take here
331 # because torch.index_select is much faster than np.take for large
332 # tensors.
336 - block_numbers = (self.input_batch.block_table_cpu_tensor.flatten()
337 - [block_table_indices].numpy())
333 + block_table_cpu = self.input_batch.block_table.get_cpu_tensor()
334 + block_numbers = block_table_cpu.flatten()[block_table_indices].numpy()
335 block_offsets = positions_np % self.block_size
336 np.add(block_numbers * self.block_size,
337 block_offsets,
@@ -450,7 +447,8 @@ class GPUModelRunner:
447 query_start_loc=query_start_loc,
448 max_seq_len=max_seq_len,
449 seq_start_loc=seq_start_loc,
453 - block_table=self.input_batch.block_table[:num_reqs],
450 + block_table=(
451 + self.input_batch.block_table.get_device_tensor()[:num_reqs]),
452 slot_mapping=slot_mapping,
453 use_cascade=use_cascade,
454 common_prefix_len=common_prefix_len,