A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Add numpy implementation of `compute_slot_mapping` (#7377)
Antoni Baum committed
Aug 9, 2024 at 15:52 UTC
999ef0b917aa00166e10bc4252de0463604265b3
1 file changed
+41
-12
vllm/attention/backends/utils.py
+41
-12
@@ -1,6 +1,7 @@
1
"""Attention backend utils"""
2
from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
3
4
+import numpy as np
5
import torch
6
7
from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
@@ -13,6 +14,10 @@ STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
14
15
PAD_SLOT_ID = -1
16
17
+# Switch to numpy implementation of compute_slot_mapping
18
+# if we have at least this many elements. Could be tuned further.
19
+_COMPUTE_SLOT_MAPPING_NUMPY_NUMEL = 256
20
+
21
if TYPE_CHECKING:
22
from vllm.worker.model_runner import ModelInputForGPUBuilder
23
@@ -46,6 +51,29 @@ def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
51
return start_idx
52
53
54
+def _compute_slot_mapping_python(slot_mapping: List[int],
55
+ block_table: List[int], range_start: int,
56
+ range_end: int, block_size: int):
57
+ for i in range(range_start, range_end):
58
+ block_number = block_table[i // block_size]
59
+ block_offset = i % block_size
60
+ slot = block_number * block_size + block_offset
61
+ slot_mapping.append(slot)
62
+
63
+
64
+def _compute_slot_mapping_numpy(slot_mapping: List[int],
65
+ block_table: List[int], range_start: int,
66
+ range_end: int, block_size: int):
67
+ block_table_array = np.array(block_table)
68
+ idx = np.arange(range_start, range_end)
69
+ block_offset = idx % block_size
70
+ idx //= block_size
71
+ seq_slot_mapping_array = block_table_array[idx]
72
+ seq_slot_mapping_array *= block_size
73
+ seq_slot_mapping_array += block_offset
74
+ slot_mapping.extend(seq_slot_mapping_array)
75
+
76
+
77
def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
78
seq_id: int, seq_len: int, context_len: int,
79
start_idx: int, block_size: int,
@@ -67,21 +95,22 @@ def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
95
# sliding window is 8, and block size is 4, the first two
96
# tokens are masked and the slot mapping will be
97
# [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
70
- block_table = block_tables[seq_id]
98
+ padding_mask_len = max(0, start_idx - context_len)
99
+ slot_mapping.extend([PAD_SLOT_ID] * padding_mask_len)
100
72
- def add_slot(i):
73
- block_number = block_table[i // block_size]
74
- block_offset = i % block_size
75
- slot = block_number * block_size + block_offset
76
- slot_mapping.append(slot)
101
+ range_start = max(start_idx, context_len)
102
+ range_end = seq_len
103
+ numel = range_end - range_start
104
+ block_table = block_tables[seq_id]
105
78
- if start_idx == 0 and (seq_len - context_len) == 1:
79
- # Optimization for common-case of decoding next token
80
- add_slot(seq_len - 1)
106
+ # numpy implementation will be faster than python if we have
107
+ # many elements, otherwise it will be slower.
108
+ if numel < _COMPUTE_SLOT_MAPPING_NUMPY_NUMEL:
109
+ _compute_slot_mapping_python(slot_mapping, block_table, range_start,
110
+ range_end, block_size)
111
else:
82
- slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
83
- for i in range(max(start_idx, context_len), seq_len):
84
- add_slot(i)
112
+ _compute_slot_mapping_numpy(slot_mapping, block_table, range_start,
113
+ range_end, block_size)
114
115
116
TAttentionMetadata = TypeVar("TAttentionMetadata", bound='AttentionMetadata')