@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 999ef0b917
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Add numpy implementation of `compute_slot_mapping` (#7377)

Antoni Baum committed Aug 9, 2024 at 15:52 UTC 999ef0b917aa00166e10bc4252de0463604265b3
1 file changed +41 -12
vllm/attention/backends/utils.py
+41 -12
@@ -1,6 +1,7 @@
1 """Attention backend utils"""
2 from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
3
4 +import numpy as np
5 import torch
6
7 from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
@@ -13,6 +14,10 @@ STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
14
15 PAD_SLOT_ID = -1
16
17 +# Switch to numpy implementation of compute_slot_mapping
18 +# if we have at least this many elements. Could be tuned further.
19 +_COMPUTE_SLOT_MAPPING_NUMPY_NUMEL = 256
20 +
21 if TYPE_CHECKING:
22 from vllm.worker.model_runner import ModelInputForGPUBuilder
23
@@ -46,6 +51,29 @@ def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
51 return start_idx
52
53
54 +def _compute_slot_mapping_python(slot_mapping: List[int],
55 + block_table: List[int], range_start: int,
56 + range_end: int, block_size: int):
57 + for i in range(range_start, range_end):
58 + block_number = block_table[i // block_size]
59 + block_offset = i % block_size
60 + slot = block_number * block_size + block_offset
61 + slot_mapping.append(slot)
62 +
63 +
64 +def _compute_slot_mapping_numpy(slot_mapping: List[int],
65 + block_table: List[int], range_start: int,
66 + range_end: int, block_size: int):
67 + block_table_array = np.array(block_table)
68 + idx = np.arange(range_start, range_end)
69 + block_offset = idx % block_size
70 + idx //= block_size
71 + seq_slot_mapping_array = block_table_array[idx]
72 + seq_slot_mapping_array *= block_size
73 + seq_slot_mapping_array += block_offset
74 + slot_mapping.extend(seq_slot_mapping_array)
75 +
76 +
77 def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
78 seq_id: int, seq_len: int, context_len: int,
79 start_idx: int, block_size: int,
@@ -67,21 +95,22 @@ def compute_slot_mapping(is_profile_run: bool, slot_mapping: List[int],
95 # sliding window is 8, and block size is 4, the first two
96 # tokens are masked and the slot mapping will be
97 # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
70 - block_table = block_tables[seq_id]
98 + padding_mask_len = max(0, start_idx - context_len)
99 + slot_mapping.extend([PAD_SLOT_ID] * padding_mask_len)
100
72 - def add_slot(i):
73 - block_number = block_table[i // block_size]
74 - block_offset = i % block_size
75 - slot = block_number * block_size + block_offset
76 - slot_mapping.append(slot)
101 + range_start = max(start_idx, context_len)
102 + range_end = seq_len
103 + numel = range_end - range_start
104 + block_table = block_tables[seq_id]
105
78 - if start_idx == 0 and (seq_len - context_len) == 1:
79 - # Optimization for common-case of decoding next token
80 - add_slot(seq_len - 1)
106 + # numpy implementation will be faster than python if we have
107 + # many elements, otherwise it will be slower.
108 + if numel < _COMPUTE_SLOT_MAPPING_NUMPY_NUMEL:
109 + _compute_slot_mapping_python(slot_mapping, block_table, range_start,
110 + range_end, block_size)
111 else:
82 - slot_mapping.extend([PAD_SLOT_ID] * max(0, start_idx - context_len))
83 - for i in range(max(start_idx, context_len), seq_len):
84 - add_slot(i)
112 + _compute_slot_mapping_numpy(slot_mapping, block_table, range_start,
113 + range_end, block_size)
114
115
116 TAttentionMetadata = TypeVar("TAttentionMetadata", bound='AttentionMetadata')