A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Change scheduler & input tensor shape (#1381)
Woosuk Kwon committed
Oct 16, 2023 at 17:48 UTC
c1376e0f825e88e32b5aca85c676fe547bcb03c9
13 files changed
+181
-179
csrc/activation_kernels.cu
+14
-14
@@ -13,8 +13,8 @@ __device__ __forceinline__ T silu(const T& x) {
13
14
template<typename scalar_t>
15
__global__ void silu_and_mul_kernel(
16
- scalar_t* __restrict__ out, // [num_tokens, d]
17
- const scalar_t* __restrict__ input, // [num_tokens, 2, d]
16
+ scalar_t* __restrict__ out, // [..., d]
17
+ const scalar_t* __restrict__ input, // [..., 2, d]
18
const int d) {
19
const int token_idx = blockIdx.x;
20
for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
@@ -27,11 +27,11 @@ __global__ void silu_and_mul_kernel(
27
} // namespace vllm
28
29
void silu_and_mul(
30
- torch::Tensor& out, // [num_tokens, d]
31
- torch::Tensor& input) // [num_tokens, 2 * d]
30
+ torch::Tensor& out, // [..., d]
31
+ torch::Tensor& input) // [..., 2 * d]
32
{
33
- int num_tokens = input.size(0);
34
- int d = input.size(1) / 2;
33
+ int num_tokens = input.numel() / input.size(-1);
34
+ int d = input.size(-1) / 2;
35
36
dim3 grid(num_tokens);
37
dim3 block(std::min(d, 1024));
@@ -52,8 +52,8 @@ namespace vllm {
52
// Element-wise activation kernel template.
53
template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
54
__global__ void activation_kernel(
55
- scalar_t* __restrict__ out, // [num_tokens, d]
56
- const scalar_t* __restrict__ input, // [num_tokens, d]
55
+ scalar_t* __restrict__ out, // [..., d]
56
+ const scalar_t* __restrict__ input, // [..., d]
57
const int d) {
58
const int token_idx = blockIdx.x;
59
for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
@@ -66,8 +66,8 @@ __global__ void activation_kernel(
66
67
// Launch element-wise activation kernel.
68
#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
69
- int num_tokens = input.size(0); \
70
- int d = input.size(1); \
69
+ int d = input.size(-1); \
70
+ int num_tokens = input.numel() / d; \
71
dim3 grid(num_tokens); \
72
dim3 block(std::min(d, 1024)); \
73
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
@@ -100,15 +100,15 @@ __device__ __forceinline__ T gelu_fast_kernel(const T& x) {
100
} // namespace vllm
101
102
void gelu_new(
103
- torch::Tensor& out, // [num_tokens, d]
104
- torch::Tensor& input) // [num_tokens, d]
103
+ torch::Tensor& out, // [..., d]
104
+ torch::Tensor& input) // [..., d]
105
{
106
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_new_kernel);
107
}
108
109
void gelu_fast(
110
- torch::Tensor& out, // [num_tokens, d]
111
- torch::Tensor& input) // [num_tokens, d]
110
+ torch::Tensor& out, // [..., d]
111
+ torch::Tensor& input) // [..., d]
112
{
113
LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
114
}
csrc/cache_kernels.cu
+7
-2
@@ -154,6 +154,11 @@ __global__ void reshape_and_cache_kernel(
154
const int x) {
155
const int token_idx = blockIdx.x;
156
const int slot_idx = slot_mapping[token_idx];
157
+ if (slot_idx < 0) {
158
+ // Padding token that should be ignored.
159
+ return;
160
+ }
161
+
162
const int block_idx = slot_idx / block_size;
163
const int block_offset = slot_idx % block_size;
164
@@ -176,8 +181,8 @@ __global__ void reshape_and_cache_kernel(
181
+ head_idx * head_size * block_size
182
+ head_offset * block_size
183
+ block_offset;
179
- key_cache[tgt_key_idx] = __ldg(&key[src_key_idx]);
180
- value_cache[tgt_value_idx] = __ldg(&value[src_value_idx]);
184
+ key_cache[tgt_key_idx] = key[src_key_idx];
185
+ value_cache[tgt_value_idx] = value[src_value_idx];
186
}
187
}
188
csrc/layernorm_kernels.cu
+6
-6
@@ -9,8 +9,8 @@ namespace vllm {
9
// TODO(woosuk): Further optimize this kernel.
10
template<typename scalar_t>
11
__global__ void rms_norm_kernel(
12
- scalar_t* __restrict__ out, // [num_tokens, hidden_size]
13
- const scalar_t* __restrict__ input, // [num_tokens, hidden_size]
12
+ scalar_t* __restrict__ out, // [..., hidden_size]
13
+ const scalar_t* __restrict__ input, // [..., hidden_size]
14
const scalar_t* __restrict__ weight, // [hidden_size]
15
const float epsilon,
16
const int num_tokens,
@@ -37,12 +37,12 @@ __global__ void rms_norm_kernel(
37
} // namespace vllm
38
39
void rms_norm(
40
- torch::Tensor& out, // [num_tokens, hidden_size]
41
- torch::Tensor& input, // [num_tokens, hidden_size]
40
+ torch::Tensor& out, // [..., hidden_size]
41
+ torch::Tensor& input, // [..., hidden_size]
42
torch::Tensor& weight, // [hidden_size]
43
float epsilon) {
44
- int num_tokens = input.size(0);
45
- int hidden_size = input.size(1);
44
+ int hidden_size = input.size(-1);
45
+ int num_tokens = input.numel() / hidden_size;
46
47
dim3 grid(num_tokens);
48
dim3 block(std::min(hidden_size, 1024));
csrc/pos_encoding_kernels.cu
+11
-11
@@ -37,9 +37,9 @@ inline __device__ void apply_rotary_embedding(
37
38
template<typename scalar_t, bool IS_NEOX>
39
__global__ void rotary_embedding_kernel(
40
- const int64_t* __restrict__ positions, // [num_tokens]
41
- scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
42
- scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
40
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
41
+ scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
42
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
43
const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
44
const int rot_dim,
45
const int query_stride,
@@ -78,18 +78,18 @@ __global__ void rotary_embedding_kernel(
78
} // namespace vllm
79
80
void rotary_embedding(
81
- torch::Tensor& positions, // [num_tokens]
82
- torch::Tensor& query, // [num_tokens, num_heads * head_size]
83
- torch::Tensor& key, // [num_tokens, num_kv_heads * head_size]
81
+ torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
82
+ torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
83
+ torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
84
int head_size,
85
torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
86
bool is_neox) {
87
- int num_tokens = query.size(0);
87
+ int num_tokens = query.numel() / query.size(-1);
88
int rot_dim = cos_sin_cache.size(1);
89
- int num_heads = query.size(1) / head_size;
90
- int num_kv_heads = key.size(1) / head_size;
91
- int query_stride = query.stride(0);
92
- int key_stride = key.stride(0);
89
+ int num_heads = query.size(-1) / head_size;
90
+ int num_kv_heads = key.size(-1) / head_size;
91
+ int query_stride = query.stride(-2);
92
+ int key_stride = key.stride(-2);
93
94
dim3 grid(num_tokens);
95
dim3 block(std::min(num_heads * rot_dim / 2, 512));
vllm/config.py
+3
@@ -268,6 +268,7 @@ class SchedulerConfig:
268
iteration.
269
max_model_len: Maximum length of a sequence (including prompt
270
and generated text).
271
+ max_paddings: Maximum number of paddings to be added to a batch.
272
"""
273
274
def __init__(
@@ -275,6 +276,7 @@ class SchedulerConfig:
276
max_num_batched_tokens: Optional[int],
277
max_num_seqs: int,
278
max_model_len: int,
279
+ max_paddings: int,
280
) -> None:
281
if max_num_batched_tokens is not None:
282
self.max_num_batched_tokens = max_num_batched_tokens
@@ -284,6 +286,7 @@ class SchedulerConfig:
286
self.max_num_batched_tokens = max(max_model_len, 2048)
287
self.max_num_seqs = max_num_seqs
288
self.max_model_len = max_model_len
289
+ self.max_paddings = max_paddings
290
self._verify_args()
291
292
def _verify_args(self) -> None:
vllm/core/scheduler.py
+11
-4
@@ -131,7 +131,8 @@ class Scheduler:
131
# requests in the generation phase.
132
num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
133
for seq_group in self.running)
134
- num_batched_tokens = 0
134
+ seq_lens: List[int] = []
135
+
136
# Optimization: We do not sort the waiting queue since the preempted
137
# sequence groups are added to the front and the new sequence groups
138
# are added to the back.
@@ -157,7 +158,9 @@ class Scheduler:
158
break
159
160
# If the number of batched tokens exceeds the limit, stop.
160
- if (num_batched_tokens + num_prompt_tokens >
161
+ new_seq_lens = seq_lens + [num_prompt_tokens]
162
+ num_batched_tokens = len(new_seq_lens) * max(new_seq_lens)
163
+ if (num_batched_tokens >
164
self.scheduler_config.max_num_batched_tokens):
165
break
166
@@ -168,10 +171,14 @@ class Scheduler:
171
self.scheduler_config.max_num_seqs):
172
break
173
174
+ num_paddings = num_batched_tokens - sum(new_seq_lens)
175
+ if num_paddings > self.scheduler_config.max_paddings:
176
+ break
177
+ seq_lens = new_seq_lens
178
+
179
seq_group = self.waiting.pop(0)
180
self._allocate(seq_group)
181
self.running.append(seq_group)
174
- num_batched_tokens += num_prompt_tokens
182
num_curr_seqs += num_new_seqs
183
scheduled.append(seq_group)
184
@@ -179,7 +186,7 @@ class Scheduler:
186
scheduler_outputs = SchedulerOutputs(
187
scheduled_seq_groups=scheduled,
188
prompt_run=True,
182
- num_batched_tokens=num_batched_tokens,
189
+ num_batched_tokens=len(seq_lens) * max(seq_lens),
190
blocks_to_swap_in=blocks_to_swap_in,
191
blocks_to_swap_out=blocks_to_swap_out,
192
blocks_to_copy=blocks_to_copy,
vllm/engine/arg_utils.py
+7
-1
@@ -27,6 +27,7 @@ class EngineArgs:
27
gpu_memory_utilization: float = 0.90
28
max_num_batched_tokens: Optional[int] = None
29
max_num_seqs: int = 256
30
+ max_paddings: int = 256
31
disable_log_stats: bool = False
32
revision: Optional[str] = None
33
tokenizer_revision: Optional[str] = None
@@ -156,6 +157,10 @@ class EngineArgs:
157
type=int,
158
default=EngineArgs.max_num_seqs,
159
help='maximum number of sequences per iteration')
160
+ parser.add_argument('--max-paddings',
161
+ type=int,
162
+ default=EngineArgs.max_paddings,
163
+ help='maximum number of paddings in a batch')
164
parser.add_argument('--disable-log-stats',
165
action='store_true',
166
help='disable logging statistics')
@@ -193,7 +198,8 @@ class EngineArgs:
198
self.worker_use_ray)
199
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
200
self.max_num_seqs,
196
- model_config.max_model_len)
201
+ model_config.max_model_len,
202
+ self.max_paddings)
203
return model_config, cache_config, parallel_config, scheduler_config
204
205
vllm/model_executor/input_metadata.py
+5
-6
@@ -39,11 +39,12 @@ class InputMetadata:
39
self.max_context_len = max_context_len
40
self.block_tables = block_tables
41
42
+ self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
43
self.to_cache = None
44
if sliding_window is not None:
45
# We need to keep the positions of sliding windows within
46
# the key / value tables, this is helpful to know which
46
- # elements we need to cache and where
47
+ # elements we need to cache.
48
to_cache, start_idx = [], 0
49
for prompt_len in self.prompt_lens:
50
to_cache.extend(
@@ -51,16 +52,15 @@ class InputMetadata:
52
start_idx + max(0, prompt_len - sliding_window),
53
start_idx + prompt_len,
54
))
54
- start_idx += prompt_len
55
+ start_idx += self.max_prompt_len
56
to_cache.extend(range(start_idx, slot_mapping.shape[0]))
57
self.to_cache = torch.tensor(to_cache,
58
dtype=torch.int32,
59
device=self.slot_mapping.device)
60
61
self.num_prompts = len(prompt_lens)
61
- self.num_prompt_tokens = sum(prompt_lens)
62
+ self.num_prompt_tokens = self.num_prompts * self.max_prompt_len
63
self.num_generation_tokens = context_lens.shape[0]
63
- self.num_valid_tokens = slot_mapping.shape[0]
64
if block_tables.numel() > 0:
65
self.max_num_blocks_per_seq = block_tables.shape[1]
66
else:
@@ -69,12 +69,11 @@ class InputMetadata:
69
assert context_lens.shape[0] == self.num_generation_tokens
70
71
# Set during the execution of the first attention op.
72
- self.attn_bias: List[AttentionBias] = []
72
+ self.attn_bias: Optional[AttentionBias] = None
73
74
def __repr__(self) -> str:
75
# Print only useful metadata.
76
return (f'InputMetadata('
77
- f'num_valid_tokens={self.num_valid_tokens}, '
77
f'num_prompt_tokens={self.num_prompt_tokens}, '
78
f'num_prompts={self.num_prompts}, '
79
f'prompt_lens={self.prompt_lens}, '
vllm/model_executor/layers/activation.py
+8
-12
@@ -8,17 +8,17 @@ from vllm import activation_ops
8
class SiluAndMul(nn.Module):
9
"""An activation function for SwiGLU.
10
11
- The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
11
+ The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
12
13
Shapes:
14
- x: (num_tokens, 2 * d)
15
- return: (num_tokens, d)
14
+ x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
15
+ return: (batch_size, seq_len, d) or (num_tokens, d)
16
"""
17
18
def forward(self, x: torch.Tensor) -> torch.Tensor:
19
- num_tokens = x.shape[0]
20
- d = x.shape[1] // 2
21
- out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
19
+ d = x.shape[-1] // 2
20
+ output_shape = (x.shape[:-1] + (d, ))
21
+ out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
22
activation_ops.silu_and_mul(out, x)
23
return out
24
@@ -26,9 +26,7 @@ class SiluAndMul(nn.Module):
26
class NewGELU(nn.Module):
27
28
def forward(self, x: torch.Tensor) -> torch.Tensor:
29
- num_tokens = x.shape[0]
30
- d = x.shape[1]
31
- out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
29
+ out = torch.empty_like(x)
30
activation_ops.gelu_new(out, x)
31
return out
32
@@ -36,9 +34,7 @@ class NewGELU(nn.Module):
34
class FastGELU(nn.Module):
35
36
def forward(self, x: torch.Tensor) -> torch.Tensor:
39
- num_tokens = x.shape[0]
40
- d = x.shape[1]
41
- out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
37
+ out = torch.empty_like(x)
38
activation_ops.gelu_fast(out, x)
39
return out
40
vllm/model_executor/layers/attention.py
+71
-95
@@ -23,25 +23,9 @@ class PagedAttention(nn.Module):
23
# pylint: disable=line-too-long
24
"""GPT-style multi-head PagedAttention.
25
26
- This class takes flattened 1D query, key, and value tensors as input. The
27
- input 1D tensors can either contain prompt tokens or generation tokens, in
28
- addition to paddings.
29
-
30
- If the input tensors contain prompt tokens, the layout is as follows:
31
-
32
- |<---------------------- num_valid_tokens ---------------------->|
33
- |<--------------- num_prompt_tokens -------------->|
34
- |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--padding-->|
35
-
36
- Otherwise, the layout is as follows:
37
-
38
- |<------------------ num_valid_tokens ------------------->|
39
- |<------- num_generation_tokens (M) ------->|
40
- |<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
41
-
42
- The prompts might have different lengths, while the generation tokens always
43
- have length 1. The paddings are appended to make the input length a multiple
44
- of 8, which is desirable for Tensor Cores.
26
+ This class takes query, key, and value tensors as input. The input tensors
27
+ can either contain prompt tokens or generation tokens, in addition to
28
+ paddings.
29
30
The class does the following:
31
1. Perform multi_query_kv_attention for the prompts. This operation does
@@ -53,7 +37,7 @@ class PagedAttention(nn.Module):
37
4. Perform single_query_cached_kv_attention for the generation tokens.
38
This operation reads the previous key and value tensors from the KV
39
cache.
56
- 5. Output a flattened 1D tensor.
40
+ 5. Return the output tensor.
41
"""
42
43
def __init__(self,
@@ -85,14 +69,15 @@ class PagedAttention(nn.Module):
69
dtype: torch.dtype,
70
) -> None:
71
del dtype # Unused.
88
- if input_metadata.attn_bias:
72
+ if input_metadata.attn_bias is not None:
73
# Already set by a previous layer.
74
return
91
- prompt_lens = input_metadata.prompt_lens
75
+ prompt_lens = [input_metadata.max_prompt_len
76
+ ] * input_metadata.num_prompts
77
attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
78
if self.sliding_window is not None:
79
attn_bias = attn_bias.make_local_attention(self.sliding_window)
95
- input_metadata.attn_bias.append(attn_bias)
80
+ input_metadata.attn_bias = attn_bias
81
82
def multi_query_kv_attention(
83
self,
@@ -111,7 +96,6 @@ class PagedAttention(nn.Module):
96
value: shape = [num_prompt_tokens, num_kv_heads, head_size]
97
input_metadata: metadata for paged attention.
98
"""
114
-
99
if self.num_kv_heads != self.num_heads:
100
# Project the key and value tensors to the desired number of heads.
101
key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
@@ -124,7 +108,7 @@ class PagedAttention(nn.Module):
108
query.unsqueeze(0),
109
key.unsqueeze(0),
110
value.unsqueeze(0),
127
- attn_bias=input_metadata.attn_bias[0],
111
+ attn_bias=input_metadata.attn_bias,
112
p=0.0,
113
scale=self.scale,
114
)
@@ -232,12 +216,12 @@ class PagedAttention(nn.Module):
216
"""PagedAttention forward pass.
217
218
NOTE: The query, key, and value tensors must be sliced from a qkv
235
- tensor of shape [num_tokens, 3 * num_heads * head_size].
219
+ tensor of shape [batch_size, seq_len, 3 * num_heads * head_size].
220
221
Args:
238
- query: shape = [num_tokens, num_heads * head_size]
239
- key: shape = [num_tokens, num_kv_heads * head_size]
240
- value: shape = [num_tokens, num_kv_heads * head_size]
222
+ query: shape = [batch_size, seq_len, num_heads * head_size]
223
+ key: shape = [batch_size, seq_len, num_kv_heads * head_size]
224
+ value: shape = [batch_size, num_kv_heads * head_size]
225
key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
226
block_size, x]
227
value_cache: shape = [num_blocks, num_kv_heads, head_size,
@@ -246,9 +230,9 @@ class PagedAttention(nn.Module):
230
cache_event: event to wait for the cache operations to finish.
231
232
Returns:
249
- shape = [num_tokens, num_heads * head_size]
233
+ shape = [batch_size, seq_len, num_heads * head_size]
234
"""
251
-
235
+ batch_size, seq_len, _ = query.shape
236
# Reshape the query, key, and value tensors.
237
query = query.view(-1, self.num_heads, self.head_size)
238
key = key.view(-1, self.num_kv_heads, self.head_size)
@@ -264,10 +248,10 @@ class PagedAttention(nn.Module):
248
assert input_metadata.num_generation_tokens == 0
249
self.set_attn_bias(input_metadata, dtype=query.dtype)
250
self.multi_query_kv_attention(
267
- output[:num_prompt_tokens],
268
- query[:num_prompt_tokens],
269
- key[:num_prompt_tokens],
270
- value[:num_prompt_tokens],
251
+ output,
252
+ query,
253
+ key,
254
+ value,
255
input_metadata,
256
)
257
@@ -278,13 +262,10 @@ class PagedAttention(nn.Module):
262
# Reshape the keys and values and store them in the cache.
263
# When key_cache and value_cache are not provided, the new key
264
# and value vectors will not be cached.
281
- num_valid_tokens = input_metadata.num_valid_tokens
282
- if (num_valid_tokens > 0 and key_cache is not None
283
- and value_cache is not None):
284
- # The stride is 3 because the key and value are sliced from qkv.
285
- key_to_cache = key[:num_valid_tokens]
286
- value_to_cache = value[:num_valid_tokens]
287
- slot_mapping = input_metadata.slot_mapping
265
+ if key_cache is not None and value_cache is not None:
266
+ key_to_cache = key
267
+ value_to_cache = value
268
+ slot_mapping = input_metadata.slot_mapping.view(-1)
269
if input_metadata.to_cache is not None:
270
key_to_cache = key_to_cache[input_metadata.to_cache]
271
value_to_cache = value_to_cache[input_metadata.to_cache]
@@ -305,14 +286,14 @@ class PagedAttention(nn.Module):
286
"key_cache and value_cache must be provided when "
287
"generating tokens.")
288
# Compute the attention op for generation tokens.
308
- self.single_query_cached_kv_attention(
309
- output[num_prompt_tokens:num_valid_tokens],
310
- query[num_prompt_tokens:num_valid_tokens], key_cache,
311
- value_cache, input_metadata, self.get_alibi_slopes())
289
+ self.single_query_cached_kv_attention(output, query, key_cache,
290
+ value_cache, input_metadata,
291
+ self.get_alibi_slopes())
292
293
# Reshape the output tensor.
294
# NOTE(woosuk): The output tensor may include paddings.
315
- return output.view(-1, self.num_heads * self.head_size)
295
+ return output.view(batch_size, seq_len,
296
+ self.num_heads * self.head_size)
297
298
299
class PagedAttentionWithRoPE(PagedAttention):
@@ -368,10 +349,10 @@ class PagedAttentionWithRoPE(PagedAttention):
349
""" PagedAttention forward pass with rotary embedding.
350
351
Args:
371
- positions: shape = [num_tokens]
372
- query: shape = [num_tokens, num_heads * head_size]
373
- key: shape = [num_tokens, num_kv_heads * head_size]
374
- value: shape = [num_tokens, num_kv_heads * head_size]
352
+ positions: shape = [batch_size, seq_len]
353
+ query: shape = [batch_size, seq_len, num_heads * head_size]
354
+ key: shape = [batch_size, seq_len, num_kv_heads * head_size]
355
+ value: shape = [batch_size, seq_len, num_kv_heads * head_size]
356
key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
357
block_size, x]
358
value_cache: shape = [num_blocks, num_kv_heads, head_size,
@@ -380,7 +361,7 @@ class PagedAttentionWithRoPE(PagedAttention):
361
cache_event: event to wait for the cache operations to finish.
362
363
Returns:
383
- shape = [num_tokens, num_heads * head_size]
364
+ shape = [batch_size, seq_len, num_heads * head_size]
365
"""
366
367
# Apply rotary embedding to the query and key before passing them
@@ -414,34 +395,34 @@ class PagedAttentionWithALiBi(PagedAttention):
395
396
def set_attn_bias(self, input_metadata: InputMetadata,
397
dtype: torch.dtype) -> None:
417
- if input_metadata.attn_bias:
398
+ if input_metadata.attn_bias is not None:
399
# Already set by a previous layer.
400
return
420
- # Generates ALiBi mask for each prompt.
421
- for prompt_len in input_metadata.prompt_lens:
422
- bias = torch.arange(prompt_len, dtype=dtype)
423
- # NOTE(zhuohan): HF uses
424
- # `bias = bias[None, :].repeat(prompt_len, 1)`
425
- # here. We find that both biases give the same results, but
426
- # the bias below more accurately follows the original ALiBi
427
- # paper.
428
- bias = bias[None, :] - bias[:, None]
429
- bias = bias.to(self.alibi_slopes.device)
430
-
431
- # When using custom attention bias, xformers requires the bias to
432
- # be sliced from a tensor whose length is a multiple of 8.
433
- padded_len = (prompt_len + 7) // 8 * 8
434
- bias = torch.empty(
435
- 1, # batch_size
436
- self.num_heads,
437
- prompt_len,
438
- padded_len,
439
- device=self.alibi_slopes.device,
440
- dtype=dtype,
441
- )[:, :, :, :prompt_len].copy_(bias)
442
- bias.mul_(self.alibi_slopes[:, None, None])
443
- attn_bias = LowerTriangularMaskWithTensorBias(bias)
444
- input_metadata.attn_bias.append(attn_bias)
401
+ # Generates ALiBi mask based on the max prompt length.
402
+ max_prompt_len = input_metadata.max_prompt_len
403
+ bias = torch.arange(max_prompt_len, dtype=dtype)
404
+ # NOTE(zhuohan): HF uses
405
+ # `bias = bias[None, :].repeat(prompt_len, 1)`
406
+ # here. We find that both biases give the same results, but
407
+ # the bias below more accurately follows the original ALiBi
408
+ # paper.
409
+ bias = bias[None, :] - bias[:, None]
410
+ bias = bias.to(self.alibi_slopes.device)
411
+
412
+ # When using custom attention bias, xformers requires the bias to
413
+ # be sliced from a tensor whose length is a multiple of 8.
414
+ padded_len = (max_prompt_len + 7) // 8 * 8
415
+ bias = torch.empty(
416
+ input_metadata.num_prompts,
417
+ self.num_heads,
418
+ max_prompt_len,
419
+ padded_len,
420
+ device=self.alibi_slopes.device,
421
+ dtype=dtype,
422
+ )[:, :, :, :max_prompt_len].copy_(bias)
423
+ bias.mul_(self.alibi_slopes[:, None, None])
424
+ attn_bias = LowerTriangularMaskWithTensorBias(bias)
425
+ input_metadata.attn_bias = attn_bias
426
427
def multi_query_kv_attention(
428
self,
@@ -466,24 +447,19 @@ class PagedAttentionWithALiBi(PagedAttention):
447
value = torch.repeat_interleave(value,
448
self.num_queries_per_kv,
449
dim=1)
450
+ batch_size = input_metadata.num_prompts
451
+ seq_len = input_metadata.max_prompt_len
452
470
- # FIXME(woosuk): Because xformers does not support dynamic sequence
471
- # lengths with custom attention bias, we process each prompt one by
472
- # one. This is inefficient, especially when we have many short prompts.
473
- start = 0
474
- for i, prompt_len in enumerate(input_metadata.prompt_lens):
475
- end = start + prompt_len
476
- out = xops.memory_efficient_attention_forward(
477
- query[None, start:end],
478
- key[None, start:end],
479
- value[None, start:end],
480
- attn_bias=input_metadata.attn_bias[i],
481
- p=0.0,
482
- scale=self.scale,
483
- )
484
- # TODO(woosuk): Unnecessary copy. Optimize.
485
- output[start:end].copy_(out.squeeze(0))
486
- start += prompt_len
453
+ out = xops.memory_efficient_attention_forward(
454
+ query.view(batch_size, seq_len, self.num_heads, self.head_size),
455
+ key.view(batch_size, seq_len, self.num_heads, self.head_size),
456
+ value.view(batch_size, seq_len, self.num_heads, self.head_size),
457
+ attn_bias=input_metadata.attn_bias,
458
+ p=0.0,
459
+ scale=self.scale,
460
+ )
461
+ # TODO(woosuk): Unnecessary copy. Optimize.
462
+ output.copy_(out.view(-1, self.num_heads, self.head_size))
463
return output
464
465
def get_alibi_slopes(self) -> Optional[torch.Tensor]:
vllm/model_executor/layers/quantized_linear/awq.py
+2
-2
@@ -50,7 +50,7 @@ class AWQColumnParallelLinear(ColumnParallelLinear):
50
bias: Optional[torch.Tensor],
51
) -> torch.Tensor:
52
pack_factor = self.quant_config.pack_factor
53
- out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
53
+ out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
54
reshaped_x = x.reshape(-1, x.shape[-1])
55
out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
56
self.qzeros, pack_factor)
@@ -95,7 +95,7 @@ class AWQRowParallelLinear(RowParallelLinear):
95
96
def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
97
pack_factor = self.quant_config.pack_factor
98
- out_shape = (x.shape[-2], self.qweight.shape[-1] * pack_factor)
98
+ out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
99
reshaped_x = x.reshape(-1, x.shape[-1])
100
out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
101
self.qzeros, pack_factor)
vllm/model_executor/layers/sampler.py
+2
-1
@@ -119,7 +119,7 @@ def _prune_hidden_states(
119
selected_token_indices.extend(
120
range(start_idx, start_idx + prompt_len - 1))
121
selected_token_indices.append(start_idx + prompt_len - 1)
122
- start_idx += prompt_len
122
+ start_idx += input_metadata.max_prompt_len
123
else:
124
num_seqs = len(seq_ids)
125
selected_token_indices.extend(
@@ -129,6 +129,7 @@ def _prune_hidden_states(
129
selected_token_indices = torch.tensor(selected_token_indices,
130
dtype=torch.long,
131
device=hidden_states.device)
132
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
133
return hidden_states.index_select(0, selected_token_indices)
134
135
vllm/worker/worker.py
+34
-25
@@ -158,9 +158,9 @@ class Worker:
158
seq_group_metadata_list: List[SequenceGroupMetadata],
159
) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
160
seq_groups: List[Tuple[List[int], SamplingParams]] = []
161
- input_tokens: List[int] = []
162
- input_positions: List[int] = []
163
- slot_mapping: List[int] = []
161
+ input_tokens: List[List[int]] = []
162
+ input_positions: List[List[int]] = []
163
+ slot_mapping: List[List[int]] = []
164
165
# Add prompt tokens.
166
prompt_lens: List[int] = []
@@ -180,24 +180,25 @@ class Worker:
180
prompt_len = len(prompt_tokens)
181
prompt_lens.append(prompt_len)
182
183
- input_tokens.extend(prompt_tokens)
183
+ input_tokens.append(prompt_tokens)
184
# NOTE(woosuk): Here we assume that the first token in the prompt
185
# is always the first token in the sequence.
186
- input_positions.extend(range(len(prompt_tokens)))
186
+ input_positions.append(list(range(prompt_len)))
187
188
if seq_group_metadata.block_tables is None:
189
# During memory profiling, the block tables are not initialized
190
# yet. In this case, we just use a dummy slot mapping.
191
- slot_mapping.extend([0] * prompt_len)
191
+ slot_mapping.append([0] * prompt_len)
192
continue
193
194
# Compute the slot mapping.
195
+ slot_mapping.append([])
196
block_table = seq_group_metadata.block_tables[seq_id]
197
for i in range(prompt_len):
198
block_number = block_table[i // self.block_size]
199
block_offset = i % self.block_size
200
slot = block_number * self.block_size + block_offset
200
- slot_mapping.append(slot)
201
+ slot_mapping[-1].append(slot)
202
203
# Add generation tokens.
204
max_context_len = 0
@@ -215,13 +216,13 @@ class Worker:
216
for seq_id in seq_ids:
217
seq_data = seq_group_metadata.seq_data[seq_id]
218
generation_token = seq_data.get_last_token_id()
218
- input_tokens.append(generation_token)
219
+ input_tokens.append([generation_token])
220
221
context_len = seq_data.get_len()
222
position = context_len - 1
223
if self.sliding_window is not None:
224
context_len = min(context_len, self.sliding_window)
224
- input_positions.append(position)
225
+ input_positions.append([position])
226
227
block_table = seq_group_metadata.block_tables[seq_id]
228
@@ -233,7 +234,7 @@ class Worker:
234
block_number = block_table[position // self.block_size]
235
block_offset = position % self.block_size
236
slot = block_number * self.block_size + block_offset
236
- slot_mapping.append(slot)
237
+ slot_mapping.append([slot])
238
239
if self.sliding_window is not None:
240
sliding_window_blocks = (self.sliding_window //
@@ -241,28 +242,36 @@ class Worker:
242
block_table = block_table[-sliding_window_blocks:]
243
generation_block_tables.append(block_table)
244
244
- # Optimization: Pad the input length to be a multiple of 8.
245
- # This is required for utilizing the Tensor Cores in NVIDIA GPUs.
246
- input_tokens = _pad_to_alignment(input_tokens, multiple_of=8)
247
- input_positions = _pad_to_alignment(input_positions, multiple_of=8)
245
+ max_seq_len = max(prompt_lens) if prompt_lens else 1
246
+ padded_input_tokens = [
247
+ _pad_to_max(tokens, max_seq_len, pad=0) for tokens in input_tokens
248
+ ]
249
+ padded_input_positions = [
250
+ _pad_to_max(positions, max_seq_len, pad=0)
251
+ for positions in input_positions
252
+ ]
253
+ padded_slot_mapping = [
254
+ _pad_to_max(mapping, max_seq_len, pad=-1)
255
+ for mapping in slot_mapping
256
+ ]
257
+ padded_block_tables = [
258
+ _pad_to_max(block_table, max_num_blocks_per_seq, pad=0)
259
+ for block_table in generation_block_tables
260
+ ]
261
262
# Convert to tensors.
250
- tokens_tensor = torch.tensor(input_tokens,
263
+ tokens_tensor = torch.tensor(padded_input_tokens,
264
dtype=torch.long,
265
device="cuda")
253
- positions_tensor = torch.tensor(input_positions,
266
+ positions_tensor = torch.tensor(padded_input_positions,
267
dtype=torch.long,
268
device="cuda")
256
- slot_mapping_tensor = torch.tensor(slot_mapping,
269
+ slot_mapping_tensor = torch.tensor(padded_slot_mapping,
270
dtype=torch.int,
271
device="cuda")
272
context_lens_tensor = torch.tensor(context_lens,
273
dtype=torch.int,
274
device="cuda")
262
- padded_block_tables = [
263
- _pad_to_max(block_table, max_num_blocks_per_seq)
264
- for block_table in generation_block_tables
265
- ]
275
block_tables_tensor = torch.tensor(padded_block_tables,
276
dtype=torch.int,
277
device="cuda")
@@ -361,12 +370,12 @@ def _init_distributed_environment(
370
parallel_config.pipeline_parallel_size)
371
372
364
-def _pad_to_alignment(x: List[int], multiple_of: int) -> List[int]:
365
- return x + [0] * ((-len(x)) % multiple_of)
373
+def _pad_to_alignment(x: List[int], multiple_of: int, pad: int) -> List[int]:
374
+ return x + [pad] * ((-len(x)) % multiple_of)
375
376
368
-def _pad_to_max(x: List[int], max_len: int) -> List[int]:
369
- return x + [0] * (max_len - len(x))
377
+def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
378
+ return x + [pad] * (max_len - len(x))
379
380
381
def _check_if_can_support_max_seq_len(max_seq_len: int,