A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Model runner refactoring 1/N] Refactor attn metadata term (#4518)
SangBin Cho committed
May 4, 2024 at 02:20 UTC
3521ba4f2554bcf246a95a9fb2d1b80990a6835b
27 files changed
+554
-525
benchmarks/kernels/benchmark_paged_attention.py
+12
-13
@@ -16,7 +16,7 @@ PARTITION_SIZE = 512
16
def main(
17
version: str,
18
num_seqs: int,
19
- context_len: int,
19
+ seq_len: int,
20
num_query_heads: int,
21
num_kv_heads: int,
22
head_size: int,
@@ -48,12 +48,12 @@ def main(
48
dtype=torch.float,
49
device=device)
50
51
- context_lens = [context_len for _ in range(num_seqs)]
52
- max_context_len = max(context_lens)
53
- context_lens = torch.tensor(context_lens, dtype=torch.int, device=device)
51
+ seq_lens = [seq_len for _ in range(num_seqs)]
52
+ max_seq_len = max(seq_lens)
53
+ seq_lens = torch.tensor(seq_lens, dtype=torch.int, device=device)
54
55
# Create the block tables.
56
- max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
56
+ max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
57
block_tables = []
58
for _ in range(num_seqs):
59
block_table = [
@@ -77,8 +77,7 @@ def main(
77
# Prepare for the paged attention kernel.
78
output = torch.empty_like(query)
79
if version == "v2":
80
- num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
81
- PARTITION_SIZE)
80
+ num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
81
tmp_output = torch.empty(
82
size=(num_seqs, num_query_heads, num_partitions, head_size),
83
dtype=output.dtype,
@@ -110,9 +109,9 @@ def main(
109
num_kv_heads,
110
scale,
111
block_tables,
113
- context_lens,
112
+ seq_lens,
113
block_size,
115
- max_context_len,
114
+ max_seq_len,
115
alibi_slopes,
116
kv_cache_dtype,
117
kv_scale,
@@ -129,9 +128,9 @@ def main(
128
num_kv_heads,
129
scale,
130
block_tables,
132
- context_lens,
131
+ seq_lens,
132
block_size,
134
- max_context_len,
133
+ max_seq_len,
134
alibi_slopes,
135
kv_cache_dtype,
136
kv_scale,
@@ -166,7 +165,7 @@ if __name__ == '__main__':
165
choices=["v1", "v2"],
166
default="v2")
167
parser.add_argument("--batch-size", type=int, default=8)
169
- parser.add_argument("--context-len", type=int, default=4096)
168
+ parser.add_argument("--seq_len", type=int, default=4096)
169
parser.add_argument("--num-query-heads", type=int, default=64)
170
parser.add_argument("--num-kv-heads", type=int, default=8)
171
parser.add_argument("--head-size",
@@ -199,7 +198,7 @@ if __name__ == '__main__':
198
main(
199
version=args.version,
200
num_seqs=args.batch_size,
202
- context_len=args.context_len,
201
+ seq_len=args.seq_len,
202
num_query_heads=args.num_query_heads,
203
num_kv_heads=args.num_kv_heads,
204
head_size=args.head_size,
csrc/attention/attention_kernels.cu
+38
-38
@@ -104,7 +104,7 @@ __device__ void paged_attention_kernel(
104
const int num_kv_heads, // [num_heads]
105
const float scale,
106
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
107
- const int* __restrict__ context_lens, // [num_seqs]
107
+ const int* __restrict__ seq_lens, // [num_seqs]
108
const int max_num_blocks_per_seq,
109
const float* __restrict__ alibi_slopes, // [num_heads]
110
const int q_stride,
@@ -115,23 +115,23 @@ __device__ void paged_attention_kernel(
115
const int partition_idx = blockIdx.z;
116
const int max_num_partitions = gridDim.z;
117
constexpr bool USE_PARTITIONING = PARTITION_SIZE > 0;
118
- const int context_len = context_lens[seq_idx];
119
- if (USE_PARTITIONING && partition_idx * PARTITION_SIZE >= context_len) {
118
+ const int seq_len = seq_lens[seq_idx];
119
+ if (USE_PARTITIONING && partition_idx * PARTITION_SIZE >= seq_len) {
120
// No work to do. Terminate the thread block.
121
return;
122
}
123
124
- const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE);
125
- const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_context_blocks;
124
+ const int num_seq_blocks = DIVIDE_ROUND_UP(seq_len, BLOCK_SIZE);
125
+ const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_seq_blocks;
126
127
// [start_block_idx, end_block_idx) is the range of blocks to process.
128
const int start_block_idx = USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
129
- const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_context_blocks);
129
+ const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_seq_blocks);
130
const int num_blocks = end_block_idx - start_block_idx;
131
132
// [start_token_idx, end_token_idx) is the range of tokens to process.
133
const int start_token_idx = start_block_idx * BLOCK_SIZE;
134
- const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, context_len);
134
+ const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, seq_len);
135
const int num_tokens = end_token_idx - start_token_idx;
136
137
constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
@@ -245,12 +245,12 @@ __device__ void paged_attention_kernel(
245
// This includes a reduction across the threads in the same thread group.
246
float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
247
// Add the ALiBi bias if slopes are given.
248
- qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len + 1) : 0;
248
+ qk += (alibi_slope != 0) ? alibi_slope * (token_idx - seq_len + 1) : 0;
249
250
if (thread_group_offset == 0) {
251
// Store the partial reductions to shared memory.
252
// NOTE(woosuk): It is required to zero out the masked logits.
253
- const bool mask = token_idx >= context_len;
253
+ const bool mask = token_idx >= seq_len;
254
logits[token_idx - start_token_idx] = mask ? 0.f : qk;
255
// Update the max value.
256
qk_max = mask ? qk_max : fmaxf(qk_max, qk);
@@ -364,14 +364,14 @@ __device__ void paged_attention_kernel(
364
} else {
365
v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
366
}
367
- if (block_idx == num_context_blocks - 1) {
367
+ if (block_idx == num_seq_blocks - 1) {
368
// NOTE(woosuk): When v_vec contains the tokens that are out of the context,
369
// we should explicitly zero out the values since they may contain NaNs.
370
// See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
371
scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
372
#pragma unroll
373
for (int j = 0; j < V_VEC_SIZE; j++) {
374
- v_vec_ptr[j] = token_idx + j < context_len ? v_vec_ptr[j] : zero_value;
374
+ v_vec_ptr[j] = token_idx + j < seq_len ? v_vec_ptr[j] : zero_value;
375
}
376
}
377
accs[i] += dot(logits_vec, v_vec);
@@ -457,7 +457,7 @@ __global__ void paged_attention_v1_kernel(
457
const int num_kv_heads, // [num_heads]
458
const float scale,
459
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
460
- const int* __restrict__ context_lens, // [num_seqs]
460
+ const int* __restrict__ seq_lens, // [num_seqs]
461
const int max_num_blocks_per_seq,
462
const float* __restrict__ alibi_slopes, // [num_heads]
463
const int q_stride,
@@ -466,7 +466,7 @@ __global__ void paged_attention_v1_kernel(
466
const float kv_scale) {
467
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE>(
468
/* exp_sums */ nullptr, /* max_logits */ nullptr,
469
- out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
469
+ out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, seq_lens,
470
max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
471
}
472
@@ -489,7 +489,7 @@ __global__ void paged_attention_v2_kernel(
489
const int num_kv_heads, // [num_heads]
490
const float scale,
491
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
492
- const int* __restrict__ context_lens, // [num_seqs]
492
+ const int* __restrict__ seq_lens, // [num_seqs]
493
const int max_num_blocks_per_seq,
494
const float* __restrict__ alibi_slopes, // [num_heads]
495
const int q_stride,
@@ -498,7 +498,7 @@ __global__ void paged_attention_v2_kernel(
498
const float kv_scale) {
499
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE, PARTITION_SIZE>(
500
exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
501
- block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
501
+ block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes,
502
q_stride, kv_block_stride, kv_head_stride, kv_scale);
503
}
504
@@ -513,13 +513,13 @@ __global__ void paged_attention_v2_reduce_kernel(
513
const float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
514
const float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
515
const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
516
- const int* __restrict__ context_lens, // [num_seqs]
516
+ const int* __restrict__ seq_lens, // [num_seqs]
517
const int max_num_partitions) {
518
const int num_heads = gridDim.x;
519
const int head_idx = blockIdx.x;
520
const int seq_idx = blockIdx.y;
521
- const int context_len = context_lens[seq_idx];
522
- const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE);
521
+ const int seq_len = seq_lens[seq_idx];
522
+ const int num_partitions = DIVIDE_ROUND_UP(seq_len, PARTITION_SIZE);
523
if (num_partitions == 1) {
524
// No need to reduce. Only copy tmp_out to out.
525
scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
@@ -616,7 +616,7 @@ __global__ void paged_attention_v2_reduce_kernel(
616
num_kv_heads, \
617
scale, \
618
block_tables_ptr, \
619
- context_lens_ptr, \
619
+ seq_lens_ptr, \
620
max_num_blocks_per_seq, \
621
alibi_slopes_ptr, \
622
q_stride, \
@@ -639,8 +639,8 @@ void paged_attention_v1_launcher(
639
int num_kv_heads,
640
float scale,
641
torch::Tensor& block_tables,
642
- torch::Tensor& context_lens,
643
- int max_context_len,
642
+ torch::Tensor& seq_lens,
643
+ int max_seq_len,
644
const c10::optional<torch::Tensor>& alibi_slopes,
645
float kv_scale) {
646
int num_seqs = query.size(0);
@@ -664,11 +664,11 @@ void paged_attention_v1_launcher(
664
CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
665
CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
666
int* block_tables_ptr = block_tables.data_ptr<int>();
667
- int* context_lens_ptr = context_lens.data_ptr<int>();
667
+ int* seq_lens_ptr = seq_lens.data_ptr<int>();
668
669
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
670
- int padded_max_context_len = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE) * BLOCK_SIZE;
671
- int logits_size = padded_max_context_len * sizeof(float);
670
+ int padded_max_seq_len = DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
671
+ int logits_size = padded_max_seq_len * sizeof(float);
672
int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
673
// Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
674
// Keep that in sync with the logic here!
@@ -715,8 +715,8 @@ void paged_attention_v1_launcher(
715
num_kv_heads, \
716
scale, \
717
block_tables, \
718
- context_lens, \
719
- max_context_len, \
718
+ seq_lens, \
719
+ max_seq_len, \
720
alibi_slopes, \
721
kv_scale);
722
@@ -746,9 +746,9 @@ void paged_attention_v1(
746
int num_kv_heads, // [num_heads]
747
float scale,
748
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
749
- torch::Tensor& context_lens, // [num_seqs]
749
+ torch::Tensor& seq_lens, // [num_seqs]
750
int block_size,
751
- int max_context_len,
751
+ int max_seq_len,
752
const c10::optional<torch::Tensor>& alibi_slopes,
753
const std::string& kv_cache_dtype,
754
float kv_scale) {
@@ -790,7 +790,7 @@ void paged_attention_v1(
790
num_kv_heads, \
791
scale, \
792
block_tables_ptr, \
793
- context_lens_ptr, \
793
+ seq_lens_ptr, \
794
max_num_blocks_per_seq, \
795
alibi_slopes_ptr, \
796
q_stride, \
@@ -803,7 +803,7 @@ void paged_attention_v1(
803
exp_sums_ptr, \
804
max_logits_ptr, \
805
tmp_out_ptr, \
806
- context_lens_ptr, \
806
+ seq_lens_ptr, \
807
max_num_partitions);
808
809
template<
@@ -824,8 +824,8 @@ void paged_attention_v2_launcher(
824
int num_kv_heads,
825
float scale,
826
torch::Tensor& block_tables,
827
- torch::Tensor& context_lens,
828
- int max_context_len,
827
+ torch::Tensor& seq_lens,
828
+ int max_seq_len,
829
const c10::optional<torch::Tensor>& alibi_slopes,
830
float kv_scale) {
831
int num_seqs = query.size(0);
@@ -852,10 +852,10 @@ void paged_attention_v2_launcher(
852
CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
853
CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
854
int* block_tables_ptr = block_tables.data_ptr<int>();
855
- int* context_lens_ptr = context_lens.data_ptr<int>();
855
+ int* seq_lens_ptr = seq_lens.data_ptr<int>();
856
857
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
858
- int max_num_partitions = DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE);
858
+ int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
859
int logits_size = PARTITION_SIZE * sizeof(float);
860
int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
861
@@ -909,8 +909,8 @@ void paged_attention_v2_launcher(
909
num_kv_heads, \
910
scale, \
911
block_tables, \
912
- context_lens, \
913
- max_context_len, \
912
+ seq_lens, \
913
+ max_seq_len, \
914
alibi_slopes, \
915
kv_scale);
916
@@ -943,9 +943,9 @@ void paged_attention_v2(
943
int num_kv_heads, // [num_heads]
944
float scale,
945
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
946
- torch::Tensor& context_lens, // [num_seqs]
946
+ torch::Tensor& seq_lens, // [num_seqs]
947
int block_size,
948
- int max_context_len,
948
+ int max_seq_len,
949
const c10::optional<torch::Tensor>& alibi_slopes,
950
const std::string& kv_cache_dtype,
951
float kv_scale) {
csrc/cpu/attention.cpp
+46
-46
@@ -70,11 +70,11 @@ template <typename T>
70
FORCE_INLINE std::pair<T, T>
71
reduceSoftmaxAlibi(T *data, const int size, const int capacity,
72
const float alibi_slope, const int start_index,
73
- const int context_len) {
74
- data[0] += alibi_slope * (start_index - context_len + 1);
73
+ const int seq_len) {
74
+ data[0] += alibi_slope * (start_index - seq_len + 1);
75
T max = data[0];
76
for (int i = 1; i < size; ++i) {
77
- T qk = data[i] + alibi_slope * (start_index + i - context_len + 1);
77
+ T qk = data[i] + alibi_slope * (start_index + i - seq_len + 1);
78
data[i] = qk;
79
max = max >= qk ? max : qk;
80
}
@@ -225,7 +225,7 @@ struct paged_attention_v1_impl {
225
const int num_kv_heads, const float scale,
226
const int
227
*__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
228
- const int *__restrict__ context_lens, // [num_seqs]
228
+ const int *__restrict__ seq_lens, // [num_seqs]
229
const int max_num_blocks_per_seq,
230
const float *__restrict__ alibi_slopes, // [num_heads]
231
const int q_stride, const int kv_block_stride, const int kv_head_stride,
@@ -235,32 +235,32 @@ struct paged_attention_v1_impl {
235
236
static_assert(BLOCK_SIZE == 16);
237
238
- int max_context_len = max_num_blocks_per_seq * BLOCK_SIZE;
239
- int max_context_len_padded = (max_context_len + 15) & 0xFFFFFFF0;
240
- TORCH_CHECK((max_context_len_padded * sizeof(float)) % 64 == 0);
238
+ int max_seq_len = max_num_blocks_per_seq * BLOCK_SIZE;
239
+ int max_seq_len_padded = (max_seq_len + 15) & 0xFFFFFFF0;
240
+ TORCH_CHECK((max_seq_len_padded * sizeof(float)) % 64 == 0);
241
242
const int parallel_work_item_num = omp_get_max_threads();
243
244
size_t logits_bytes =
245
- parallel_work_item_num * max_context_len_padded * sizeof(float);
245
+ parallel_work_item_num * max_seq_len_padded * sizeof(float);
246
float *logits = (float *)std::aligned_alloc(
247
64, logits_bytes); // Cacheline alignment for each context token.
248
- // [parallel_work_item_num, max_context_len_padded]
248
+ // [parallel_work_item_num, max_seq_len_padded]
249
250
#pragma omp parallel for collapse(2) schedule(dynamic, 1)
251
for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
252
for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
253
- int context_len = context_lens[seq_idx];
253
+ int seq_len = seq_lens[seq_idx];
254
const int *seq_block_table =
255
block_tables + max_num_blocks_per_seq * seq_idx;
256
- const int block_num = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
256
+ const int block_num = (seq_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
257
const int64_t kv_head_idx = head_idx / num_queries_per_kv;
258
const scalar_t *__restrict__ q_vec_ptr =
259
q + seq_idx * q_stride + head_idx * HEAD_SIZE;
260
const int last_block_token_num =
261
- context_len - (block_num - 1) * BLOCK_SIZE;
261
+ seq_len - (block_num - 1) * BLOCK_SIZE;
262
float *__restrict__ thread_block_logits =
263
- logits + omp_get_thread_num() * max_context_len_padded;
263
+ logits + omp_get_thread_num() * max_seq_len_padded;
264
265
// Compute logits
266
for (int block_idx = 0; block_idx < block_num; ++block_idx) {
@@ -278,11 +278,11 @@ struct paged_attention_v1_impl {
278
279
// Compute softmax
280
if (alibi_slopes) {
281
- reduceSoftmaxAlibi(thread_block_logits, context_len,
281
+ reduceSoftmaxAlibi(thread_block_logits, seq_len,
282
block_num * BLOCK_SIZE, alibi_slopes[head_idx], 0,
283
- context_len);
283
+ seq_len);
284
} else {
285
- reduceSoftmax(thread_block_logits, context_len,
285
+ reduceSoftmax(thread_block_logits, seq_len,
286
block_num * BLOCK_SIZE);
287
}
288
@@ -340,7 +340,7 @@ struct paged_attention_v1_impl {
340
#define LAUNCH_V1_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
341
paged_attention_v1_impl<T, HEAD_SIZE, BLOCK_SIZE>::call( \
342
out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
343
- block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
343
+ block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
344
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, num_seqs, \
345
num_heads);
346
@@ -348,8 +348,8 @@ template <typename T, int BLOCK_SIZE>
348
void paged_attention_v1_impl_launcher(
349
torch::Tensor &out, torch::Tensor &query, torch::Tensor &key_cache,
350
torch::Tensor &value_cache, int num_kv_heads, float scale,
351
- torch::Tensor &block_tables, torch::Tensor &context_lens,
352
- int max_context_len, const c10::optional<torch::Tensor> &alibi_slopes) {
351
+ torch::Tensor &block_tables, torch::Tensor &seq_lens,
352
+ int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
353
int num_seqs = query.size(0);
354
int num_heads = query.size(1);
355
int head_size = query.size(2);
@@ -369,7 +369,7 @@ void paged_attention_v1_impl_launcher(
369
T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
370
T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
371
int *block_tables_ptr = block_tables.data_ptr<int>();
372
- int *context_lens_ptr = context_lens.data_ptr<int>();
372
+ int *seq_lens_ptr = seq_lens.data_ptr<int>();
373
374
switch (head_size) {
375
case 64:
@@ -399,7 +399,7 @@ void paged_attention_v1_impl_launcher(
399
#define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
400
paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
401
out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
402
- context_lens, max_context_len, alibi_slopes);
402
+ seq_lens, max_seq_len, alibi_slopes);
403
404
#define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
405
switch (block_size) { \
@@ -416,8 +416,8 @@ void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
416
torch::Tensor &key_cache, torch::Tensor &value_cache,
417
int num_kv_heads, float scale,
418
torch::Tensor &block_tables,
419
- torch::Tensor &context_lens, int block_size,
420
- int max_context_len,
419
+ torch::Tensor &seq_lens, int block_size,
420
+ int max_seq_len,
421
const c10::optional<torch::Tensor> &alibi_slopes,
422
const std::string &kv_cache_dtype, float kv_scale) {
423
TORCH_CHECK(kv_scale == 1.0f);
@@ -448,7 +448,7 @@ struct paged_attention_v2_impl {
448
const int num_kv_heads, const float scale,
449
const int
450
*__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
451
- const int *__restrict__ context_lens, // [num_seqs]
451
+ const int *__restrict__ seq_lens, // [num_seqs]
452
const int max_num_blocks_per_seq,
453
const float *__restrict__ alibi_slopes, // [num_heads]
454
const int q_stride, const int kv_block_stride, const int kv_head_stride,
@@ -465,22 +465,22 @@ struct paged_attention_v2_impl {
465
for (int partition_idx = 0; partition_idx < max_num_partitions;
466
++partition_idx) {
467
for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
468
- const int context_len = context_lens[seq_idx];
468
+ const int seq_len = seq_lens[seq_idx];
469
const int start_token_idx = partition_idx * PARTITION_SIZE;
470
471
- if (start_token_idx >= context_len)
471
+ if (start_token_idx >= seq_len)
472
continue;
473
474
const int partition_num =
475
- (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
475
+ (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
476
const bool no_reduce = (partition_num == 1);
477
- const int context_token_num =
478
- (std::min(context_len, start_token_idx + PARTITION_SIZE) -
477
+ const int token_num =
478
+ (std::min(seq_len, start_token_idx + PARTITION_SIZE) -
479
start_token_idx);
480
const int block_num =
481
- (context_token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
481
+ (token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
482
const int last_block_token_num =
483
- context_token_num - (block_num - 1) * BLOCK_SIZE;
483
+ token_num - (block_num - 1) * BLOCK_SIZE;
484
const int *seq_block_table = block_tables +
485
max_num_blocks_per_seq * seq_idx +
486
start_token_idx / BLOCK_SIZE;
@@ -507,10 +507,10 @@ struct paged_attention_v2_impl {
507
std::pair<float, float> max_and_sum;
508
if (alibi_slopes) {
509
max_and_sum = reduceSoftmaxAlibi(
510
- logits, context_token_num, block_num * BLOCK_SIZE,
511
- alibi_slopes[head_idx], start_token_idx, context_len);
510
+ logits, token_num, block_num * BLOCK_SIZE,
511
+ alibi_slopes[head_idx], start_token_idx, seq_len);
512
} else {
513
- max_and_sum = reduceSoftmax(logits, context_token_num,
513
+ max_and_sum = reduceSoftmax(logits, token_num,
514
block_num * BLOCK_SIZE);
515
}
516
@@ -583,9 +583,9 @@ struct paged_attention_v2_impl {
583
#pragma omp parallel for collapse(2) schedule(static, 1)
584
for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
585
for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
586
- const int context_len = context_lens[seq_idx];
586
+ const int seq_len = seq_lens[seq_idx];
587
const int partition_num =
588
- (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
588
+ (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
589
590
if (partition_num == 1)
591
continue;
@@ -612,9 +612,9 @@ struct paged_attention_v2_impl {
612
for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
613
for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
614
for (int group_idx = 0; group_idx < head_group_num; ++group_idx) {
615
- const int context_len = context_lens[seq_idx];
615
+ const int seq_len = seq_lens[seq_idx];
616
const int partition_num =
617
- (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
617
+ (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
618
619
if (partition_num == 1)
620
continue;
@@ -649,7 +649,7 @@ struct paged_attention_v2_impl {
649
paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
650
out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
651
key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
652
- context_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
652
+ seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
653
kv_block_stride, kv_head_stride, num_seqs, num_heads, \
654
max_num_partitions);
655
@@ -658,8 +658,8 @@ void paged_attention_v2_impl_launcher(
658
torch::Tensor &out, torch::Tensor &exp_sums, torch::Tensor &max_logits,
659
torch::Tensor &tmp_out, torch::Tensor &query, torch::Tensor &key_cache,
660
torch::Tensor &value_cache, int num_kv_heads, float scale,
661
- torch::Tensor &block_tables, torch::Tensor &context_lens, int block_size,
662
- int max_context_len, const c10::optional<torch::Tensor> &alibi_slopes) {
661
+ torch::Tensor &block_tables, torch::Tensor &seq_lens, int block_size,
662
+ int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
663
int num_seqs = query.size(0);
664
int num_heads = query.size(1);
665
int head_size = query.size(2);
@@ -683,7 +683,7 @@ void paged_attention_v2_impl_launcher(
683
T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
684
T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
685
int *block_tables_ptr = block_tables.data_ptr<int>();
686
- int *context_lens_ptr = context_lens.data_ptr<int>();
686
+ int *seq_lens_ptr = seq_lens.data_ptr<int>();
687
688
switch (head_size) {
689
case 64:
@@ -713,8 +713,8 @@ void paged_attention_v2_impl_launcher(
713
#define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
714
paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
715
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
716
- num_kv_heads, scale, block_tables, context_lens, block_size, \
717
- max_context_len, alibi_slopes);
716
+ num_kv_heads, scale, block_tables, seq_lens, block_size, \
717
+ max_seq_len, alibi_slopes);
718
719
#define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
720
switch (block_size) { \
@@ -732,8 +732,8 @@ void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
732
torch::Tensor &query, torch::Tensor &key_cache,
733
torch::Tensor &value_cache, int num_kv_heads,
734
float scale, torch::Tensor &block_tables,
735
- torch::Tensor &context_lens, int block_size,
736
- int max_context_len,
735
+ torch::Tensor &seq_lens, int block_size,
736
+ int max_seq_len,
737
const c10::optional<torch::Tensor> &alibi_slopes,
738
const std::string &kv_cache_dtype, float kv_scale) {
739
TORCH_CHECK(kv_scale == 1.0f);
csrc/ops.h
+4
-4
@@ -10,9 +10,9 @@ void paged_attention_v1(
10
int num_kv_heads,
11
float scale,
12
torch::Tensor& block_tables,
13
- torch::Tensor& context_lens,
13
+ torch::Tensor& seq_lens,
14
int block_size,
15
- int max_context_len,
15
+ int max_seq_len,
16
const c10::optional<torch::Tensor>& alibi_slopes,
17
const std::string& kv_cache_dtype,
18
float kv_scale);
@@ -28,9 +28,9 @@ void paged_attention_v2(
28
int num_kv_heads,
29
float scale,
30
torch::Tensor& block_tables,
31
- torch::Tensor& context_lens,
31
+ torch::Tensor& seq_lens,
32
int block_size,
33
- int max_context_len,
33
+ int max_seq_len,
34
const c10::optional<torch::Tensor>& alibi_slopes,
35
const std::string& kv_cache_dtype,
36
float kv_scale);
tests/kernels/test_attention.py
+17
-18
@@ -61,7 +61,7 @@ def ref_single_query_cached_kv_attention(
61
key_cache: torch.Tensor,
62
value_cache: torch.Tensor,
63
block_tables: torch.Tensor,
64
- context_lens: torch.Tensor,
64
+ seq_lens: torch.Tensor,
65
scale: float,
66
alibi_slopes: Optional[torch.Tensor],
67
) -> None:
@@ -72,15 +72,15 @@ def ref_single_query_cached_kv_attention(
72
num_seqs = query.shape[0]
73
74
block_tables = block_tables.cpu().tolist()
75
- context_lens = context_lens.cpu().tolist()
75
+ seq_lens = seq_lens.cpu().tolist()
76
for i in range(num_seqs):
77
q = query[i].unsqueeze(0)
78
block_table = block_tables[i]
79
- context_len = int(context_lens[i])
79
+ seq_len = int(seq_lens[i])
80
81
keys = []
82
values = []
83
- for j in range(context_len):
83
+ for j in range(seq_len):
84
block_number = int(block_table[j // block_size])
85
block_offset = j % block_size
86
@@ -100,8 +100,8 @@ def ref_single_query_cached_kv_attention(
100
alibi_bias = None
101
if alibi_slopes is not None:
102
# Create the ALiBi bias used in the paged attention kernel.
103
- position_ids = torch.arange(context_len).int()
104
- alibi_bias = (position_ids - context_len + 1).float()
103
+ position_ids = torch.arange(seq_len).int()
104
+ alibi_bias = (position_ids - seq_len + 1).float()
105
alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
106
1, 1, -1)
107
@@ -149,13 +149,13 @@ def test_paged_attention(
149
if use_alibi:
150
alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
151
152
- context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
153
- context_lens[-1] = MAX_SEQ_LEN
154
- max_context_len = max(context_lens)
155
- context_lens = torch.tensor(context_lens, dtype=torch.int)
152
+ seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
153
+ seq_lens[-1] = MAX_SEQ_LEN
154
+ max_seq_len = max(seq_lens)
155
+ seq_lens = torch.tensor(seq_lens, dtype=torch.int)
156
157
# Create the block tables.
158
- max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
158
+ max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
159
block_tables = []
160
for _ in range(num_seqs):
161
block_table = [
@@ -186,16 +186,15 @@ def test_paged_attention(
186
num_kv_heads,
187
scale,
188
block_tables,
189
- context_lens,
189
+ seq_lens,
190
block_size,
191
- max_context_len,
191
+ max_seq_len,
192
alibi_slopes,
193
kv_cache_dtype,
194
kv_scale,
195
)
196
elif version == "v2":
197
- num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
198
- PARTITION_SIZE)
197
+ num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
198
assert PARTITION_SIZE % block_size == 0
199
num_seqs, num_heads, head_size = output.shape
200
tmp_output = torch.empty(
@@ -218,9 +217,9 @@ def test_paged_attention(
217
num_kv_heads,
218
scale,
219
block_tables,
221
- context_lens,
220
+ seq_lens,
221
block_size,
223
- max_context_len,
222
+ max_seq_len,
223
alibi_slopes,
224
kv_cache_dtype,
225
kv_scale,
@@ -255,7 +254,7 @@ def test_paged_attention(
254
key_cache,
255
value_cache,
256
block_tables,
258
- context_lens,
257
+ seq_lens,
258
scale,
259
alibi_slopes,
260
)
tests/kernels/test_prefix_prefill.py
+8
-8
@@ -51,12 +51,12 @@ def test_contexted_kv_attention(
51
cache_size = 640
52
block_size = 32
53
max_block_per_request = 64
54
- subquery_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
54
+ query_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
55
ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
56
- seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
56
+ seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
57
num_kv_heads = num_heads // num_queries_per_kv
58
59
- num_tokens = sum(subquery_lens)
59
+ num_tokens = sum(query_lens)
60
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
61
query.uniform_(-1e-3, 1e-3)
62
output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
@@ -75,15 +75,15 @@ def test_contexted_kv_attention(
75
num_kv_heads,
76
head_size,
77
dtype=dtype)
78
- k = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
79
- v = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
78
+ k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
79
+ v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
80
values = torch.arange(0, cache_size, dtype=torch.long)
81
values = values[torch.randperm(cache_size)]
82
block_table = values[:BS * max_block_per_request].view(
83
BS, max_block_per_request)
84
b_seq_len = torch.tensor(seq_lens, dtype=torch.long)
85
b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
86
- b_start_loc = torch.cumsum(torch.tensor([0] + subquery_lens[:-1],
86
+ b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
87
dtype=torch.long),
88
dim=0)
89
max_input_len = MAX_SEQ_LEN
@@ -92,7 +92,7 @@ def test_contexted_kv_attention(
92
dtype=torch.long),
93
dim=0)
94
for i in range(BS):
95
- for j in range(subquery_lens[i]):
95
+ for j in range(query_lens[i]):
96
k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
97
j])
98
v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
@@ -178,7 +178,7 @@ def test_contexted_kv_attention(
178
value = value.unsqueeze(0)
179
180
attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
181
- subquery_lens, seq_lens)
181
+ query_lens, seq_lens)
182
if sliding_window > 0:
183
attn_bias = attn_bias.make_local_attention_from_bottomright(
184
sliding_window)
tests/samplers/test_sampler.py
+17
-17
@@ -58,7 +58,7 @@ def _do_sample(
58
device: str,
59
):
60
seq_group_metadata_list = []
61
- prompt_lens = []
61
+ seq_lens = []
62
for i in range(batch_size):
63
seq_group_metadata_list.append(
64
SequenceGroupMetadata(
@@ -68,12 +68,12 @@ def _do_sample(
68
sampling_params=sampling_params,
69
block_tables={0: [1]},
70
))
71
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
71
+ seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
72
73
sampling_metadata = SamplingMetadata.prepare(
74
seq_group_metadata_list,
75
- prompt_lens,
76
- subquery_lens=prompt_lens,
75
+ seq_lens,
76
+ query_lens=seq_lens,
77
device=device,
78
pin_memory=model_runner.pin_memory)
79
return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
@@ -421,7 +421,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
421
"Invalid test case, need seq_group_metadata_list"
422
423
batch_size = 0
424
- prompt_lens = []
424
+ seq_lens = []
425
sampling_params_per_row = []
426
for sgm in seq_group_metadata_list:
427
sampling_params = sgm.sampling_params
@@ -431,7 +431,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
431
# a prompt seq_group has only one sequence
432
seq_data = next(iter(sgm.seq_data.values()))
433
prompt_len = seq_data.get_prompt_len()
434
- prompt_lens.append(prompt_len)
434
+ seq_lens.append(prompt_len)
435
436
if sgm.sampling_params.prompt_logprobs:
437
# with prompt_logprobs each token in the prompt has a row in
@@ -451,8 +451,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
451
_, fake_logits, sampler, model_runner = _prepare_test(batch_size)
452
sampling_metadata = SamplingMetadata.prepare(
453
seq_group_metadata_list,
454
- prompt_lens=prompt_lens if prompt_lens else None,
455
- subquery_lens=prompt_lens if prompt_lens else None,
454
+ seq_lens=seq_lens if seq_lens else None,
455
+ query_lens=seq_lens if seq_lens else None,
456
device=device,
457
pin_memory=model_runner.pin_memory)
458
# the logits tensor is modified in-place by the sampler
@@ -497,7 +497,7 @@ def test_sampler_mixed(seed: int, device: str):
497
498
seq_group_metadata_list = []
499
expected_tokens: List[Optional[List[int]]] = []
500
- prompt_lens = []
500
+ seq_lens = []
501
for i in range(batch_size):
502
expected: Optional[List[int]] = None
503
sampling_type = random.randint(0, 3)
@@ -532,13 +532,13 @@ def test_sampler_mixed(seed: int, device: str):
532
sampling_params=sampling_params,
533
block_tables={0: [1]},
534
))
535
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
535
+ seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
536
537
def test_sampling(model_runner: ModelRunner):
538
sampling_metadata = SamplingMetadata.prepare(
539
seq_group_metadata_list,
540
- prompt_lens,
541
- subquery_lens=prompt_lens,
540
+ seq_lens,
541
+ query_lens=seq_lens,
542
device=device,
543
pin_memory=model_runner.pin_memory)
544
sampler_output = sampler(logits=fake_logits,
@@ -575,7 +575,7 @@ def test_sampler_mixed(seed: int, device: str):
575
# Shuffle the batch and resample
576
target_index = list(range(batch_size))
577
for list_to_shuffle in (target_index, seq_group_metadata_list,
578
- expected_tokens, prompt_lens):
578
+ expected_tokens, seq_lens):
579
random.Random(seed).shuffle(list_to_shuffle)
580
target_index = torch.tensor(target_index)
581
input_tensor.data = input_tensor.index_select(0, target_index)
@@ -620,7 +620,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
620
assert len(warpers) == 2 # top_p and top_k
621
622
seq_group_metadata_list = []
623
- prompt_lens = []
623
+ seq_lens = []
624
for i in range(batch_size):
625
seq_group_metadata_list.append(
626
SequenceGroupMetadata(
@@ -634,12 +634,12 @@ def test_sampler_top_k_top_p(seed: int, device: str):
634
),
635
block_tables={0: [1]},
636
))
637
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
637
+ seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
638
639
sampling_metadata = SamplingMetadata.prepare(
640
seq_group_metadata_list,
641
- prompt_lens,
642
- subquery_lens=prompt_lens,
641
+ seq_lens,
642
+ query_lens=seq_lens,
643
device=device,
644
pin_memory=model_runner.pin_memory)
645
tests/spec_decode/e2e/conftest.py
+2
-2
@@ -45,7 +45,7 @@ class AsyncLLM:
45
gpu_memory_utilization: float = 0.9,
46
swap_space: int = 4,
47
enforce_eager: bool = False,
48
- max_context_len_to_capture: int = 8192,
48
+ max_seq_len_to_capture: int = 8192,
49
disable_custom_all_reduce: bool = False,
50
**kwargs,
51
) -> None:
@@ -66,7 +66,7 @@ class AsyncLLM:
66
gpu_memory_utilization=gpu_memory_utilization,
67
swap_space=swap_space,
68
enforce_eager=enforce_eager,
69
- max_context_len_to_capture=max_context_len_to_capture,
69
+ max_seq_len_to_capture=max_seq_len_to_capture,
70
engine_use_ray=True,
71
disable_custom_all_reduce=disable_custom_all_reduce,
72
**kwargs,
tests/spec_decode/test_multi_step_worker.py
+14
-10
@@ -34,7 +34,7 @@ def test_assert_enough_kv_space(num_steps: int):
34
list(range(block_size * 2)),
35
]
36
37
- final_seq_lens = [
37
+ final_prompt_lens = [
38
len(prompt + output) + num_steps
39
for prompt, output in zip(prompts, prev_output_tokens)
40
]
@@ -43,7 +43,7 @@ def test_assert_enough_kv_space(num_steps: int):
43
prompts,
44
num_gpu_blocks,
45
block_size,
46
- final_seq_lens,
46
+ final_prompt_lens,
47
continuations=prev_output_tokens)
48
49
assert_enough_kv_space = MultiStepWorker._assert_enough_kv_space # pylint: disable=protected-access
@@ -103,17 +103,21 @@ def test_same_output_for_single_step():
103
[6, 7, 8, 9, 10],
104
]
105
106
- final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
106
+ final_prompt_lens = [len(prompt) + num_steps for prompt in prompts]
107
108
multi_step_execute_model_data = create_execute_model_data(
109
seq_group_metadata_list=create_seq_group_metadata_from_prompts(
110
- prompts, num_gpu_blocks, block_size,
111
- final_seq_lens=final_seq_lens))
110
+ prompts,
111
+ num_gpu_blocks,
112
+ block_size,
113
+ final_prompt_lens=final_prompt_lens))
114
115
single_step_execute_model_data = create_execute_model_data(
116
seq_group_metadata_list=create_seq_group_metadata_from_prompts(
115
- prompts, num_gpu_blocks, block_size,
116
- final_seq_lens=final_seq_lens))
117
+ prompts,
118
+ num_gpu_blocks,
119
+ block_size,
120
+ final_prompt_lens=final_prompt_lens))
121
122
zero_kv_cache(multi_step_worker.cache_engine)
123
set_random_seed(seed)
@@ -181,7 +185,7 @@ def test_same_output_for_multi_step():
185
random.randint(0, 1000) for _ in range(random.randint(10, 20))
186
] for _ in range(10)]
187
184
- final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
188
+ final_prompt_lens = [len(prompt) + num_steps for prompt in prompts]
189
190
rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
191
multi_step_worker.execute_model = patch_execute_model_with_seeds(
@@ -195,7 +199,7 @@ def test_same_output_for_multi_step():
199
num_gpu_blocks,
200
block_size,
201
continuations=continuations,
198
- final_seq_lens=final_seq_lens), )
202
+ final_prompt_lens=final_prompt_lens), )
203
204
# Run multi-step.
205
zero_kv_cache(multi_step_worker.cache_engine)
@@ -217,7 +221,7 @@ def test_same_output_for_multi_step():
221
num_gpu_blocks,
222
block_size,
223
continuations=continuations,
220
- final_seq_lens=final_seq_lens))
224
+ final_prompt_lens=final_prompt_lens))
225
226
single_step_output.extend(
227
worker.execute_model(**execute_model_data.to_dict(), ))
tests/spec_decode/test_ngram_worker.py
+15
-9
@@ -43,11 +43,13 @@ def test_ngram_algo_correctness_for_single_no_match():
43
]
44
45
proposal_len = 5
46
- final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
46
+ final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
47
ngram_sampler_output_data = create_execute_model_data(
48
seq_group_metadata_list=create_seq_group_metadata_from_prompts(
49
- prompts, num_gpu_blocks, block_size,
50
- final_seq_lens=final_seq_lens))
49
+ prompts,
50
+ num_gpu_blocks,
51
+ block_size,
52
+ final_prompt_lens=final_prompt_lens))
53
54
proposals = proposer.get_proposals(
55
**ngram_sampler_output_data.to_dict(),
@@ -110,11 +112,13 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
112
]
113
114
proposal_len = 5
113
- final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
115
+ final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
116
ngram_sampler_output_data = create_execute_model_data(
117
seq_group_metadata_list=create_seq_group_metadata_from_prompts(
116
- prompts, num_gpu_blocks, block_size,
117
- final_seq_lens=final_seq_lens))
118
+ prompts,
119
+ num_gpu_blocks,
120
+ block_size,
121
+ final_prompt_lens=final_prompt_lens))
122
123
proposals = proposer.get_proposals(
124
**ngram_sampler_output_data.to_dict(),
@@ -180,11 +184,13 @@ def test_ngram_algo_correctness_for_batches_match_all():
184
]
185
186
proposal_len = 5
183
- final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
187
+ final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
188
ngram_sampler_output_data = create_execute_model_data(
189
seq_group_metadata_list=create_seq_group_metadata_from_prompts(
186
- prompts, num_gpu_blocks, block_size,
187
- final_seq_lens=final_seq_lens))
190
+ prompts,
191
+ num_gpu_blocks,
192
+ block_size,
193
+ final_prompt_lens=final_prompt_lens))
194
195
proposals = proposer.get_proposals(
196
**ngram_sampler_output_data.to_dict(),
tests/spec_decode/utils.py
+4
-4
@@ -144,7 +144,7 @@ def create_seq_group_metadata_from_prompts(
144
prompts: List[List[int]],
145
num_gpu_blocks: int,
146
block_size: int,
147
- final_seq_lens: List[int],
147
+ final_prompt_lens: List[int],
148
continuations: Optional[List[List[int]]] = None,
149
seq_ids: Optional[List[int]] = None,
150
) -> List[SequenceGroupMetadata]:
@@ -162,7 +162,7 @@ def create_seq_group_metadata_from_prompts(
162
free_gpu_blocks.pop()
163
for _ in range(round_up_to_next_block(final_len, block_size))
164
]
165
- for i, final_len in enumerate(final_seq_lens)
165
+ for i, final_len in enumerate(final_prompt_lens)
166
}
167
168
return [
@@ -251,13 +251,13 @@ def create_batch(batch_size,
251
prev_output_tokens = [[
252
next(iterator) for _ in range(prev_output_token_len)
253
] for _ in range(batch_size)]
254
- final_seq_lens = [
254
+ final_prompt_lens = [
255
len(prompt) + len(prev_output_token) + k + 1
256
for prompt, prev_output_token in zip(prompts, prev_output_tokens)
257
]
258
259
execute_model_data = create_execute_model_data(
260
create_seq_group_metadata_from_prompts(prompts, num_gpu_blocks,
261
- block_size, final_seq_lens,
261
+ block_size, final_prompt_lens,
262
prev_output_tokens, seq_ids), )
263
return execute_model_data, prompts, prev_output_tokens
tests/test_logits_processor.py
+4
-4
@@ -70,7 +70,7 @@ def test_logits_processors(seed: int, device: str):
70
return logits
71
72
seq_group_metadata_list = []
73
- prompt_lens = []
73
+ seq_lens = []
74
for i in range(batch_size):
75
seq_group_metadata_list.append(
76
SequenceGroupMetadata(
@@ -81,12 +81,12 @@ def test_logits_processors(seed: int, device: str):
81
logits_processors=[pick_ith]),
82
block_tables={0: [1]},
83
))
84
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
84
+ seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
85
86
sampling_metadata = SamplingMetadata.prepare(
87
seq_group_metadata_list,
88
- prompt_lens,
89
- subquery_lens=prompt_lens,
88
+ seq_lens,
89
+ query_lens=seq_lens,
90
device=model_runner.device,
91
pin_memory=model_runner.pin_memory)
92
logits_processor_output = logits_processor(
tests/worker/test_model_runner.py
+48
-51
@@ -23,14 +23,14 @@ def test_prepare_prompt(batch_size):
23
lora_config=None)
24
model_runner.set_block_size(16)
25
26
- prompt_lens = []
26
+ seq_lens = []
27
seq_group_metadata_list = []
28
block_tables = {0: [1]}
29
for i in range(batch_size):
30
# make sure all tokens fit into one block
31
- prompt_len = i % (model_runner.block_size - 1) + 1
32
- prompt_lens.append(prompt_len)
33
- seq_data = SequenceData(list(range(prompt_len)))
31
+ seq_len = i % (model_runner.block_size - 1) + 1
32
+ seq_lens.append(seq_len)
33
+ seq_data = SequenceData(list(range(seq_len)))
34
seq_group_metadata = SequenceGroupMetadata(
35
request_id=f"test_{i}",
36
is_prompt=True,
@@ -43,29 +43,29 @@ def test_prepare_prompt(batch_size):
43
44
expected_selected_token_indices = []
45
selected_token_start_idx = 0
46
- for prompt_len in prompt_lens:
46
+ for seq_len in seq_lens:
47
expected_selected_token_indices.append(selected_token_start_idx +
48
- prompt_len - 1)
49
- selected_token_start_idx += prompt_len
50
- (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
51
- _, _,
52
- slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
53
- assert return_prompt_lens == prompt_lens
48
+ seq_len - 1)
49
+ selected_token_start_idx += seq_len
50
+ (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
51
+ _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
52
+ assert return_seq_lens == seq_lens
53
assert len(slot_mapping) == len(input_tokens)
54
55
# Verify input metadata is correct for prompts.
56
device = model_runner.device
57
assert attn_metadata.is_prompt is True
59
- assert torch.allclose(attn_metadata.prompt_lens_tensor,
60
- torch.tensor(prompt_lens, device=device))
61
- assert attn_metadata.prompt_lens == prompt_lens
62
- assert attn_metadata.max_prompt_len == max(prompt_lens)
58
+ assert torch.allclose(
59
+ attn_metadata.seq_lens_tensor,
60
+ torch.tensor(seq_lens, device=device, dtype=torch.int))
61
+ assert attn_metadata.seq_lens == seq_lens
62
+ assert attn_metadata.max_seq_len == max(seq_lens)
63
64
# Test subquery start locs.
65
start_idx = 0
66
start_loc = [start_idx]
67
- for prompt_len in prompt_lens:
68
- start_idx += prompt_len
67
+ for seq_len in seq_lens:
68
+ start_idx += seq_len
69
start_loc.append(start_idx)
70
assert torch.allclose(
71
attn_metadata.subquery_start_loc,
@@ -75,17 +75,16 @@ def test_prepare_prompt(batch_size):
75
# equivalent to subquery_start_loc.
76
start_idx = 0
77
seq_start_loc = [start_idx]
78
- for prompt_len in prompt_lens:
79
- start_idx += prompt_len
78
+ for seq_len in seq_lens:
79
+ start_idx += seq_len
80
seq_start_loc.append(start_idx)
81
82
assert torch.allclose(
83
attn_metadata.seq_start_loc,
84
torch.tensor(start_loc, dtype=torch.int32, device=device))
85
- assert attn_metadata.max_context_len is None
85
assert torch.allclose(
87
- attn_metadata.context_lens,
88
- torch.zeros(attn_metadata.context_lens.shape[0],
86
+ attn_metadata.context_lens_tensor,
87
+ torch.zeros(attn_metadata.context_lens_tensor.shape[0],
88
dtype=torch.int,
89
device=device))
90
@@ -96,18 +95,18 @@ def test_prepare_prompt(batch_size):
95
# Cuda graph should not be used for prerill.
96
assert attn_metadata.use_cuda_graph is False
97
99
- assert len(input_tokens) == sum(prompt_lens)
100
- assert len(input_positions) == sum(prompt_lens)
98
+ assert len(input_tokens) == sum(seq_lens)
99
+ assert len(input_positions) == sum(seq_lens)
100
torch.testing.assert_close(input_tokens, input_positions)
101
102
sampling_metadata = SamplingMetadata.prepare(
103
seq_group_metadata_list,
105
- prompt_lens,
106
- subquery_lens=prompt_lens,
104
+ seq_lens,
105
+ query_lens=seq_lens,
106
device=model_runner.device,
107
pin_memory=model_runner.pin_memory)
109
- assert len(input_tokens) == sum(prompt_lens)
110
- assert len(input_positions) == sum(prompt_lens)
108
+ assert len(input_tokens) == sum(seq_lens)
109
+ assert len(input_positions) == sum(seq_lens)
110
actual = sampling_metadata.selected_token_indices
111
expected = torch.tensor(expected_selected_token_indices,
112
device=actual.device,
@@ -146,13 +145,13 @@ def test_prepare_decode_cuda_graph(batch_size):
145
lora_config=None)
146
model_runner.set_block_size(16)
147
149
- prompt_lens = []
148
+ seq_lens = []
149
seq_group_metadata_list = []
150
for i in range(batch_size):
151
# make sure all tokens fit into one block
153
- prompt_len = i % (model_runner.block_size - 1) + 1
154
- prompt_lens.append(prompt_len)
155
- seq_data = list(range(prompt_len))
152
+ seq_len = i % (model_runner.block_size - 1) + 1
153
+ seq_lens.append(seq_len)
154
+ seq_data = list(range(seq_len))
155
seq_data = SequenceData(seq_data)
156
seq_group_metadata = SequenceGroupMetadata(
157
request_id=f"test_{i}",
@@ -172,14 +171,13 @@ def test_prepare_decode_cuda_graph(batch_size):
171
# Verify input metadata is correct for prompts.
172
device = model_runner.device
173
assert attn_metadata.is_prompt is False
175
- assert attn_metadata.prompt_lens is None
176
- assert attn_metadata.max_prompt_len is None
174
+ assert attn_metadata.seq_lens is None
175
assert attn_metadata.subquery_start_loc is None
176
assert attn_metadata.seq_start_loc is None
179
- assert attn_metadata.max_context_len == max(prompt_lens)
177
+ assert attn_metadata.max_seq_len == max(seq_lens)
178
assert torch.allclose(
181
- attn_metadata.context_lens[:len(prompt_lens)],
182
- torch.tensor(prompt_lens, dtype=torch.int, device=device))
179
+ attn_metadata.seq_lens_tensor[:len(seq_lens)],
180
+ torch.tensor(seq_lens, dtype=torch.int, device=device))
181
182
# block table's first index corresponds to each batch, meaning in
183
# decoding it is each token.
@@ -198,13 +196,13 @@ def test_prepare_decode_cuda_graph(batch_size):
196
# Verify Sampling
197
expected_selected_token_indices = []
198
selected_token_start_idx = 0
201
- for prompt_len in prompt_lens:
199
+ for seq_len in seq_lens:
200
expected_selected_token_indices.append(selected_token_start_idx)
201
selected_token_start_idx += 1
202
sampling_metadata = SamplingMetadata.prepare(
203
seq_group_metadata_list,
206
- prompt_lens,
207
- subquery_lens=prompt_lens,
204
+ seq_lens,
205
+ query_lens=seq_lens,
206
device=model_runner.device,
207
pin_memory=model_runner.pin_memory)
208
actual = sampling_metadata.selected_token_indices
@@ -241,14 +239,13 @@ def test_empty_seq_group():
239
assert attn_metadata is None
240
assert len(slot_mapping) == 0
241
244
- (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
245
- _, _,
246
- slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
242
+ (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
243
+ _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
244
assert len(input_tokens) == 0
245
assert len(input_positions) == 0
246
assert attn_metadata is None
247
assert len(slot_mapping) == 0
251
- assert len(return_prompt_lens) == 0
248
+ assert len(return_seq_lens) == 0
249
250
251
@pytest.fixture
@@ -288,7 +285,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
285
model_runner.set_block_size(16)
286
287
# Add prefill requests.
291
- prompt_lens = []
288
+ seq_lens = []
289
seq_group_metadata_list = []
290
prefill_metadata_list = []
291
decode_metadata_list = []
@@ -297,9 +294,9 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
294
decode_batch_size = batch_size - prefill_batch_size
295
for i in range(prefill_batch_size):
296
# make sure all tokens fit into one block
300
- prompt_len = i % (model_runner.block_size - 1) + 1
301
- prompt_lens.append(prompt_len)
302
- seq_data = SequenceData(list(range(prompt_len)))
297
+ seq_len = i % (model_runner.block_size - 1) + 1
298
+ seq_lens.append(seq_len)
299
+ seq_data = SequenceData(list(range(seq_len)))
300
seq_group_metadata = SequenceGroupMetadata(
301
request_id=f"test_{i}",
302
is_prompt=True,
@@ -314,8 +311,8 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
311
# Add decode requests
312
for i in range(prefill_batch_size, batch_size):
313
# make sure all tokens fit into one block
317
- prompt_len = i % (model_runner.block_size - 1) + 1
318
- prompt_toks = list(range(prompt_len))
314
+ seq_len = i % (model_runner.block_size - 1) + 1
315
+ prompt_toks = list(range(seq_len))
316
seq_data = SequenceData(prompt_toks)
317
seq_group_metadata = SequenceGroupMetadata(
318
request_id=f"test_{i}",
@@ -343,7 +340,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
340
else:
341
assert attn_metadata.num_decode_tokens == _get_graph_batch_size(
342
decode_batch_size)
346
- assert attn_metadata.num_prefill_tokens == sum(prompt_lens)
343
+ assert attn_metadata.num_prefill_tokens == sum(seq_lens)
344
345
# Verify attn metadata is consistent. We don't need to test individual
346
# values here because they are tested above.
vllm/_custom_ops.py
+9
-9
@@ -39,17 +39,17 @@ def paged_attention_v1(
39
num_kv_heads: int,
40
scale: float,
41
block_tables: torch.Tensor,
42
- context_lens: torch.Tensor,
42
+ seq_lens: torch.Tensor,
43
block_size: int,
44
- max_context_len: int,
44
+ max_seq_len: int,
45
alibi_slopes: Optional[torch.Tensor],
46
kv_cache_dtype: str,
47
kv_scale: float,
48
) -> None:
49
vllm_ops.paged_attention_v1(out, query, key_cache, value_cache,
50
- num_kv_heads, scale, block_tables,
51
- context_lens, block_size, max_context_len,
52
- alibi_slopes, kv_cache_dtype, kv_scale)
50
+ num_kv_heads, scale, block_tables, seq_lens,
51
+ block_size, max_seq_len, alibi_slopes,
52
+ kv_cache_dtype, kv_scale)
53
54
55
def paged_attention_v2(
@@ -63,17 +63,17 @@ def paged_attention_v2(
63
num_kv_heads: int,
64
scale: float,
65
block_tables: torch.Tensor,
66
- context_lens: torch.Tensor,
66
+ seq_lens: torch.Tensor,
67
block_size: int,
68
- max_context_len: int,
68
+ max_seq_len: int,
69
alibi_slopes: Optional[torch.Tensor],
70
kv_cache_dtype: str,
71
kv_scale: float,
72
) -> None:
73
vllm_ops.paged_attention_v2(out, exp_sum, max_logits, tmp_out, query,
74
key_cache, value_cache, num_kv_heads, scale,
75
- block_tables, context_lens, block_size,
76
- max_context_len, alibi_slopes, kv_cache_dtype,
75
+ block_tables, seq_lens, block_size,
76
+ max_seq_len, alibi_slopes, kv_cache_dtype,
77
kv_scale)
78
79
vllm/attention/backends/flash_attn.py
+22
-22
@@ -66,27 +66,24 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
66
# Currently, input sequences can only contain all prompts
67
# or all decoding. True if all sequences are prompts.
68
is_prompt: bool
69
- # (batch_size,). The prompt length per sequence. None if it is a decoding.
70
- prompt_lens: Optional[List[int]]
71
- # prompt_lens stored as a tensor.
72
- prompt_lens_tensor: Optional[torch.Tensor]
69
+ # (batch_size,). The sequence length per sequence. Sequence length means
70
+ # the computed tokens + new tokens None if it is a decoding.
71
+ seq_lens: Optional[List[int]]
72
+ # seq_lens stored as a tensor.
73
+ seq_lens_tensor: Optional[torch.Tensor]
74
74
- # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75
+ # NOTE(sang): Definition of context_len, query_len, and seq_len.
76
# |---------- N-1 iteration --------|
77
# |---------------- N iteration ---------------------|
78
# |- tokenA -|......................|-- newTokens ---|
79
# |---------- context_len ----------|
79
- # |-------------------- seqlen ----------------------|
80
- # |- subquery_len -|
80
+ # |-------------------- seq_len ----------------------|
81
+ # |-- query_len ---|
82
82
- # WARNING(sang): context_len has different definition depending on if it is
83
- # prefill vs decoding. When it is prefill, it doesn't include new tokens.
84
- # When it is for decoding, it includes a new token.
85
-
86
- # Maximum subquery length in the batch.
87
- max_subquery_len: Optional[int]
88
- # Maximum prompt length in the batch.
89
- max_prompt_len: Optional[int]
83
+ # Maximum query length in the batch.
84
+ max_query_len: Optional[int]
85
+ # Maximum sequence length in the batch.
86
+ max_seq_len: Optional[int]
87
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
88
# the batch, used to index into subquery. E.g., if the subquery length
89
# is [4, 6], it is [0, 4, 10].
@@ -95,6 +92,9 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
92
# the batch, used to index into sequence. E.g., if the sequence length is
93
# [4, 6], it is [0, 4, 10].
94
seq_start_loc: Optional[torch.Tensor]
95
+ # (batch_size,) A tensor of context lengths (tokens that are computed
96
+ # so far).
97
+ context_lens_tensor: Optional[torch.Tensor]
98
99
# Whether or not if cuda graph is enabled.
100
# Cuda-graph is currently enabled for decoding only.
@@ -223,8 +223,8 @@ class FlashAttentionImpl(AttentionImpl):
223
v=value,
224
cu_seqlens_q=prefill_meta.seq_start_loc,
225
cu_seqlens_k=prefill_meta.seq_start_loc,
226
- max_seqlen_q=prefill_meta.max_prompt_len,
227
- max_seqlen_k=prefill_meta.max_prompt_len,
226
+ max_seqlen_q=prefill_meta.max_seq_len,
227
+ max_seqlen_k=prefill_meta.max_seq_len,
228
softmax_scale=self.scale,
229
causal=True,
230
window_size=self.sliding_window,
@@ -245,9 +245,9 @@ class FlashAttentionImpl(AttentionImpl):
245
value_cache,
246
prefill_meta.block_tables,
247
prefill_meta.subquery_start_loc,
248
- prefill_meta.prompt_lens_tensor,
249
- prefill_meta.context_lens,
250
- prefill_meta.max_subquery_len,
248
+ prefill_meta.seq_lens_tensor,
249
+ prefill_meta.context_lens_tensor,
250
+ prefill_meta.max_query_len,
251
self.alibi_slopes,
252
self.sliding_window[0],
253
)
@@ -258,8 +258,8 @@ class FlashAttentionImpl(AttentionImpl):
258
key_cache,
259
value_cache,
260
decode_meta.block_tables,
261
- decode_meta.context_lens,
262
- decode_meta.max_context_len,
261
+ decode_meta.seq_lens_tensor,
262
+ decode_meta.max_seq_len,
263
attn_metadata.kv_cache_dtype,
264
self.num_kv_heads,
265
self.scale,
vllm/attention/backends/rocm_flash_attn.py
+30
-30
@@ -64,27 +64,24 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
64
# Currently, input sequences can only contain all prompts
65
# or all decoding. True if all sequences are prompts.
66
is_prompt: bool
67
- # (batch_size,). The prompt length per sequence. None if it is a decoding.
68
- prompt_lens: Optional[List[int]]
69
- # prompt_lens stored as a tensor.
70
- prompt_lens_tensor: Optional[torch.Tensor]
67
+ # (batch_size,). The sequence length per sequence. Sequence length means
68
+ # the computed tokens + new tokens None if it is a decoding.
69
+ seq_lens: Optional[List[int]]
70
+ # seq_lens stored as a tensor.
71
+ seq_lens_tensor: Optional[torch.Tensor]
72
72
- # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
73
+ # NOTE(sang): Definition of context_len, query_len, and seq_len.
74
# |---------- N-1 iteration --------|
75
# |---------------- N iteration ---------------------|
76
# |- tokenA -|......................|-- newTokens ---|
77
# |---------- context_len ----------|
77
- # |-------------------- seqlen ----------------------|
78
- # |- subquery_len -|
78
+ # |-------------------- seq_len ----------------------|
79
+ # |-- query_len ---|
80
80
- # WARNING(sang): context_len has different definition depending on if it is
81
- # prefill vs decoding. When it is prefill, it doesn't include new tokens.
82
- # When it is for decoding, it includes a new token.
83
-
84
- # Maximum subquery length in the batch.
85
- max_subquery_len: Optional[int]
86
- # Maximum prompt length in the batch.
87
- max_prompt_len: Optional[int]
81
+ # Maximum query length in the batch.
82
+ max_query_len: Optional[int]
83
+ # Maximum sequence length in the batch.
84
+ max_seq_len: Optional[int]
85
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
86
# the batch, used to index into subquery. E.g., if the subquery length
87
# is [4, 6], it is [0, 4, 10].
@@ -98,6 +95,9 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
95
# Cuda-graph is currently enabled for decoding only.
96
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
97
use_cuda_graph: bool
98
+ # (batch_size,) A tensor of context lengths (tokens that are computed
99
+ # so far).
100
+ context_lens_tensor: Optional[torch.Tensor]
101
102
103
class ROCmFlashAttentionImpl(AttentionImpl):
@@ -247,7 +247,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
247
248
if prefill_meta := attn_metadata.prefill_metadata:
249
# Prompt run.
250
- assert prefill_meta.prompt_lens is not None
250
+ assert prefill_meta.seq_lens is not None
251
if kv_cache is None or prefill_meta.block_tables.numel() == 0:
252
# triton attention
253
# When block_tables are not filled, it means q and k are the
@@ -260,8 +260,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
260
None,
261
prefill_meta.seq_start_loc,
262
prefill_meta.seq_start_loc,
263
- prefill_meta.max_prompt_len,
264
- prefill_meta.max_prompt_len,
263
+ prefill_meta.max_seq_len,
264
+ prefill_meta.max_seq_len,
265
True,
266
self.scale,
267
)
@@ -274,7 +274,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
274
query,
275
key,
276
value,
277
- prefill_meta.prompt_lens,
277
+ prefill_meta.seq_lens,
278
self.scale,
279
)
280
else:
@@ -284,8 +284,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
284
v=value,
285
cu_seqlens_q=prefill_meta.seq_start_loc,
286
cu_seqlens_k=prefill_meta.seq_start_loc,
287
- max_seqlen_q=prefill_meta.max_prompt_len,
288
- max_seqlen_k=prefill_meta.max_prompt_len,
287
+ max_seqlen_q=prefill_meta.max_seq_len,
288
+ max_seqlen_k=prefill_meta.max_seq_len,
289
softmax_scale=self.scale,
290
causal=True,
291
)
@@ -303,9 +303,9 @@ class ROCmFlashAttentionImpl(AttentionImpl):
303
value_cache,
304
prefill_meta.block_tables,
305
prefill_meta.subquery_start_loc,
306
- prefill_meta.prompt_lens_tensor,
307
- prefill_meta.context_lens,
308
- prefill_meta.max_subquery_len,
306
+ prefill_meta.seq_lens_tensor,
307
+ prefill_meta.context_lens_tensor,
308
+ prefill_meta.max_query_len,
309
self.alibi_slopes,
310
self.sliding_window[0],
311
)
@@ -317,8 +317,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
317
key_cache,
318
value_cache,
319
decode_meta.block_tables,
320
- decode_meta.context_lens,
321
- decode_meta.max_context_len,
320
+ decode_meta.seq_lens_tensor,
321
+ decode_meta.max_seq_len,
322
attn_metadata.kv_cache_dtype,
323
self.num_kv_heads,
324
self.scale,
@@ -334,13 +334,13 @@ def _naive_attention(
334
query: torch.Tensor,
335
key: torch.Tensor,
336
value: torch.Tensor,
337
- prompt_lens: List[int],
337
+ seq_lens: List[int],
338
scale: float,
339
) -> torch.Tensor:
340
output = torch.empty_like(query)
341
start = 0
342
- for _, prompt_len in enumerate(prompt_lens):
343
- end = start + prompt_len
342
+ for _, seq_len in enumerate(seq_lens):
343
+ end = start + seq_len
344
out = _naive_masked_attention(
345
query[start:end],
346
key[start:end],
@@ -349,7 +349,7 @@ def _naive_attention(
349
)
350
# TODO(woosuk): Unnecessary copy. Optimize.
351
output[start:end].copy_(out)
352
- start += prompt_len
352
+ start += seq_len
353
354
return output
355
vllm/attention/backends/torch_sdpa.py
+18
-18
@@ -58,7 +58,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
58
# or all decoding. True if all sequences are prompts.
59
is_prompt: bool
60
slot_mapping: torch.Tensor
61
- prompt_lens: Optional[List[int]]
61
+ seq_lens: Optional[List[int]]
62
63
def __post_init__(self):
64
# Set during the execution of the first attention op.
@@ -136,7 +136,7 @@ class TorchSDPABackendImpl(AttentionImpl):
136
kv_scale)
137
138
if attn_metadata.is_prompt:
139
- assert attn_metadata.prompt_lens is not None
139
+ assert attn_metadata.seq_lens is not None
140
if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
141
if self.num_kv_heads != self.num_heads:
142
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
@@ -147,13 +147,13 @@ class TorchSDPABackendImpl(AttentionImpl):
147
if self.alibi_slopes is not None:
148
att_masks = _make_alibi_bias(
149
self.alibi_slopes, query.dtype,
150
- attn_metadata.prompt_lens) # type: ignore
150
+ attn_metadata.seq_lens) # type: ignore
151
elif self.sliding_window is not None:
152
att_masks = _make_sliding_window_bias(
153
- attn_metadata.prompt_lens, self.sliding_window,
153
+ attn_metadata.seq_lens, self.sliding_window,
154
query.dtype) # type: ignore
155
else:
156
- att_masks = [None] * len(attn_metadata.prompt_lens)
156
+ att_masks = [None] * len(attn_metadata.seq_lens)
157
attn_metadata.attn_bias = att_masks
158
159
query = query.movedim(0, query.dim() - 2)
@@ -164,9 +164,9 @@ class TorchSDPABackendImpl(AttentionImpl):
164
output = torch.empty(
165
(num_tokens, self.num_heads, self.head_size),
166
dtype=query.dtype)
167
- for prompt_len, mask in zip(attn_metadata.prompt_lens,
168
- attn_metadata.attn_bias):
169
- end = start + prompt_len
167
+ for seq_len, mask in zip(attn_metadata.seq_lens,
168
+ attn_metadata.attn_bias):
169
+ end = start + seq_len
170
sub_out = scaled_dot_product_attention(
171
query[:, start:end, :],
172
key[:, start:end, :],
@@ -189,8 +189,8 @@ class TorchSDPABackendImpl(AttentionImpl):
189
key_cache,
190
value_cache,
191
attn_metadata.block_tables,
192
- attn_metadata.context_lens,
193
- attn_metadata.max_context_len,
192
+ attn_metadata.seq_lens_tensor,
193
+ attn_metadata.max_seq_len,
194
attn_metadata.kv_cache_dtype,
195
self.num_kv_heads,
196
self.scale,
@@ -205,13 +205,13 @@ class TorchSDPABackendImpl(AttentionImpl):
205
def _make_alibi_bias(
206
alibi_slopes: torch.Tensor,
207
dtype: torch.dtype,
208
- prompt_lens: List[int],
208
+ seq_lens: List[int],
209
) -> List[torch.Tensor]:
210
attn_biases = []
211
- for prompt_len in prompt_lens:
212
- bias = torch.arange(prompt_len, dtype=dtype)
211
+ for seq_len in seq_lens:
212
+ bias = torch.arange(seq_len, dtype=dtype)
213
# NOTE(zhuohan): HF uses
214
- # `bias = bias[None, :].repeat(prompt_len, 1)`
214
+ # `bias = bias[None, :].repeat(seq_len, 1)`
215
# here. We find that both biases give the same results, but
216
# the bias below more accurately follows the original ALiBi
217
# paper.
@@ -221,7 +221,7 @@ def _make_alibi_bias(
221
bias = bias[None, :].repeat((num_heads, 1, 1))
222
bias.mul_(alibi_slopes[:, None, None])
223
inf_mask = torch.empty(
224
- (1, prompt_len, prompt_len),
224
+ (1, seq_len, seq_len),
225
dtype=bias.dtype).fill_(-torch.inf).triu_(diagonal=1)
226
attn_biases.append((bias + inf_mask).to(dtype))
227
@@ -229,14 +229,14 @@ def _make_alibi_bias(
229
230
231
def _make_sliding_window_bias(
232
- prompt_lens: List[int],
232
+ seq_lens: List[int],
233
window_size: Optional[int],
234
dtype: torch.dtype,
235
) -> List[torch.Tensor]:
236
attn_biases = []
237
- for prompt_len in prompt_lens:
237
+ for seq_len in seq_lens:
238
tensor = torch.full(
239
- (1, prompt_len, prompt_len),
239
+ (1, seq_len, seq_len),
240
dtype=dtype,
241
fill_value=1,
242
)
vllm/attention/backends/xformers.py
+32
-33
@@ -66,28 +66,24 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
66
# Currently, input sequences can only contain all prompts
67
# or all decoding. True if all sequences are prompts.
68
is_prompt: bool
69
- # (batch_size,). The prompt length per sequence. None if it is a decoding.
70
- prompt_lens: Optional[List[int]]
71
- # prompt_lens stored as a tensor.
72
- prompt_lens_tensor: Optional[torch.Tensor]
69
+ # (batch_size,). The sequence length per sequence. Sequence length means
70
+ # the computed tokens + new tokens None if it is a decoding.
71
+ seq_lens: Optional[List[int]]
72
+ # seq_lens stored as a tensor.
73
+ seq_lens_tensor: Optional[torch.Tensor]
74
74
- # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75
# |---------- N-1 iteration --------|
76
# |---------------- N iteration ---------------------|
77
# |- tokenA -|......................|-- newTokens ---|
78
# |---------- context_len ----------|
79
- # |-------------------- seqlen ----------------------|
80
- # |- subquery_len -|
79
+ # |-------------------- seq_len ----------------------|
80
+ # |-- query_len ---|
81
82
- # WARNING(sang): context_len has different definition depending on if it is
83
- # prefill vs decoding. When it is prefill, it doesn't include new tokens.
84
- # When it is for decoding, it includes a new token.
85
-
86
- # Maximum subquery length in the batch.
87
- max_subquery_len: Optional[int]
82
+ # Maximum query length in the batch.
83
+ max_query_len: Optional[int]
84
# FIXME: It is for flash attn.
89
- # Maximum prompt length in the batch.
90
- max_prompt_len: Optional[int]
85
+ # Maximum sequence length in the batch.
86
+ max_seq_len: Optional[int]
87
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
88
# the batch, used to index into subquery. E.g., if the subquery length
89
# is [4, 6], it is [0, 4, 10].
@@ -97,6 +93,9 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
93
# the batch, used to index into sequence. E.g., if the sequence length is
94
# [4, 6], it is [0, 4, 10].
95
seq_start_loc: Optional[torch.Tensor]
96
+ # (batch_size,) A tensor of context lengths (tokens that are computed
97
+ # so far).
98
+ context_lens_tensor: Optional[torch.Tensor]
99
100
# Whether or not if cuda graph is enabled.
101
# Cuda-graph is currently enabled for decoding only.
@@ -242,9 +241,9 @@ class XFormersImpl(AttentionImpl):
241
value_cache,
242
prefill_meta.block_tables,
243
prefill_meta.subquery_start_loc,
245
- prefill_meta.prompt_lens_tensor,
246
- prefill_meta.context_lens,
247
- prefill_meta.max_subquery_len,
244
+ prefill_meta.seq_lens_tensor,
245
+ prefill_meta.context_lens_tensor,
246
+ prefill_meta.max_query_len,
247
self.alibi_slopes,
248
self.sliding_window,
249
)
@@ -257,8 +256,8 @@ class XFormersImpl(AttentionImpl):
256
key_cache,
257
value_cache,
258
decode_meta.block_tables,
260
- decode_meta.context_lens,
261
- decode_meta.max_context_len,
259
+ decode_meta.seq_lens_tensor,
260
+ decode_meta.max_seq_len,
261
attn_metadata.kv_cache_dtype,
262
self.num_kv_heads,
263
self.scale,
@@ -289,7 +288,7 @@ class XFormersImpl(AttentionImpl):
288
value: shape = [num_prefill_tokens, num_kv_heads, head_size]
289
attn_metadata: Metadata for attention.
290
"""
292
- assert attn_metadata.prompt_lens is not None
291
+ assert attn_metadata.seq_lens is not None
292
original_query = query
293
if self.num_kv_heads != self.num_heads:
294
# GQA/MQA requires the shape [B, M, G, H, K].
@@ -310,7 +309,7 @@ class XFormersImpl(AttentionImpl):
309
if attn_metadata.attn_bias is None:
310
if self.alibi_slopes is None:
311
attn_bias = BlockDiagonalCausalMask.from_seqlens(
313
- attn_metadata.prompt_lens)
312
+ attn_metadata.seq_lens)
313
if self.sliding_window is not None:
314
attn_bias = attn_bias.make_local_attention(
315
self.sliding_window)
@@ -318,7 +317,7 @@ class XFormersImpl(AttentionImpl):
317
else:
318
attn_metadata.attn_bias = _make_alibi_bias(
319
self.alibi_slopes, self.num_kv_heads, query.dtype,
321
- attn_metadata.prompt_lens)
320
+ attn_metadata.seq_lens)
321
322
# No alibi slopes.
323
# TODO(woosuk): Too many view operations. Let's try to reduce
@@ -343,8 +342,8 @@ class XFormersImpl(AttentionImpl):
342
# one. This is inefficient, especially when we have many short prompts.
343
output = torch.empty_like(original_query)
344
start = 0
346
- for i, prompt_len in enumerate(attn_metadata.prompt_lens):
347
- end = start + prompt_len
345
+ for i, seq_len in enumerate(attn_metadata.seq_lens):
346
+ end = start + seq_len
347
out = xops.memory_efficient_attention_forward(
348
query[None, start:end],
349
key[None, start:end],
@@ -354,7 +353,7 @@ class XFormersImpl(AttentionImpl):
353
scale=self.scale)
354
# TODO(woosuk): Unnecessary copy. Optimize.
355
output[start:end].copy_(out.view_as(original_query[start:end]))
357
- start += prompt_len
356
+ start += seq_len
357
return output
358
359
@@ -362,13 +361,13 @@ def _make_alibi_bias(
361
alibi_slopes: torch.Tensor,
362
num_kv_heads: int,
363
dtype: torch.dtype,
365
- prompt_lens: List[int],
364
+ seq_lens: List[int],
365
) -> LowerTriangularMaskWithTensorBias:
366
attn_biases = []
368
- for prompt_len in prompt_lens:
369
- bias = torch.arange(prompt_len, dtype=dtype)
367
+ for seq_len in seq_lens:
368
+ bias = torch.arange(seq_len, dtype=dtype)
369
# NOTE(zhuohan): HF uses
371
- # `bias = bias[None, :].repeat(prompt_len, 1)`
370
+ # `bias = bias[None, :].repeat(seq_len, 1)`
371
# here. We find that both biases give the same results, but
372
# the bias below more accurately follows the original ALiBi
373
# paper.
@@ -376,16 +375,16 @@ def _make_alibi_bias(
375
# element.
376
bias = bias[None, :] - bias[:, None]
377
379
- padded_len = (prompt_len + 7) // 8 * 8
378
+ padded_len = (seq_len + 7) // 8 * 8
379
num_heads = alibi_slopes.shape[0]
380
bias = torch.empty(
381
1, # batch size
382
num_heads,
384
- prompt_len,
383
+ seq_len,
384
padded_len,
385
device=alibi_slopes.device,
386
dtype=dtype,
388
- )[:, :, :, :prompt_len].copy_(bias)
387
+ )[:, :, :, :seq_len].copy_(bias)
388
bias.mul_(alibi_slopes[:, None, None])
389
if num_heads != num_kv_heads:
390
bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
vllm/attention/ops/paged_attn.py
+17
-18
@@ -13,12 +13,11 @@ _PARTITION_SIZE = 512
13
@dataclass
14
class PagedAttentionMetadata:
15
"""Metadata for PagedAttention."""
16
- # (batch_size,). The length of context (tokens stored in KV cache) per
17
- # sequence. WARNING: When it is a prefill request, it doesn't include new
18
- # tokens. When it is for decoding, it includes a new token.
19
- context_lens: Optional[torch.Tensor]
20
- # Maximum context length in the batch.
21
- max_context_len: Optional[int]
16
+ # (batch_size,). The length of sequences (entire tokens seen so far) per
17
+ # sequence.
18
+ seq_lens_tensor: Optional[torch.Tensor]
19
+ # Maximum sequence length in the batch.
20
+ max_seq_len: Optional[int]
21
# (batch_size, max_blocks_per_seq).
22
# Block addresses per sequence. (Seq id -> list of physical block)
23
# E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
@@ -85,8 +84,8 @@ class PagedAttention:
84
key_cache: torch.Tensor,
85
value_cache: torch.Tensor,
86
block_tables: torch.Tensor,
88
- context_lens: torch.Tensor,
89
- max_context_len: int,
87
+ seq_lens: torch.Tensor,
88
+ max_seq_len: int,
89
kv_cache_dtype: str,
90
num_kv_heads: int,
91
scale: float,
@@ -97,7 +96,7 @@ class PagedAttention:
96
97
block_size = value_cache.shape[3]
98
num_seqs, num_heads, head_size = query.shape
100
- max_num_partitions = ((max_context_len + _PARTITION_SIZE - 1) //
99
+ max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
100
_PARTITION_SIZE)
101
# NOTE(woosuk): We use a simple heuristic to decide whether to use
102
# PagedAttention V1 or V2. If the number of partitions is 1, we use
@@ -106,7 +105,7 @@ class PagedAttention:
105
# to parallelize.
106
# TODO(woosuk): Tune this heuristic.
107
# For context len > 8192, use V2 kernel to avoid shared memory shortage.
109
- use_v1 = (max_context_len <= 8192
108
+ use_v1 = (max_seq_len <= 8192
109
and (max_num_partitions == 1 or num_seqs * num_heads > 512))
110
if use_v1:
111
# Run PagedAttention V1.
@@ -118,9 +117,9 @@ class PagedAttention:
117
num_kv_heads,
118
scale,
119
block_tables,
121
- context_lens,
120
+ seq_lens,
121
block_size,
123
- max_context_len,
122
+ max_seq_len,
123
alibi_slopes,
124
kv_cache_dtype,
125
kv_scale,
@@ -150,9 +149,9 @@ class PagedAttention:
149
num_kv_heads,
150
scale,
151
block_tables,
153
- context_lens,
152
+ seq_lens,
153
block_size,
155
- max_context_len,
154
+ max_seq_len,
155
alibi_slopes,
156
kv_cache_dtype,
157
kv_scale,
@@ -168,9 +167,9 @@ class PagedAttention:
167
value_cache: torch.Tensor,
168
block_tables: torch.Tensor,
169
subquery_start_loc: torch.Tensor,
171
- prompt_lens_tensor: torch.Tensor,
170
+ seq_lens_tensor: torch.Tensor,
171
context_lens: torch.Tensor,
173
- max_subquery_len: int,
172
+ max_query_len: int,
173
alibi_slopes: Optional[torch.Tensor],
174
sliding_window: Optional[int],
175
) -> torch.Tensor:
@@ -185,9 +184,9 @@ class PagedAttention:
184
block_tables,
185
# subquery_start_loc is (batch_size + 1,)
186
subquery_start_loc[:-1],
188
- prompt_lens_tensor,
187
+ seq_lens_tensor,
188
context_lens,
190
- max_subquery_len,
189
+ max_query_len,
190
alibi_slopes,
191
sliding_window,
192
)
vllm/config.py
+16
-7
@@ -63,7 +63,10 @@ class ModelConfig:
63
If False, we will use CUDA graph and eager execution in hybrid.
64
max_context_len_to_capture: Maximum context len covered by CUDA graphs.
65
When a sequence has context length larger than this, we fall back
66
- to eager mode.
66
+ to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
67
+ max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
68
+ When a sequence has context length larger than this, we fall back
69
+ to eager mode
70
skip_tokenizer_init: If true, skip initialization of tokenizer and
71
detokenizer.
72
"""
@@ -84,6 +87,7 @@ class ModelConfig:
87
quantization_param_path: Optional[str] = None,
88
enforce_eager: bool = False,
89
max_context_len_to_capture: Optional[int] = None,
90
+ max_seq_len_to_capture: Optional[int] = None,
91
max_logprobs: int = 5,
92
skip_tokenizer_init: bool = False,
93
) -> None:
@@ -99,6 +103,11 @@ class ModelConfig:
103
self.quantization_param_path = quantization_param_path
104
self.enforce_eager = enforce_eager
105
self.max_context_len_to_capture = max_context_len_to_capture
106
+ if self.max_context_len_to_capture is not None:
107
+ raise ValueError("`max_context_len_to_capture` is deprecated. "
108
+ "Use `max_seq_len_to_capture` instead.")
109
+ self.max_seq_len_to_capture = (max_seq_len_to_capture
110
+ or max_context_len_to_capture)
111
self.max_logprobs = max_logprobs
112
self.skip_tokenizer_init = skip_tokenizer_init
113
@@ -190,10 +199,10 @@ class ModelConfig:
199
"non-quantized models.", self.quantization)
200
201
def _verify_cuda_graph(self) -> None:
193
- if self.max_context_len_to_capture is None:
194
- self.max_context_len_to_capture = self.max_model_len
195
- self.max_context_len_to_capture = min(self.max_context_len_to_capture,
196
- self.max_model_len)
202
+ if self.max_seq_len_to_capture is None:
203
+ self.max_seq_len_to_capture = self.max_model_len
204
+ self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
205
+ self.max_model_len)
206
207
def verify_with_parallel_config(
208
self,
@@ -772,8 +781,8 @@ class SpeculativeConfig:
781
max_model_len=None,
782
quantization=draft_quantization,
783
enforce_eager=target_model_config.enforce_eager,
775
- max_context_len_to_capture=target_model_config.
776
- max_context_len_to_capture,
784
+ max_seq_len_to_capture=target_model_config.
785
+ max_seq_len_to_capture,
786
max_logprobs=target_model_config.max_logprobs,
787
)
788
vllm/engine/arg_utils.py
+12
-2
@@ -44,7 +44,8 @@ class EngineArgs:
44
tokenizer_revision: Optional[str] = None
45
quantization: Optional[str] = None
46
enforce_eager: bool = False
47
- max_context_len_to_capture: int = 8192
47
+ max_context_len_to_capture: Optional[int] = None
48
+ max_seq_len_to_capture: int = 8192
49
disable_custom_all_reduce: bool = False
50
tokenizer_pool_size: int = 0
51
tokenizer_pool_type: str = "ray"
@@ -322,6 +323,14 @@ class EngineArgs:
323
default=EngineArgs.max_context_len_to_capture,
324
help='Maximum context length covered by CUDA '
325
'graphs. When a sequence has context length '
326
+ 'larger than this, we fall back to eager mode. '
327
+ '(DEPRECATED. Use --max-seq_len-to-capture instead'
328
+ ')')
329
+ parser.add_argument('--max-seq_len-to-capture',
330
+ type=int,
331
+ default=EngineArgs.max_seq_len_to_capture,
332
+ help='Maximum sequence length covered by CUDA '
333
+ 'graphs. When a sequence has context length '
334
'larger than this, we fall back to eager mode.')
335
parser.add_argument('--disable-custom-all-reduce',
336
action='store_true',
@@ -492,7 +501,8 @@ class EngineArgs:
501
self.code_revision, self.tokenizer_revision, self.max_model_len,
502
self.quantization, self.quantization_param_path,
503
self.enforce_eager, self.max_context_len_to_capture,
495
- self.max_logprobs, self.skip_tokenizer_init)
504
+ self.max_seq_len_to_capture, self.max_logprobs,
505
+ self.skip_tokenizer_init)
506
cache_config = CacheConfig(self.block_size,
507
self.gpu_memory_utilization,
508
self.swap_space, self.kv_cache_dtype,
vllm/entrypoints/llm.py
+6
-1
@@ -69,6 +69,9 @@ class LLM:
69
disable CUDA graph and always execute the model in eager mode.
70
If False, we will use CUDA graph and eager execution in hybrid.
71
max_context_len_to_capture: Maximum context len covered by CUDA graphs.
72
+ When a sequence has context length larger than this, we fall back
73
+ to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
74
+ max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
75
When a sequence has context length larger than this, we fall back
76
to eager mode.
77
disable_custom_all_reduce: See ParallelConfig
@@ -90,7 +93,8 @@ class LLM:
93
gpu_memory_utilization: float = 0.9,
94
swap_space: int = 4,
95
enforce_eager: bool = False,
93
- max_context_len_to_capture: int = 8192,
96
+ max_context_len_to_capture: Optional[int] = None,
97
+ max_seq_len_to_capture: int = 8192,
98
disable_custom_all_reduce: bool = False,
99
**kwargs,
100
) -> None:
@@ -112,6 +116,7 @@ class LLM:
116
swap_space=swap_space,
117
enforce_eager=enforce_eager,
118
max_context_len_to_capture=max_context_len_to_capture,
119
+ max_seq_len_to_capture=max_seq_len_to_capture,
120
disable_custom_all_reduce=disable_custom_all_reduce,
121
**kwargs,
122
)
vllm/model_executor/layers/sampler.py
+3
-3
@@ -1033,8 +1033,8 @@ def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
1033
assert seq_group.is_prompt, (
1034
"Caller should ensure the sequence group is in a prefill stage.")
1035
seq_ids = seq_group.seq_ids
1036
- subquery_len = seq_group.subquery_len
1037
- assert subquery_len is not None
1036
+ query_len = seq_group.query_len
1037
+ assert query_len is not None
1038
# prompt has only 1 seq id.
1039
assert len(seq_ids) == 1
1040
seq_data = seq_group.seq_data[seq_ids[0]]
@@ -1042,7 +1042,7 @@ def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
1042
prompt_tokens = seq_data.prompt_token_ids
1043
# +1 because we are looking for a next prompt token.
1044
next_token_index_start = computed_len + 1
1045
- next_token_index_end = min(computed_len + subquery_len + 1,
1045
+ next_token_index_end = min(computed_len + query_len + 1,
1046
len(prompt_tokens))
1047
next_prompt_tokens = prompt_tokens[
1048
next_token_index_start:next_token_index_end]
vllm/model_executor/sampling_metadata.py
+36
-27
@@ -16,17 +16,26 @@ _SEED_0_REPLACEMENT = 3403598558
16
17
@dataclass
18
class SequenceGroupToSample:
19
+ # |---------- N-1 iteration --------|
20
+ # |---------------- N iteration ---------------------|
21
+ # |- tokenA -|......................|-- newTokens ---|
22
+ # |---------- context_len ----------|
23
+ # |-------------------- seq_len ----------------------|
24
+ # |-- query_len ---|
25
+
26
# Sequence ids for the sequence group in a previous step.
27
seq_ids: List[int]
28
sampling_params: SamplingParams
29
# seq_id -> sequence data.
30
seq_data: Dict[int, SequenceData]
24
- # The length of the prompt of the sequence group. None if it is in a decode
31
+ # The length of the sequence (all tokens seen in the past + new token to
32
+ # compute attention) of the sequence group. None if it is in a decode
33
# stage.
26
- prompt_len: Optional[int]
27
- # The length of the query tokens to compute in the current step. None if it
28
- # is in a decode stage. The length of subquery_len <= prompt_len.
29
- subquery_len: Optional[int]
34
+ seq_len: Optional[int]
35
+ # The length of new query tokens to compute in the current step. None if it
36
+ # is in a decode stage. The length of query_len <= seq_len if chunked
37
+ # prefill is enabled.
38
+ query_len: Optional[int]
39
# A random number generator for sampling.
40
generator: Optional[torch.Generator]
41
# True if the sequence group is in prefill stage. False if it is in a
@@ -46,8 +55,8 @@ class SequenceGroupToSample:
55
if len(self.prompt_logprob_indices) > 0:
56
assert self.sampling_params.prompt_logprobs is not None
57
if self.is_prompt:
49
- assert self.prompt_len is not None
50
- assert self.subquery_len is not None
58
+ assert self.seq_len is not None
59
+ assert self.query_len is not None
60
61
62
class SamplingMetadata:
@@ -94,8 +103,8 @@ class SamplingMetadata:
103
@staticmethod
104
def prepare(
105
seq_group_metadata_list: List[SequenceGroupMetadata],
97
- prompt_lens: List[int],
98
- subquery_lens: Optional[List[int]],
106
+ seq_lens: List[int],
107
+ query_lens: Optional[List[int]],
108
device: str,
109
pin_memory: bool,
110
) -> "SamplingMetadata":
@@ -104,8 +113,8 @@ class SamplingMetadata:
113
selected_token_indices,
114
categorized_sample_indices,
115
num_prompts,
107
- ) = _prepare_seq_groups(seq_group_metadata_list, prompt_lens,
108
- subquery_lens, device)
116
+ ) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
117
+ device)
118
selected_token_indices = async_tensor_h2d(selected_token_indices,
119
dtype=torch.long,
120
target_device=device,
@@ -137,8 +146,8 @@ class SamplingMetadata:
146
147
def _prepare_seq_groups(
148
seq_group_metadata_list: List[SequenceGroupMetadata],
140
- prompt_lens: List[int],
141
- subquery_lens: Optional[List[int]],
149
+ seq_lens: List[int],
150
+ query_lens: Optional[List[int]],
151
device: str,
152
) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
153
SamplingType, List[Tuple[int, int]]], int]:
@@ -146,9 +155,9 @@ def _prepare_seq_groups(
155
156
Args:
157
seq_group_metadata_list: A list of sequence group to batch.
149
- prompt_lens: A list of prompt lens per sequence group.
158
+ seq_lens: A list of sequence lens per sequence group.
159
Index of prompt len should match with seq_group_metadata_list.
151
- subquery_lens: A list of query lengths. Prompt lens include the length
160
+ query_lens: A list of query lengths. Prompt lens include the length
161
of entire prompt tokens, and it could be shorter.
162
device: A device to use for random number generator,
163
`SequenceGroupToSample.generator`.
@@ -189,8 +198,8 @@ def _prepare_seq_groups(
198
is_prompt = seq_group_metadata.is_prompt
199
generator: Optional[torch.Generator] = None
200
# If the current seq group is in decode stage, it is None.
192
- prompt_len: Optional[int] = None
193
- subquery_len: Optional[int] = None
201
+ seq_len: Optional[int] = None
202
+ query_len: Optional[int] = None
203
prompt_logprob_indices: List[int] = []
204
sample_indices: List[int] = []
205
do_sample = seq_group_metadata.do_sample
@@ -203,12 +212,12 @@ def _prepare_seq_groups(
212
num_prompts += 1
213
num_prefill_sample = len(seq_ids)
214
assert num_prefill_sample == 1
206
- assert subquery_lens is not None and prompt_lens is not None
207
- subquery_len, prompt_len = subquery_lens[i], prompt_lens[i]
215
+ assert query_lens is not None and seq_lens is not None
216
+ query_len, seq_len = query_lens[i], seq_lens[i]
217
# If we need sampling, exclude num_prefill_sample tokens from
218
# prompt logprob.
210
- prompt_logprob_len = (subquery_len - num_prefill_sample
211
- if do_sample else subquery_len)
219
+ prompt_logprob_len = (query_len - num_prefill_sample
220
+ if do_sample else query_len)
221
sample_len = num_prefill_sample if do_sample else 0
222
else:
223
# Decode
@@ -267,8 +276,8 @@ def _prepare_seq_groups(
276
seq_ids=seq_ids,
277
sampling_params=sampling_params,
278
seq_data=seq_group_metadata.seq_data,
270
- prompt_len=prompt_len,
271
- subquery_len=subquery_len,
279
+ seq_len=seq_len,
280
+ query_len=query_len,
281
generator=generator,
282
is_prompt=is_prompt,
283
prompt_logprob_indices=list(prompt_logprob_indices),
@@ -367,8 +376,8 @@ class SamplingTensors:
376
and sampling_params.prompt_logprobs is not None):
377
# For tokens in the prompt that we only need to get
378
# their logprobs
370
- subquery_len = seq_group.subquery_len
371
- assert subquery_len is not None
379
+ query_len = seq_group.query_len
380
+ assert query_len is not None
381
prefill_len = len(seq_group.prompt_logprob_indices)
382
temperatures += [temperature] * prefill_len
383
top_ps += [top_p] * prefill_len
@@ -397,8 +406,8 @@ class SamplingTensors:
406
407
if is_prompt:
408
prompt_best_of.append(sampling_params.best_of)
400
- subquery_len = seq_group.subquery_len
401
- assert subquery_len is not None
409
+ query_len = seq_group.query_len
410
+ assert query_len is not None
411
412
for seq_id in seq_ids:
413
seq_data = seq_group.seq_data[seq_id]
vllm/worker/cpu_model_runner.py
+29
-29
@@ -80,7 +80,7 @@ class CPUModelRunner:
80
input_tokens: List[int] = []
81
input_positions: List[int] = []
82
slot_mapping: List[int] = []
83
- prompt_lens: List[int] = []
83
+ seq_lens: List[int] = []
84
multi_modal_input_list: List[torch.Tensor] = []
85
86
for seq_group_metadata in seq_group_metadata_list:
@@ -92,15 +92,15 @@ class CPUModelRunner:
92
seq_data = seq_group_metadata.seq_data[seq_id]
93
prompt_tokens = seq_data.get_token_ids()
94
computed_len = seq_data.get_num_computed_tokens()
95
- prompt_len = len(prompt_tokens)
95
+ seq_len = len(prompt_tokens)
96
97
- prompt_lens.append(prompt_len) # Prompt token num
97
+ seq_lens.append(seq_len) # Prompt token num
98
input_tokens.extend(prompt_tokens) # Token ids
99
100
# Token position ids
101
# NOTE(woosuk): Here we assume that the first token in the prompt
102
# is always the first token in the sequence.
103
- input_positions.extend(list(range(computed_len, prompt_len)))
103
+ input_positions.extend(list(range(computed_len, seq_len)))
104
105
if seq_group_metadata.multi_modal_data:
106
multi_modal_input_list.append(
@@ -109,15 +109,15 @@ class CPUModelRunner:
109
# Compute the slot mapping.
110
block_table = seq_group_metadata.block_tables[seq_id]
111
# Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
112
- # where start_idx is max(0, prompt_len - sliding_window).
112
+ # where start_idx is max(0, seq_len - sliding_window).
113
# For example, if the prompt len is 10, sliding window is 8, and
114
# block size is 4, the first two tokens are masked and the slot
115
# mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
116
start_idx = 0
117
if self.sliding_window is not None:
118
- start_idx = max(0, prompt_len - self.sliding_window)
118
+ start_idx = max(0, seq_len - self.sliding_window)
119
120
- for i in range(computed_len, prompt_len):
120
+ for i in range(computed_len, seq_len):
121
if i < start_idx:
122
slot_mapping.append(_PAD_SLOT_ID)
123
continue
@@ -151,19 +151,19 @@ class CPUModelRunner:
151
152
attn_metadata = self.attn_backend.make_metadata(
153
is_prompt=True,
154
- prompt_lens=prompt_lens,
155
- num_prefills=len(prompt_lens),
154
+ seq_lens=seq_lens,
155
+ seq_lens_tensor=None,
156
+ max_seq_len=None,
157
+ num_prefills=len(seq_lens),
158
num_prefill_tokens=num_prompt_tokens,
159
num_decode_tokens=0,
160
prefill_metadata=None,
161
decode_metadata=None,
160
- max_context_len=None,
161
- context_lens=None,
162
block_tables=torch.tensor([]),
163
slot_mapping=slot_mapping,
164
kv_cache_dtype=self.kv_cache_dtype,
165
)
166
- return (input_tokens, input_positions, attn_metadata, prompt_lens,
166
+ return (input_tokens, input_positions, attn_metadata, seq_lens,
167
multi_modal_input)
168
169
def _prepare_decode(
@@ -174,7 +174,7 @@ class CPUModelRunner:
174
input_tokens: List[int] = []
175
input_positions: List[int] = []
176
slot_mapping: List[int] = []
177
- context_lens: List[int] = []
177
+ seq_lens: List[int] = []
178
block_tables: List[List[int]] = []
179
180
for seq_group_metadata in seq_group_metadata_list:
@@ -192,9 +192,9 @@ class CPUModelRunner:
192
position = seq_len - 1
193
input_positions.append(position)
194
195
- context_len = seq_len if self.sliding_window is None else min(
195
+ seq_len = seq_len if self.sliding_window is None else min(
196
seq_len, self.sliding_window)
197
- context_lens.append(context_len)
197
+ seq_lens.append(seq_len)
198
199
block_table = seq_group_metadata.block_tables[seq_id]
200
block_number = block_table[position // self.block_size]
@@ -208,7 +208,7 @@ class CPUModelRunner:
208
block_table = block_table[-sliding_window_blocks:]
209
block_tables.append(block_table)
210
211
- max_context_len = max(context_lens)
211
+ max_seq_len = max(seq_lens)
212
213
input_tokens = torch.tensor(input_tokens,
214
dtype=torch.long,
@@ -219,9 +219,9 @@ class CPUModelRunner:
219
slot_mapping = torch.tensor(slot_mapping,
220
dtype=torch.long,
221
device=self.device)
222
- context_lens = torch.tensor(context_lens,
223
- dtype=torch.int,
224
- device=self.device)
222
+ seq_lens_tensor = torch.tensor(seq_lens,
223
+ dtype=torch.int,
224
+ device=self.device)
225
226
max_block_table_len = max(
227
len(block_table) for block_table in block_tables)
@@ -236,14 +236,14 @@ class CPUModelRunner:
236
attn_metadata = self.attn_backend.make_metadata(
237
is_prompt=False,
238
slot_mapping=slot_mapping,
239
- prompt_lens=None,
239
+ seq_lens=seq_lens,
240
+ seq_lens_tensor=seq_lens_tensor,
241
+ max_seq_len=max_seq_len,
242
num_prefill_tokens=0,
243
num_decode_tokens=len(input_tokens),
242
- max_context_len=max_context_len,
244
num_prefills=0,
245
prefill_metadata=None,
246
decode_metadata=None,
246
- context_lens=context_lens,
247
block_tables=block_tables,
248
kv_cache_dtype=self.kv_cache_dtype,
249
)
@@ -265,20 +265,20 @@ class CPUModelRunner:
265
is_prompt = seq_group_metadata_list[0].is_prompt
266
# Prepare input tensors.
267
if is_prompt:
268
- (input_tokens, input_positions, attn_metadata, prompt_lens,
268
+ (input_tokens, input_positions, attn_metadata, seq_lens,
269
multi_modal_input
270
) = self._prepare_prompt(seq_group_metadata_list)
271
else:
272
(input_tokens, input_positions,
273
attn_metadata) = self._prepare_decode(seq_group_metadata_list)
274
- prompt_lens = []
274
+ seq_lens = []
275
sampling_metadata = SamplingMetadata.prepare(
276
seq_group_metadata_list,
277
- prompt_lens,
278
- # subquery_lens is not needed if chunked prefill is not
277
+ seq_lens,
278
+ # query_lens is not needed if chunked prefill is not
279
# supported. Since CPU worker doesn't support chunked prefill
280
- # just use prompt_lens instead.
281
- prompt_lens,
280
+ # just use seq_lens instead.
281
+ seq_lens,
282
self.device,
283
pin_memory=False)
284
# Broadcast the metadata.
@@ -300,7 +300,7 @@ class CPUModelRunner:
300
sampling_metadata = SamplingMetadata(
301
seq_groups=None,
302
seq_data=None,
303
- prompt_lens=None,
303
+ seq_lens=None,
304
selected_token_indices=selected_token_indices,
305
categorized_sample_indices=None,
306
generators=None,
vllm/worker/model_runner.py
+80
-87
@@ -42,8 +42,8 @@ class PreparePromptMetadata(NamedTuple):
42
input_tokens: List[int]
43
input_positions: List[int]
44
attn_metadata: Optional[AttentionMetadataPerStage]
45
- prompt_lens: List[int]
46
- subquery_lens: List[int]
45
+ seq_lens: List[int]
46
+ query_lens: List[int]
47
lora_index_mapping: List[int]
48
lora_prompt_mapping: List[int]
49
lora_requests: Set[LoRARequest]
@@ -56,8 +56,8 @@ class PreparePromptMetadata(NamedTuple):
56
input_tokens=[],
57
input_positions=[],
58
attn_metadata=None,
59
- prompt_lens=[],
60
- subquery_lens=[],
59
+ seq_lens=[],
60
+ query_lens=[],
61
lora_index_mapping=[],
62
lora_prompt_mapping=[],
63
lora_requests=set(),
@@ -134,9 +134,8 @@ class ModelRunner:
134
self.graph_memory_pool: Optional[Tuple[
135
int, int]] = None # Set during graph capture.
136
137
- self.max_context_len_to_capture = (
138
- self.model_config.max_context_len_to_capture
139
- if self.model_config is not None else 0)
137
+ self.max_seq_len_to_capture = (self.model_config.max_seq_len_to_capture
138
+ if self.model_config is not None else 0)
139
140
self.pin_memory = is_pin_memory_available()
141
self.kv_cache_dtype = kv_cache_dtype
@@ -149,7 +148,7 @@ class ModelRunner:
148
self.model: torch.nn.Module # Set after load_model
149
self.block_size: int # Set after initial profiling.
150
# When using CUDA graph, the input block tables must be padded to
152
- # max_context_len_to_capture. However, creating the block table in
151
+ # max_seq_len_to_capture. However, creating the block table in
152
# Python can be expensive. To optimize this, we cache the block table
153
# in numpy and only copy the actual input content at every iteration.
154
# The shape of the cached block table will be
@@ -218,7 +217,7 @@ class ModelRunner:
217
218
def get_max_block_per_batch(self) -> int:
219
block_size = self.block_size
221
- return (self.max_context_len_to_capture + block_size - 1) // block_size
220
+ return (self.max_seq_len_to_capture + block_size - 1) // block_size
221
222
def _prepare_prompt(
223
self,
@@ -231,9 +230,9 @@ class ModelRunner:
230
lora_prompt_mapping: List[int] = []
231
lora_requests: Set[LoRARequest] = set()
232
234
- prompt_lens: List[int] = []
233
+ seq_lens: List[int] = []
234
context_lens: List[int] = []
236
- subquery_lens: List[int] = []
235
+ query_lens: List[int] = []
236
prefix_block_tables: List[List[int]] = []
237
multi_modal_input_list: List[torch.Tensor] = []
238
@@ -257,21 +256,19 @@ class ModelRunner:
256
257
token_chunk_size = seq_group_metadata.token_chunk_size
258
seq_data = seq_group_metadata.seq_data[seq_id]
260
- computed_len = seq_data.get_num_computed_tokens()
259
+ context_len = seq_data.get_num_computed_tokens()
260
# We should use get_len here because in case of preemption
261
# it contains output tokens.
263
- prefill_end = min(seq_data.get_len(),
264
- computed_len + token_chunk_size)
265
- prompt_tokens = seq_data.get_token_ids()[computed_len:prefill_end]
266
- prompt_len = prefill_end
267
- prompt_lens.append(prompt_len)
262
+ seq_len = min(seq_data.get_len(), context_len + token_chunk_size)
263
+ prompt_tokens = seq_data.get_token_ids()[context_len:seq_len]
264
+ seq_lens.append(seq_len)
265
266
# NOTE: This only works for oooooooxxx style attention.
267
if computed_block_nums is not None and len(
268
computed_block_nums) > 0 and self.sliding_window is None:
269
# Prefix is not supported with sliding_window
273
- computed_len = len(computed_block_nums) * self.block_size
274
- prompt_tokens = prompt_tokens[computed_len:]
270
+ context_len = len(computed_block_nums) * self.block_size
271
+ prompt_tokens = prompt_tokens[context_len:]
272
prefix_block_tables.append(computed_block_nums)
273
elif self.scheduler_config.chunked_prefill_enabled:
274
if seq_group_metadata.block_tables is not None:
@@ -285,25 +282,25 @@ class ModelRunner:
282
prefix_block_tables.append([])
283
# Right now, prefill start is always 0. However, this
284
# assumption can be changed once chunked prefill is introduced.
288
- assert computed_len == 0
285
+ assert context_len == 0
286
287
# actual prompt lens
291
- context_lens.append(computed_len)
292
- subquery_lens.append(prompt_len - computed_len)
288
+ context_lens.append(context_len)
289
+ query_lens.append(seq_len - context_len)
290
291
input_tokens.extend(prompt_tokens)
292
# NOTE(woosuk): Here we assume that the first token in the prompt
293
# is always the first token in the sequence.
297
- input_positions.extend(list(range(computed_len, prefill_end)))
294
+ input_positions.extend(list(range(context_len, seq_len)))
295
lora_id = seq_group_metadata.lora_int_id
296
297
if lora_id > 0:
298
lora_requests.add(seq_group_metadata.lora_request)
299
303
- lora_index_mapping += [lora_id] * (prompt_len - computed_len)
300
+ lora_index_mapping += [lora_id] * (seq_len - context_len)
301
lora_prompt_mapping.extend(
302
[lora_id] *
306
- (prompt_len - computed_len
303
+ (seq_len - context_len
304
if seq_group_metadata.sampling_params.prompt_logprobs else 1))
305
306
if seq_group_metadata.multi_modal_data:
@@ -313,24 +310,24 @@ class ModelRunner:
310
if seq_group_metadata.block_tables is None:
311
# During memory profiling, the block tables are not initialized
312
# yet. In this case, we just use a dummy slot mapping.
316
- slot_mapping.extend([_PAD_SLOT_ID] * prompt_len)
313
+ slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
314
continue
315
316
# Compute the slot mapping.
317
block_table = seq_group_metadata.block_tables[seq_id]
318
# Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
322
- # where start_idx is max(0, prompt_len - sliding_window).
319
+ # where start_idx is max(0, seq_len - sliding_window).
320
# For example, if the prompt len is 10, sliding window is 8, and
321
# block size is 4, the first two tokens are masked and the slot
322
# mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
323
start_idx = 0
324
if self.sliding_window is not None:
328
- assert computed_len == 0, (
325
+ assert context_len == 0, (
326
"Prefix caching is currently not supported with "
327
"sliding window attention")
331
- start_idx = max(0, prompt_len - self.sliding_window)
328
+ start_idx = max(0, seq_len - self.sliding_window)
329
333
- for i in range(computed_len, prefill_end):
330
+ for i in range(context_len, seq_len):
331
if i < start_idx:
332
slot_mapping.append(_PAD_SLOT_ID)
333
continue
@@ -340,9 +337,9 @@ class ModelRunner:
337
slot = block_number * self.block_size + block_offset
338
slot_mapping.append(slot)
339
343
- max_subquery_len = max(subquery_lens)
344
- max_prompt_len = max(prompt_lens)
345
- assert max_subquery_len > 0
340
+ max_query_len = max(query_lens)
341
+ max_seq_len = max(seq_lens)
342
+ assert max_query_len > 0
343
344
context_lens_tensor = torch.tensor(context_lens,
345
dtype=torch.int,
@@ -369,40 +366,39 @@ class ModelRunner:
366
367
# Query length can be shorter than key (i.e., prompt) when prefill
368
# is chunked or prefix cached.
372
- subquery_lens_tensor = torch.tensor(subquery_lens,
373
- dtype=torch.long,
374
- device=self.device)
375
- subquery_start_loc = torch.zeros(subquery_lens_tensor.shape[0] + 1,
369
+ query_lens_tensor = torch.tensor(query_lens,
370
+ dtype=torch.long,
371
+ device=self.device)
372
+ subquery_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
373
dtype=torch.int32,
374
device=self.device)
375
379
- prompt_lens_tensor = torch.tensor(prompt_lens,
380
- dtype=torch.long,
381
- device=self.device)
382
- seq_start_loc = torch.zeros(prompt_lens_tensor.shape[0] + 1,
376
+ seq_lens_tensor = torch.tensor(seq_lens,
377
+ dtype=torch.int,
378
+ device=self.device)
379
+ seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
380
dtype=torch.int32,
381
device=self.device)
382
386
- torch.cumsum(subquery_lens_tensor,
383
+ torch.cumsum(query_lens_tensor,
384
dim=0,
385
dtype=subquery_start_loc.dtype,
386
out=subquery_start_loc[1:])
387
391
- torch.cumsum(prompt_lens_tensor,
388
+ torch.cumsum(seq_lens_tensor,
389
dim=0,
390
dtype=seq_start_loc.dtype,
391
out=seq_start_loc[1:])
392
393
attn_metadata = self.attn_backend.make_metadata(
394
is_prompt=True,
398
- prompt_lens=prompt_lens,
399
- prompt_lens_tensor=prompt_lens_tensor,
400
- max_subquery_len=max_subquery_len,
401
- max_context_len=None,
402
- max_prompt_len=max_prompt_len,
395
+ seq_lens=seq_lens,
396
+ seq_lens_tensor=seq_lens_tensor,
397
+ max_query_len=max_query_len,
398
+ max_seq_len=max_seq_len,
399
subquery_start_loc=subquery_start_loc,
400
seq_start_loc=seq_start_loc,
405
- context_lens=context_lens_tensor,
401
+ context_lens_tensor=context_lens_tensor,
402
block_tables=block_tables,
403
use_cuda_graph=False,
404
)
@@ -411,8 +407,8 @@ class ModelRunner:
407
input_tokens=input_tokens,
408
input_positions=input_positions,
409
attn_metadata=attn_metadata,
414
- prompt_lens=prompt_lens,
415
- subquery_lens=subquery_lens,
410
+ seq_lens=seq_lens,
411
+ query_lens=query_lens,
412
lora_index_mapping=lora_index_mapping,
413
lora_prompt_mapping=lora_prompt_mapping,
414
lora_requests=lora_requests,
@@ -427,7 +423,7 @@ class ModelRunner:
423
input_tokens: List[int] = []
424
input_positions: List[int] = []
425
slot_mapping: List[int] = []
430
- context_lens: List[int] = []
426
+ seq_lens: List[int] = []
427
block_tables: List[List[int]] = []
428
lora_index_mapping: List[int] = []
429
lora_prompt_mapping: List[int] = []
@@ -455,9 +451,9 @@ class ModelRunner:
451
position = seq_len - 1
452
input_positions.append(position)
453
458
- context_len = seq_len if self.sliding_window is None else min(
454
+ seq_len = seq_len if self.sliding_window is None else min(
455
seq_len, self.sliding_window)
460
- context_lens.append(context_len)
456
+ seq_lens.append(seq_len)
457
458
block_table = seq_group_metadata.block_tables[seq_id]
459
block_number = block_table[position // self.block_size]
@@ -477,11 +473,10 @@ class ModelRunner:
473
# See `capture_model` API for more details.
474
# For decoding requests, batch_size == input_tokens.
475
batch_size = len(input_tokens)
480
- max_context_len = max(context_lens)
481
- use_captured_graph = (
482
- not self.model_config.enforce_eager
483
- and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
484
- and max_context_len <= self.max_context_len_to_capture)
476
+ max_seq_len = max(seq_lens)
477
+ use_captured_graph = (not self.model_config.enforce_eager
478
+ and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
479
+ and max_seq_len <= self.max_seq_len_to_capture)
480
if use_captured_graph:
481
graph_batch_size = _get_graph_batch_size(batch_size)
482
assert graph_batch_size >= batch_size
@@ -489,21 +484,21 @@ class ModelRunner:
484
input_tokens.append(0)
485
input_positions.append(0)
486
slot_mapping.append(_PAD_SLOT_ID)
492
- context_lens.append(1)
487
+ seq_lens.append(1)
488
block_tables.append([])
489
lora_index_mapping.append(0)
490
batch_size = graph_batch_size
491
497
- context_lens_tensor = torch.tensor(context_lens,
498
- dtype=torch.int,
499
- device=self.device)
492
+ seq_lens_tensor = torch.tensor(seq_lens,
493
+ dtype=torch.int,
494
+ device=self.device)
495
496
if use_captured_graph:
497
# When using cuda-graph all these tensors should be
498
# padded.
504
- assert context_lens_tensor.shape[0] == len(input_tokens)
505
- assert context_lens_tensor.shape[0] == len(input_positions)
506
- assert context_lens_tensor.shape[0] == len(slot_mapping)
499
+ assert seq_lens_tensor.shape[0] == len(input_tokens)
500
+ assert seq_lens_tensor.shape[0] == len(input_positions)
501
+ assert seq_lens_tensor.shape[0] == len(slot_mapping)
502
503
# The shape of graph_block_tables is
504
# [max batch size, max context len // block size].
@@ -525,14 +520,13 @@ class ModelRunner:
520
521
attn_metadata = self.attn_backend.make_metadata(
522
is_prompt=False,
528
- prompt_lens=None,
529
- prompt_lens_tensor=None,
530
- max_subquery_len=None,
531
- max_context_len=max_context_len,
532
- max_prompt_len=None,
523
+ seq_lens=None,
524
+ seq_lens_tensor=seq_lens_tensor,
525
+ max_query_len=None,
526
+ max_seq_len=max_seq_len,
527
subquery_start_loc=None,
528
seq_start_loc=None,
535
- context_lens=context_lens_tensor,
529
+ context_lens_tensor=None,
530
block_tables=block_tables,
531
use_cuda_graph=use_captured_graph,
532
)
@@ -565,8 +559,8 @@ class ModelRunner:
559
input_tokens,
560
input_positions,
561
prefill_attn_metadata,
568
- prompt_lens,
569
- subquery_lens,
562
+ seq_lens,
563
+ query_lens,
564
lora_index_mapping,
565
lora_prompt_mapping,
566
lora_requests,
@@ -583,13 +577,13 @@ class ModelRunner:
577
decode_slot_mapping,
578
) = self._prepare_decode(decode_reqs)
579
sampling_metadata = SamplingMetadata.prepare(
586
- seq_group_metadata_list, prompt_lens, subquery_lens,
587
- self.device, self.pin_memory)
580
+ seq_group_metadata_list, seq_lens, query_lens, self.device,
581
+ self.pin_memory)
582
583
if not self.scheduler_config.chunked_prefill_enabled:
584
assert (len(prefill_reqs) and len(decode_reqs)) == 0
585
592
- num_prefills = len(prompt_lens)
586
+ num_prefills = len(seq_lens)
587
num_prefill_tokens = len(input_tokens)
588
num_decode_tokens = len(decode_input_tokens)
589
@@ -886,7 +880,7 @@ class ModelRunner:
880
input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
881
slot_mapping = torch.empty(max_batch_size, dtype=torch.long).cuda()
882
slot_mapping.fill_(_PAD_SLOT_ID)
889
- context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
883
+ seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
884
block_tables = torch.from_numpy(self.graph_block_tables).cuda()
885
886
graph_batch_size = _get_graph_batch_size(
@@ -908,14 +902,13 @@ class ModelRunner:
902
# Create dummy attn_metadata.
903
decode_metadata = self.attn_backend.make_metadata(
904
is_prompt=False,
911
- prompt_lens=None,
912
- prompt_lens_tensor=None,
913
- max_subquery_len=None,
914
- max_context_len=self.max_context_len_to_capture,
915
- max_prompt_len=None,
905
+ seq_lens=None,
906
+ seq_lens_tensor=seq_lens[:batch_size],
907
+ max_query_len=None,
908
+ max_seq_len=self.max_seq_len_to_capture,
909
subquery_start_loc=None,
910
seq_start_loc=None,
918
- context_lens=context_lens[:batch_size],
911
+ context_lens_tensor=None,
912
block_tables=block_tables[:batch_size],
913
use_cuda_graph=True,
914
)
@@ -1025,7 +1018,7 @@ class CUDAGraphRunner:
1018
"positions": positions,
1019
"kv_caches": kv_caches,
1020
"slot_mapping": attn_metadata.slot_mapping,
1028
- "context_lens": attn_metadata.decode_metadata.context_lens,
1021
+ "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
1022
"block_tables": attn_metadata.decode_metadata.block_tables,
1023
}
1024
self.output_buffers = {"hidden_states": hidden_states}
@@ -1047,8 +1040,8 @@ class CUDAGraphRunner:
1040
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1041
self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1042
non_blocking=True)
1050
- self.input_buffers["context_lens"].copy_(
1051
- attn_metadata.decode_metadata.context_lens, non_blocking=True)
1043
+ self.input_buffers["seq_lens_tensor"].copy_(
1044
+ attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
1045
self.input_buffers["block_tables"].copy_(
1046
attn_metadata.decode_metadata.block_tables, non_blocking=True)
1047
# Run the graph.
vllm/worker/neuron_model_runner.py
+15
-15
@@ -52,7 +52,7 @@ class NeuronModelRunner:
52
input_positions: List[List[int]] = []
53
input_block_ids: List[int] = []
54
55
- prompt_lens: List[int] = []
55
+ seq_lens: List[int] = []
56
for seq_group_metadata in seq_group_metadata_list:
57
assert seq_group_metadata.is_prompt
58
seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -61,26 +61,26 @@ class NeuronModelRunner:
61
62
seq_data = seq_group_metadata.seq_data[seq_id]
63
prompt_tokens = seq_data.get_token_ids()
64
- prompt_len = len(prompt_tokens)
65
- prompt_lens.append(prompt_len)
64
+ seq_len = len(prompt_tokens)
65
+ seq_lens.append(seq_len)
66
67
input_tokens.append(prompt_tokens)
68
- input_positions.append(list(range(prompt_len)))
68
+ input_positions.append(list(range(seq_len)))
69
70
assert seq_group_metadata.block_tables is not None
71
block_table = seq_group_metadata.block_tables[seq_id]
72
assert len(block_table) == 1
73
input_block_ids.append(block_table[0])
74
75
- max_prompt_len = max(prompt_lens)
76
- assert max_prompt_len > 0
75
+ max_seq_len = max(seq_lens)
76
+ assert max_seq_len > 0
77
input_tokens = make_tensor_with_pad(input_tokens,
78
- max_prompt_len,
78
+ max_seq_len,
79
pad=0,
80
dtype=torch.long,
81
device=self.device)
82
input_positions = make_tensor_with_pad(input_positions,
83
- max_prompt_len,
83
+ max_seq_len,
84
pad=0,
85
dtype=torch.long,
86
device=self.device)
@@ -88,7 +88,7 @@ class NeuronModelRunner:
88
dtype=torch.long,
89
device=self.device)
90
91
- return input_tokens, input_positions, input_block_ids, prompt_lens
91
+ return input_tokens, input_positions, input_block_ids, seq_lens
92
93
def _prepare_decode(
94
self,
@@ -149,18 +149,18 @@ class NeuronModelRunner:
149
# Prepare input tensors.
150
if is_prompt:
151
(input_tokens, input_positions, input_block_ids,
152
- prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
152
+ seq_lens) = self._prepare_prompt(seq_group_metadata_list)
153
else:
154
(input_tokens, input_positions,
155
input_block_ids) = self._prepare_decode(seq_group_metadata_list)
156
- prompt_lens = []
156
+ seq_lens = []
157
sampling_metadata = SamplingMetadata.prepare(
158
seq_group_metadata_list,
159
- prompt_lens,
160
- # subquery_lens is not needed if chunked prefill is not
159
+ seq_lens,
160
+ # query_lens is not needed if chunked prefill is not
161
# supported. Since neuron worker doesn't support chunked prefill
162
- # just use prompt_lens instead.
163
- prompt_lens,
162
+ # just use seq_lens instead.
163
+ seq_lens,
164
self.device,
165
self.pin_memory)
166