@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3521ba4f25
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Model runner refactoring 1/N] Refactor attn metadata term (#4518)

SangBin Cho committed May 4, 2024 at 02:20 UTC 3521ba4f2554bcf246a95a9fb2d1b80990a6835b
27 files changed +554 -525
benchmarks/kernels/benchmark_paged_attention.py
+12 -13
@@ -16,7 +16,7 @@ PARTITION_SIZE = 512
16 def main(
17 version: str,
18 num_seqs: int,
19 - context_len: int,
19 + seq_len: int,
20 num_query_heads: int,
21 num_kv_heads: int,
22 head_size: int,
@@ -48,12 +48,12 @@ def main(
48 dtype=torch.float,
49 device=device)
50
51 - context_lens = [context_len for _ in range(num_seqs)]
52 - max_context_len = max(context_lens)
53 - context_lens = torch.tensor(context_lens, dtype=torch.int, device=device)
51 + seq_lens = [seq_len for _ in range(num_seqs)]
52 + max_seq_len = max(seq_lens)
53 + seq_lens = torch.tensor(seq_lens, dtype=torch.int, device=device)
54
55 # Create the block tables.
56 - max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
56 + max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
57 block_tables = []
58 for _ in range(num_seqs):
59 block_table = [
@@ -77,8 +77,7 @@ def main(
77 # Prepare for the paged attention kernel.
78 output = torch.empty_like(query)
79 if version == "v2":
80 - num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
81 - PARTITION_SIZE)
80 + num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
81 tmp_output = torch.empty(
82 size=(num_seqs, num_query_heads, num_partitions, head_size),
83 dtype=output.dtype,
@@ -110,9 +109,9 @@ def main(
109 num_kv_heads,
110 scale,
111 block_tables,
113 - context_lens,
112 + seq_lens,
113 block_size,
115 - max_context_len,
114 + max_seq_len,
115 alibi_slopes,
116 kv_cache_dtype,
117 kv_scale,
@@ -129,9 +128,9 @@ def main(
128 num_kv_heads,
129 scale,
130 block_tables,
132 - context_lens,
131 + seq_lens,
132 block_size,
134 - max_context_len,
133 + max_seq_len,
134 alibi_slopes,
135 kv_cache_dtype,
136 kv_scale,
@@ -166,7 +165,7 @@ if __name__ == '__main__':
165 choices=["v1", "v2"],
166 default="v2")
167 parser.add_argument("--batch-size", type=int, default=8)
169 - parser.add_argument("--context-len", type=int, default=4096)
168 + parser.add_argument("--seq_len", type=int, default=4096)
169 parser.add_argument("--num-query-heads", type=int, default=64)
170 parser.add_argument("--num-kv-heads", type=int, default=8)
171 parser.add_argument("--head-size",
@@ -199,7 +198,7 @@ if __name__ == '__main__':
198 main(
199 version=args.version,
200 num_seqs=args.batch_size,
202 - context_len=args.context_len,
201 + seq_len=args.seq_len,
202 num_query_heads=args.num_query_heads,
203 num_kv_heads=args.num_kv_heads,
204 head_size=args.head_size,
csrc/attention/attention_kernels.cu
+38 -38
@@ -104,7 +104,7 @@ __device__ void paged_attention_kernel(
104 const int num_kv_heads, // [num_heads]
105 const float scale,
106 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
107 - const int* __restrict__ context_lens, // [num_seqs]
107 + const int* __restrict__ seq_lens, // [num_seqs]
108 const int max_num_blocks_per_seq,
109 const float* __restrict__ alibi_slopes, // [num_heads]
110 const int q_stride,
@@ -115,23 +115,23 @@ __device__ void paged_attention_kernel(
115 const int partition_idx = blockIdx.z;
116 const int max_num_partitions = gridDim.z;
117 constexpr bool USE_PARTITIONING = PARTITION_SIZE > 0;
118 - const int context_len = context_lens[seq_idx];
119 - if (USE_PARTITIONING && partition_idx * PARTITION_SIZE >= context_len) {
118 + const int seq_len = seq_lens[seq_idx];
119 + if (USE_PARTITIONING && partition_idx * PARTITION_SIZE >= seq_len) {
120 // No work to do. Terminate the thread block.
121 return;
122 }
123
124 - const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE);
125 - const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_context_blocks;
124 + const int num_seq_blocks = DIVIDE_ROUND_UP(seq_len, BLOCK_SIZE);
125 + const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_seq_blocks;
126
127 // [start_block_idx, end_block_idx) is the range of blocks to process.
128 const int start_block_idx = USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
129 - const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_context_blocks);
129 + const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_seq_blocks);
130 const int num_blocks = end_block_idx - start_block_idx;
131
132 // [start_token_idx, end_token_idx) is the range of tokens to process.
133 const int start_token_idx = start_block_idx * BLOCK_SIZE;
134 - const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, context_len);
134 + const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, seq_len);
135 const int num_tokens = end_token_idx - start_token_idx;
136
137 constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
@@ -245,12 +245,12 @@ __device__ void paged_attention_kernel(
245 // This includes a reduction across the threads in the same thread group.
246 float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
247 // Add the ALiBi bias if slopes are given.
248 - qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len + 1) : 0;
248 + qk += (alibi_slope != 0) ? alibi_slope * (token_idx - seq_len + 1) : 0;
249
250 if (thread_group_offset == 0) {
251 // Store the partial reductions to shared memory.
252 // NOTE(woosuk): It is required to zero out the masked logits.
253 - const bool mask = token_idx >= context_len;
253 + const bool mask = token_idx >= seq_len;
254 logits[token_idx - start_token_idx] = mask ? 0.f : qk;
255 // Update the max value.
256 qk_max = mask ? qk_max : fmaxf(qk_max, qk);
@@ -364,14 +364,14 @@ __device__ void paged_attention_kernel(
364 } else {
365 v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
366 }
367 - if (block_idx == num_context_blocks - 1) {
367 + if (block_idx == num_seq_blocks - 1) {
368 // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
369 // we should explicitly zero out the values since they may contain NaNs.
370 // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
371 scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
372 #pragma unroll
373 for (int j = 0; j < V_VEC_SIZE; j++) {
374 - v_vec_ptr[j] = token_idx + j < context_len ? v_vec_ptr[j] : zero_value;
374 + v_vec_ptr[j] = token_idx + j < seq_len ? v_vec_ptr[j] : zero_value;
375 }
376 }
377 accs[i] += dot(logits_vec, v_vec);
@@ -457,7 +457,7 @@ __global__ void paged_attention_v1_kernel(
457 const int num_kv_heads, // [num_heads]
458 const float scale,
459 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
460 - const int* __restrict__ context_lens, // [num_seqs]
460 + const int* __restrict__ seq_lens, // [num_seqs]
461 const int max_num_blocks_per_seq,
462 const float* __restrict__ alibi_slopes, // [num_heads]
463 const int q_stride,
@@ -466,7 +466,7 @@ __global__ void paged_attention_v1_kernel(
466 const float kv_scale) {
467 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE>(
468 /* exp_sums */ nullptr, /* max_logits */ nullptr,
469 - out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
469 + out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, seq_lens,
470 max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
471 }
472
@@ -489,7 +489,7 @@ __global__ void paged_attention_v2_kernel(
489 const int num_kv_heads, // [num_heads]
490 const float scale,
491 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
492 - const int* __restrict__ context_lens, // [num_seqs]
492 + const int* __restrict__ seq_lens, // [num_seqs]
493 const int max_num_blocks_per_seq,
494 const float* __restrict__ alibi_slopes, // [num_heads]
495 const int q_stride,
@@ -498,7 +498,7 @@ __global__ void paged_attention_v2_kernel(
498 const float kv_scale) {
499 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE, PARTITION_SIZE>(
500 exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
501 - block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
501 + block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes,
502 q_stride, kv_block_stride, kv_head_stride, kv_scale);
503 }
504
@@ -513,13 +513,13 @@ __global__ void paged_attention_v2_reduce_kernel(
513 const float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
514 const float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
515 const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
516 - const int* __restrict__ context_lens, // [num_seqs]
516 + const int* __restrict__ seq_lens, // [num_seqs]
517 const int max_num_partitions) {
518 const int num_heads = gridDim.x;
519 const int head_idx = blockIdx.x;
520 const int seq_idx = blockIdx.y;
521 - const int context_len = context_lens[seq_idx];
522 - const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE);
521 + const int seq_len = seq_lens[seq_idx];
522 + const int num_partitions = DIVIDE_ROUND_UP(seq_len, PARTITION_SIZE);
523 if (num_partitions == 1) {
524 // No need to reduce. Only copy tmp_out to out.
525 scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
@@ -616,7 +616,7 @@ __global__ void paged_attention_v2_reduce_kernel(
616 num_kv_heads, \
617 scale, \
618 block_tables_ptr, \
619 - context_lens_ptr, \
619 + seq_lens_ptr, \
620 max_num_blocks_per_seq, \
621 alibi_slopes_ptr, \
622 q_stride, \
@@ -639,8 +639,8 @@ void paged_attention_v1_launcher(
639 int num_kv_heads,
640 float scale,
641 torch::Tensor& block_tables,
642 - torch::Tensor& context_lens,
643 - int max_context_len,
642 + torch::Tensor& seq_lens,
643 + int max_seq_len,
644 const c10::optional<torch::Tensor>& alibi_slopes,
645 float kv_scale) {
646 int num_seqs = query.size(0);
@@ -664,11 +664,11 @@ void paged_attention_v1_launcher(
664 CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
665 CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
666 int* block_tables_ptr = block_tables.data_ptr<int>();
667 - int* context_lens_ptr = context_lens.data_ptr<int>();
667 + int* seq_lens_ptr = seq_lens.data_ptr<int>();
668
669 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
670 - int padded_max_context_len = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE) * BLOCK_SIZE;
671 - int logits_size = padded_max_context_len * sizeof(float);
670 + int padded_max_seq_len = DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
671 + int logits_size = padded_max_seq_len * sizeof(float);
672 int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
673 // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
674 // Keep that in sync with the logic here!
@@ -715,8 +715,8 @@ void paged_attention_v1_launcher(
715 num_kv_heads, \
716 scale, \
717 block_tables, \
718 - context_lens, \
719 - max_context_len, \
718 + seq_lens, \
719 + max_seq_len, \
720 alibi_slopes, \
721 kv_scale);
722
@@ -746,9 +746,9 @@ void paged_attention_v1(
746 int num_kv_heads, // [num_heads]
747 float scale,
748 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
749 - torch::Tensor& context_lens, // [num_seqs]
749 + torch::Tensor& seq_lens, // [num_seqs]
750 int block_size,
751 - int max_context_len,
751 + int max_seq_len,
752 const c10::optional<torch::Tensor>& alibi_slopes,
753 const std::string& kv_cache_dtype,
754 float kv_scale) {
@@ -790,7 +790,7 @@ void paged_attention_v1(
790 num_kv_heads, \
791 scale, \
792 block_tables_ptr, \
793 - context_lens_ptr, \
793 + seq_lens_ptr, \
794 max_num_blocks_per_seq, \
795 alibi_slopes_ptr, \
796 q_stride, \
@@ -803,7 +803,7 @@ void paged_attention_v1(
803 exp_sums_ptr, \
804 max_logits_ptr, \
805 tmp_out_ptr, \
806 - context_lens_ptr, \
806 + seq_lens_ptr, \
807 max_num_partitions);
808
809 template<
@@ -824,8 +824,8 @@ void paged_attention_v2_launcher(
824 int num_kv_heads,
825 float scale,
826 torch::Tensor& block_tables,
827 - torch::Tensor& context_lens,
828 - int max_context_len,
827 + torch::Tensor& seq_lens,
828 + int max_seq_len,
829 const c10::optional<torch::Tensor>& alibi_slopes,
830 float kv_scale) {
831 int num_seqs = query.size(0);
@@ -852,10 +852,10 @@ void paged_attention_v2_launcher(
852 CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
853 CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
854 int* block_tables_ptr = block_tables.data_ptr<int>();
855 - int* context_lens_ptr = context_lens.data_ptr<int>();
855 + int* seq_lens_ptr = seq_lens.data_ptr<int>();
856
857 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
858 - int max_num_partitions = DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE);
858 + int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
859 int logits_size = PARTITION_SIZE * sizeof(float);
860 int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
861
@@ -909,8 +909,8 @@ void paged_attention_v2_launcher(
909 num_kv_heads, \
910 scale, \
911 block_tables, \
912 - context_lens, \
913 - max_context_len, \
912 + seq_lens, \
913 + max_seq_len, \
914 alibi_slopes, \
915 kv_scale);
916
@@ -943,9 +943,9 @@ void paged_attention_v2(
943 int num_kv_heads, // [num_heads]
944 float scale,
945 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
946 - torch::Tensor& context_lens, // [num_seqs]
946 + torch::Tensor& seq_lens, // [num_seqs]
947 int block_size,
948 - int max_context_len,
948 + int max_seq_len,
949 const c10::optional<torch::Tensor>& alibi_slopes,
950 const std::string& kv_cache_dtype,
951 float kv_scale) {
csrc/cpu/attention.cpp
+46 -46
@@ -70,11 +70,11 @@ template <typename T>
70 FORCE_INLINE std::pair<T, T>
71 reduceSoftmaxAlibi(T *data, const int size, const int capacity,
72 const float alibi_slope, const int start_index,
73 - const int context_len) {
74 - data[0] += alibi_slope * (start_index - context_len + 1);
73 + const int seq_len) {
74 + data[0] += alibi_slope * (start_index - seq_len + 1);
75 T max = data[0];
76 for (int i = 1; i < size; ++i) {
77 - T qk = data[i] + alibi_slope * (start_index + i - context_len + 1);
77 + T qk = data[i] + alibi_slope * (start_index + i - seq_len + 1);
78 data[i] = qk;
79 max = max >= qk ? max : qk;
80 }
@@ -225,7 +225,7 @@ struct paged_attention_v1_impl {
225 const int num_kv_heads, const float scale,
226 const int
227 *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
228 - const int *__restrict__ context_lens, // [num_seqs]
228 + const int *__restrict__ seq_lens, // [num_seqs]
229 const int max_num_blocks_per_seq,
230 const float *__restrict__ alibi_slopes, // [num_heads]
231 const int q_stride, const int kv_block_stride, const int kv_head_stride,
@@ -235,32 +235,32 @@ struct paged_attention_v1_impl {
235
236 static_assert(BLOCK_SIZE == 16);
237
238 - int max_context_len = max_num_blocks_per_seq * BLOCK_SIZE;
239 - int max_context_len_padded = (max_context_len + 15) & 0xFFFFFFF0;
240 - TORCH_CHECK((max_context_len_padded * sizeof(float)) % 64 == 0);
238 + int max_seq_len = max_num_blocks_per_seq * BLOCK_SIZE;
239 + int max_seq_len_padded = (max_seq_len + 15) & 0xFFFFFFF0;
240 + TORCH_CHECK((max_seq_len_padded * sizeof(float)) % 64 == 0);
241
242 const int parallel_work_item_num = omp_get_max_threads();
243
244 size_t logits_bytes =
245 - parallel_work_item_num * max_context_len_padded * sizeof(float);
245 + parallel_work_item_num * max_seq_len_padded * sizeof(float);
246 float *logits = (float *)std::aligned_alloc(
247 64, logits_bytes); // Cacheline alignment for each context token.
248 - // [parallel_work_item_num, max_context_len_padded]
248 + // [parallel_work_item_num, max_seq_len_padded]
249
250 #pragma omp parallel for collapse(2) schedule(dynamic, 1)
251 for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
252 for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
253 - int context_len = context_lens[seq_idx];
253 + int seq_len = seq_lens[seq_idx];
254 const int *seq_block_table =
255 block_tables + max_num_blocks_per_seq * seq_idx;
256 - const int block_num = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
256 + const int block_num = (seq_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
257 const int64_t kv_head_idx = head_idx / num_queries_per_kv;
258 const scalar_t *__restrict__ q_vec_ptr =
259 q + seq_idx * q_stride + head_idx * HEAD_SIZE;
260 const int last_block_token_num =
261 - context_len - (block_num - 1) * BLOCK_SIZE;
261 + seq_len - (block_num - 1) * BLOCK_SIZE;
262 float *__restrict__ thread_block_logits =
263 - logits + omp_get_thread_num() * max_context_len_padded;
263 + logits + omp_get_thread_num() * max_seq_len_padded;
264
265 // Compute logits
266 for (int block_idx = 0; block_idx < block_num; ++block_idx) {
@@ -278,11 +278,11 @@ struct paged_attention_v1_impl {
278
279 // Compute softmax
280 if (alibi_slopes) {
281 - reduceSoftmaxAlibi(thread_block_logits, context_len,
281 + reduceSoftmaxAlibi(thread_block_logits, seq_len,
282 block_num * BLOCK_SIZE, alibi_slopes[head_idx], 0,
283 - context_len);
283 + seq_len);
284 } else {
285 - reduceSoftmax(thread_block_logits, context_len,
285 + reduceSoftmax(thread_block_logits, seq_len,
286 block_num * BLOCK_SIZE);
287 }
288
@@ -340,7 +340,7 @@ struct paged_attention_v1_impl {
340 #define LAUNCH_V1_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
341 paged_attention_v1_impl<T, HEAD_SIZE, BLOCK_SIZE>::call( \
342 out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
343 - block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
343 + block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
344 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, num_seqs, \
345 num_heads);
346
@@ -348,8 +348,8 @@ template <typename T, int BLOCK_SIZE>
348 void paged_attention_v1_impl_launcher(
349 torch::Tensor &out, torch::Tensor &query, torch::Tensor &key_cache,
350 torch::Tensor &value_cache, int num_kv_heads, float scale,
351 - torch::Tensor &block_tables, torch::Tensor &context_lens,
352 - int max_context_len, const c10::optional<torch::Tensor> &alibi_slopes) {
351 + torch::Tensor &block_tables, torch::Tensor &seq_lens,
352 + int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
353 int num_seqs = query.size(0);
354 int num_heads = query.size(1);
355 int head_size = query.size(2);
@@ -369,7 +369,7 @@ void paged_attention_v1_impl_launcher(
369 T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
370 T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
371 int *block_tables_ptr = block_tables.data_ptr<int>();
372 - int *context_lens_ptr = context_lens.data_ptr<int>();
372 + int *seq_lens_ptr = seq_lens.data_ptr<int>();
373
374 switch (head_size) {
375 case 64:
@@ -399,7 +399,7 @@ void paged_attention_v1_impl_launcher(
399 #define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
400 paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
401 out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
402 - context_lens, max_context_len, alibi_slopes);
402 + seq_lens, max_seq_len, alibi_slopes);
403
404 #define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
405 switch (block_size) { \
@@ -416,8 +416,8 @@ void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
416 torch::Tensor &key_cache, torch::Tensor &value_cache,
417 int num_kv_heads, float scale,
418 torch::Tensor &block_tables,
419 - torch::Tensor &context_lens, int block_size,
420 - int max_context_len,
419 + torch::Tensor &seq_lens, int block_size,
420 + int max_seq_len,
421 const c10::optional<torch::Tensor> &alibi_slopes,
422 const std::string &kv_cache_dtype, float kv_scale) {
423 TORCH_CHECK(kv_scale == 1.0f);
@@ -448,7 +448,7 @@ struct paged_attention_v2_impl {
448 const int num_kv_heads, const float scale,
449 const int
450 *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
451 - const int *__restrict__ context_lens, // [num_seqs]
451 + const int *__restrict__ seq_lens, // [num_seqs]
452 const int max_num_blocks_per_seq,
453 const float *__restrict__ alibi_slopes, // [num_heads]
454 const int q_stride, const int kv_block_stride, const int kv_head_stride,
@@ -465,22 +465,22 @@ struct paged_attention_v2_impl {
465 for (int partition_idx = 0; partition_idx < max_num_partitions;
466 ++partition_idx) {
467 for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
468 - const int context_len = context_lens[seq_idx];
468 + const int seq_len = seq_lens[seq_idx];
469 const int start_token_idx = partition_idx * PARTITION_SIZE;
470
471 - if (start_token_idx >= context_len)
471 + if (start_token_idx >= seq_len)
472 continue;
473
474 const int partition_num =
475 - (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
475 + (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
476 const bool no_reduce = (partition_num == 1);
477 - const int context_token_num =
478 - (std::min(context_len, start_token_idx + PARTITION_SIZE) -
477 + const int token_num =
478 + (std::min(seq_len, start_token_idx + PARTITION_SIZE) -
479 start_token_idx);
480 const int block_num =
481 - (context_token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
481 + (token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
482 const int last_block_token_num =
483 - context_token_num - (block_num - 1) * BLOCK_SIZE;
483 + token_num - (block_num - 1) * BLOCK_SIZE;
484 const int *seq_block_table = block_tables +
485 max_num_blocks_per_seq * seq_idx +
486 start_token_idx / BLOCK_SIZE;
@@ -507,10 +507,10 @@ struct paged_attention_v2_impl {
507 std::pair<float, float> max_and_sum;
508 if (alibi_slopes) {
509 max_and_sum = reduceSoftmaxAlibi(
510 - logits, context_token_num, block_num * BLOCK_SIZE,
511 - alibi_slopes[head_idx], start_token_idx, context_len);
510 + logits, token_num, block_num * BLOCK_SIZE,
511 + alibi_slopes[head_idx], start_token_idx, seq_len);
512 } else {
513 - max_and_sum = reduceSoftmax(logits, context_token_num,
513 + max_and_sum = reduceSoftmax(logits, token_num,
514 block_num * BLOCK_SIZE);
515 }
516
@@ -583,9 +583,9 @@ struct paged_attention_v2_impl {
583 #pragma omp parallel for collapse(2) schedule(static, 1)
584 for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
585 for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
586 - const int context_len = context_lens[seq_idx];
586 + const int seq_len = seq_lens[seq_idx];
587 const int partition_num =
588 - (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
588 + (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
589
590 if (partition_num == 1)
591 continue;
@@ -612,9 +612,9 @@ struct paged_attention_v2_impl {
612 for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
613 for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
614 for (int group_idx = 0; group_idx < head_group_num; ++group_idx) {
615 - const int context_len = context_lens[seq_idx];
615 + const int seq_len = seq_lens[seq_idx];
616 const int partition_num =
617 - (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
617 + (seq_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
618
619 if (partition_num == 1)
620 continue;
@@ -649,7 +649,7 @@ struct paged_attention_v2_impl {
649 paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
650 out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
651 key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
652 - context_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
652 + seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
653 kv_block_stride, kv_head_stride, num_seqs, num_heads, \
654 max_num_partitions);
655
@@ -658,8 +658,8 @@ void paged_attention_v2_impl_launcher(
658 torch::Tensor &out, torch::Tensor &exp_sums, torch::Tensor &max_logits,
659 torch::Tensor &tmp_out, torch::Tensor &query, torch::Tensor &key_cache,
660 torch::Tensor &value_cache, int num_kv_heads, float scale,
661 - torch::Tensor &block_tables, torch::Tensor &context_lens, int block_size,
662 - int max_context_len, const c10::optional<torch::Tensor> &alibi_slopes) {
661 + torch::Tensor &block_tables, torch::Tensor &seq_lens, int block_size,
662 + int max_seq_len, const c10::optional<torch::Tensor> &alibi_slopes) {
663 int num_seqs = query.size(0);
664 int num_heads = query.size(1);
665 int head_size = query.size(2);
@@ -683,7 +683,7 @@ void paged_attention_v2_impl_launcher(
683 T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
684 T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
685 int *block_tables_ptr = block_tables.data_ptr<int>();
686 - int *context_lens_ptr = context_lens.data_ptr<int>();
686 + int *seq_lens_ptr = seq_lens.data_ptr<int>();
687
688 switch (head_size) {
689 case 64:
@@ -713,8 +713,8 @@ void paged_attention_v2_impl_launcher(
713 #define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
714 paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
715 out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
716 - num_kv_heads, scale, block_tables, context_lens, block_size, \
717 - max_context_len, alibi_slopes);
716 + num_kv_heads, scale, block_tables, seq_lens, block_size, \
717 + max_seq_len, alibi_slopes);
718
719 #define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
720 switch (block_size) { \
@@ -732,8 +732,8 @@ void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
732 torch::Tensor &query, torch::Tensor &key_cache,
733 torch::Tensor &value_cache, int num_kv_heads,
734 float scale, torch::Tensor &block_tables,
735 - torch::Tensor &context_lens, int block_size,
736 - int max_context_len,
735 + torch::Tensor &seq_lens, int block_size,
736 + int max_seq_len,
737 const c10::optional<torch::Tensor> &alibi_slopes,
738 const std::string &kv_cache_dtype, float kv_scale) {
739 TORCH_CHECK(kv_scale == 1.0f);
csrc/ops.h
+4 -4
@@ -10,9 +10,9 @@ void paged_attention_v1(
10 int num_kv_heads,
11 float scale,
12 torch::Tensor& block_tables,
13 - torch::Tensor& context_lens,
13 + torch::Tensor& seq_lens,
14 int block_size,
15 - int max_context_len,
15 + int max_seq_len,
16 const c10::optional<torch::Tensor>& alibi_slopes,
17 const std::string& kv_cache_dtype,
18 float kv_scale);
@@ -28,9 +28,9 @@ void paged_attention_v2(
28 int num_kv_heads,
29 float scale,
30 torch::Tensor& block_tables,
31 - torch::Tensor& context_lens,
31 + torch::Tensor& seq_lens,
32 int block_size,
33 - int max_context_len,
33 + int max_seq_len,
34 const c10::optional<torch::Tensor>& alibi_slopes,
35 const std::string& kv_cache_dtype,
36 float kv_scale);
tests/kernels/test_attention.py
+17 -18
@@ -61,7 +61,7 @@ def ref_single_query_cached_kv_attention(
61 key_cache: torch.Tensor,
62 value_cache: torch.Tensor,
63 block_tables: torch.Tensor,
64 - context_lens: torch.Tensor,
64 + seq_lens: torch.Tensor,
65 scale: float,
66 alibi_slopes: Optional[torch.Tensor],
67 ) -> None:
@@ -72,15 +72,15 @@ def ref_single_query_cached_kv_attention(
72 num_seqs = query.shape[0]
73
74 block_tables = block_tables.cpu().tolist()
75 - context_lens = context_lens.cpu().tolist()
75 + seq_lens = seq_lens.cpu().tolist()
76 for i in range(num_seqs):
77 q = query[i].unsqueeze(0)
78 block_table = block_tables[i]
79 - context_len = int(context_lens[i])
79 + seq_len = int(seq_lens[i])
80
81 keys = []
82 values = []
83 - for j in range(context_len):
83 + for j in range(seq_len):
84 block_number = int(block_table[j // block_size])
85 block_offset = j % block_size
86
@@ -100,8 +100,8 @@ def ref_single_query_cached_kv_attention(
100 alibi_bias = None
101 if alibi_slopes is not None:
102 # Create the ALiBi bias used in the paged attention kernel.
103 - position_ids = torch.arange(context_len).int()
104 - alibi_bias = (position_ids - context_len + 1).float()
103 + position_ids = torch.arange(seq_len).int()
104 + alibi_bias = (position_ids - seq_len + 1).float()
105 alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
106 1, 1, -1)
107
@@ -149,13 +149,13 @@ def test_paged_attention(
149 if use_alibi:
150 alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
151
152 - context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
153 - context_lens[-1] = MAX_SEQ_LEN
154 - max_context_len = max(context_lens)
155 - context_lens = torch.tensor(context_lens, dtype=torch.int)
152 + seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
153 + seq_lens[-1] = MAX_SEQ_LEN
154 + max_seq_len = max(seq_lens)
155 + seq_lens = torch.tensor(seq_lens, dtype=torch.int)
156
157 # Create the block tables.
158 - max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
158 + max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
159 block_tables = []
160 for _ in range(num_seqs):
161 block_table = [
@@ -186,16 +186,15 @@ def test_paged_attention(
186 num_kv_heads,
187 scale,
188 block_tables,
189 - context_lens,
189 + seq_lens,
190 block_size,
191 - max_context_len,
191 + max_seq_len,
192 alibi_slopes,
193 kv_cache_dtype,
194 kv_scale,
195 )
196 elif version == "v2":
197 - num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
198 - PARTITION_SIZE)
197 + num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
198 assert PARTITION_SIZE % block_size == 0
199 num_seqs, num_heads, head_size = output.shape
200 tmp_output = torch.empty(
@@ -218,9 +217,9 @@ def test_paged_attention(
217 num_kv_heads,
218 scale,
219 block_tables,
221 - context_lens,
220 + seq_lens,
221 block_size,
223 - max_context_len,
222 + max_seq_len,
223 alibi_slopes,
224 kv_cache_dtype,
225 kv_scale,
@@ -255,7 +254,7 @@ def test_paged_attention(
254 key_cache,
255 value_cache,
256 block_tables,
258 - context_lens,
257 + seq_lens,
258 scale,
259 alibi_slopes,
260 )
tests/kernels/test_prefix_prefill.py
+8 -8
@@ -51,12 +51,12 @@ def test_contexted_kv_attention(
51 cache_size = 640
52 block_size = 32
53 max_block_per_request = 64
54 - subquery_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
54 + query_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
55 ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
56 - seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
56 + seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
57 num_kv_heads = num_heads // num_queries_per_kv
58
59 - num_tokens = sum(subquery_lens)
59 + num_tokens = sum(query_lens)
60 query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
61 query.uniform_(-1e-3, 1e-3)
62 output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
@@ -75,15 +75,15 @@ def test_contexted_kv_attention(
75 num_kv_heads,
76 head_size,
77 dtype=dtype)
78 - k = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
79 - v = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
78 + k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
79 + v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
80 values = torch.arange(0, cache_size, dtype=torch.long)
81 values = values[torch.randperm(cache_size)]
82 block_table = values[:BS * max_block_per_request].view(
83 BS, max_block_per_request)
84 b_seq_len = torch.tensor(seq_lens, dtype=torch.long)
85 b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
86 - b_start_loc = torch.cumsum(torch.tensor([0] + subquery_lens[:-1],
86 + b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
87 dtype=torch.long),
88 dim=0)
89 max_input_len = MAX_SEQ_LEN
@@ -92,7 +92,7 @@ def test_contexted_kv_attention(
92 dtype=torch.long),
93 dim=0)
94 for i in range(BS):
95 - for j in range(subquery_lens[i]):
95 + for j in range(query_lens[i]):
96 k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
97 j])
98 v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
@@ -178,7 +178,7 @@ def test_contexted_kv_attention(
178 value = value.unsqueeze(0)
179
180 attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
181 - subquery_lens, seq_lens)
181 + query_lens, seq_lens)
182 if sliding_window > 0:
183 attn_bias = attn_bias.make_local_attention_from_bottomright(
184 sliding_window)
tests/samplers/test_sampler.py
+17 -17
@@ -58,7 +58,7 @@ def _do_sample(
58 device: str,
59 ):
60 seq_group_metadata_list = []
61 - prompt_lens = []
61 + seq_lens = []
62 for i in range(batch_size):
63 seq_group_metadata_list.append(
64 SequenceGroupMetadata(
@@ -68,12 +68,12 @@ def _do_sample(
68 sampling_params=sampling_params,
69 block_tables={0: [1]},
70 ))
71 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
71 + seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
72
73 sampling_metadata = SamplingMetadata.prepare(
74 seq_group_metadata_list,
75 - prompt_lens,
76 - subquery_lens=prompt_lens,
75 + seq_lens,
76 + query_lens=seq_lens,
77 device=device,
78 pin_memory=model_runner.pin_memory)
79 return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
@@ -421,7 +421,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
421 "Invalid test case, need seq_group_metadata_list"
422
423 batch_size = 0
424 - prompt_lens = []
424 + seq_lens = []
425 sampling_params_per_row = []
426 for sgm in seq_group_metadata_list:
427 sampling_params = sgm.sampling_params
@@ -431,7 +431,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
431 # a prompt seq_group has only one sequence
432 seq_data = next(iter(sgm.seq_data.values()))
433 prompt_len = seq_data.get_prompt_len()
434 - prompt_lens.append(prompt_len)
434 + seq_lens.append(prompt_len)
435
436 if sgm.sampling_params.prompt_logprobs:
437 # with prompt_logprobs each token in the prompt has a row in
@@ -451,8 +451,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
451 _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
452 sampling_metadata = SamplingMetadata.prepare(
453 seq_group_metadata_list,
454 - prompt_lens=prompt_lens if prompt_lens else None,
455 - subquery_lens=prompt_lens if prompt_lens else None,
454 + seq_lens=seq_lens if seq_lens else None,
455 + query_lens=seq_lens if seq_lens else None,
456 device=device,
457 pin_memory=model_runner.pin_memory)
458 # the logits tensor is modified in-place by the sampler
@@ -497,7 +497,7 @@ def test_sampler_mixed(seed: int, device: str):
497
498 seq_group_metadata_list = []
499 expected_tokens: List[Optional[List[int]]] = []
500 - prompt_lens = []
500 + seq_lens = []
501 for i in range(batch_size):
502 expected: Optional[List[int]] = None
503 sampling_type = random.randint(0, 3)
@@ -532,13 +532,13 @@ def test_sampler_mixed(seed: int, device: str):
532 sampling_params=sampling_params,
533 block_tables={0: [1]},
534 ))
535 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
535 + seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
536
537 def test_sampling(model_runner: ModelRunner):
538 sampling_metadata = SamplingMetadata.prepare(
539 seq_group_metadata_list,
540 - prompt_lens,
541 - subquery_lens=prompt_lens,
540 + seq_lens,
541 + query_lens=seq_lens,
542 device=device,
543 pin_memory=model_runner.pin_memory)
544 sampler_output = sampler(logits=fake_logits,
@@ -575,7 +575,7 @@ def test_sampler_mixed(seed: int, device: str):
575 # Shuffle the batch and resample
576 target_index = list(range(batch_size))
577 for list_to_shuffle in (target_index, seq_group_metadata_list,
578 - expected_tokens, prompt_lens):
578 + expected_tokens, seq_lens):
579 random.Random(seed).shuffle(list_to_shuffle)
580 target_index = torch.tensor(target_index)
581 input_tensor.data = input_tensor.index_select(0, target_index)
@@ -620,7 +620,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
620 assert len(warpers) == 2 # top_p and top_k
621
622 seq_group_metadata_list = []
623 - prompt_lens = []
623 + seq_lens = []
624 for i in range(batch_size):
625 seq_group_metadata_list.append(
626 SequenceGroupMetadata(
@@ -634,12 +634,12 @@ def test_sampler_top_k_top_p(seed: int, device: str):
634 ),
635 block_tables={0: [1]},
636 ))
637 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
637 + seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
638
639 sampling_metadata = SamplingMetadata.prepare(
640 seq_group_metadata_list,
641 - prompt_lens,
642 - subquery_lens=prompt_lens,
641 + seq_lens,
642 + query_lens=seq_lens,
643 device=device,
644 pin_memory=model_runner.pin_memory)
645
tests/spec_decode/e2e/conftest.py
+2 -2
@@ -45,7 +45,7 @@ class AsyncLLM:
45 gpu_memory_utilization: float = 0.9,
46 swap_space: int = 4,
47 enforce_eager: bool = False,
48 - max_context_len_to_capture: int = 8192,
48 + max_seq_len_to_capture: int = 8192,
49 disable_custom_all_reduce: bool = False,
50 **kwargs,
51 ) -> None:
@@ -66,7 +66,7 @@ class AsyncLLM:
66 gpu_memory_utilization=gpu_memory_utilization,
67 swap_space=swap_space,
68 enforce_eager=enforce_eager,
69 - max_context_len_to_capture=max_context_len_to_capture,
69 + max_seq_len_to_capture=max_seq_len_to_capture,
70 engine_use_ray=True,
71 disable_custom_all_reduce=disable_custom_all_reduce,
72 **kwargs,
tests/spec_decode/test_multi_step_worker.py
+14 -10
@@ -34,7 +34,7 @@ def test_assert_enough_kv_space(num_steps: int):
34 list(range(block_size * 2)),
35 ]
36
37 - final_seq_lens = [
37 + final_prompt_lens = [
38 len(prompt + output) + num_steps
39 for prompt, output in zip(prompts, prev_output_tokens)
40 ]
@@ -43,7 +43,7 @@ def test_assert_enough_kv_space(num_steps: int):
43 prompts,
44 num_gpu_blocks,
45 block_size,
46 - final_seq_lens,
46 + final_prompt_lens,
47 continuations=prev_output_tokens)
48
49 assert_enough_kv_space = MultiStepWorker._assert_enough_kv_space # pylint: disable=protected-access
@@ -103,17 +103,21 @@ def test_same_output_for_single_step():
103 [6, 7, 8, 9, 10],
104 ]
105
106 - final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
106 + final_prompt_lens = [len(prompt) + num_steps for prompt in prompts]
107
108 multi_step_execute_model_data = create_execute_model_data(
109 seq_group_metadata_list=create_seq_group_metadata_from_prompts(
110 - prompts, num_gpu_blocks, block_size,
111 - final_seq_lens=final_seq_lens))
110 + prompts,
111 + num_gpu_blocks,
112 + block_size,
113 + final_prompt_lens=final_prompt_lens))
114
115 single_step_execute_model_data = create_execute_model_data(
116 seq_group_metadata_list=create_seq_group_metadata_from_prompts(
115 - prompts, num_gpu_blocks, block_size,
116 - final_seq_lens=final_seq_lens))
117 + prompts,
118 + num_gpu_blocks,
119 + block_size,
120 + final_prompt_lens=final_prompt_lens))
121
122 zero_kv_cache(multi_step_worker.cache_engine)
123 set_random_seed(seed)
@@ -181,7 +185,7 @@ def test_same_output_for_multi_step():
185 random.randint(0, 1000) for _ in range(random.randint(10, 20))
186 ] for _ in range(10)]
187
184 - final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
188 + final_prompt_lens = [len(prompt) + num_steps for prompt in prompts]
189
190 rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
191 multi_step_worker.execute_model = patch_execute_model_with_seeds(
@@ -195,7 +199,7 @@ def test_same_output_for_multi_step():
199 num_gpu_blocks,
200 block_size,
201 continuations=continuations,
198 - final_seq_lens=final_seq_lens), )
202 + final_prompt_lens=final_prompt_lens), )
203
204 # Run multi-step.
205 zero_kv_cache(multi_step_worker.cache_engine)
@@ -217,7 +221,7 @@ def test_same_output_for_multi_step():
221 num_gpu_blocks,
222 block_size,
223 continuations=continuations,
220 - final_seq_lens=final_seq_lens))
224 + final_prompt_lens=final_prompt_lens))
225
226 single_step_output.extend(
227 worker.execute_model(**execute_model_data.to_dict(), ))
tests/spec_decode/test_ngram_worker.py
+15 -9
@@ -43,11 +43,13 @@ def test_ngram_algo_correctness_for_single_no_match():
43 ]
44
45 proposal_len = 5
46 - final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
46 + final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
47 ngram_sampler_output_data = create_execute_model_data(
48 seq_group_metadata_list=create_seq_group_metadata_from_prompts(
49 - prompts, num_gpu_blocks, block_size,
50 - final_seq_lens=final_seq_lens))
49 + prompts,
50 + num_gpu_blocks,
51 + block_size,
52 + final_prompt_lens=final_prompt_lens))
53
54 proposals = proposer.get_proposals(
55 **ngram_sampler_output_data.to_dict(),
@@ -110,11 +112,13 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
112 ]
113
114 proposal_len = 5
113 - final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
115 + final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
116 ngram_sampler_output_data = create_execute_model_data(
117 seq_group_metadata_list=create_seq_group_metadata_from_prompts(
116 - prompts, num_gpu_blocks, block_size,
117 - final_seq_lens=final_seq_lens))
118 + prompts,
119 + num_gpu_blocks,
120 + block_size,
121 + final_prompt_lens=final_prompt_lens))
122
123 proposals = proposer.get_proposals(
124 **ngram_sampler_output_data.to_dict(),
@@ -180,11 +184,13 @@ def test_ngram_algo_correctness_for_batches_match_all():
184 ]
185
186 proposal_len = 5
183 - final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
187 + final_prompt_lens = [len(prompt) + proposal_len for prompt in prompts]
188 ngram_sampler_output_data = create_execute_model_data(
189 seq_group_metadata_list=create_seq_group_metadata_from_prompts(
186 - prompts, num_gpu_blocks, block_size,
187 - final_seq_lens=final_seq_lens))
190 + prompts,
191 + num_gpu_blocks,
192 + block_size,
193 + final_prompt_lens=final_prompt_lens))
194
195 proposals = proposer.get_proposals(
196 **ngram_sampler_output_data.to_dict(),
tests/spec_decode/utils.py
+4 -4
@@ -144,7 +144,7 @@ def create_seq_group_metadata_from_prompts(
144 prompts: List[List[int]],
145 num_gpu_blocks: int,
146 block_size: int,
147 - final_seq_lens: List[int],
147 + final_prompt_lens: List[int],
148 continuations: Optional[List[List[int]]] = None,
149 seq_ids: Optional[List[int]] = None,
150 ) -> List[SequenceGroupMetadata]:
@@ -162,7 +162,7 @@ def create_seq_group_metadata_from_prompts(
162 free_gpu_blocks.pop()
163 for _ in range(round_up_to_next_block(final_len, block_size))
164 ]
165 - for i, final_len in enumerate(final_seq_lens)
165 + for i, final_len in enumerate(final_prompt_lens)
166 }
167
168 return [
@@ -251,13 +251,13 @@ def create_batch(batch_size,
251 prev_output_tokens = [[
252 next(iterator) for _ in range(prev_output_token_len)
253 ] for _ in range(batch_size)]
254 - final_seq_lens = [
254 + final_prompt_lens = [
255 len(prompt) + len(prev_output_token) + k + 1
256 for prompt, prev_output_token in zip(prompts, prev_output_tokens)
257 ]
258
259 execute_model_data = create_execute_model_data(
260 create_seq_group_metadata_from_prompts(prompts, num_gpu_blocks,
261 - block_size, final_seq_lens,
261 + block_size, final_prompt_lens,
262 prev_output_tokens, seq_ids), )
263 return execute_model_data, prompts, prev_output_tokens
tests/test_logits_processor.py
+4 -4
@@ -70,7 +70,7 @@ def test_logits_processors(seed: int, device: str):
70 return logits
71
72 seq_group_metadata_list = []
73 - prompt_lens = []
73 + seq_lens = []
74 for i in range(batch_size):
75 seq_group_metadata_list.append(
76 SequenceGroupMetadata(
@@ -81,12 +81,12 @@ def test_logits_processors(seed: int, device: str):
81 logits_processors=[pick_ith]),
82 block_tables={0: [1]},
83 ))
84 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
84 + seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
85
86 sampling_metadata = SamplingMetadata.prepare(
87 seq_group_metadata_list,
88 - prompt_lens,
89 - subquery_lens=prompt_lens,
88 + seq_lens,
89 + query_lens=seq_lens,
90 device=model_runner.device,
91 pin_memory=model_runner.pin_memory)
92 logits_processor_output = logits_processor(
tests/worker/test_model_runner.py
+48 -51
@@ -23,14 +23,14 @@ def test_prepare_prompt(batch_size):
23 lora_config=None)
24 model_runner.set_block_size(16)
25
26 - prompt_lens = []
26 + seq_lens = []
27 seq_group_metadata_list = []
28 block_tables = {0: [1]}
29 for i in range(batch_size):
30 # make sure all tokens fit into one block
31 - prompt_len = i % (model_runner.block_size - 1) + 1
32 - prompt_lens.append(prompt_len)
33 - seq_data = SequenceData(list(range(prompt_len)))
31 + seq_len = i % (model_runner.block_size - 1) + 1
32 + seq_lens.append(seq_len)
33 + seq_data = SequenceData(list(range(seq_len)))
34 seq_group_metadata = SequenceGroupMetadata(
35 request_id=f"test_{i}",
36 is_prompt=True,
@@ -43,29 +43,29 @@ def test_prepare_prompt(batch_size):
43
44 expected_selected_token_indices = []
45 selected_token_start_idx = 0
46 - for prompt_len in prompt_lens:
46 + for seq_len in seq_lens:
47 expected_selected_token_indices.append(selected_token_start_idx +
48 - prompt_len - 1)
49 - selected_token_start_idx += prompt_len
50 - (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
51 - _, _,
52 - slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
53 - assert return_prompt_lens == prompt_lens
48 + seq_len - 1)
49 + selected_token_start_idx += seq_len
50 + (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
51 + _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
52 + assert return_seq_lens == seq_lens
53 assert len(slot_mapping) == len(input_tokens)
54
55 # Verify input metadata is correct for prompts.
56 device = model_runner.device
57 assert attn_metadata.is_prompt is True
59 - assert torch.allclose(attn_metadata.prompt_lens_tensor,
60 - torch.tensor(prompt_lens, device=device))
61 - assert attn_metadata.prompt_lens == prompt_lens
62 - assert attn_metadata.max_prompt_len == max(prompt_lens)
58 + assert torch.allclose(
59 + attn_metadata.seq_lens_tensor,
60 + torch.tensor(seq_lens, device=device, dtype=torch.int))
61 + assert attn_metadata.seq_lens == seq_lens
62 + assert attn_metadata.max_seq_len == max(seq_lens)
63
64 # Test subquery start locs.
65 start_idx = 0
66 start_loc = [start_idx]
67 - for prompt_len in prompt_lens:
68 - start_idx += prompt_len
67 + for seq_len in seq_lens:
68 + start_idx += seq_len
69 start_loc.append(start_idx)
70 assert torch.allclose(
71 attn_metadata.subquery_start_loc,
@@ -75,17 +75,16 @@ def test_prepare_prompt(batch_size):
75 # equivalent to subquery_start_loc.
76 start_idx = 0
77 seq_start_loc = [start_idx]
78 - for prompt_len in prompt_lens:
79 - start_idx += prompt_len
78 + for seq_len in seq_lens:
79 + start_idx += seq_len
80 seq_start_loc.append(start_idx)
81
82 assert torch.allclose(
83 attn_metadata.seq_start_loc,
84 torch.tensor(start_loc, dtype=torch.int32, device=device))
85 - assert attn_metadata.max_context_len is None
85 assert torch.allclose(
87 - attn_metadata.context_lens,
88 - torch.zeros(attn_metadata.context_lens.shape[0],
86 + attn_metadata.context_lens_tensor,
87 + torch.zeros(attn_metadata.context_lens_tensor.shape[0],
88 dtype=torch.int,
89 device=device))
90
@@ -96,18 +95,18 @@ def test_prepare_prompt(batch_size):
95 # Cuda graph should not be used for prerill.
96 assert attn_metadata.use_cuda_graph is False
97
99 - assert len(input_tokens) == sum(prompt_lens)
100 - assert len(input_positions) == sum(prompt_lens)
98 + assert len(input_tokens) == sum(seq_lens)
99 + assert len(input_positions) == sum(seq_lens)
100 torch.testing.assert_close(input_tokens, input_positions)
101
102 sampling_metadata = SamplingMetadata.prepare(
103 seq_group_metadata_list,
105 - prompt_lens,
106 - subquery_lens=prompt_lens,
104 + seq_lens,
105 + query_lens=seq_lens,
106 device=model_runner.device,
107 pin_memory=model_runner.pin_memory)
109 - assert len(input_tokens) == sum(prompt_lens)
110 - assert len(input_positions) == sum(prompt_lens)
108 + assert len(input_tokens) == sum(seq_lens)
109 + assert len(input_positions) == sum(seq_lens)
110 actual = sampling_metadata.selected_token_indices
111 expected = torch.tensor(expected_selected_token_indices,
112 device=actual.device,
@@ -146,13 +145,13 @@ def test_prepare_decode_cuda_graph(batch_size):
145 lora_config=None)
146 model_runner.set_block_size(16)
147
149 - prompt_lens = []
148 + seq_lens = []
149 seq_group_metadata_list = []
150 for i in range(batch_size):
151 # make sure all tokens fit into one block
153 - prompt_len = i % (model_runner.block_size - 1) + 1
154 - prompt_lens.append(prompt_len)
155 - seq_data = list(range(prompt_len))
152 + seq_len = i % (model_runner.block_size - 1) + 1
153 + seq_lens.append(seq_len)
154 + seq_data = list(range(seq_len))
155 seq_data = SequenceData(seq_data)
156 seq_group_metadata = SequenceGroupMetadata(
157 request_id=f"test_{i}",
@@ -172,14 +171,13 @@ def test_prepare_decode_cuda_graph(batch_size):
171 # Verify input metadata is correct for prompts.
172 device = model_runner.device
173 assert attn_metadata.is_prompt is False
175 - assert attn_metadata.prompt_lens is None
176 - assert attn_metadata.max_prompt_len is None
174 + assert attn_metadata.seq_lens is None
175 assert attn_metadata.subquery_start_loc is None
176 assert attn_metadata.seq_start_loc is None
179 - assert attn_metadata.max_context_len == max(prompt_lens)
177 + assert attn_metadata.max_seq_len == max(seq_lens)
178 assert torch.allclose(
181 - attn_metadata.context_lens[:len(prompt_lens)],
182 - torch.tensor(prompt_lens, dtype=torch.int, device=device))
179 + attn_metadata.seq_lens_tensor[:len(seq_lens)],
180 + torch.tensor(seq_lens, dtype=torch.int, device=device))
181
182 # block table's first index corresponds to each batch, meaning in
183 # decoding it is each token.
@@ -198,13 +196,13 @@ def test_prepare_decode_cuda_graph(batch_size):
196 # Verify Sampling
197 expected_selected_token_indices = []
198 selected_token_start_idx = 0
201 - for prompt_len in prompt_lens:
199 + for seq_len in seq_lens:
200 expected_selected_token_indices.append(selected_token_start_idx)
201 selected_token_start_idx += 1
202 sampling_metadata = SamplingMetadata.prepare(
203 seq_group_metadata_list,
206 - prompt_lens,
207 - subquery_lens=prompt_lens,
204 + seq_lens,
205 + query_lens=seq_lens,
206 device=model_runner.device,
207 pin_memory=model_runner.pin_memory)
208 actual = sampling_metadata.selected_token_indices
@@ -241,14 +239,13 @@ def test_empty_seq_group():
239 assert attn_metadata is None
240 assert len(slot_mapping) == 0
241
244 - (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
245 - _, _,
246 - slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
242 + (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
243 + _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
244 assert len(input_tokens) == 0
245 assert len(input_positions) == 0
246 assert attn_metadata is None
247 assert len(slot_mapping) == 0
251 - assert len(return_prompt_lens) == 0
248 + assert len(return_seq_lens) == 0
249
250
251 @pytest.fixture
@@ -288,7 +285,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
285 model_runner.set_block_size(16)
286
287 # Add prefill requests.
291 - prompt_lens = []
288 + seq_lens = []
289 seq_group_metadata_list = []
290 prefill_metadata_list = []
291 decode_metadata_list = []
@@ -297,9 +294,9 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
294 decode_batch_size = batch_size - prefill_batch_size
295 for i in range(prefill_batch_size):
296 # make sure all tokens fit into one block
300 - prompt_len = i % (model_runner.block_size - 1) + 1
301 - prompt_lens.append(prompt_len)
302 - seq_data = SequenceData(list(range(prompt_len)))
297 + seq_len = i % (model_runner.block_size - 1) + 1
298 + seq_lens.append(seq_len)
299 + seq_data = SequenceData(list(range(seq_len)))
300 seq_group_metadata = SequenceGroupMetadata(
301 request_id=f"test_{i}",
302 is_prompt=True,
@@ -314,8 +311,8 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
311 # Add decode requests
312 for i in range(prefill_batch_size, batch_size):
313 # make sure all tokens fit into one block
317 - prompt_len = i % (model_runner.block_size - 1) + 1
318 - prompt_toks = list(range(prompt_len))
314 + seq_len = i % (model_runner.block_size - 1) + 1
315 + prompt_toks = list(range(seq_len))
316 seq_data = SequenceData(prompt_toks)
317 seq_group_metadata = SequenceGroupMetadata(
318 request_id=f"test_{i}",
@@ -343,7 +340,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
340 else:
341 assert attn_metadata.num_decode_tokens == _get_graph_batch_size(
342 decode_batch_size)
346 - assert attn_metadata.num_prefill_tokens == sum(prompt_lens)
343 + assert attn_metadata.num_prefill_tokens == sum(seq_lens)
344
345 # Verify attn metadata is consistent. We don't need to test individual
346 # values here because they are tested above.
vllm/_custom_ops.py
+9 -9
@@ -39,17 +39,17 @@ def paged_attention_v1(
39 num_kv_heads: int,
40 scale: float,
41 block_tables: torch.Tensor,
42 - context_lens: torch.Tensor,
42 + seq_lens: torch.Tensor,
43 block_size: int,
44 - max_context_len: int,
44 + max_seq_len: int,
45 alibi_slopes: Optional[torch.Tensor],
46 kv_cache_dtype: str,
47 kv_scale: float,
48 ) -> None:
49 vllm_ops.paged_attention_v1(out, query, key_cache, value_cache,
50 - num_kv_heads, scale, block_tables,
51 - context_lens, block_size, max_context_len,
52 - alibi_slopes, kv_cache_dtype, kv_scale)
50 + num_kv_heads, scale, block_tables, seq_lens,
51 + block_size, max_seq_len, alibi_slopes,
52 + kv_cache_dtype, kv_scale)
53
54
55 def paged_attention_v2(
@@ -63,17 +63,17 @@ def paged_attention_v2(
63 num_kv_heads: int,
64 scale: float,
65 block_tables: torch.Tensor,
66 - context_lens: torch.Tensor,
66 + seq_lens: torch.Tensor,
67 block_size: int,
68 - max_context_len: int,
68 + max_seq_len: int,
69 alibi_slopes: Optional[torch.Tensor],
70 kv_cache_dtype: str,
71 kv_scale: float,
72 ) -> None:
73 vllm_ops.paged_attention_v2(out, exp_sum, max_logits, tmp_out, query,
74 key_cache, value_cache, num_kv_heads, scale,
75 - block_tables, context_lens, block_size,
76 - max_context_len, alibi_slopes, kv_cache_dtype,
75 + block_tables, seq_lens, block_size,
76 + max_seq_len, alibi_slopes, kv_cache_dtype,
77 kv_scale)
78
79
vllm/attention/backends/flash_attn.py
+22 -22
@@ -66,27 +66,24 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
66 # Currently, input sequences can only contain all prompts
67 # or all decoding. True if all sequences are prompts.
68 is_prompt: bool
69 - # (batch_size,). The prompt length per sequence. None if it is a decoding.
70 - prompt_lens: Optional[List[int]]
71 - # prompt_lens stored as a tensor.
72 - prompt_lens_tensor: Optional[torch.Tensor]
69 + # (batch_size,). The sequence length per sequence. Sequence length means
70 + # the computed tokens + new tokens None if it is a decoding.
71 + seq_lens: Optional[List[int]]
72 + # seq_lens stored as a tensor.
73 + seq_lens_tensor: Optional[torch.Tensor]
74
74 - # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75 + # NOTE(sang): Definition of context_len, query_len, and seq_len.
76 # |---------- N-1 iteration --------|
77 # |---------------- N iteration ---------------------|
78 # |- tokenA -|......................|-- newTokens ---|
79 # |---------- context_len ----------|
79 - # |-------------------- seqlen ----------------------|
80 - # |- subquery_len -|
80 + # |-------------------- seq_len ----------------------|
81 + # |-- query_len ---|
82
82 - # WARNING(sang): context_len has different definition depending on if it is
83 - # prefill vs decoding. When it is prefill, it doesn't include new tokens.
84 - # When it is for decoding, it includes a new token.
85 -
86 - # Maximum subquery length in the batch.
87 - max_subquery_len: Optional[int]
88 - # Maximum prompt length in the batch.
89 - max_prompt_len: Optional[int]
83 + # Maximum query length in the batch.
84 + max_query_len: Optional[int]
85 + # Maximum sequence length in the batch.
86 + max_seq_len: Optional[int]
87 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
88 # the batch, used to index into subquery. E.g., if the subquery length
89 # is [4, 6], it is [0, 4, 10].
@@ -95,6 +92,9 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
92 # the batch, used to index into sequence. E.g., if the sequence length is
93 # [4, 6], it is [0, 4, 10].
94 seq_start_loc: Optional[torch.Tensor]
95 + # (batch_size,) A tensor of context lengths (tokens that are computed
96 + # so far).
97 + context_lens_tensor: Optional[torch.Tensor]
98
99 # Whether or not if cuda graph is enabled.
100 # Cuda-graph is currently enabled for decoding only.
@@ -223,8 +223,8 @@ class FlashAttentionImpl(AttentionImpl):
223 v=value,
224 cu_seqlens_q=prefill_meta.seq_start_loc,
225 cu_seqlens_k=prefill_meta.seq_start_loc,
226 - max_seqlen_q=prefill_meta.max_prompt_len,
227 - max_seqlen_k=prefill_meta.max_prompt_len,
226 + max_seqlen_q=prefill_meta.max_seq_len,
227 + max_seqlen_k=prefill_meta.max_seq_len,
228 softmax_scale=self.scale,
229 causal=True,
230 window_size=self.sliding_window,
@@ -245,9 +245,9 @@ class FlashAttentionImpl(AttentionImpl):
245 value_cache,
246 prefill_meta.block_tables,
247 prefill_meta.subquery_start_loc,
248 - prefill_meta.prompt_lens_tensor,
249 - prefill_meta.context_lens,
250 - prefill_meta.max_subquery_len,
248 + prefill_meta.seq_lens_tensor,
249 + prefill_meta.context_lens_tensor,
250 + prefill_meta.max_query_len,
251 self.alibi_slopes,
252 self.sliding_window[0],
253 )
@@ -258,8 +258,8 @@ class FlashAttentionImpl(AttentionImpl):
258 key_cache,
259 value_cache,
260 decode_meta.block_tables,
261 - decode_meta.context_lens,
262 - decode_meta.max_context_len,
261 + decode_meta.seq_lens_tensor,
262 + decode_meta.max_seq_len,
263 attn_metadata.kv_cache_dtype,
264 self.num_kv_heads,
265 self.scale,
vllm/attention/backends/rocm_flash_attn.py
+30 -30
@@ -64,27 +64,24 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
64 # Currently, input sequences can only contain all prompts
65 # or all decoding. True if all sequences are prompts.
66 is_prompt: bool
67 - # (batch_size,). The prompt length per sequence. None if it is a decoding.
68 - prompt_lens: Optional[List[int]]
69 - # prompt_lens stored as a tensor.
70 - prompt_lens_tensor: Optional[torch.Tensor]
67 + # (batch_size,). The sequence length per sequence. Sequence length means
68 + # the computed tokens + new tokens None if it is a decoding.
69 + seq_lens: Optional[List[int]]
70 + # seq_lens stored as a tensor.
71 + seq_lens_tensor: Optional[torch.Tensor]
72
72 - # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
73 + # NOTE(sang): Definition of context_len, query_len, and seq_len.
74 # |---------- N-1 iteration --------|
75 # |---------------- N iteration ---------------------|
76 # |- tokenA -|......................|-- newTokens ---|
77 # |---------- context_len ----------|
77 - # |-------------------- seqlen ----------------------|
78 - # |- subquery_len -|
78 + # |-------------------- seq_len ----------------------|
79 + # |-- query_len ---|
80
80 - # WARNING(sang): context_len has different definition depending on if it is
81 - # prefill vs decoding. When it is prefill, it doesn't include new tokens.
82 - # When it is for decoding, it includes a new token.
83 -
84 - # Maximum subquery length in the batch.
85 - max_subquery_len: Optional[int]
86 - # Maximum prompt length in the batch.
87 - max_prompt_len: Optional[int]
81 + # Maximum query length in the batch.
82 + max_query_len: Optional[int]
83 + # Maximum sequence length in the batch.
84 + max_seq_len: Optional[int]
85 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
86 # the batch, used to index into subquery. E.g., if the subquery length
87 # is [4, 6], it is [0, 4, 10].
@@ -98,6 +95,9 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
95 # Cuda-graph is currently enabled for decoding only.
96 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
97 use_cuda_graph: bool
98 + # (batch_size,) A tensor of context lengths (tokens that are computed
99 + # so far).
100 + context_lens_tensor: Optional[torch.Tensor]
101
102
103 class ROCmFlashAttentionImpl(AttentionImpl):
@@ -247,7 +247,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
247
248 if prefill_meta := attn_metadata.prefill_metadata:
249 # Prompt run.
250 - assert prefill_meta.prompt_lens is not None
250 + assert prefill_meta.seq_lens is not None
251 if kv_cache is None or prefill_meta.block_tables.numel() == 0:
252 # triton attention
253 # When block_tables are not filled, it means q and k are the
@@ -260,8 +260,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
260 None,
261 prefill_meta.seq_start_loc,
262 prefill_meta.seq_start_loc,
263 - prefill_meta.max_prompt_len,
264 - prefill_meta.max_prompt_len,
263 + prefill_meta.max_seq_len,
264 + prefill_meta.max_seq_len,
265 True,
266 self.scale,
267 )
@@ -274,7 +274,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
274 query,
275 key,
276 value,
277 - prefill_meta.prompt_lens,
277 + prefill_meta.seq_lens,
278 self.scale,
279 )
280 else:
@@ -284,8 +284,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
284 v=value,
285 cu_seqlens_q=prefill_meta.seq_start_loc,
286 cu_seqlens_k=prefill_meta.seq_start_loc,
287 - max_seqlen_q=prefill_meta.max_prompt_len,
288 - max_seqlen_k=prefill_meta.max_prompt_len,
287 + max_seqlen_q=prefill_meta.max_seq_len,
288 + max_seqlen_k=prefill_meta.max_seq_len,
289 softmax_scale=self.scale,
290 causal=True,
291 )
@@ -303,9 +303,9 @@ class ROCmFlashAttentionImpl(AttentionImpl):
303 value_cache,
304 prefill_meta.block_tables,
305 prefill_meta.subquery_start_loc,
306 - prefill_meta.prompt_lens_tensor,
307 - prefill_meta.context_lens,
308 - prefill_meta.max_subquery_len,
306 + prefill_meta.seq_lens_tensor,
307 + prefill_meta.context_lens_tensor,
308 + prefill_meta.max_query_len,
309 self.alibi_slopes,
310 self.sliding_window[0],
311 )
@@ -317,8 +317,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
317 key_cache,
318 value_cache,
319 decode_meta.block_tables,
320 - decode_meta.context_lens,
321 - decode_meta.max_context_len,
320 + decode_meta.seq_lens_tensor,
321 + decode_meta.max_seq_len,
322 attn_metadata.kv_cache_dtype,
323 self.num_kv_heads,
324 self.scale,
@@ -334,13 +334,13 @@ def _naive_attention(
334 query: torch.Tensor,
335 key: torch.Tensor,
336 value: torch.Tensor,
337 - prompt_lens: List[int],
337 + seq_lens: List[int],
338 scale: float,
339 ) -> torch.Tensor:
340 output = torch.empty_like(query)
341 start = 0
342 - for _, prompt_len in enumerate(prompt_lens):
343 - end = start + prompt_len
342 + for _, seq_len in enumerate(seq_lens):
343 + end = start + seq_len
344 out = _naive_masked_attention(
345 query[start:end],
346 key[start:end],
@@ -349,7 +349,7 @@ def _naive_attention(
349 )
350 # TODO(woosuk): Unnecessary copy. Optimize.
351 output[start:end].copy_(out)
352 - start += prompt_len
352 + start += seq_len
353
354 return output
355
vllm/attention/backends/torch_sdpa.py
+18 -18
@@ -58,7 +58,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
58 # or all decoding. True if all sequences are prompts.
59 is_prompt: bool
60 slot_mapping: torch.Tensor
61 - prompt_lens: Optional[List[int]]
61 + seq_lens: Optional[List[int]]
62
63 def __post_init__(self):
64 # Set during the execution of the first attention op.
@@ -136,7 +136,7 @@ class TorchSDPABackendImpl(AttentionImpl):
136 kv_scale)
137
138 if attn_metadata.is_prompt:
139 - assert attn_metadata.prompt_lens is not None
139 + assert attn_metadata.seq_lens is not None
140 if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
141 if self.num_kv_heads != self.num_heads:
142 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
@@ -147,13 +147,13 @@ class TorchSDPABackendImpl(AttentionImpl):
147 if self.alibi_slopes is not None:
148 att_masks = _make_alibi_bias(
149 self.alibi_slopes, query.dtype,
150 - attn_metadata.prompt_lens) # type: ignore
150 + attn_metadata.seq_lens) # type: ignore
151 elif self.sliding_window is not None:
152 att_masks = _make_sliding_window_bias(
153 - attn_metadata.prompt_lens, self.sliding_window,
153 + attn_metadata.seq_lens, self.sliding_window,
154 query.dtype) # type: ignore
155 else:
156 - att_masks = [None] * len(attn_metadata.prompt_lens)
156 + att_masks = [None] * len(attn_metadata.seq_lens)
157 attn_metadata.attn_bias = att_masks
158
159 query = query.movedim(0, query.dim() - 2)
@@ -164,9 +164,9 @@ class TorchSDPABackendImpl(AttentionImpl):
164 output = torch.empty(
165 (num_tokens, self.num_heads, self.head_size),
166 dtype=query.dtype)
167 - for prompt_len, mask in zip(attn_metadata.prompt_lens,
168 - attn_metadata.attn_bias):
169 - end = start + prompt_len
167 + for seq_len, mask in zip(attn_metadata.seq_lens,
168 + attn_metadata.attn_bias):
169 + end = start + seq_len
170 sub_out = scaled_dot_product_attention(
171 query[:, start:end, :],
172 key[:, start:end, :],
@@ -189,8 +189,8 @@ class TorchSDPABackendImpl(AttentionImpl):
189 key_cache,
190 value_cache,
191 attn_metadata.block_tables,
192 - attn_metadata.context_lens,
193 - attn_metadata.max_context_len,
192 + attn_metadata.seq_lens_tensor,
193 + attn_metadata.max_seq_len,
194 attn_metadata.kv_cache_dtype,
195 self.num_kv_heads,
196 self.scale,
@@ -205,13 +205,13 @@ class TorchSDPABackendImpl(AttentionImpl):
205 def _make_alibi_bias(
206 alibi_slopes: torch.Tensor,
207 dtype: torch.dtype,
208 - prompt_lens: List[int],
208 + seq_lens: List[int],
209 ) -> List[torch.Tensor]:
210 attn_biases = []
211 - for prompt_len in prompt_lens:
212 - bias = torch.arange(prompt_len, dtype=dtype)
211 + for seq_len in seq_lens:
212 + bias = torch.arange(seq_len, dtype=dtype)
213 # NOTE(zhuohan): HF uses
214 - # `bias = bias[None, :].repeat(prompt_len, 1)`
214 + # `bias = bias[None, :].repeat(seq_len, 1)`
215 # here. We find that both biases give the same results, but
216 # the bias below more accurately follows the original ALiBi
217 # paper.
@@ -221,7 +221,7 @@ def _make_alibi_bias(
221 bias = bias[None, :].repeat((num_heads, 1, 1))
222 bias.mul_(alibi_slopes[:, None, None])
223 inf_mask = torch.empty(
224 - (1, prompt_len, prompt_len),
224 + (1, seq_len, seq_len),
225 dtype=bias.dtype).fill_(-torch.inf).triu_(diagonal=1)
226 attn_biases.append((bias + inf_mask).to(dtype))
227
@@ -229,14 +229,14 @@ def _make_alibi_bias(
229
230
231 def _make_sliding_window_bias(
232 - prompt_lens: List[int],
232 + seq_lens: List[int],
233 window_size: Optional[int],
234 dtype: torch.dtype,
235 ) -> List[torch.Tensor]:
236 attn_biases = []
237 - for prompt_len in prompt_lens:
237 + for seq_len in seq_lens:
238 tensor = torch.full(
239 - (1, prompt_len, prompt_len),
239 + (1, seq_len, seq_len),
240 dtype=dtype,
241 fill_value=1,
242 )
vllm/attention/backends/xformers.py
+32 -33
@@ -66,28 +66,24 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
66 # Currently, input sequences can only contain all prompts
67 # or all decoding. True if all sequences are prompts.
68 is_prompt: bool
69 - # (batch_size,). The prompt length per sequence. None if it is a decoding.
70 - prompt_lens: Optional[List[int]]
71 - # prompt_lens stored as a tensor.
72 - prompt_lens_tensor: Optional[torch.Tensor]
69 + # (batch_size,). The sequence length per sequence. Sequence length means
70 + # the computed tokens + new tokens None if it is a decoding.
71 + seq_lens: Optional[List[int]]
72 + # seq_lens stored as a tensor.
73 + seq_lens_tensor: Optional[torch.Tensor]
74
74 - # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
75 # |---------- N-1 iteration --------|
76 # |---------------- N iteration ---------------------|
77 # |- tokenA -|......................|-- newTokens ---|
78 # |---------- context_len ----------|
79 - # |-------------------- seqlen ----------------------|
80 - # |- subquery_len -|
79 + # |-------------------- seq_len ----------------------|
80 + # |-- query_len ---|
81
82 - # WARNING(sang): context_len has different definition depending on if it is
83 - # prefill vs decoding. When it is prefill, it doesn't include new tokens.
84 - # When it is for decoding, it includes a new token.
85 -
86 - # Maximum subquery length in the batch.
87 - max_subquery_len: Optional[int]
82 + # Maximum query length in the batch.
83 + max_query_len: Optional[int]
84 # FIXME: It is for flash attn.
89 - # Maximum prompt length in the batch.
90 - max_prompt_len: Optional[int]
85 + # Maximum sequence length in the batch.
86 + max_seq_len: Optional[int]
87 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
88 # the batch, used to index into subquery. E.g., if the subquery length
89 # is [4, 6], it is [0, 4, 10].
@@ -97,6 +93,9 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
93 # the batch, used to index into sequence. E.g., if the sequence length is
94 # [4, 6], it is [0, 4, 10].
95 seq_start_loc: Optional[torch.Tensor]
96 + # (batch_size,) A tensor of context lengths (tokens that are computed
97 + # so far).
98 + context_lens_tensor: Optional[torch.Tensor]
99
100 # Whether or not if cuda graph is enabled.
101 # Cuda-graph is currently enabled for decoding only.
@@ -242,9 +241,9 @@ class XFormersImpl(AttentionImpl):
241 value_cache,
242 prefill_meta.block_tables,
243 prefill_meta.subquery_start_loc,
245 - prefill_meta.prompt_lens_tensor,
246 - prefill_meta.context_lens,
247 - prefill_meta.max_subquery_len,
244 + prefill_meta.seq_lens_tensor,
245 + prefill_meta.context_lens_tensor,
246 + prefill_meta.max_query_len,
247 self.alibi_slopes,
248 self.sliding_window,
249 )
@@ -257,8 +256,8 @@ class XFormersImpl(AttentionImpl):
256 key_cache,
257 value_cache,
258 decode_meta.block_tables,
260 - decode_meta.context_lens,
261 - decode_meta.max_context_len,
259 + decode_meta.seq_lens_tensor,
260 + decode_meta.max_seq_len,
261 attn_metadata.kv_cache_dtype,
262 self.num_kv_heads,
263 self.scale,
@@ -289,7 +288,7 @@ class XFormersImpl(AttentionImpl):
288 value: shape = [num_prefill_tokens, num_kv_heads, head_size]
289 attn_metadata: Metadata for attention.
290 """
292 - assert attn_metadata.prompt_lens is not None
291 + assert attn_metadata.seq_lens is not None
292 original_query = query
293 if self.num_kv_heads != self.num_heads:
294 # GQA/MQA requires the shape [B, M, G, H, K].
@@ -310,7 +309,7 @@ class XFormersImpl(AttentionImpl):
309 if attn_metadata.attn_bias is None:
310 if self.alibi_slopes is None:
311 attn_bias = BlockDiagonalCausalMask.from_seqlens(
313 - attn_metadata.prompt_lens)
312 + attn_metadata.seq_lens)
313 if self.sliding_window is not None:
314 attn_bias = attn_bias.make_local_attention(
315 self.sliding_window)
@@ -318,7 +317,7 @@ class XFormersImpl(AttentionImpl):
317 else:
318 attn_metadata.attn_bias = _make_alibi_bias(
319 self.alibi_slopes, self.num_kv_heads, query.dtype,
321 - attn_metadata.prompt_lens)
320 + attn_metadata.seq_lens)
321
322 # No alibi slopes.
323 # TODO(woosuk): Too many view operations. Let's try to reduce
@@ -343,8 +342,8 @@ class XFormersImpl(AttentionImpl):
342 # one. This is inefficient, especially when we have many short prompts.
343 output = torch.empty_like(original_query)
344 start = 0
346 - for i, prompt_len in enumerate(attn_metadata.prompt_lens):
347 - end = start + prompt_len
345 + for i, seq_len in enumerate(attn_metadata.seq_lens):
346 + end = start + seq_len
347 out = xops.memory_efficient_attention_forward(
348 query[None, start:end],
349 key[None, start:end],
@@ -354,7 +353,7 @@ class XFormersImpl(AttentionImpl):
353 scale=self.scale)
354 # TODO(woosuk): Unnecessary copy. Optimize.
355 output[start:end].copy_(out.view_as(original_query[start:end]))
357 - start += prompt_len
356 + start += seq_len
357 return output
358
359
@@ -362,13 +361,13 @@ def _make_alibi_bias(
361 alibi_slopes: torch.Tensor,
362 num_kv_heads: int,
363 dtype: torch.dtype,
365 - prompt_lens: List[int],
364 + seq_lens: List[int],
365 ) -> LowerTriangularMaskWithTensorBias:
366 attn_biases = []
368 - for prompt_len in prompt_lens:
369 - bias = torch.arange(prompt_len, dtype=dtype)
367 + for seq_len in seq_lens:
368 + bias = torch.arange(seq_len, dtype=dtype)
369 # NOTE(zhuohan): HF uses
371 - # `bias = bias[None, :].repeat(prompt_len, 1)`
370 + # `bias = bias[None, :].repeat(seq_len, 1)`
371 # here. We find that both biases give the same results, but
372 # the bias below more accurately follows the original ALiBi
373 # paper.
@@ -376,16 +375,16 @@ def _make_alibi_bias(
375 # element.
376 bias = bias[None, :] - bias[:, None]
377
379 - padded_len = (prompt_len + 7) // 8 * 8
378 + padded_len = (seq_len + 7) // 8 * 8
379 num_heads = alibi_slopes.shape[0]
380 bias = torch.empty(
381 1, # batch size
382 num_heads,
384 - prompt_len,
383 + seq_len,
384 padded_len,
385 device=alibi_slopes.device,
386 dtype=dtype,
388 - )[:, :, :, :prompt_len].copy_(bias)
387 + )[:, :, :, :seq_len].copy_(bias)
388 bias.mul_(alibi_slopes[:, None, None])
389 if num_heads != num_kv_heads:
390 bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
vllm/attention/ops/paged_attn.py
+17 -18
@@ -13,12 +13,11 @@ _PARTITION_SIZE = 512
13 @dataclass
14 class PagedAttentionMetadata:
15 """Metadata for PagedAttention."""
16 - # (batch_size,). The length of context (tokens stored in KV cache) per
17 - # sequence. WARNING: When it is a prefill request, it doesn't include new
18 - # tokens. When it is for decoding, it includes a new token.
19 - context_lens: Optional[torch.Tensor]
20 - # Maximum context length in the batch.
21 - max_context_len: Optional[int]
16 + # (batch_size,). The length of sequences (entire tokens seen so far) per
17 + # sequence.
18 + seq_lens_tensor: Optional[torch.Tensor]
19 + # Maximum sequence length in the batch.
20 + max_seq_len: Optional[int]
21 # (batch_size, max_blocks_per_seq).
22 # Block addresses per sequence. (Seq id -> list of physical block)
23 # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
@@ -85,8 +84,8 @@ class PagedAttention:
84 key_cache: torch.Tensor,
85 value_cache: torch.Tensor,
86 block_tables: torch.Tensor,
88 - context_lens: torch.Tensor,
89 - max_context_len: int,
87 + seq_lens: torch.Tensor,
88 + max_seq_len: int,
89 kv_cache_dtype: str,
90 num_kv_heads: int,
91 scale: float,
@@ -97,7 +96,7 @@ class PagedAttention:
96
97 block_size = value_cache.shape[3]
98 num_seqs, num_heads, head_size = query.shape
100 - max_num_partitions = ((max_context_len + _PARTITION_SIZE - 1) //
99 + max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
100 _PARTITION_SIZE)
101 # NOTE(woosuk): We use a simple heuristic to decide whether to use
102 # PagedAttention V1 or V2. If the number of partitions is 1, we use
@@ -106,7 +105,7 @@ class PagedAttention:
105 # to parallelize.
106 # TODO(woosuk): Tune this heuristic.
107 # For context len > 8192, use V2 kernel to avoid shared memory shortage.
109 - use_v1 = (max_context_len <= 8192
108 + use_v1 = (max_seq_len <= 8192
109 and (max_num_partitions == 1 or num_seqs * num_heads > 512))
110 if use_v1:
111 # Run PagedAttention V1.
@@ -118,9 +117,9 @@ class PagedAttention:
117 num_kv_heads,
118 scale,
119 block_tables,
121 - context_lens,
120 + seq_lens,
121 block_size,
123 - max_context_len,
122 + max_seq_len,
123 alibi_slopes,
124 kv_cache_dtype,
125 kv_scale,
@@ -150,9 +149,9 @@ class PagedAttention:
149 num_kv_heads,
150 scale,
151 block_tables,
153 - context_lens,
152 + seq_lens,
153 block_size,
155 - max_context_len,
154 + max_seq_len,
155 alibi_slopes,
156 kv_cache_dtype,
157 kv_scale,
@@ -168,9 +167,9 @@ class PagedAttention:
167 value_cache: torch.Tensor,
168 block_tables: torch.Tensor,
169 subquery_start_loc: torch.Tensor,
171 - prompt_lens_tensor: torch.Tensor,
170 + seq_lens_tensor: torch.Tensor,
171 context_lens: torch.Tensor,
173 - max_subquery_len: int,
172 + max_query_len: int,
173 alibi_slopes: Optional[torch.Tensor],
174 sliding_window: Optional[int],
175 ) -> torch.Tensor:
@@ -185,9 +184,9 @@ class PagedAttention:
184 block_tables,
185 # subquery_start_loc is (batch_size + 1,)
186 subquery_start_loc[:-1],
188 - prompt_lens_tensor,
187 + seq_lens_tensor,
188 context_lens,
190 - max_subquery_len,
189 + max_query_len,
190 alibi_slopes,
191 sliding_window,
192 )
vllm/config.py
+16 -7
@@ -63,7 +63,10 @@ class ModelConfig:
63 If False, we will use CUDA graph and eager execution in hybrid.
64 max_context_len_to_capture: Maximum context len covered by CUDA graphs.
65 When a sequence has context length larger than this, we fall back
66 - to eager mode.
66 + to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
67 + max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
68 + When a sequence has context length larger than this, we fall back
69 + to eager mode
70 skip_tokenizer_init: If true, skip initialization of tokenizer and
71 detokenizer.
72 """
@@ -84,6 +87,7 @@ class ModelConfig:
87 quantization_param_path: Optional[str] = None,
88 enforce_eager: bool = False,
89 max_context_len_to_capture: Optional[int] = None,
90 + max_seq_len_to_capture: Optional[int] = None,
91 max_logprobs: int = 5,
92 skip_tokenizer_init: bool = False,
93 ) -> None:
@@ -99,6 +103,11 @@ class ModelConfig:
103 self.quantization_param_path = quantization_param_path
104 self.enforce_eager = enforce_eager
105 self.max_context_len_to_capture = max_context_len_to_capture
106 + if self.max_context_len_to_capture is not None:
107 + raise ValueError("`max_context_len_to_capture` is deprecated. "
108 + "Use `max_seq_len_to_capture` instead.")
109 + self.max_seq_len_to_capture = (max_seq_len_to_capture
110 + or max_context_len_to_capture)
111 self.max_logprobs = max_logprobs
112 self.skip_tokenizer_init = skip_tokenizer_init
113
@@ -190,10 +199,10 @@ class ModelConfig:
199 "non-quantized models.", self.quantization)
200
201 def _verify_cuda_graph(self) -> None:
193 - if self.max_context_len_to_capture is None:
194 - self.max_context_len_to_capture = self.max_model_len
195 - self.max_context_len_to_capture = min(self.max_context_len_to_capture,
196 - self.max_model_len)
202 + if self.max_seq_len_to_capture is None:
203 + self.max_seq_len_to_capture = self.max_model_len
204 + self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
205 + self.max_model_len)
206
207 def verify_with_parallel_config(
208 self,
@@ -772,8 +781,8 @@ class SpeculativeConfig:
781 max_model_len=None,
782 quantization=draft_quantization,
783 enforce_eager=target_model_config.enforce_eager,
775 - max_context_len_to_capture=target_model_config.
776 - max_context_len_to_capture,
784 + max_seq_len_to_capture=target_model_config.
785 + max_seq_len_to_capture,
786 max_logprobs=target_model_config.max_logprobs,
787 )
788
vllm/engine/arg_utils.py
+12 -2
@@ -44,7 +44,8 @@ class EngineArgs:
44 tokenizer_revision: Optional[str] = None
45 quantization: Optional[str] = None
46 enforce_eager: bool = False
47 - max_context_len_to_capture: int = 8192
47 + max_context_len_to_capture: Optional[int] = None
48 + max_seq_len_to_capture: int = 8192
49 disable_custom_all_reduce: bool = False
50 tokenizer_pool_size: int = 0
51 tokenizer_pool_type: str = "ray"
@@ -322,6 +323,14 @@ class EngineArgs:
323 default=EngineArgs.max_context_len_to_capture,
324 help='Maximum context length covered by CUDA '
325 'graphs. When a sequence has context length '
326 + 'larger than this, we fall back to eager mode. '
327 + '(DEPRECATED. Use --max-seq_len-to-capture instead'
328 + ')')
329 + parser.add_argument('--max-seq_len-to-capture',
330 + type=int,
331 + default=EngineArgs.max_seq_len_to_capture,
332 + help='Maximum sequence length covered by CUDA '
333 + 'graphs. When a sequence has context length '
334 'larger than this, we fall back to eager mode.')
335 parser.add_argument('--disable-custom-all-reduce',
336 action='store_true',
@@ -492,7 +501,8 @@ class EngineArgs:
501 self.code_revision, self.tokenizer_revision, self.max_model_len,
502 self.quantization, self.quantization_param_path,
503 self.enforce_eager, self.max_context_len_to_capture,
495 - self.max_logprobs, self.skip_tokenizer_init)
504 + self.max_seq_len_to_capture, self.max_logprobs,
505 + self.skip_tokenizer_init)
506 cache_config = CacheConfig(self.block_size,
507 self.gpu_memory_utilization,
508 self.swap_space, self.kv_cache_dtype,
vllm/entrypoints/llm.py
+6 -1
@@ -69,6 +69,9 @@ class LLM:
69 disable CUDA graph and always execute the model in eager mode.
70 If False, we will use CUDA graph and eager execution in hybrid.
71 max_context_len_to_capture: Maximum context len covered by CUDA graphs.
72 + When a sequence has context length larger than this, we fall back
73 + to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
74 + max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
75 When a sequence has context length larger than this, we fall back
76 to eager mode.
77 disable_custom_all_reduce: See ParallelConfig
@@ -90,7 +93,8 @@ class LLM:
93 gpu_memory_utilization: float = 0.9,
94 swap_space: int = 4,
95 enforce_eager: bool = False,
93 - max_context_len_to_capture: int = 8192,
96 + max_context_len_to_capture: Optional[int] = None,
97 + max_seq_len_to_capture: int = 8192,
98 disable_custom_all_reduce: bool = False,
99 **kwargs,
100 ) -> None:
@@ -112,6 +116,7 @@ class LLM:
116 swap_space=swap_space,
117 enforce_eager=enforce_eager,
118 max_context_len_to_capture=max_context_len_to_capture,
119 + max_seq_len_to_capture=max_seq_len_to_capture,
120 disable_custom_all_reduce=disable_custom_all_reduce,
121 **kwargs,
122 )
vllm/model_executor/layers/sampler.py
+3 -3
@@ -1033,8 +1033,8 @@ def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
1033 assert seq_group.is_prompt, (
1034 "Caller should ensure the sequence group is in a prefill stage.")
1035 seq_ids = seq_group.seq_ids
1036 - subquery_len = seq_group.subquery_len
1037 - assert subquery_len is not None
1036 + query_len = seq_group.query_len
1037 + assert query_len is not None
1038 # prompt has only 1 seq id.
1039 assert len(seq_ids) == 1
1040 seq_data = seq_group.seq_data[seq_ids[0]]
@@ -1042,7 +1042,7 @@ def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
1042 prompt_tokens = seq_data.prompt_token_ids
1043 # +1 because we are looking for a next prompt token.
1044 next_token_index_start = computed_len + 1
1045 - next_token_index_end = min(computed_len + subquery_len + 1,
1045 + next_token_index_end = min(computed_len + query_len + 1,
1046 len(prompt_tokens))
1047 next_prompt_tokens = prompt_tokens[
1048 next_token_index_start:next_token_index_end]
vllm/model_executor/sampling_metadata.py
+36 -27
@@ -16,17 +16,26 @@ _SEED_0_REPLACEMENT = 3403598558
16
17 @dataclass
18 class SequenceGroupToSample:
19 + # |---------- N-1 iteration --------|
20 + # |---------------- N iteration ---------------------|
21 + # |- tokenA -|......................|-- newTokens ---|
22 + # |---------- context_len ----------|
23 + # |-------------------- seq_len ----------------------|
24 + # |-- query_len ---|
25 +
26 # Sequence ids for the sequence group in a previous step.
27 seq_ids: List[int]
28 sampling_params: SamplingParams
29 # seq_id -> sequence data.
30 seq_data: Dict[int, SequenceData]
24 - # The length of the prompt of the sequence group. None if it is in a decode
31 + # The length of the sequence (all tokens seen in the past + new token to
32 + # compute attention) of the sequence group. None if it is in a decode
33 # stage.
26 - prompt_len: Optional[int]
27 - # The length of the query tokens to compute in the current step. None if it
28 - # is in a decode stage. The length of subquery_len <= prompt_len.
29 - subquery_len: Optional[int]
34 + seq_len: Optional[int]
35 + # The length of new query tokens to compute in the current step. None if it
36 + # is in a decode stage. The length of query_len <= seq_len if chunked
37 + # prefill is enabled.
38 + query_len: Optional[int]
39 # A random number generator for sampling.
40 generator: Optional[torch.Generator]
41 # True if the sequence group is in prefill stage. False if it is in a
@@ -46,8 +55,8 @@ class SequenceGroupToSample:
55 if len(self.prompt_logprob_indices) > 0:
56 assert self.sampling_params.prompt_logprobs is not None
57 if self.is_prompt:
49 - assert self.prompt_len is not None
50 - assert self.subquery_len is not None
58 + assert self.seq_len is not None
59 + assert self.query_len is not None
60
61
62 class SamplingMetadata:
@@ -94,8 +103,8 @@ class SamplingMetadata:
103 @staticmethod
104 def prepare(
105 seq_group_metadata_list: List[SequenceGroupMetadata],
97 - prompt_lens: List[int],
98 - subquery_lens: Optional[List[int]],
106 + seq_lens: List[int],
107 + query_lens: Optional[List[int]],
108 device: str,
109 pin_memory: bool,
110 ) -> "SamplingMetadata":
@@ -104,8 +113,8 @@ class SamplingMetadata:
113 selected_token_indices,
114 categorized_sample_indices,
115 num_prompts,
107 - ) = _prepare_seq_groups(seq_group_metadata_list, prompt_lens,
108 - subquery_lens, device)
116 + ) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
117 + device)
118 selected_token_indices = async_tensor_h2d(selected_token_indices,
119 dtype=torch.long,
120 target_device=device,
@@ -137,8 +146,8 @@ class SamplingMetadata:
146
147 def _prepare_seq_groups(
148 seq_group_metadata_list: List[SequenceGroupMetadata],
140 - prompt_lens: List[int],
141 - subquery_lens: Optional[List[int]],
149 + seq_lens: List[int],
150 + query_lens: Optional[List[int]],
151 device: str,
152 ) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
153 SamplingType, List[Tuple[int, int]]], int]:
@@ -146,9 +155,9 @@ def _prepare_seq_groups(
155
156 Args:
157 seq_group_metadata_list: A list of sequence group to batch.
149 - prompt_lens: A list of prompt lens per sequence group.
158 + seq_lens: A list of sequence lens per sequence group.
159 Index of prompt len should match with seq_group_metadata_list.
151 - subquery_lens: A list of query lengths. Prompt lens include the length
160 + query_lens: A list of query lengths. Prompt lens include the length
161 of entire prompt tokens, and it could be shorter.
162 device: A device to use for random number generator,
163 `SequenceGroupToSample.generator`.
@@ -189,8 +198,8 @@ def _prepare_seq_groups(
198 is_prompt = seq_group_metadata.is_prompt
199 generator: Optional[torch.Generator] = None
200 # If the current seq group is in decode stage, it is None.
192 - prompt_len: Optional[int] = None
193 - subquery_len: Optional[int] = None
201 + seq_len: Optional[int] = None
202 + query_len: Optional[int] = None
203 prompt_logprob_indices: List[int] = []
204 sample_indices: List[int] = []
205 do_sample = seq_group_metadata.do_sample
@@ -203,12 +212,12 @@ def _prepare_seq_groups(
212 num_prompts += 1
213 num_prefill_sample = len(seq_ids)
214 assert num_prefill_sample == 1
206 - assert subquery_lens is not None and prompt_lens is not None
207 - subquery_len, prompt_len = subquery_lens[i], prompt_lens[i]
215 + assert query_lens is not None and seq_lens is not None
216 + query_len, seq_len = query_lens[i], seq_lens[i]
217 # If we need sampling, exclude num_prefill_sample tokens from
218 # prompt logprob.
210 - prompt_logprob_len = (subquery_len - num_prefill_sample
211 - if do_sample else subquery_len)
219 + prompt_logprob_len = (query_len - num_prefill_sample
220 + if do_sample else query_len)
221 sample_len = num_prefill_sample if do_sample else 0
222 else:
223 # Decode
@@ -267,8 +276,8 @@ def _prepare_seq_groups(
276 seq_ids=seq_ids,
277 sampling_params=sampling_params,
278 seq_data=seq_group_metadata.seq_data,
270 - prompt_len=prompt_len,
271 - subquery_len=subquery_len,
279 + seq_len=seq_len,
280 + query_len=query_len,
281 generator=generator,
282 is_prompt=is_prompt,
283 prompt_logprob_indices=list(prompt_logprob_indices),
@@ -367,8 +376,8 @@ class SamplingTensors:
376 and sampling_params.prompt_logprobs is not None):
377 # For tokens in the prompt that we only need to get
378 # their logprobs
370 - subquery_len = seq_group.subquery_len
371 - assert subquery_len is not None
379 + query_len = seq_group.query_len
380 + assert query_len is not None
381 prefill_len = len(seq_group.prompt_logprob_indices)
382 temperatures += [temperature] * prefill_len
383 top_ps += [top_p] * prefill_len
@@ -397,8 +406,8 @@ class SamplingTensors:
406
407 if is_prompt:
408 prompt_best_of.append(sampling_params.best_of)
400 - subquery_len = seq_group.subquery_len
401 - assert subquery_len is not None
409 + query_len = seq_group.query_len
410 + assert query_len is not None
411
412 for seq_id in seq_ids:
413 seq_data = seq_group.seq_data[seq_id]
vllm/worker/cpu_model_runner.py
+29 -29
@@ -80,7 +80,7 @@ class CPUModelRunner:
80 input_tokens: List[int] = []
81 input_positions: List[int] = []
82 slot_mapping: List[int] = []
83 - prompt_lens: List[int] = []
83 + seq_lens: List[int] = []
84 multi_modal_input_list: List[torch.Tensor] = []
85
86 for seq_group_metadata in seq_group_metadata_list:
@@ -92,15 +92,15 @@ class CPUModelRunner:
92 seq_data = seq_group_metadata.seq_data[seq_id]
93 prompt_tokens = seq_data.get_token_ids()
94 computed_len = seq_data.get_num_computed_tokens()
95 - prompt_len = len(prompt_tokens)
95 + seq_len = len(prompt_tokens)
96
97 - prompt_lens.append(prompt_len) # Prompt token num
97 + seq_lens.append(seq_len) # Prompt token num
98 input_tokens.extend(prompt_tokens) # Token ids
99
100 # Token position ids
101 # NOTE(woosuk): Here we assume that the first token in the prompt
102 # is always the first token in the sequence.
103 - input_positions.extend(list(range(computed_len, prompt_len)))
103 + input_positions.extend(list(range(computed_len, seq_len)))
104
105 if seq_group_metadata.multi_modal_data:
106 multi_modal_input_list.append(
@@ -109,15 +109,15 @@ class CPUModelRunner:
109 # Compute the slot mapping.
110 block_table = seq_group_metadata.block_tables[seq_id]
111 # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
112 - # where start_idx is max(0, prompt_len - sliding_window).
112 + # where start_idx is max(0, seq_len - sliding_window).
113 # For example, if the prompt len is 10, sliding window is 8, and
114 # block size is 4, the first two tokens are masked and the slot
115 # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
116 start_idx = 0
117 if self.sliding_window is not None:
118 - start_idx = max(0, prompt_len - self.sliding_window)
118 + start_idx = max(0, seq_len - self.sliding_window)
119
120 - for i in range(computed_len, prompt_len):
120 + for i in range(computed_len, seq_len):
121 if i < start_idx:
122 slot_mapping.append(_PAD_SLOT_ID)
123 continue
@@ -151,19 +151,19 @@ class CPUModelRunner:
151
152 attn_metadata = self.attn_backend.make_metadata(
153 is_prompt=True,
154 - prompt_lens=prompt_lens,
155 - num_prefills=len(prompt_lens),
154 + seq_lens=seq_lens,
155 + seq_lens_tensor=None,
156 + max_seq_len=None,
157 + num_prefills=len(seq_lens),
158 num_prefill_tokens=num_prompt_tokens,
159 num_decode_tokens=0,
160 prefill_metadata=None,
161 decode_metadata=None,
160 - max_context_len=None,
161 - context_lens=None,
162 block_tables=torch.tensor([]),
163 slot_mapping=slot_mapping,
164 kv_cache_dtype=self.kv_cache_dtype,
165 )
166 - return (input_tokens, input_positions, attn_metadata, prompt_lens,
166 + return (input_tokens, input_positions, attn_metadata, seq_lens,
167 multi_modal_input)
168
169 def _prepare_decode(
@@ -174,7 +174,7 @@ class CPUModelRunner:
174 input_tokens: List[int] = []
175 input_positions: List[int] = []
176 slot_mapping: List[int] = []
177 - context_lens: List[int] = []
177 + seq_lens: List[int] = []
178 block_tables: List[List[int]] = []
179
180 for seq_group_metadata in seq_group_metadata_list:
@@ -192,9 +192,9 @@ class CPUModelRunner:
192 position = seq_len - 1
193 input_positions.append(position)
194
195 - context_len = seq_len if self.sliding_window is None else min(
195 + seq_len = seq_len if self.sliding_window is None else min(
196 seq_len, self.sliding_window)
197 - context_lens.append(context_len)
197 + seq_lens.append(seq_len)
198
199 block_table = seq_group_metadata.block_tables[seq_id]
200 block_number = block_table[position // self.block_size]
@@ -208,7 +208,7 @@ class CPUModelRunner:
208 block_table = block_table[-sliding_window_blocks:]
209 block_tables.append(block_table)
210
211 - max_context_len = max(context_lens)
211 + max_seq_len = max(seq_lens)
212
213 input_tokens = torch.tensor(input_tokens,
214 dtype=torch.long,
@@ -219,9 +219,9 @@ class CPUModelRunner:
219 slot_mapping = torch.tensor(slot_mapping,
220 dtype=torch.long,
221 device=self.device)
222 - context_lens = torch.tensor(context_lens,
223 - dtype=torch.int,
224 - device=self.device)
222 + seq_lens_tensor = torch.tensor(seq_lens,
223 + dtype=torch.int,
224 + device=self.device)
225
226 max_block_table_len = max(
227 len(block_table) for block_table in block_tables)
@@ -236,14 +236,14 @@ class CPUModelRunner:
236 attn_metadata = self.attn_backend.make_metadata(
237 is_prompt=False,
238 slot_mapping=slot_mapping,
239 - prompt_lens=None,
239 + seq_lens=seq_lens,
240 + seq_lens_tensor=seq_lens_tensor,
241 + max_seq_len=max_seq_len,
242 num_prefill_tokens=0,
243 num_decode_tokens=len(input_tokens),
242 - max_context_len=max_context_len,
244 num_prefills=0,
245 prefill_metadata=None,
246 decode_metadata=None,
246 - context_lens=context_lens,
247 block_tables=block_tables,
248 kv_cache_dtype=self.kv_cache_dtype,
249 )
@@ -265,20 +265,20 @@ class CPUModelRunner:
265 is_prompt = seq_group_metadata_list[0].is_prompt
266 # Prepare input tensors.
267 if is_prompt:
268 - (input_tokens, input_positions, attn_metadata, prompt_lens,
268 + (input_tokens, input_positions, attn_metadata, seq_lens,
269 multi_modal_input
270 ) = self._prepare_prompt(seq_group_metadata_list)
271 else:
272 (input_tokens, input_positions,
273 attn_metadata) = self._prepare_decode(seq_group_metadata_list)
274 - prompt_lens = []
274 + seq_lens = []
275 sampling_metadata = SamplingMetadata.prepare(
276 seq_group_metadata_list,
277 - prompt_lens,
278 - # subquery_lens is not needed if chunked prefill is not
277 + seq_lens,
278 + # query_lens is not needed if chunked prefill is not
279 # supported. Since CPU worker doesn't support chunked prefill
280 - # just use prompt_lens instead.
281 - prompt_lens,
280 + # just use seq_lens instead.
281 + seq_lens,
282 self.device,
283 pin_memory=False)
284 # Broadcast the metadata.
@@ -300,7 +300,7 @@ class CPUModelRunner:
300 sampling_metadata = SamplingMetadata(
301 seq_groups=None,
302 seq_data=None,
303 - prompt_lens=None,
303 + seq_lens=None,
304 selected_token_indices=selected_token_indices,
305 categorized_sample_indices=None,
306 generators=None,
vllm/worker/model_runner.py
+80 -87
@@ -42,8 +42,8 @@ class PreparePromptMetadata(NamedTuple):
42 input_tokens: List[int]
43 input_positions: List[int]
44 attn_metadata: Optional[AttentionMetadataPerStage]
45 - prompt_lens: List[int]
46 - subquery_lens: List[int]
45 + seq_lens: List[int]
46 + query_lens: List[int]
47 lora_index_mapping: List[int]
48 lora_prompt_mapping: List[int]
49 lora_requests: Set[LoRARequest]
@@ -56,8 +56,8 @@ class PreparePromptMetadata(NamedTuple):
56 input_tokens=[],
57 input_positions=[],
58 attn_metadata=None,
59 - prompt_lens=[],
60 - subquery_lens=[],
59 + seq_lens=[],
60 + query_lens=[],
61 lora_index_mapping=[],
62 lora_prompt_mapping=[],
63 lora_requests=set(),
@@ -134,9 +134,8 @@ class ModelRunner:
134 self.graph_memory_pool: Optional[Tuple[
135 int, int]] = None # Set during graph capture.
136
137 - self.max_context_len_to_capture = (
138 - self.model_config.max_context_len_to_capture
139 - if self.model_config is not None else 0)
137 + self.max_seq_len_to_capture = (self.model_config.max_seq_len_to_capture
138 + if self.model_config is not None else 0)
139
140 self.pin_memory = is_pin_memory_available()
141 self.kv_cache_dtype = kv_cache_dtype
@@ -149,7 +148,7 @@ class ModelRunner:
148 self.model: torch.nn.Module # Set after load_model
149 self.block_size: int # Set after initial profiling.
150 # When using CUDA graph, the input block tables must be padded to
152 - # max_context_len_to_capture. However, creating the block table in
151 + # max_seq_len_to_capture. However, creating the block table in
152 # Python can be expensive. To optimize this, we cache the block table
153 # in numpy and only copy the actual input content at every iteration.
154 # The shape of the cached block table will be
@@ -218,7 +217,7 @@ class ModelRunner:
217
218 def get_max_block_per_batch(self) -> int:
219 block_size = self.block_size
221 - return (self.max_context_len_to_capture + block_size - 1) // block_size
220 + return (self.max_seq_len_to_capture + block_size - 1) // block_size
221
222 def _prepare_prompt(
223 self,
@@ -231,9 +230,9 @@ class ModelRunner:
230 lora_prompt_mapping: List[int] = []
231 lora_requests: Set[LoRARequest] = set()
232
234 - prompt_lens: List[int] = []
233 + seq_lens: List[int] = []
234 context_lens: List[int] = []
236 - subquery_lens: List[int] = []
235 + query_lens: List[int] = []
236 prefix_block_tables: List[List[int]] = []
237 multi_modal_input_list: List[torch.Tensor] = []
238
@@ -257,21 +256,19 @@ class ModelRunner:
256
257 token_chunk_size = seq_group_metadata.token_chunk_size
258 seq_data = seq_group_metadata.seq_data[seq_id]
260 - computed_len = seq_data.get_num_computed_tokens()
259 + context_len = seq_data.get_num_computed_tokens()
260 # We should use get_len here because in case of preemption
261 # it contains output tokens.
263 - prefill_end = min(seq_data.get_len(),
264 - computed_len + token_chunk_size)
265 - prompt_tokens = seq_data.get_token_ids()[computed_len:prefill_end]
266 - prompt_len = prefill_end
267 - prompt_lens.append(prompt_len)
262 + seq_len = min(seq_data.get_len(), context_len + token_chunk_size)
263 + prompt_tokens = seq_data.get_token_ids()[context_len:seq_len]
264 + seq_lens.append(seq_len)
265
266 # NOTE: This only works for oooooooxxx style attention.
267 if computed_block_nums is not None and len(
268 computed_block_nums) > 0 and self.sliding_window is None:
269 # Prefix is not supported with sliding_window
273 - computed_len = len(computed_block_nums) * self.block_size
274 - prompt_tokens = prompt_tokens[computed_len:]
270 + context_len = len(computed_block_nums) * self.block_size
271 + prompt_tokens = prompt_tokens[context_len:]
272 prefix_block_tables.append(computed_block_nums)
273 elif self.scheduler_config.chunked_prefill_enabled:
274 if seq_group_metadata.block_tables is not None:
@@ -285,25 +282,25 @@ class ModelRunner:
282 prefix_block_tables.append([])
283 # Right now, prefill start is always 0. However, this
284 # assumption can be changed once chunked prefill is introduced.
288 - assert computed_len == 0
285 + assert context_len == 0
286
287 # actual prompt lens
291 - context_lens.append(computed_len)
292 - subquery_lens.append(prompt_len - computed_len)
288 + context_lens.append(context_len)
289 + query_lens.append(seq_len - context_len)
290
291 input_tokens.extend(prompt_tokens)
292 # NOTE(woosuk): Here we assume that the first token in the prompt
293 # is always the first token in the sequence.
297 - input_positions.extend(list(range(computed_len, prefill_end)))
294 + input_positions.extend(list(range(context_len, seq_len)))
295 lora_id = seq_group_metadata.lora_int_id
296
297 if lora_id > 0:
298 lora_requests.add(seq_group_metadata.lora_request)
299
303 - lora_index_mapping += [lora_id] * (prompt_len - computed_len)
300 + lora_index_mapping += [lora_id] * (seq_len - context_len)
301 lora_prompt_mapping.extend(
302 [lora_id] *
306 - (prompt_len - computed_len
303 + (seq_len - context_len
304 if seq_group_metadata.sampling_params.prompt_logprobs else 1))
305
306 if seq_group_metadata.multi_modal_data:
@@ -313,24 +310,24 @@ class ModelRunner:
310 if seq_group_metadata.block_tables is None:
311 # During memory profiling, the block tables are not initialized
312 # yet. In this case, we just use a dummy slot mapping.
316 - slot_mapping.extend([_PAD_SLOT_ID] * prompt_len)
313 + slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
314 continue
315
316 # Compute the slot mapping.
317 block_table = seq_group_metadata.block_tables[seq_id]
318 # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
322 - # where start_idx is max(0, prompt_len - sliding_window).
319 + # where start_idx is max(0, seq_len - sliding_window).
320 # For example, if the prompt len is 10, sliding window is 8, and
321 # block size is 4, the first two tokens are masked and the slot
322 # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
323 start_idx = 0
324 if self.sliding_window is not None:
328 - assert computed_len == 0, (
325 + assert context_len == 0, (
326 "Prefix caching is currently not supported with "
327 "sliding window attention")
331 - start_idx = max(0, prompt_len - self.sliding_window)
328 + start_idx = max(0, seq_len - self.sliding_window)
329
333 - for i in range(computed_len, prefill_end):
330 + for i in range(context_len, seq_len):
331 if i < start_idx:
332 slot_mapping.append(_PAD_SLOT_ID)
333 continue
@@ -340,9 +337,9 @@ class ModelRunner:
337 slot = block_number * self.block_size + block_offset
338 slot_mapping.append(slot)
339
343 - max_subquery_len = max(subquery_lens)
344 - max_prompt_len = max(prompt_lens)
345 - assert max_subquery_len > 0
340 + max_query_len = max(query_lens)
341 + max_seq_len = max(seq_lens)
342 + assert max_query_len > 0
343
344 context_lens_tensor = torch.tensor(context_lens,
345 dtype=torch.int,
@@ -369,40 +366,39 @@ class ModelRunner:
366
367 # Query length can be shorter than key (i.e., prompt) when prefill
368 # is chunked or prefix cached.
372 - subquery_lens_tensor = torch.tensor(subquery_lens,
373 - dtype=torch.long,
374 - device=self.device)
375 - subquery_start_loc = torch.zeros(subquery_lens_tensor.shape[0] + 1,
369 + query_lens_tensor = torch.tensor(query_lens,
370 + dtype=torch.long,
371 + device=self.device)
372 + subquery_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
373 dtype=torch.int32,
374 device=self.device)
375
379 - prompt_lens_tensor = torch.tensor(prompt_lens,
380 - dtype=torch.long,
381 - device=self.device)
382 - seq_start_loc = torch.zeros(prompt_lens_tensor.shape[0] + 1,
376 + seq_lens_tensor = torch.tensor(seq_lens,
377 + dtype=torch.int,
378 + device=self.device)
379 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
380 dtype=torch.int32,
381 device=self.device)
382
386 - torch.cumsum(subquery_lens_tensor,
383 + torch.cumsum(query_lens_tensor,
384 dim=0,
385 dtype=subquery_start_loc.dtype,
386 out=subquery_start_loc[1:])
387
391 - torch.cumsum(prompt_lens_tensor,
388 + torch.cumsum(seq_lens_tensor,
389 dim=0,
390 dtype=seq_start_loc.dtype,
391 out=seq_start_loc[1:])
392
393 attn_metadata = self.attn_backend.make_metadata(
394 is_prompt=True,
398 - prompt_lens=prompt_lens,
399 - prompt_lens_tensor=prompt_lens_tensor,
400 - max_subquery_len=max_subquery_len,
401 - max_context_len=None,
402 - max_prompt_len=max_prompt_len,
395 + seq_lens=seq_lens,
396 + seq_lens_tensor=seq_lens_tensor,
397 + max_query_len=max_query_len,
398 + max_seq_len=max_seq_len,
399 subquery_start_loc=subquery_start_loc,
400 seq_start_loc=seq_start_loc,
405 - context_lens=context_lens_tensor,
401 + context_lens_tensor=context_lens_tensor,
402 block_tables=block_tables,
403 use_cuda_graph=False,
404 )
@@ -411,8 +407,8 @@ class ModelRunner:
407 input_tokens=input_tokens,
408 input_positions=input_positions,
409 attn_metadata=attn_metadata,
414 - prompt_lens=prompt_lens,
415 - subquery_lens=subquery_lens,
410 + seq_lens=seq_lens,
411 + query_lens=query_lens,
412 lora_index_mapping=lora_index_mapping,
413 lora_prompt_mapping=lora_prompt_mapping,
414 lora_requests=lora_requests,
@@ -427,7 +423,7 @@ class ModelRunner:
423 input_tokens: List[int] = []
424 input_positions: List[int] = []
425 slot_mapping: List[int] = []
430 - context_lens: List[int] = []
426 + seq_lens: List[int] = []
427 block_tables: List[List[int]] = []
428 lora_index_mapping: List[int] = []
429 lora_prompt_mapping: List[int] = []
@@ -455,9 +451,9 @@ class ModelRunner:
451 position = seq_len - 1
452 input_positions.append(position)
453
458 - context_len = seq_len if self.sliding_window is None else min(
454 + seq_len = seq_len if self.sliding_window is None else min(
455 seq_len, self.sliding_window)
460 - context_lens.append(context_len)
456 + seq_lens.append(seq_len)
457
458 block_table = seq_group_metadata.block_tables[seq_id]
459 block_number = block_table[position // self.block_size]
@@ -477,11 +473,10 @@ class ModelRunner:
473 # See `capture_model` API for more details.
474 # For decoding requests, batch_size == input_tokens.
475 batch_size = len(input_tokens)
480 - max_context_len = max(context_lens)
481 - use_captured_graph = (
482 - not self.model_config.enforce_eager
483 - and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
484 - and max_context_len <= self.max_context_len_to_capture)
476 + max_seq_len = max(seq_lens)
477 + use_captured_graph = (not self.model_config.enforce_eager
478 + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
479 + and max_seq_len <= self.max_seq_len_to_capture)
480 if use_captured_graph:
481 graph_batch_size = _get_graph_batch_size(batch_size)
482 assert graph_batch_size >= batch_size
@@ -489,21 +484,21 @@ class ModelRunner:
484 input_tokens.append(0)
485 input_positions.append(0)
486 slot_mapping.append(_PAD_SLOT_ID)
492 - context_lens.append(1)
487 + seq_lens.append(1)
488 block_tables.append([])
489 lora_index_mapping.append(0)
490 batch_size = graph_batch_size
491
497 - context_lens_tensor = torch.tensor(context_lens,
498 - dtype=torch.int,
499 - device=self.device)
492 + seq_lens_tensor = torch.tensor(seq_lens,
493 + dtype=torch.int,
494 + device=self.device)
495
496 if use_captured_graph:
497 # When using cuda-graph all these tensors should be
498 # padded.
504 - assert context_lens_tensor.shape[0] == len(input_tokens)
505 - assert context_lens_tensor.shape[0] == len(input_positions)
506 - assert context_lens_tensor.shape[0] == len(slot_mapping)
499 + assert seq_lens_tensor.shape[0] == len(input_tokens)
500 + assert seq_lens_tensor.shape[0] == len(input_positions)
501 + assert seq_lens_tensor.shape[0] == len(slot_mapping)
502
503 # The shape of graph_block_tables is
504 # [max batch size, max context len // block size].
@@ -525,14 +520,13 @@ class ModelRunner:
520
521 attn_metadata = self.attn_backend.make_metadata(
522 is_prompt=False,
528 - prompt_lens=None,
529 - prompt_lens_tensor=None,
530 - max_subquery_len=None,
531 - max_context_len=max_context_len,
532 - max_prompt_len=None,
523 + seq_lens=None,
524 + seq_lens_tensor=seq_lens_tensor,
525 + max_query_len=None,
526 + max_seq_len=max_seq_len,
527 subquery_start_loc=None,
528 seq_start_loc=None,
535 - context_lens=context_lens_tensor,
529 + context_lens_tensor=None,
530 block_tables=block_tables,
531 use_cuda_graph=use_captured_graph,
532 )
@@ -565,8 +559,8 @@ class ModelRunner:
559 input_tokens,
560 input_positions,
561 prefill_attn_metadata,
568 - prompt_lens,
569 - subquery_lens,
562 + seq_lens,
563 + query_lens,
564 lora_index_mapping,
565 lora_prompt_mapping,
566 lora_requests,
@@ -583,13 +577,13 @@ class ModelRunner:
577 decode_slot_mapping,
578 ) = self._prepare_decode(decode_reqs)
579 sampling_metadata = SamplingMetadata.prepare(
586 - seq_group_metadata_list, prompt_lens, subquery_lens,
587 - self.device, self.pin_memory)
580 + seq_group_metadata_list, seq_lens, query_lens, self.device,
581 + self.pin_memory)
582
583 if not self.scheduler_config.chunked_prefill_enabled:
584 assert (len(prefill_reqs) and len(decode_reqs)) == 0
585
592 - num_prefills = len(prompt_lens)
586 + num_prefills = len(seq_lens)
587 num_prefill_tokens = len(input_tokens)
588 num_decode_tokens = len(decode_input_tokens)
589
@@ -886,7 +880,7 @@ class ModelRunner:
880 input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
881 slot_mapping = torch.empty(max_batch_size, dtype=torch.long).cuda()
882 slot_mapping.fill_(_PAD_SLOT_ID)
889 - context_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
883 + seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
884 block_tables = torch.from_numpy(self.graph_block_tables).cuda()
885
886 graph_batch_size = _get_graph_batch_size(
@@ -908,14 +902,13 @@ class ModelRunner:
902 # Create dummy attn_metadata.
903 decode_metadata = self.attn_backend.make_metadata(
904 is_prompt=False,
911 - prompt_lens=None,
912 - prompt_lens_tensor=None,
913 - max_subquery_len=None,
914 - max_context_len=self.max_context_len_to_capture,
915 - max_prompt_len=None,
905 + seq_lens=None,
906 + seq_lens_tensor=seq_lens[:batch_size],
907 + max_query_len=None,
908 + max_seq_len=self.max_seq_len_to_capture,
909 subquery_start_loc=None,
910 seq_start_loc=None,
918 - context_lens=context_lens[:batch_size],
911 + context_lens_tensor=None,
912 block_tables=block_tables[:batch_size],
913 use_cuda_graph=True,
914 )
@@ -1025,7 +1018,7 @@ class CUDAGraphRunner:
1018 "positions": positions,
1019 "kv_caches": kv_caches,
1020 "slot_mapping": attn_metadata.slot_mapping,
1028 - "context_lens": attn_metadata.decode_metadata.context_lens,
1021 + "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
1022 "block_tables": attn_metadata.decode_metadata.block_tables,
1023 }
1024 self.output_buffers = {"hidden_states": hidden_states}
@@ -1047,8 +1040,8 @@ class CUDAGraphRunner:
1040 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1041 self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1042 non_blocking=True)
1050 - self.input_buffers["context_lens"].copy_(
1051 - attn_metadata.decode_metadata.context_lens, non_blocking=True)
1043 + self.input_buffers["seq_lens_tensor"].copy_(
1044 + attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
1045 self.input_buffers["block_tables"].copy_(
1046 attn_metadata.decode_metadata.block_tables, non_blocking=True)
1047 # Run the graph.
vllm/worker/neuron_model_runner.py
+15 -15
@@ -52,7 +52,7 @@ class NeuronModelRunner:
52 input_positions: List[List[int]] = []
53 input_block_ids: List[int] = []
54
55 - prompt_lens: List[int] = []
55 + seq_lens: List[int] = []
56 for seq_group_metadata in seq_group_metadata_list:
57 assert seq_group_metadata.is_prompt
58 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -61,26 +61,26 @@ class NeuronModelRunner:
61
62 seq_data = seq_group_metadata.seq_data[seq_id]
63 prompt_tokens = seq_data.get_token_ids()
64 - prompt_len = len(prompt_tokens)
65 - prompt_lens.append(prompt_len)
64 + seq_len = len(prompt_tokens)
65 + seq_lens.append(seq_len)
66
67 input_tokens.append(prompt_tokens)
68 - input_positions.append(list(range(prompt_len)))
68 + input_positions.append(list(range(seq_len)))
69
70 assert seq_group_metadata.block_tables is not None
71 block_table = seq_group_metadata.block_tables[seq_id]
72 assert len(block_table) == 1
73 input_block_ids.append(block_table[0])
74
75 - max_prompt_len = max(prompt_lens)
76 - assert max_prompt_len > 0
75 + max_seq_len = max(seq_lens)
76 + assert max_seq_len > 0
77 input_tokens = make_tensor_with_pad(input_tokens,
78 - max_prompt_len,
78 + max_seq_len,
79 pad=0,
80 dtype=torch.long,
81 device=self.device)
82 input_positions = make_tensor_with_pad(input_positions,
83 - max_prompt_len,
83 + max_seq_len,
84 pad=0,
85 dtype=torch.long,
86 device=self.device)
@@ -88,7 +88,7 @@ class NeuronModelRunner:
88 dtype=torch.long,
89 device=self.device)
90
91 - return input_tokens, input_positions, input_block_ids, prompt_lens
91 + return input_tokens, input_positions, input_block_ids, seq_lens
92
93 def _prepare_decode(
94 self,
@@ -149,18 +149,18 @@ class NeuronModelRunner:
149 # Prepare input tensors.
150 if is_prompt:
151 (input_tokens, input_positions, input_block_ids,
152 - prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
152 + seq_lens) = self._prepare_prompt(seq_group_metadata_list)
153 else:
154 (input_tokens, input_positions,
155 input_block_ids) = self._prepare_decode(seq_group_metadata_list)
156 - prompt_lens = []
156 + seq_lens = []
157 sampling_metadata = SamplingMetadata.prepare(
158 seq_group_metadata_list,
159 - prompt_lens,
160 - # subquery_lens is not needed if chunked prefill is not
159 + seq_lens,
160 + # query_lens is not needed if chunked prefill is not
161 # supported. Since neuron worker doesn't support chunked prefill
162 - # just use prompt_lens instead.
163 - prompt_lens,
162 + # just use seq_lens instead.
163 + seq_lens,
164 self.device,
165 self.pin_memory)
166