A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Compute query_start_loc/seq_start_loc on CPU (#9447)
Co-authored-by: Yang Zheng(SW)(Alex) <you@example.com>
Yang Zheng committed
Nov 4, 2024 at 16:54 UTC
4dbcbbeb09628eb3181dedb6789f0ccb05e83957
2 files changed
+20
-36
vllm/attention/backends/flash_attn.py
+10
-18
@@ -1,6 +1,7 @@
1
"""Attention layer with FlashAttention."""
2
from collections import defaultdict
3
from dataclasses import dataclass
4
+from itertools import accumulate
5
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
6
7
import torch
@@ -503,6 +504,8 @@ class FlashAttentionMetadataBuilder(
504
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
505
max_decode_seq_len = max(self.curr_seq_lens, default=0)
506
num_decode_tokens = self.num_decode_tokens
507
+ query_start_loc = list(accumulate(query_lens, initial=0))
508
+ seq_start_loc = list(accumulate(seq_lens, initial=0))
509
510
num_seqs = len(seq_lens)
511
if use_captured_graph:
@@ -525,29 +528,18 @@ class FlashAttentionMetadataBuilder(
528
device, self.runner.pin_memory)
529
seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
530
self.runner.pin_memory)
528
- query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
529
- self.runner.pin_memory)
531
slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
532
device, self.runner.pin_memory)
532
- query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
533
- dtype=torch.int32,
534
- device=device)
535
- seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
536
- dtype=torch.int32,
537
- device=device)
533
+ query_start_loc_tensor = async_tensor_h2d(query_start_loc, torch.int32,
534
+ device,
535
+ self.runner.pin_memory)
536
+ seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
537
+ device, self.runner.pin_memory)
538
placeholder_index_maps = {
539
modality: placeholder_map.index_map()
540
for modality, placeholder_map in
541
self.multimodal_placeholder_maps.items()
542
}
543
- torch.cumsum(seq_lens_tensor,
544
- dim=0,
545
- dtype=seq_start_loc.dtype,
546
- out=seq_start_loc[1:])
547
- torch.cumsum(query_lens_tensor,
548
- dim=0,
549
- dtype=query_start_loc.dtype,
550
- out=query_start_loc[1:])
543
544
return FlashAttentionMetadata(
545
num_prefills=self.num_prefills,
@@ -561,8 +553,8 @@ class FlashAttentionMetadataBuilder(
553
max_decode_query_len=max_decode_query_len,
554
max_prefill_seq_len=max_prefill_seq_len,
555
max_decode_seq_len=max_decode_seq_len,
564
- query_start_loc=query_start_loc,
565
- seq_start_loc=seq_start_loc,
556
+ query_start_loc=query_start_loc_tensor,
557
+ seq_start_loc=seq_start_loc_tensor,
558
context_lens_tensor=context_lens_tensor,
559
block_tables=block_tables,
560
use_cuda_graph=use_captured_graph,
vllm/attention/backends/utils.py
+10
-18
@@ -1,6 +1,7 @@
1
"""Attention backend utils"""
2
from collections import defaultdict
3
from contextlib import contextmanager
4
+from itertools import accumulate
5
from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, TypeVar, Union
6
7
import numpy as np
@@ -216,6 +217,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
217
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
218
max_decode_seq_len = max(self.curr_seq_lens, default=0)
219
num_decode_tokens = self.num_decode_tokens
220
+ query_start_loc = list(accumulate(query_lens, initial=0))
221
+ seq_start_loc = list(accumulate(seq_lens, initial=0))
222
223
if use_captured_graph:
224
self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
@@ -244,29 +247,18 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
247
device, self.runner.pin_memory)
248
seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
249
self.runner.pin_memory)
247
- query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
248
- self.runner.pin_memory)
250
slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
251
device, self.runner.pin_memory)
251
- query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
252
- dtype=torch.int32,
253
- device=device)
254
- seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
255
- dtype=torch.int32,
256
- device=device)
252
+ query_start_loc_tensor = async_tensor_h2d(query_start_loc, torch.int32,
253
+ device,
254
+ self.runner.pin_memory)
255
+ seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
256
+ device, self.runner.pin_memory)
257
placeholder_index_maps = {
258
modality: placeholder_map.index_map()
259
for modality, placeholder_map in
260
self.multimodal_placeholder_maps.items()
261
}
262
- torch.cumsum(seq_lens_tensor,
263
- dim=0,
264
- dtype=seq_start_loc.dtype,
265
- out=seq_start_loc[1:])
266
- torch.cumsum(query_lens_tensor,
267
- dim=0,
268
- dtype=query_start_loc.dtype,
269
- out=query_start_loc[1:])
262
263
return self._metadata_cls( # type: ignore
264
num_prefills=self.num_prefills,
@@ -279,8 +271,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
271
max_query_len=max_query_len,
272
max_prefill_seq_len=max_prefill_seq_len,
273
max_decode_seq_len=max_decode_seq_len,
282
- query_start_loc=query_start_loc,
283
- seq_start_loc=seq_start_loc,
274
+ query_start_loc=query_start_loc_tensor,
275
+ seq_start_loc=seq_start_loc_tensor,
276
context_lens_tensor=context_lens_tensor,
277
block_tables=block_tables,
278
use_cuda_graph=use_captured_graph,