@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4dbcbbeb09
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Compute query_start_loc/seq_start_loc on CPU (#9447)

Co-authored-by: Yang Zheng(SW)(Alex) <you@example.com>

Yang Zheng committed Nov 4, 2024 at 16:54 UTC 4dbcbbeb09628eb3181dedb6789f0ccb05e83957
2 files changed +20 -36
vllm/attention/backends/flash_attn.py
+10 -18
@@ -1,6 +1,7 @@
1 """Attention layer with FlashAttention."""
2 from collections import defaultdict
3 from dataclasses import dataclass
4 +from itertools import accumulate
5 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
6
7 import torch
@@ -503,6 +504,8 @@ class FlashAttentionMetadataBuilder(
504 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
505 max_decode_seq_len = max(self.curr_seq_lens, default=0)
506 num_decode_tokens = self.num_decode_tokens
507 + query_start_loc = list(accumulate(query_lens, initial=0))
508 + seq_start_loc = list(accumulate(seq_lens, initial=0))
509
510 num_seqs = len(seq_lens)
511 if use_captured_graph:
@@ -525,29 +528,18 @@ class FlashAttentionMetadataBuilder(
528 device, self.runner.pin_memory)
529 seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
530 self.runner.pin_memory)
528 - query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
529 - self.runner.pin_memory)
531 slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
532 device, self.runner.pin_memory)
532 - query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
533 - dtype=torch.int32,
534 - device=device)
535 - seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
536 - dtype=torch.int32,
537 - device=device)
533 + query_start_loc_tensor = async_tensor_h2d(query_start_loc, torch.int32,
534 + device,
535 + self.runner.pin_memory)
536 + seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
537 + device, self.runner.pin_memory)
538 placeholder_index_maps = {
539 modality: placeholder_map.index_map()
540 for modality, placeholder_map in
541 self.multimodal_placeholder_maps.items()
542 }
543 - torch.cumsum(seq_lens_tensor,
544 - dim=0,
545 - dtype=seq_start_loc.dtype,
546 - out=seq_start_loc[1:])
547 - torch.cumsum(query_lens_tensor,
548 - dim=0,
549 - dtype=query_start_loc.dtype,
550 - out=query_start_loc[1:])
543
544 return FlashAttentionMetadata(
545 num_prefills=self.num_prefills,
@@ -561,8 +553,8 @@ class FlashAttentionMetadataBuilder(
553 max_decode_query_len=max_decode_query_len,
554 max_prefill_seq_len=max_prefill_seq_len,
555 max_decode_seq_len=max_decode_seq_len,
564 - query_start_loc=query_start_loc,
565 - seq_start_loc=seq_start_loc,
556 + query_start_loc=query_start_loc_tensor,
557 + seq_start_loc=seq_start_loc_tensor,
558 context_lens_tensor=context_lens_tensor,
559 block_tables=block_tables,
560 use_cuda_graph=use_captured_graph,
vllm/attention/backends/utils.py
+10 -18
@@ -1,6 +1,7 @@
1 """Attention backend utils"""
2 from collections import defaultdict
3 from contextlib import contextmanager
4 +from itertools import accumulate
5 from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, TypeVar, Union
6
7 import numpy as np
@@ -216,6 +217,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
217 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
218 max_decode_seq_len = max(self.curr_seq_lens, default=0)
219 num_decode_tokens = self.num_decode_tokens
220 + query_start_loc = list(accumulate(query_lens, initial=0))
221 + seq_start_loc = list(accumulate(seq_lens, initial=0))
222
223 if use_captured_graph:
224 self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
@@ -244,29 +247,18 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
247 device, self.runner.pin_memory)
248 seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
249 self.runner.pin_memory)
247 - query_lens_tensor = async_tensor_h2d(query_lens, torch.long, device,
248 - self.runner.pin_memory)
250 slot_mapping_tensor = async_tensor_h2d(self.slot_mapping, torch.long,
251 device, self.runner.pin_memory)
251 - query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
252 - dtype=torch.int32,
253 - device=device)
254 - seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
255 - dtype=torch.int32,
256 - device=device)
252 + query_start_loc_tensor = async_tensor_h2d(query_start_loc, torch.int32,
253 + device,
254 + self.runner.pin_memory)
255 + seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
256 + device, self.runner.pin_memory)
257 placeholder_index_maps = {
258 modality: placeholder_map.index_map()
259 for modality, placeholder_map in
260 self.multimodal_placeholder_maps.items()
261 }
262 - torch.cumsum(seq_lens_tensor,
263 - dim=0,
264 - dtype=seq_start_loc.dtype,
265 - out=seq_start_loc[1:])
266 - torch.cumsum(query_lens_tensor,
267 - dim=0,
268 - dtype=query_start_loc.dtype,
269 - out=query_start_loc[1:])
262
263 return self._metadata_cls( # type: ignore
264 num_prefills=self.num_prefills,
@@ -279,8 +271,8 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
271 max_query_len=max_query_len,
272 max_prefill_seq_len=max_prefill_seq_len,
273 max_decode_seq_len=max_decode_seq_len,
282 - query_start_loc=query_start_loc,
283 - seq_start_loc=seq_start_loc,
274 + query_start_loc=query_start_loc_tensor,
275 + seq_start_loc=seq_start_loc_tensor,
276 context_lens_tensor=context_lens_tensor,
277 block_tables=block_tables,
278 use_cuda_graph=use_captured_graph,