@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ff4511e43
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] Add chunked-prefill support on FlashInfer. (#9781)

Elfie Guo committed Oct 30, 2024 at 09:33 UTC 9ff4511e43bb95efefd4e28048ca257e408277fb
2 files changed +72 -28
tests/basic_correctness/test_chunked_prefill.py
+12
@@ -11,6 +11,8 @@ from contextlib import nullcontext
11
12 import pytest
13
14 +from tests.kernels.utils import override_backend_env_variable
15 +
16 from ..models.utils import check_logprobs_close, check_outputs_equal
17 from ..utils import multi_gpu_test
18
@@ -28,6 +30,7 @@ MODELS = [
30 # NOTE: Increasing this in this suite will fail CI because we currently cannot
31 # reset distributed env properly. Use a value > 1 just when you test.
32 @pytest.mark.parametrize("tensor_parallel_size", [1])
33 +@pytest.mark.parametrize("attention_backend", ["FLASHINFER", "FLASH_ATTN"])
34 def test_models(
35 hf_runner,
36 vllm_runner,
@@ -38,11 +41,15 @@ def test_models(
41 chunked_prefill_token_size: int,
42 enforce_eager: bool,
43 tensor_parallel_size: int,
44 + attention_backend: str,
45 + monkeypatch,
46 ) -> None:
47 """
48 Checks exact match decode between huggingface model and vllm runner with
49 chunked prefill.
50 """
51 + override_backend_env_variable(monkeypatch, attention_backend)
52 +
53 max_num_seqs = chunked_prefill_token_size
54 max_num_batched_tokens = chunked_prefill_token_size
55
@@ -71,13 +78,18 @@ def test_models(
78 @multi_gpu_test(num_gpus=2)
79 @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
80 @pytest.mark.parametrize("model", MODELS)
81 +@pytest.mark.parametrize("attention_backend", ["FLASHINFER", "FLASH_ATTN"])
82 def test_models_distributed(
83 hf_runner,
84 vllm_runner,
85 example_prompts,
86 model: str,
87 distributed_executor_backend: str,
88 + attention_backend: str,
89 + monkeypatch,
90 ) -> None:
91 + override_backend_env_variable(monkeypatch, attention_backend)
92 +
93 if (model == "meta-llama/Llama-2-7b-hf"
94 and distributed_executor_backend == "ray"):
95 # test ray adag
vllm/attention/backends/flashinfer.py
+60 -28
@@ -268,6 +268,11 @@ class FlashInferMetadata(AttentionMetadata):
268 # Maximum sequence length among prefill batch. 0 if there are decoding
269 # requests only.
270 max_prefill_seq_len: int
271 + # Number of query tokens for each request in the batch.
272 + # Currently, we require that all requests have the same number of query
273 + # tokens during the decoding phase. When speculavie decoding is enabled,
274 + # decode_query_len might be greater than 1. In all other cases, it is 1.
275 + decode_query_len: Optional[int] = 1
276
277 use_cuda_graph: bool = True
278
@@ -335,6 +340,7 @@ class FlashInferMetadata(AttentionMetadata):
340 assert self.paged_kv_last_page_len is not None
341 assert self.block_table_bound is not None
342 assert self.seq_lens_tensor is not None
343 + self.query_start_loc = self.query_start_loc[:self.num_prefills + 1]
344 batch_size = self.query_start_loc.shape[0] - 1
345 assert batch_size >= 0
346 # We will use flash attention for profiling to
@@ -349,11 +355,13 @@ class FlashInferMetadata(AttentionMetadata):
355 self.paged_kv_indices = self.paged_kv_indices.to(self.device)
356 self.prefill_wrapper.end_forward()
357 self.prefill_wrapper.begin_forward(
352 - self.query_start_loc, self.paged_kv_indptr,
353 - self.paged_kv_indices, self.paged_kv_last_page_len,
358 + self.query_start_loc,
359 + self.paged_kv_indptr[:self.num_prefills + 1],
360 + self.paged_kv_indices,
361 + self.paged_kv_last_page_len[:self.num_prefills],
362 self.num_qo_heads, self.num_kv_heads, self.head_dim,
363 self.page_size)
356 - else:
364 + if self.num_decode_tokens > 0:
365 assert self.paged_kv_indices is not None
366 assert self.paged_kv_indptr is not None
367 assert self.paged_kv_last_page_len is not None
@@ -370,9 +378,9 @@ class FlashInferMetadata(AttentionMetadata):
378 assert self.decode_wrapper is not None
379 self.decode_wrapper.end_forward()
380 self.decode_wrapper.begin_forward(
373 - self.paged_kv_indptr,
381 + self.paged_kv_indptr[self.num_prefills:],
382 self.paged_kv_indices,
375 - self.paged_kv_last_page_len,
383 + self.paged_kv_last_page_len[self.num_prefills:],
384 self.num_qo_heads,
385 self.num_kv_heads,
386 self.head_dim,
@@ -397,21 +405,14 @@ class FlashInferMetadata(AttentionMetadata):
405
406 @property
407 def prefill_metadata(self) -> Optional["FlashInferMetadata"]:
400 - # Currently chunked prefill is not supported
401 - if self.num_decode_tokens == 0:
402 - assert self.num_prefills > 0
403 - return self
404 -
405 - return None
408 + if self.num_prefills == 0:
409 + return None
410 + return self
411
412 @property
413 def decode_metadata(self) -> Optional["FlashInferMetadata"]:
409 - # Currently chunked prefill is not supported
410 - if self.num_prefills > 0:
411 - assert self.num_decode_tokens == 0, (
412 - "Chunked prefill is not supported with flashinfer yet.")
414 + if self.num_decode_tokens == 0:
415 return None
414 -
416 return self
417
418 def advance_step(self,
@@ -599,11 +600,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
600
601 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
602 num_decode_tokens = self.num_decode_tokens
603 + decode_query_len = max(query_lens[self.num_prefills:], default=1)
604
605 if use_captured_graph:
606 self.slot_mapping.extend([PAD_SLOT_ID] * cuda_graph_pad_size)
607 self.block_tables.extend([] * cuda_graph_pad_size)
606 - num_decode_tokens = batch_size
608 + num_decode_tokens = batch_size - self.num_prefill_tokens
609
610 # The shape of graph_block_tables is
611 # [max batch size, max context len // block size].
@@ -689,6 +691,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
691 self.runner.kv_cache_dtype, self.runner.model_config.dtype)
692
693 return FlashInferMetadata(
694 + decode_query_len=decode_query_len,
695 num_prefills=self.num_prefills,
696 slot_mapping=slot_mapping_tensor,
697 num_prefill_tokens=self.num_prefill_tokens,
@@ -811,12 +814,6 @@ def unified_flash_infer(
814 key = key.view(-1, num_kv_heads, head_size)
815 value = value.view(-1, num_kv_heads, head_size)
816
814 - if attn_metadata.num_prefill_tokens > 0:
815 - assert attn_metadata.num_decode_tokens == 0, (
816 - "Chunked prefill is not supported with flashinfer yet.")
817 - if attn_metadata.num_decode_tokens > 0:
818 - assert attn_metadata.num_prefill_tokens == 0, (
819 - "Chunked prefill is not supported with flashinfer yet.")
817 if kv_cache.numel() > 0:
818 # Use the same reshape and cache kernel as flash attention.
819 ops.reshape_and_cache_flash(
@@ -836,14 +833,33 @@ def unified_flash_infer(
833 kv_cache_dtype)
834 kv_cache = kv_cache.view(torch_dtype)
835
836 + num_prefill_tokens = attn_metadata.num_prefill_tokens
837 + num_decode_tokens = attn_metadata.num_decode_tokens
838 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
839 + f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
840 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
841 + f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
842 query = query.contiguous() # Flashinfer requires query to be contiguous
843 + # Query for decode. KV is not needed because it is already cached.
844 + # QKV for prefill.
845 + decode_query = query[num_prefill_tokens:]
846 + query = query[:num_prefill_tokens]
847 +
848 + key = key[:num_prefill_tokens]
849 + value = value[:num_prefill_tokens]
850 +
851 + assert query.shape[0] == num_prefill_tokens
852 + assert decode_query.shape[0] == num_decode_tokens
853 +
854 + prefill_output: Optional[torch.Tensor] = None
855 + decode_output: Optional[torch.Tensor] = None
856 if prefill_meta := attn_metadata.prefill_metadata:
857 # We will use flash attention for prefill
858 # when kv_cache is not provided.
859 # This happens when vllm runs the profiling to
860 # determine the number of blocks.
861 if kv_cache.numel() == 0:
846 - output = flash_attn_varlen_func(
862 + prefill_output = flash_attn_varlen_func(
863 q=query,
864 k=key,
865 v=value,
@@ -859,18 +875,34 @@ def unified_flash_infer(
875 else:
876 assert prefill_meta is not None
877 assert prefill_meta.prefill_wrapper is not None
862 - output = prefill_meta.prefill_wrapper.forward(
878 + prefill_output = prefill_meta.prefill_wrapper.forward(
879 query, kv_cache, logits_soft_cap=logits_soft_cap, causal=True)
864 - else:
880 + if decode_meta := attn_metadata.decode_metadata:
881 assert attn_metadata.decode_metadata is not None
882 assert attn_metadata.decode_metadata.decode_wrapper is not None
867 - output = attn_metadata.decode_metadata.decode_wrapper.forward(
868 - query,
883 + decode_output = attn_metadata.decode_metadata.decode_wrapper.forward(
884 + decode_query,
885 kv_cache,
886 sm_scale=softmax_scale,
887 logits_soft_cap=logits_soft_cap,
888 k_scale=k_scale,
889 v_scale=v_scale)
890 +
891 + if prefill_output is None and decode_output is not None:
892 + # Decode only batch.
893 + output, num_tokens = decode_output, num_decode_tokens
894 + elif decode_output is None and prefill_output is not None:
895 + # Prefill only batch.
896 + output, num_tokens = prefill_output, num_prefill_tokens
897 + else:
898 + # Chunked prefill batch does not work with speculative decoding in
899 + # FlashInfer backend, so the query length for decode should be 1.
900 + assert prefill_output is not None
901 + assert decode_output is not None
902 + assert decode_meta is not None
903 + assert decode_meta.decode_query_len == 1
904 + decode_output = decode_output.squeeze(1)
905 + output = torch.cat([prefill_output, decode_output], dim=0)
906 return output.view(num_tokens, hidden_size)
907
908