@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0794e7446e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Add multipstep chunked-prefill support for FlashInfer (#10467)

Elfie Guo committed Jan 14, 2025 at 20:47 UTC 0794e7446efca1fd7b8ea1cde96777897660cdea
5 files changed +169 -109
csrc/prepare_inputs/advance_step.cu
+10
@@ -95,6 +95,16 @@ __global__ void advance_step_flashinfer_kernel(
95 long* input_positions_ptr, int* seq_lens_ptr, long* slot_mapping_ptr,
96 int const* block_tables_ptr, int64_t const block_tables_stride,
97 int* paged_kv_last_page_len_ptr, int* block_table_bound_ptr) {
98 + int const n_pad = num_seqs - num_queries;
99 + if (n_pad && blockIdx.x == 0) {
100 + // Handle cuda graph padding
101 + int const offset = num_queries;
102 + for (int i = threadIdx.x; i < n_pad; i += blockDim.x) {
103 + input_tokens_ptr[offset + i] = 0;
104 + input_positions_ptr[offset + i] = 0;
105 + slot_mapping_ptr[offset + i] = -1;
106 + }
107 + }
108 int num_query_blocks = div_ceil(num_queries, num_threads);
109
110 if (blockIdx.x < num_query_blocks) {
tests/multi_step/test_correctness_llm.py
+16 -1
@@ -5,6 +5,8 @@ from typing import Optional
5
6 import pytest
7
8 +from tests.kernels.utils import override_backend_env_variable
9 +
10 from ..models.utils import check_logprobs_close, check_outputs_equal
11
12 MODELS = [
@@ -19,10 +21,11 @@ NUM_PROMPTS = [10]
21 @pytest.mark.parametrize("tp_size", [1])
22 @pytest.mark.parametrize("enable_chunked_prefill", [False, True])
23 @pytest.mark.parametrize("max_tokens", [5])
22 -@pytest.mark.parametrize("enforce_eager", [True])
24 +@pytest.mark.parametrize("enforce_eager", [True, False])
25 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
26 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
27 @pytest.mark.parametrize("num_logprobs", [None, 5])
28 +@pytest.mark.parametrize("attention_backend", ["FLASH_ATTN", "FLASHINFER"])
29 def test_multi_step_llm(
30 hf_runner,
31 vllm_runner,
@@ -36,6 +39,8 @@ def test_multi_step_llm(
39 num_scheduler_steps: int,
40 num_prompts: int,
41 num_logprobs: Optional[int],
42 + attention_backend: str,
43 + monkeypatch,
44 ) -> None:
45 """Test vLLM engine with multi-step scheduling via sync LLM Engine.
46
@@ -63,6 +68,7 @@ def test_multi_step_llm(
68 num_logprobs: corresponds to the `logprobs` argument to the OpenAI
69 completions endpoint; `None` -> 1 logprob returned.
70 """
71 + override_backend_env_variable(monkeypatch, attention_backend)
72
73 prompts = example_prompts
74 if len(prompts) < num_prompts:
@@ -114,6 +120,7 @@ def test_multi_step_llm(
120 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
121 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
122 @pytest.mark.parametrize("num_logprobs,num_prompt_logprobs", [(5, 5)])
123 +@pytest.mark.parametrize("attention_backend", ["FLASH_ATTN"])
124 def test_multi_step_llm_w_prompt_logprobs(
125 vllm_runner,
126 example_prompts,
@@ -126,6 +133,8 @@ def test_multi_step_llm_w_prompt_logprobs(
133 num_prompts: int,
134 num_logprobs: Optional[int],
135 num_prompt_logprobs: Optional[int],
136 + attention_backend: str,
137 + monkeypatch,
138 ) -> None:
139 """Test prompt logprobs with multi-step scheduling via sync LLM Engine.
140
@@ -155,6 +164,7 @@ def test_multi_step_llm_w_prompt_logprobs(
164 note that this argument is not supported by the
165 OpenAI completions endpoint.
166 """
167 + override_backend_env_variable(monkeypatch, attention_backend)
168
169 prompts = example_prompts
170 if len(prompts) < num_prompts:
@@ -205,6 +215,7 @@ def test_multi_step_llm_w_prompt_logprobs(
215 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
216 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
217 @pytest.mark.parametrize("num_logprobs", [None, 5])
218 +@pytest.mark.parametrize("attention_backend", ["FLASH_ATTN"])
219 def test_multi_step_llm_chunked_prefill_prefix_cache(
220 vllm_runner,
221 example_prompts,
@@ -216,6 +227,8 @@ def test_multi_step_llm_chunked_prefill_prefix_cache(
227 num_scheduler_steps: int,
228 num_prompts: int,
229 num_logprobs: Optional[int],
230 + attention_backend: str,
231 + monkeypatch,
232 ) -> None:
233 """Test vLLM engine with multi-step+"single-step chunked prefill"+APC.
234
@@ -278,6 +291,8 @@ def test_multi_step_llm_chunked_prefill_prefix_cache(
291 #
292 # The Incorrect scheduling behavior - if it occurs - will cause an exception
293 # in the model runner resulting from `do_sample=False`.
294 + override_backend_env_variable(monkeypatch, attention_backend)
295 +
296 assert len(example_prompts) >= 2
297 challenge_prompts = copy.deepcopy(example_prompts)
298 challenge_prompts[0] = ('vLLM is a high-throughput and memory-efficient '
vllm/attention/backends/flashinfer.py
+24 -5
@@ -256,7 +256,12 @@ class FlashInferState(AttentionState):
256 def begin_forward(self, model_input):
257 assert not self._is_graph_capturing
258 state = self
259 - if model_input.attn_metadata.use_cuda_graph:
259 + use_cuda_graph = model_input.attn_metadata.use_cuda_graph
260 + is_decode = model_input.attn_metadata.num_prefills == 0
261 + # In case of multistep chunked-prefill, there might be prefill requests
262 + # scheduled while CUDA graph mode is enabled. We don't run graph in that
263 + # case.
264 + if use_cuda_graph and is_decode:
265 batch_size = model_input.input_tokens.shape[0]
266 state = (self.runner.graph_runners[model_input.virtual_engine]
267 [batch_size].attn_state)
@@ -429,10 +434,24 @@ class FlashInferMetadata(AttentionMetadata):
434 Update metadata in-place to advance one decode step.
435 """
436
432 - assert not turn_prefills_into_decodes, \
433 - ("Chunked prefill is not supported with flashinfer yet."
434 - "turn_prefills_into_decodes is a Multi-Step + Chunked-Prefill "
435 - "specific parameter.")
437 + if turn_prefills_into_decodes:
438 + # When Multi-Step is enabled with Chunked-Prefill, prefills and
439 + # decodes are scheduled together. In the first step, all the
440 + # prefills turn into decodes. This update reflects that
441 + # conversion.
442 + assert self.num_decode_tokens + self.num_prefills == num_seqs
443 + # Flashinfer doesn't support speculative decoding + chunked-prefill
444 + # + multi-step scheduling yet.
445 + assert self.decode_query_len == 1
446 + self.num_decode_tokens += self.num_prefills
447 + self.num_prefills = 0
448 + self.num_prefill_tokens = 0
449 + self.max_prefill_seq_len = 0
450 + self.max_query_len = 1
451 +
452 + self.slot_mapping = self.slot_mapping[:num_seqs]
453 + else:
454 + assert self.seq_lens_tensor is not None
455
456 assert num_seqs > 0
457 assert num_queries > 0
vllm/worker/model_runner.py
+118 -102
@@ -5,6 +5,7 @@ import itertools
5 import time
6 import warnings
7 import weakref
8 +from contextlib import contextmanager
9 from dataclasses import dataclass
10 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
11 Tuple, Type, TypeVar, Union)
@@ -1028,6 +1029,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1029
1030 self.has_inner_state = model_config.has_inner_state
1031
1032 + self.in_profile_run = False
1033 +
1034 # When using CUDA graph, the input block tables must be padded to
1035 # max_seq_len_to_capture. However, creating the block table in
1036 # Python can be expensive. To optimize this, we cache the block table
@@ -1228,110 +1231,123 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1231
1232 return builder.build() # type: ignore
1233
1234 + @contextmanager
1235 + def set_in_profile_run(self):
1236 + self.in_profile_run = True
1237 + try:
1238 + yield
1239 + finally:
1240 + self.in_profile_run = False
1241 +
1242 @torch.inference_mode()
1243 def profile_run(self) -> None:
1233 - # Enable top-k sampling to reflect the accurate memory usage.
1234 - sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
1235 - max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
1236 - max_num_seqs = self.scheduler_config.max_num_seqs
1237 - # This represents the maximum number of different requests
1238 - # that will have unique loras, an therefore the max amount of memory
1239 - # consumption create dummy lora request copies from the lora request
1240 - # passed in, which contains a lora from the lora warmup path.
1241 - dummy_lora_requests: List[LoRARequest] = []
1242 - dummy_lora_requests_per_seq: List[LoRARequest] = []
1243 - if self.lora_config:
1244 - assert self.lora_manager is not None
1245 - with self.lora_manager.dummy_lora_cache():
1246 - for idx in range(self.lora_config.max_loras):
1247 - lora_id = idx + 1
1248 - dummy_lora_request = LoRARequest(
1249 - lora_name=f"warmup_{lora_id}",
1250 - lora_int_id=lora_id,
1251 - lora_path="/not/a/real/path",
1252 - )
1253 - self.lora_manager.add_dummy_lora(dummy_lora_request,
1254 - rank=LORA_WARMUP_RANK)
1255 - dummy_lora_requests.append(dummy_lora_request)
1256 - dummy_lora_requests_per_seq = [
1257 - dummy_lora_requests[idx % len(dummy_lora_requests)]
1258 - for idx in range(max_num_seqs)
1259 - ]
1260 -
1261 - # Profile memory usage with max_num_sequences sequences and the total
1262 - # number of tokens equal to max_num_batched_tokens.
1263 - seqs: List[SequenceGroupMetadata] = []
1264 - # Additional GPU memory may be needed for multi-modal encoding, which
1265 - # needs to be accounted for when calculating the GPU blocks for
1266 - # vLLM blocker manager.
1267 - # To exercise the worst scenario for GPU memory consumption,
1268 - # the number of seqs (batch_size) is chosen to maximize the number
1269 - # of images processed.
1270 -
1271 - max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
1272 - self.model_config)
1273 - if max_mm_tokens > 0:
1274 - max_num_seqs_orig = max_num_seqs
1275 - max_num_seqs = min(max_num_seqs,
1276 - max_num_batched_tokens // max_mm_tokens)
1277 - if max_num_seqs < 1:
1278 - expr = (f"min({max_num_seqs_orig}, "
1279 - f"{max_num_batched_tokens} // {max_mm_tokens})")
1280 - logger.warning(
1281 - "Computed max_num_seqs (%s) to be less than 1. "
1282 - "Setting it to the minimum value of 1.", expr)
1283 - max_num_seqs = 1
1284 -
1285 - batch_size = 0
1286 - for group_id in range(max_num_seqs):
1287 - seq_len = (max_num_batched_tokens // max_num_seqs +
1288 - (group_id < max_num_batched_tokens % max_num_seqs))
1289 - batch_size += seq_len
1290 -
1291 - dummy_data = self.input_registry \
1292 - .dummy_data_for_profiling(self.model_config,
1293 - seq_len,
1294 - self.mm_registry)
1295 -
1296 - seq = SequenceGroupMetadata(
1297 - request_id=str(group_id),
1298 - is_prompt=True,
1299 - seq_data={group_id: dummy_data.seq_data},
1300 - sampling_params=sampling_params,
1301 - block_tables=None,
1302 - lora_request=dummy_lora_requests_per_seq[group_id]
1303 - if dummy_lora_requests_per_seq else None,
1304 - multi_modal_data=dummy_data.multi_modal_data,
1305 - multi_modal_placeholders=dummy_data.multi_modal_placeholders,
1306 - )
1307 - seqs.append(seq)
1308 -
1309 - # Run the model with the dummy inputs.
1310 - num_layers = self.model_config.get_num_layers(self.parallel_config)
1311 - # use an empty tensor instead of `None`` to force Dynamo to pass
1312 - # it by reference, rather by specializing on the value ``None``.
1313 - # the `dtype` argument does not matter, and we use `float32` as
1314 - # a placeholder (it has wide hardware support).
1315 - # it is important to create tensors inside the loop, rather than
1316 - # multiplying the list, to avoid Dynamo from treating them as
1317 - # tensor aliasing.
1318 - kv_caches = [
1319 - torch.tensor([], dtype=torch.float32, device=self.device)
1320 - for _ in range(num_layers)
1321 - ]
1322 - finished_requests_ids = [seq.request_id for seq in seqs]
1323 - model_input = self.prepare_model_input(
1324 - seqs, finished_requests_ids=finished_requests_ids)
1325 - intermediate_tensors = None
1326 - if not get_pp_group().is_first_rank:
1327 - intermediate_tensors = self.model.make_empty_intermediate_tensors(
1328 - batch_size=batch_size,
1329 - dtype=self.model_config.dtype,
1330 - device=self.device)
1331 -
1332 - self.execute_model(model_input, kv_caches, intermediate_tensors)
1333 - torch.cuda.synchronize()
1334 - return
1244 + with self.set_in_profile_run():
1245 + # Enable top-k sampling to reflect the accurate memory usage.
1246 + sampling_params = \
1247 + SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
1248 + max_num_batched_tokens = \
1249 + self.scheduler_config.max_num_batched_tokens
1250 + max_num_seqs = self.scheduler_config.max_num_seqs
1251 + # This represents the maximum number of different requests
1252 + # that will have unique loras, an therefore the max amount of memory
1253 + # consumption create dummy lora request copies from the lora request
1254 + # passed in, which contains a lora from the lora warmup path.
1255 + dummy_lora_requests: List[LoRARequest] = []
1256 + dummy_lora_requests_per_seq: List[LoRARequest] = []
1257 + if self.lora_config:
1258 + assert self.lora_manager is not None
1259 + with self.lora_manager.dummy_lora_cache():
1260 + for idx in range(self.lora_config.max_loras):
1261 + lora_id = idx + 1
1262 + dummy_lora_request = LoRARequest(
1263 + lora_name=f"warmup_{lora_id}",
1264 + lora_int_id=lora_id,
1265 + lora_path="/not/a/real/path",
1266 + )
1267 + self.lora_manager.add_dummy_lora(dummy_lora_request,
1268 + rank=LORA_WARMUP_RANK)
1269 + dummy_lora_requests.append(dummy_lora_request)
1270 + dummy_lora_requests_per_seq = [
1271 + dummy_lora_requests[idx % len(dummy_lora_requests)]
1272 + for idx in range(max_num_seqs)
1273 + ]
1274 +
1275 + # Profile memory usage with max_num_sequences sequences and the
1276 + # total number of tokens equal to max_num_batched_tokens.
1277 + seqs: List[SequenceGroupMetadata] = []
1278 + # Additional GPU memory may be needed for multi-modal encoding,
1279 + # which needs to be accounted for when calculating the GPU blocks
1280 + # for vLLM blocker manager.
1281 + # To exercise the worst scenario for GPU memory consumption,
1282 + # the number of seqs (batch_size) is chosen to maximize the number
1283 + # of images processed.
1284 +
1285 + max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
1286 + self.model_config)
1287 + if max_mm_tokens > 0:
1288 + max_num_seqs_orig = max_num_seqs
1289 + max_num_seqs = min(max_num_seqs,
1290 + max_num_batched_tokens // max_mm_tokens)
1291 + if max_num_seqs < 1:
1292 + expr = (f"min({max_num_seqs_orig}, "
1293 + f"{max_num_batched_tokens} // {max_mm_tokens})")
1294 + logger.warning(
1295 + "Computed max_num_seqs (%s) to be less than 1. "
1296 + "Setting it to the minimum value of 1.", expr)
1297 + max_num_seqs = 1
1298 +
1299 + batch_size = 0
1300 + for group_id in range(max_num_seqs):
1301 + seq_len = (max_num_batched_tokens // max_num_seqs +
1302 + (group_id < max_num_batched_tokens % max_num_seqs))
1303 + batch_size += seq_len
1304 +
1305 + dummy_data = self.input_registry \
1306 + .dummy_data_for_profiling(self.model_config,
1307 + seq_len,
1308 + self.mm_registry)
1309 +
1310 + seq = SequenceGroupMetadata(
1311 + request_id=str(group_id),
1312 + is_prompt=True,
1313 + seq_data={group_id: dummy_data.seq_data},
1314 + sampling_params=sampling_params,
1315 + block_tables=None,
1316 + lora_request=dummy_lora_requests_per_seq[group_id]
1317 + if dummy_lora_requests_per_seq else None,
1318 + multi_modal_data=dummy_data.multi_modal_data,
1319 + multi_modal_placeholders=dummy_data.
1320 + multi_modal_placeholders,
1321 + )
1322 + seqs.append(seq)
1323 +
1324 + # Run the model with the dummy inputs.
1325 + num_layers = self.model_config.get_num_layers(self.parallel_config)
1326 + # use an empty tensor instead of `None`` to force Dynamo to pass
1327 + # it by reference, rather by specializing on the value ``None``.
1328 + # the `dtype` argument does not matter, and we use `float32` as
1329 + # a placeholder (it has wide hardware support).
1330 + # it is important to create tensors inside the loop, rather than
1331 + # multiplying the list, to avoid Dynamo from treating them as
1332 + # tensor aliasing.
1333 + kv_caches = [
1334 + torch.tensor([], dtype=torch.float32, device=self.device)
1335 + for _ in range(num_layers)
1336 + ]
1337 + finished_requests_ids = [seq.request_id for seq in seqs]
1338 + model_input = self.prepare_model_input(
1339 + seqs, finished_requests_ids=finished_requests_ids)
1340 + intermediate_tensors = None
1341 + if not get_pp_group().is_first_rank:
1342 + intermediate_tensors = \
1343 + self.model.make_empty_intermediate_tensors(
1344 + batch_size=batch_size,
1345 + dtype=self.model_config.dtype,
1346 + device=self.device)
1347 +
1348 + self.execute_model(model_input, kv_caches, intermediate_tensors)
1349 + torch.cuda.synchronize()
1350 + return
1351
1352 def remove_all_loras(self):
1353 if not self.lora_manager:
vllm/worker/multi_step_model_runner.py
+1 -1
@@ -32,7 +32,7 @@ logger = init_logger(__name__)
32 MULTI_STEP_ATTENTION_BACKENDS = [
33 "FLASH_ATTN", "ROCM_FLASH", "FLASHINFER", "NO_ATTENTION"
34 ]
35 -MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["FLASH_ATTN"]
35 +MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["FLASH_ATTN", "FLASHINFER"]
36
37 def _get_supported_attention_backends(chunked_prefill_enabled: bool) \
38 -> List[str]: