@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 563649aafe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core] Combined support for multi-step scheduling, chunked prefill & prefix caching (#8804)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Andrew Feldman <afeld2012@gmail.com>

afeldman-nm committed Oct 2, 2024 at 03:52 UTC 563649aafe7d4b9cb0047bba60d6f58efa53fd28
3 files changed +180 -17
tests/multi_step/test_correctness_llm.py
+158
@@ -1,5 +1,6 @@
1 # Test the LLMEngine with multi-step-decoding
2
3 +import copy
4 from typing import Optional
5
6 import pytest
@@ -196,3 +197,160 @@ def test_multi_step_llm_w_prompt_logprobs(
197 name_0="hf",
198 name_1="vllm",
199 )
200 +
201 +
202 +@pytest.mark.parametrize("model", MODELS)
203 +@pytest.mark.parametrize("dtype", ["half"])
204 +@pytest.mark.parametrize("tp_size", [1])
205 +@pytest.mark.parametrize("max_tokens", [5])
206 +@pytest.mark.parametrize("enforce_eager", [True])
207 +@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
208 +@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
209 +@pytest.mark.parametrize("num_logprobs", [None, 5])
210 +def test_multi_step_llm_chunked_prefill_prefix_cache(
211 + vllm_runner,
212 + example_prompts,
213 + model: str,
214 + dtype: str,
215 + tp_size: int,
216 + max_tokens: int,
217 + enforce_eager: int,
218 + num_scheduler_steps: int,
219 + num_prompts: int,
220 + num_logprobs: Optional[int],
221 +) -> None:
222 + """Test vLLM engine with multi-step+"single-step chunked prefill"+APC.
223 +
224 + Set up contrived scenario which tests for a possible failure mode of
225 + scheduling with multi-step+"single-step chunked prefill"+APC
226 +
227 + "single-step chunked prefill" here refers to the current vLLM multi-step+
228 + chunked-prefill implementation, which requires that a prefill may only
229 + be scheduled in the same step as decodes if the prefill prompt fits in a
230 + single chunk (note that "complete" multi-step+chunked-prefill would allow
231 + a prefill to span multiple chunks & multiple steps but that is not yet
232 + the case.)
233 +
234 + "APC" is short for "automatic prefix caching".
235 +
236 + This test creates a scenario where the scheduler must decide whether/how
237 + to schedule a prefill with a prompt that exceeds the available token budget.
238 + The correct behavior for multi-step+"single-step chunked prefill"+APC is to
239 + put off scheduling the prefill until a future step.
240 +
241 + Validate that:
242 + * Multi-step kernels do not raise an exception due to incorrect scheduler
243 + behavior
244 + * Generated tokens match between
245 + multi-step+"single-step chunked prefill"+APC and
246 + single-step scheduling.
247 + * (If logprobs are enabled) check logprobs are close enough
248 +
249 + Args:
250 + vllm_runner: vLLM model runner fixture
251 + example_prompts: test fixture providing example prompts
252 + model: model under test (same for single- and multi-step engines)
253 + dtype: tensor datatype for engine to utilize
254 + tp_size: degree of tensor-parallelism
255 + max_tokens: the maximum number of tokens to generate
256 + enforce_eager
257 + num_scheduler_steps: for multi-step scheduling, GPU-side steps per
258 + GPU -> CPU output transfer
259 + num_prompts: number of example prompts under test
260 + num_logprobs: corresponds to the `logprobs` argument to the OpenAI
261 + completions endpoint; `None` -> 1 logprob returned.
262 + """
263 +
264 + # Set up contrived test for correct scheduling behavior with
265 + # multi-step+"single-step chunked prefill"+APC.
266 + #
267 + # Assume block_size=16
268 + #
269 + # Assume max_num_batched_tokens=48
270 + # => Per-step token budget=48
271 + #
272 + # 1. Scheduler schedules 0th prompt (24 tokens)
273 + # => Remaining token budget=24
274 + # 2. Scheduler attempts to schedule 1st prompt (30 tokens)
275 + # * 30 tokens exceeds 24 token remaining budget
276 + # * Correct behavior: do not schedule this prompt in this step
277 + # * Incorrect behavior: schedule prompt chunk
278 + # * `do_sample=False` for this prompt in this step
279 + # * Chunk size = (remaining tokens // block size) * block size
280 + #
281 + # The Incorrect scheduling behavior - if it occurs - will cause an exception
282 + # in the model runner resulting from `do_sample=False`.
283 + assert len(example_prompts) >= 2
284 + challenge_prompts = copy.deepcopy(example_prompts)
285 + challenge_prompts[0] = ('vLLM is a high-throughput and memory-efficient '
286 + 'inference and serving engine for LLMs.\n'
287 + ) # 24 tok
288 + challenge_prompts[1] = (
289 + 'Briefly describe the major milestones in the '
290 + 'development of artificial intelligence from 1950 to 2020.\n'
291 + ) # 30 tok
292 +
293 + # If necessary, adjust the length of `challenge_prompts` to match
294 + # `num_prompts`
295 + if len(challenge_prompts) < num_prompts:
296 + challenge_prompts = (challenge_prompts *
297 + ((num_prompts // len(challenge_prompts)) + 1))
298 + challenge_prompts = challenge_prompts[:num_prompts]
299 + assert len(challenge_prompts) == num_prompts
300 +
301 + # Single-step scheduler baseline
302 + with vllm_runner(
303 + model,
304 + dtype=dtype,
305 + enforce_eager=enforce_eager,
306 + gpu_memory_utilization=0.7,
307 + tensor_parallel_size=tp_size,
308 + use_v2_block_manager=True,
309 + num_scheduler_steps=num_scheduler_steps,
310 + max_model_len=48,
311 + max_num_batched_tokens=48,
312 + max_num_seqs=4,
313 + block_size=16,
314 + ) as vllm_model:
315 + outputs_baseline = (vllm_model.generate_greedy(
316 + challenge_prompts, max_tokens) if num_logprobs is None else
317 + vllm_model.generate_greedy_logprobs(
318 + challenge_prompts, max_tokens, num_logprobs))
319 +
320 + # multi-step+"single-step chunked prefill"+APC
321 + with vllm_runner(
322 + model,
323 + dtype=dtype,
324 + enforce_eager=enforce_eager,
325 + gpu_memory_utilization=0.7,
326 + tensor_parallel_size=tp_size,
327 + use_v2_block_manager=True,
328 + enable_chunked_prefill=True,
329 + enable_prefix_caching=True,
330 + num_scheduler_steps=num_scheduler_steps,
331 + max_model_len=48,
332 + max_num_batched_tokens=48,
333 + max_num_seqs=4,
334 + block_size=16,
335 + ) as vllm_model:
336 + outputs_w_features = (vllm_model.generate_greedy(
337 + challenge_prompts, max_tokens) if num_logprobs is None else
338 + vllm_model.generate_greedy_logprobs(
339 + challenge_prompts, max_tokens, num_logprobs))
340 +
341 + if num_logprobs is None:
342 + # No-logprobs test
343 + check_outputs_equal(
344 + outputs_0_lst=outputs_baseline,
345 + outputs_1_lst=outputs_w_features,
346 + name_0="multi-step",
347 + name_1="multi-step+features",
348 + )
349 + else:
350 + # Yes-logprobs test
351 + check_logprobs_close(
352 + outputs_0_lst=outputs_baseline,
353 + outputs_1_lst=outputs_w_features,
354 + name_0="multi-step",
355 + name_1="multi-step+features",
356 + )
vllm/core/scheduler.py
+22 -13
@@ -1607,10 +1607,29 @@ class Scheduler:
1607 # in a decode phase. Do not chunk.
1608 if enable_chunking and len(seqs) == 1:
1609 remaining_token_budget = budget.remaining_token_budget()
1610 - if self.cache_config.enable_prefix_caching:
1610 + if self.scheduler_config.is_multi_step:
1611 + # The current multi-step + chunked prefill capability does
1612 + # not actually support chunking prompts.
1613 + #
1614 + # Therefore, `num_new_tokens` is computed in the same fashion
1615 + # for both multi-step+chunked-prefill &
1616 + # multi-step+chunked-prefill+APC
1617 + #
1618 + # Prompts with more tokens than the current remaining budget
1619 + # are postponed to future scheduler steps
1620 + if num_new_tokens > self._get_prompt_limit(seq_group):
1621 + # If the seq_group is in prompt-stage, pass the
1622 + # num_new_tokens as-is so the caller can ignore
1623 + # the sequence.
1624 + pass
1625 + else:
1626 + num_new_tokens = 0 \
1627 + if num_new_tokens > remaining_token_budget \
1628 + else num_new_tokens
1629 + elif self.cache_config.enable_prefix_caching:
1630 # When prefix caching is enabled, we always allocate
1612 - # the number of new tokens that is dividable by the block size
1613 - # to avoid partial block matching.
1631 + # the number of new tokens that is dividable by the block
1632 + # size to avoid partial block matching.
1633 block_size = self.cache_config.block_size
1634 remainder = budget.token_budget % block_size
1635 if remainder != 0:
@@ -1623,16 +1642,6 @@ class Scheduler:
1642 if remaining_token_budget < num_new_tokens:
1643 num_new_tokens = (remaining_token_budget //
1644 block_size) * block_size
1626 - elif self.scheduler_config.is_multi_step:
1627 - if num_new_tokens > self._get_prompt_limit(seq_group):
1628 - # If the seq_group is in prompt-stage, pass the
1629 - # num_new_tokens as-is so the caller can ignore
1630 - # the sequence.
1631 - pass
1632 - else:
1633 - num_new_tokens = 0 \
1634 - if num_new_tokens > remaining_token_budget \
1635 - else num_new_tokens
1645 else:
1646 num_new_tokens = min(num_new_tokens, remaining_token_budget)
1647 return num_new_tokens
vllm/engine/arg_utils.py
-4
@@ -999,10 +999,6 @@ class EngineArgs:
999 if speculative_config is not None:
1000 raise ValueError("Speculative decoding is not supported with "
1001 "multi-step (--num-scheduler-steps > 1)")
1002 - if self.enable_chunked_prefill and self.enable_prefix_caching:
1003 - raise ValueError("Multi-Step is not supported with "
1004 - "both Chunked-Prefill and Prefix-Caching "
1005 - "enabled together.")
1002 if self.enable_chunked_prefill and self.pipeline_parallel_size > 1:
1003 raise ValueError("Multi-Step Chunked-Prefill is not supported "
1004 "for pipeline-parallel-size > 1")