@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9db93de20c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Add multi-step support to LLMEngine (#7789)

Alexander Matveev committed Aug 23, 2024 at 15:45 UTC 9db93de20ca282feb4dfaabbc56032c9312bde7b
7 files changed +195 -87
.buildkite/test-pipeline.yaml
+2 -1
@@ -335,7 +335,8 @@ steps:
335 - vllm/engine
336 - tests/multi_step
337 commands:
338 - - pytest -v -s multi_step/test_correctness.py
338 + - pytest -v -s multi_step/test_correctness_async_llm.py
339 + - pytest -v -s multi_step/test_correctness_llm.py
340
341 - label: Pipeline Parallelism Test # 23min
342 working_dir: "/vllm-workspace/tests"
benchmarks/benchmark_throughput.py
+15 -2
@@ -82,6 +82,8 @@ def run_vllm(
82 max_num_batched_tokens: int,
83 distributed_executor_backend: Optional[str],
84 gpu_memory_utilization: float = 0.9,
85 + num_scheduler_steps: int = 1,
86 + use_v2_block_manager: bool = False,
87 download_dir: Optional[str] = None,
88 load_format: str = EngineArgs.load_format,
89 ) -> float:
@@ -106,6 +108,8 @@ def run_vllm(
108 max_num_batched_tokens=max_num_batched_tokens,
109 distributed_executor_backend=distributed_executor_backend,
110 load_format=load_format,
111 + num_scheduler_steps=num_scheduler_steps,
112 + use_v2_block_manager=use_v2_block_manager,
113 )
114
115 # Add the requests to the engine.
@@ -232,7 +236,8 @@ def main(args: argparse.Namespace):
236 args.quantization_param_path, args.device,
237 args.enable_prefix_caching, args.enable_chunked_prefill,
238 args.max_num_batched_tokens, args.distributed_executor_backend,
235 - args.gpu_memory_utilization, args.download_dir, args.load_format)
239 + args.gpu_memory_utilization, args.num_scheduler_steps,
240 + args.use_v2_block_manager, args.download_dir, args.load_format)
241 elif args.backend == "hf":
242 assert args.tensor_parallel_size == 1
243 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -353,10 +358,18 @@ if __name__ == "__main__":
358 choices=["auto", "cuda", "cpu", "openvino", "tpu", "xpu"],
359 help='device type for vLLM execution, supporting CUDA, OpenVINO and '
360 'CPU.')
361 + parser.add_argument(
362 + "--num-scheduler-steps",
363 + type=int,
364 + default=1,
365 + help="Maximum number of forward steps per scheduler call.")
366 + parser.add_argument("--use-v2-block-manager",
367 + action='store_true',
368 + help="Enable block manager v2.")
369 parser.add_argument(
370 "--enable-prefix-caching",
371 action='store_true',
359 - help="enable automatic prefix caching for vLLM backend.")
372 + help="Enable automatic prefix caching for vLLM backend.")
373 parser.add_argument("--enable-chunked-prefill",
374 action='store_true',
375 help="enable chunked prefill for vLLM backend.")
tests/lora/test_gemma.py
+1 -1
@@ -37,7 +37,7 @@ def test_gemma_lora(gemma_lora_files):
37 expected_lora_output = [
38 "more important than knowledge.\nAuthor: Albert Einstein\n",
39 "everyone else is already taken.\nAuthor: Oscar Wilde\n",
40 - "so little time.\nAuthor: Frank Zappa\n",
40 + "so little time\nAuthor: Frank Zappa\n",
41 ]
42
43 output1 = do_sample(llm, gemma_lora_files, lora_id=1)
tests/multi_step/test_correctness_async_llm.py renamed
tests/multi_step/test_correctness_llm.py new
+49
@@ -0,0 +1,49 @@
1 +# Test the LLMEngine with multi-step-decoding
2 +
3 +import pytest
4 +
5 +from ..models.utils import check_outputs_equal
6 +
7 +MODELS = [
8 + "JackFram/llama-160m",
9 +]
10 +NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
11 +NUM_PROMPTS = [10]
12 +
13 +
14 +@pytest.mark.parametrize("model", MODELS)
15 +@pytest.mark.parametrize("dtype", ["half"])
16 +@pytest.mark.parametrize("tp_size", [1])
17 +@pytest.mark.parametrize("max_tokens", [5])
18 +@pytest.mark.parametrize("enforce_eager", [True])
19 +@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
20 +@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
21 +def test_multi_step_llm(hf_runner, vllm_runner, example_prompts, model: str,
22 + dtype: str, tp_size: int, max_tokens: int,
23 + enforce_eager: int, num_scheduler_steps: int,
24 + num_prompts: int) -> None:
25 +
26 + prompts = example_prompts
27 + if len(prompts) < num_prompts:
28 + prompts = prompts * ((num_prompts // len(prompts)) + 1)
29 + prompts = prompts[:num_prompts]
30 + assert len(prompts) == num_prompts
31 +
32 + with vllm_runner(model,
33 + dtype=dtype,
34 + enforce_eager=enforce_eager,
35 + gpu_memory_utilization=0.7,
36 + tensor_parallel_size=tp_size,
37 + use_v2_block_manager=True,
38 + num_scheduler_steps=num_scheduler_steps) as vllm_model:
39 + vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
40 +
41 + with hf_runner(model, dtype=dtype) as hf_model:
42 + hf_outputs = hf_model.generate_greedy(prompts, max_tokens)
43 +
44 + check_outputs_equal(
45 + outputs_0_lst=hf_outputs,
46 + outputs_1_lst=vllm_outputs,
47 + name_0="hf",
48 + name_1="vllm",
49 + )
vllm/engine/async_llm_engine.py
+2 -72
@@ -1,11 +1,9 @@
1 import asyncio
2 import time
3 -from dataclasses import dataclass
3 from functools import partial
4 from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
5 Mapping, Optional, Set, Tuple, Type, Union)
6
8 -import torch
7 from typing_extensions import assert_never
8
9 import vllm.envs as envs
@@ -15,7 +13,7 @@ from vllm.core.scheduler import SchedulerOutputs
13 from vllm.engine.arg_utils import AsyncEngineArgs
14 from vllm.engine.async_timeout import asyncio_timeout
15 from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
18 - PromptComponents)
16 + PromptComponents, SchedulerOutputState)
17 from vllm.engine.metrics_types import StatLoggerBase
18 from vllm.executor.executor_base import ExecutorAsyncBase
19 from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -28,8 +26,7 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
26 from vllm.pooling_params import PoolingParams
27 from vllm.prompt_adapter.request import PromptAdapterRequest
28 from vllm.sampling_params import SamplingParams
31 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
32 - SequenceGroupMetadata)
29 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
30 from vllm.transformers_utils.tokenizer import AnyTokenizer
31 from vllm.usage.usage_lib import UsageContext
32 from vllm.utils import print_warning_once
@@ -257,24 +254,11 @@ class RequestTracker:
254 return not self._new_requests.empty()
255
256
260 -@dataclass
261 -class SchedulerOutputState:
262 - """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
263 - last_output: Optional[SamplerOutput] = None
264 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
265 - scheduler_outputs: Optional[SchedulerOutputs] = None
266 -
267 -
257 class _AsyncLLMEngine(LLMEngine):
258 """Extension of LLMEngine to add async methods."""
259
260 def __init__(self, *args, **kwargs):
261 super().__init__(*args, **kwargs)
273 - pipeline_parallel_size = \
274 - self.parallel_config.pipeline_parallel_size
275 - self.cached_scheduler_outputs = [
276 - SchedulerOutputState() for _ in range(pipeline_parallel_size)
277 - ]
262
263 async def step_async(
264 self, virtual_engine: int
@@ -367,60 +351,6 @@ class _AsyncLLMEngine(LLMEngine):
351
352 return request_outputs
353
370 - def _has_remaining_steps(
371 - self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
372 - ) -> bool:
373 - if (not self.scheduler_config.is_multi_step
374 - or not seq_group_metadata_list):
375 - return False
376 -
377 - # TODO(will) this is a sanity check for nowto make sure that all the
378 - # seqs are on the same steps. Eventually we will want to do some sort of
379 - # dynamic scheduling when doing multi-step decoding.
380 - ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps
381 - if any([
382 - seq_group.state.remaining_steps != ref_remaining_steps
383 - for seq_group in seq_group_metadata_list[1:]
384 - ]):
385 - raise AssertionError(("All running sequence groups should "
386 - "have the same remaining steps."))
387 -
388 - return ref_remaining_steps > 0
389 -
390 - def _cache_scheduler_outputs_for_multi_step(
391 - self, virtual_engine: int,
392 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
393 - scheduler_outputs: SchedulerOutputs) -> None:
394 - self.cached_scheduler_outputs[
395 - virtual_engine].seq_group_metadata_list = seq_group_metadata_list
396 - self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
397 - scheduler_outputs
398 - self.cached_scheduler_outputs[virtual_engine].last_output = None
399 -
400 - def _get_last_sampled_token_ids(
401 - self, virtual_engine: int) -> Optional[torch.Tensor]:
402 - cached_last_output = self.cached_scheduler_outputs[
403 - virtual_engine].last_output
404 - if (self.scheduler_config.is_multi_step
405 - and self.parallel_config.pipeline_parallel_size > 1
406 - and cached_last_output is not None
407 - and cached_last_output.sampled_token_ids_cpu is not None):
408 - return cached_last_output.sampled_token_ids_cpu
409 - return None
410 -
411 - def _update_cached_scheduler_output(
412 - self, virtual_engine: int,
413 - output: List[Optional[SamplerOutput]]) -> None:
414 - if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0
415 - and output[0] is not None):
416 - last_output = output[-1]
417 - assert last_output is not None
418 - assert last_output.sampled_token_ids_cpu is not None
419 - assert last_output.sampled_token_ids is None
420 - assert last_output.sampled_token_probs is None
421 - self.cached_scheduler_outputs[
422 - virtual_engine].last_output = last_output
423 -
354 async def stop_remote_worker_execution_loop_async(self) -> None:
355 """Stop the remote worker execution loop."""
356 await self.model_executor.stop_remote_worker_execution_loop_async()
vllm/engine/llm_engine.py
+126 -11
@@ -1,10 +1,12 @@
1 import time
2 from contextlib import contextmanager
3 +from dataclasses import dataclass
4 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5 Mapping, Optional)
6 from typing import Sequence as GenericSequence
7 from typing import Set, Tuple, Type, Union
8
9 +import torch
10 from typing_extensions import TypeVar, assert_never
11
12 import vllm.envs as envs
@@ -77,6 +79,14 @@ DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
79 Optional[MultiModalDataDict]]
80
81
82 +@dataclass
83 +class SchedulerOutputState:
84 + """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
85 + last_output: Optional[SamplerOutput] = None
86 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
87 + scheduler_outputs: Optional[SchedulerOutputs] = None
88 +
89 +
90 class LLMEngine:
91 """An LLM engine that receives requests and generates texts.
92
@@ -194,7 +204,7 @@ class LLMEngine:
204 "quantization_param_path=%s, device_config=%s, "
205 "decoding_config=%r, observability_config=%r, "
206 "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
197 - "enable_prefix_caching=%s)",
207 + "num_scheduler_steps=%d, enable_prefix_caching=%s)",
208 VLLM_VERSION,
209 model_config.model,
210 speculative_config,
@@ -223,6 +233,7 @@ class LLMEngine:
233 model_config.seed,
234 model_config.served_model_name,
235 scheduler_config.use_v2_block_manager,
236 + scheduler_config.num_scheduler_steps,
237 cache_config.enable_prefix_caching,
238 )
239 # TODO(woosuk): Print more configs in debug mode.
@@ -380,6 +391,11 @@ class LLMEngine:
391 ),
392 ))
393
394 + self.cached_scheduler_outputs = [
395 + SchedulerOutputState()
396 + for _ in range(self.parallel_config.pipeline_parallel_size)
397 + ]
398 +
399 def _initialize_kv_caches(self) -> None:
400 """Initialize the KV cache in the worker(s).
401
@@ -1304,16 +1320,40 @@ class LLMEngine:
1320 "Pipeline parallelism is only supported through AsyncLLMEngine "
1321 "as performance will be severely degraded otherwise.")
1322
1307 - if self.scheduler_config.num_scheduler_steps > 1:
1308 - raise NotImplementedError(
1309 - "Multiple scheduler steps (multi-step) are only supported "
1310 - "through AsyncLLMEngine. ")
1311 - seq_group_metadata_list, scheduler_outputs = self.scheduler[
1312 - 0].schedule()
1323 + # These are cached outputs from previous iterations. None if on first
1324 + # iteration
1325 + cached_outputs = self.cached_scheduler_outputs[0]
1326 + seq_group_metadata_list = cached_outputs.seq_group_metadata_list
1327 + scheduler_outputs = cached_outputs.scheduler_outputs
1328 +
1329 + # Skip the scheduler if there are any remaining steps in the seq groups.
1330 + # This ensures that the scheduler is only called again when the current
1331 + # batch has completed.
1332 + if not self._has_remaining_steps(seq_group_metadata_list):
1333 + seq_group_metadata_list, scheduler_outputs = self.scheduler[
1334 + 0].schedule()
1335 +
1336 + if (self.scheduler_config.is_multi_step
1337 + and scheduler_outputs.num_lookahead_slots > 0):
1338 + # cache the scheduler outputs for the next iteration if we have
1339 + # lookahead slots
1340 + self._cache_scheduler_outputs_for_multi_step(
1341 + 0, seq_group_metadata_list, scheduler_outputs)
1342 +
1343 + assert seq_group_metadata_list is not None
1344 + assert scheduler_outputs is not None
1345
1346 if not scheduler_outputs.is_empty():
1347 finished_requests_ids = self.scheduler[
1348 0].get_and_reset_finished_requests_ids()
1349 +
1350 + # Check if we have a cached last_output from the previous iteration.
1351 + # For supporting PP this is probably the best way to pass the
1352 + # sampled_token_ids, as a separate broadcast over all the PP stages
1353 + # will cause one virtual engine's microbatch to block the pipeline.
1354 + last_sampled_token_ids = \
1355 + self._get_last_sampled_token_ids(0)
1356 +
1357 execute_model_req = ExecuteModelRequest(
1358 seq_group_metadata_list=seq_group_metadata_list,
1359 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
@@ -1321,15 +1361,36 @@ class LLMEngine:
1361 blocks_to_copy=scheduler_outputs.blocks_to_copy,
1362 num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
1363 running_queue_size=scheduler_outputs.running_queue_size,
1324 - finished_requests_ids=finished_requests_ids)
1364 + finished_requests_ids=finished_requests_ids,
1365 + # We use ExecuteModelRequest to pass the last sampled_token_ids
1366 + # to each of the non-last PP stages for in-place prepare_input.
1367 + last_sampled_token_ids=last_sampled_token_ids)
1368 +
1369 output = self.model_executor.execute_model(
1370 execute_model_req=execute_model_req)
1371 +
1372 + # we need to do this here so that last step's sampled_token_ids can
1373 + # be passed to the next iteration for PP.
1374 + if self.scheduler_config.is_multi_step:
1375 + self._update_cached_scheduler_output(0, output)
1376 else:
1377 output = []
1378
1330 - request_outputs = self._process_model_outputs(
1331 - output, scheduler_outputs.scheduled_seq_groups,
1332 - scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
1379 + # Finish the current step for all the sequence groups.
1380 + if self.scheduler_config.is_multi_step:
1381 + for seq_group in seq_group_metadata_list:
1382 + seq_group.finish_step()
1383 +
1384 + if not self._has_remaining_steps(seq_group_metadata_list):
1385 + # clear the cache if we have finished all the steps
1386 + if self.scheduler_config.is_multi_step:
1387 + self.cached_scheduler_outputs[0] = SchedulerOutputState()
1388 + request_outputs = self._process_model_outputs(
1389 + output, scheduler_outputs.scheduled_seq_groups,
1390 + scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
1391 +
1392 + else:
1393 + request_outputs = []
1394
1395 # Log stats.
1396 self.do_log_stats(scheduler_outputs, output)
@@ -1347,6 +1408,60 @@ class LLMEngine:
1408
1409 return request_outputs
1410
1411 + def _has_remaining_steps(
1412 + self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
1413 + ) -> bool:
1414 + if (not self.scheduler_config.is_multi_step
1415 + or not seq_group_metadata_list):
1416 + return False
1417 +
1418 + # TODO(will) this is a sanity check for nowto make sure that all the
1419 + # seqs are on the same steps. Eventually we will want to do some sort of
1420 + # dynamic scheduling when doing multi-step decoding.
1421 + ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps
1422 + if any([
1423 + seq_group.state.remaining_steps != ref_remaining_steps
1424 + for seq_group in seq_group_metadata_list[1:]
1425 + ]):
1426 + raise AssertionError(("All running sequence groups should "
1427 + "have the same remaining steps."))
1428 +
1429 + return ref_remaining_steps > 0
1430 +
1431 + def _cache_scheduler_outputs_for_multi_step(
1432 + self, virtual_engine: int,
1433 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
1434 + scheduler_outputs: SchedulerOutputs) -> None:
1435 + self.cached_scheduler_outputs[
1436 + virtual_engine].seq_group_metadata_list = seq_group_metadata_list
1437 + self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
1438 + scheduler_outputs
1439 + self.cached_scheduler_outputs[virtual_engine].last_output = None
1440 +
1441 + def _update_cached_scheduler_output(
1442 + self, virtual_engine: int,
1443 + output: List[Optional[SamplerOutput]]) -> None:
1444 + if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0
1445 + and output[0] is not None):
1446 + last_output = output[-1]
1447 + assert last_output is not None
1448 + assert last_output.sampled_token_ids_cpu is not None
1449 + assert last_output.sampled_token_ids is None
1450 + assert last_output.sampled_token_probs is None
1451 + self.cached_scheduler_outputs[
1452 + virtual_engine].last_output = last_output
1453 +
1454 + def _get_last_sampled_token_ids(
1455 + self, virtual_engine: int) -> Optional[torch.Tensor]:
1456 + cached_last_output = self.cached_scheduler_outputs[
1457 + virtual_engine].last_output
1458 + if (self.scheduler_config.is_multi_step
1459 + and self.parallel_config.pipeline_parallel_size > 1
1460 + and cached_last_output is not None
1461 + and cached_last_output.sampled_token_ids_cpu is not None):
1462 + return cached_last_output.sampled_token_ids_cpu
1463 + return None
1464 +
1465 def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
1466 if logger_name in self.stat_loggers:
1467 raise KeyError(f"Logger with name {logger_name} already exists.")