A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Add multi-step support to LLMEngine (#7789)
Alexander Matveev committed
Aug 23, 2024 at 15:45 UTC
9db93de20ca282feb4dfaabbc56032c9312bde7b
7 files changed
+195
-87
.buildkite/test-pipeline.yaml
+2
-1
@@ -335,7 +335,8 @@ steps:
335
- vllm/engine
336
- tests/multi_step
337
commands:
338
- - pytest -v -s multi_step/test_correctness.py
338
+ - pytest -v -s multi_step/test_correctness_async_llm.py
339
+ - pytest -v -s multi_step/test_correctness_llm.py
340
341
- label: Pipeline Parallelism Test # 23min
342
working_dir: "/vllm-workspace/tests"
benchmarks/benchmark_throughput.py
+15
-2
@@ -82,6 +82,8 @@ def run_vllm(
82
max_num_batched_tokens: int,
83
distributed_executor_backend: Optional[str],
84
gpu_memory_utilization: float = 0.9,
85
+ num_scheduler_steps: int = 1,
86
+ use_v2_block_manager: bool = False,
87
download_dir: Optional[str] = None,
88
load_format: str = EngineArgs.load_format,
89
) -> float:
@@ -106,6 +108,8 @@ def run_vllm(
108
max_num_batched_tokens=max_num_batched_tokens,
109
distributed_executor_backend=distributed_executor_backend,
110
load_format=load_format,
111
+ num_scheduler_steps=num_scheduler_steps,
112
+ use_v2_block_manager=use_v2_block_manager,
113
)
114
115
# Add the requests to the engine.
@@ -232,7 +236,8 @@ def main(args: argparse.Namespace):
236
args.quantization_param_path, args.device,
237
args.enable_prefix_caching, args.enable_chunked_prefill,
238
args.max_num_batched_tokens, args.distributed_executor_backend,
235
- args.gpu_memory_utilization, args.download_dir, args.load_format)
239
+ args.gpu_memory_utilization, args.num_scheduler_steps,
240
+ args.use_v2_block_manager, args.download_dir, args.load_format)
241
elif args.backend == "hf":
242
assert args.tensor_parallel_size == 1
243
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -353,10 +358,18 @@ if __name__ == "__main__":
358
choices=["auto", "cuda", "cpu", "openvino", "tpu", "xpu"],
359
help='device type for vLLM execution, supporting CUDA, OpenVINO and '
360
'CPU.')
361
+ parser.add_argument(
362
+ "--num-scheduler-steps",
363
+ type=int,
364
+ default=1,
365
+ help="Maximum number of forward steps per scheduler call.")
366
+ parser.add_argument("--use-v2-block-manager",
367
+ action='store_true',
368
+ help="Enable block manager v2.")
369
parser.add_argument(
370
"--enable-prefix-caching",
371
action='store_true',
359
- help="enable automatic prefix caching for vLLM backend.")
372
+ help="Enable automatic prefix caching for vLLM backend.")
373
parser.add_argument("--enable-chunked-prefill",
374
action='store_true',
375
help="enable chunked prefill for vLLM backend.")
tests/lora/test_gemma.py
+1
-1
@@ -37,7 +37,7 @@ def test_gemma_lora(gemma_lora_files):
37
expected_lora_output = [
38
"more important than knowledge.\nAuthor: Albert Einstein\n",
39
"everyone else is already taken.\nAuthor: Oscar Wilde\n",
40
- "so little time.\nAuthor: Frank Zappa\n",
40
+ "so little time\nAuthor: Frank Zappa\n",
41
]
42
43
output1 = do_sample(llm, gemma_lora_files, lora_id=1)
tests/multi_step/test_correctness_async_llm.py
renamed
tests/multi_step/test_correctness_llm.py
new
+49
@@ -0,0 +1,49 @@
1
+# Test the LLMEngine with multi-step-decoding
2
+
3
+import pytest
4
+
5
+from ..models.utils import check_outputs_equal
6
+
7
+MODELS = [
8
+ "JackFram/llama-160m",
9
+]
10
+NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
11
+NUM_PROMPTS = [10]
12
+
13
+
14
+@pytest.mark.parametrize("model", MODELS)
15
+@pytest.mark.parametrize("dtype", ["half"])
16
+@pytest.mark.parametrize("tp_size", [1])
17
+@pytest.mark.parametrize("max_tokens", [5])
18
+@pytest.mark.parametrize("enforce_eager", [True])
19
+@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
20
+@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
21
+def test_multi_step_llm(hf_runner, vllm_runner, example_prompts, model: str,
22
+ dtype: str, tp_size: int, max_tokens: int,
23
+ enforce_eager: int, num_scheduler_steps: int,
24
+ num_prompts: int) -> None:
25
+
26
+ prompts = example_prompts
27
+ if len(prompts) < num_prompts:
28
+ prompts = prompts * ((num_prompts // len(prompts)) + 1)
29
+ prompts = prompts[:num_prompts]
30
+ assert len(prompts) == num_prompts
31
+
32
+ with vllm_runner(model,
33
+ dtype=dtype,
34
+ enforce_eager=enforce_eager,
35
+ gpu_memory_utilization=0.7,
36
+ tensor_parallel_size=tp_size,
37
+ use_v2_block_manager=True,
38
+ num_scheduler_steps=num_scheduler_steps) as vllm_model:
39
+ vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
40
+
41
+ with hf_runner(model, dtype=dtype) as hf_model:
42
+ hf_outputs = hf_model.generate_greedy(prompts, max_tokens)
43
+
44
+ check_outputs_equal(
45
+ outputs_0_lst=hf_outputs,
46
+ outputs_1_lst=vllm_outputs,
47
+ name_0="hf",
48
+ name_1="vllm",
49
+ )
vllm/engine/async_llm_engine.py
+2
-72
@@ -1,11 +1,9 @@
1
import asyncio
2
import time
3
-from dataclasses import dataclass
3
from functools import partial
4
from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
5
Mapping, Optional, Set, Tuple, Type, Union)
6
8
-import torch
7
from typing_extensions import assert_never
8
9
import vllm.envs as envs
@@ -15,7 +13,7 @@ from vllm.core.scheduler import SchedulerOutputs
13
from vllm.engine.arg_utils import AsyncEngineArgs
14
from vllm.engine.async_timeout import asyncio_timeout
15
from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
18
- PromptComponents)
16
+ PromptComponents, SchedulerOutputState)
17
from vllm.engine.metrics_types import StatLoggerBase
18
from vllm.executor.executor_base import ExecutorAsyncBase
19
from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -28,8 +26,7 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
26
from vllm.pooling_params import PoolingParams
27
from vllm.prompt_adapter.request import PromptAdapterRequest
28
from vllm.sampling_params import SamplingParams
31
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
32
- SequenceGroupMetadata)
29
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
30
from vllm.transformers_utils.tokenizer import AnyTokenizer
31
from vllm.usage.usage_lib import UsageContext
32
from vllm.utils import print_warning_once
@@ -257,24 +254,11 @@ class RequestTracker:
254
return not self._new_requests.empty()
255
256
260
-@dataclass
261
-class SchedulerOutputState:
262
- """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
263
- last_output: Optional[SamplerOutput] = None
264
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
265
- scheduler_outputs: Optional[SchedulerOutputs] = None
266
-
267
-
257
class _AsyncLLMEngine(LLMEngine):
258
"""Extension of LLMEngine to add async methods."""
259
260
def __init__(self, *args, **kwargs):
261
super().__init__(*args, **kwargs)
273
- pipeline_parallel_size = \
274
- self.parallel_config.pipeline_parallel_size
275
- self.cached_scheduler_outputs = [
276
- SchedulerOutputState() for _ in range(pipeline_parallel_size)
277
- ]
262
263
async def step_async(
264
self, virtual_engine: int
@@ -367,60 +351,6 @@ class _AsyncLLMEngine(LLMEngine):
351
352
return request_outputs
353
370
- def _has_remaining_steps(
371
- self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
372
- ) -> bool:
373
- if (not self.scheduler_config.is_multi_step
374
- or not seq_group_metadata_list):
375
- return False
376
-
377
- # TODO(will) this is a sanity check for nowto make sure that all the
378
- # seqs are on the same steps. Eventually we will want to do some sort of
379
- # dynamic scheduling when doing multi-step decoding.
380
- ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps
381
- if any([
382
- seq_group.state.remaining_steps != ref_remaining_steps
383
- for seq_group in seq_group_metadata_list[1:]
384
- ]):
385
- raise AssertionError(("All running sequence groups should "
386
- "have the same remaining steps."))
387
-
388
- return ref_remaining_steps > 0
389
-
390
- def _cache_scheduler_outputs_for_multi_step(
391
- self, virtual_engine: int,
392
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
393
- scheduler_outputs: SchedulerOutputs) -> None:
394
- self.cached_scheduler_outputs[
395
- virtual_engine].seq_group_metadata_list = seq_group_metadata_list
396
- self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
397
- scheduler_outputs
398
- self.cached_scheduler_outputs[virtual_engine].last_output = None
399
-
400
- def _get_last_sampled_token_ids(
401
- self, virtual_engine: int) -> Optional[torch.Tensor]:
402
- cached_last_output = self.cached_scheduler_outputs[
403
- virtual_engine].last_output
404
- if (self.scheduler_config.is_multi_step
405
- and self.parallel_config.pipeline_parallel_size > 1
406
- and cached_last_output is not None
407
- and cached_last_output.sampled_token_ids_cpu is not None):
408
- return cached_last_output.sampled_token_ids_cpu
409
- return None
410
-
411
- def _update_cached_scheduler_output(
412
- self, virtual_engine: int,
413
- output: List[Optional[SamplerOutput]]) -> None:
414
- if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0
415
- and output[0] is not None):
416
- last_output = output[-1]
417
- assert last_output is not None
418
- assert last_output.sampled_token_ids_cpu is not None
419
- assert last_output.sampled_token_ids is None
420
- assert last_output.sampled_token_probs is None
421
- self.cached_scheduler_outputs[
422
- virtual_engine].last_output = last_output
423
-
354
async def stop_remote_worker_execution_loop_async(self) -> None:
355
"""Stop the remote worker execution loop."""
356
await self.model_executor.stop_remote_worker_execution_loop_async()
vllm/engine/llm_engine.py
+126
-11
@@ -1,10 +1,12 @@
1
import time
2
from contextlib import contextmanager
3
+from dataclasses import dataclass
4
from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5
Mapping, Optional)
6
from typing import Sequence as GenericSequence
7
from typing import Set, Tuple, Type, Union
8
9
+import torch
10
from typing_extensions import TypeVar, assert_never
11
12
import vllm.envs as envs
@@ -77,6 +79,14 @@ DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
79
Optional[MultiModalDataDict]]
80
81
82
+@dataclass
83
+class SchedulerOutputState:
84
+ """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
85
+ last_output: Optional[SamplerOutput] = None
86
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
87
+ scheduler_outputs: Optional[SchedulerOutputs] = None
88
+
89
+
90
class LLMEngine:
91
"""An LLM engine that receives requests and generates texts.
92
@@ -194,7 +204,7 @@ class LLMEngine:
204
"quantization_param_path=%s, device_config=%s, "
205
"decoding_config=%r, observability_config=%r, "
206
"seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
197
- "enable_prefix_caching=%s)",
207
+ "num_scheduler_steps=%d, enable_prefix_caching=%s)",
208
VLLM_VERSION,
209
model_config.model,
210
speculative_config,
@@ -223,6 +233,7 @@ class LLMEngine:
233
model_config.seed,
234
model_config.served_model_name,
235
scheduler_config.use_v2_block_manager,
236
+ scheduler_config.num_scheduler_steps,
237
cache_config.enable_prefix_caching,
238
)
239
# TODO(woosuk): Print more configs in debug mode.
@@ -380,6 +391,11 @@ class LLMEngine:
391
),
392
))
393
394
+ self.cached_scheduler_outputs = [
395
+ SchedulerOutputState()
396
+ for _ in range(self.parallel_config.pipeline_parallel_size)
397
+ ]
398
+
399
def _initialize_kv_caches(self) -> None:
400
"""Initialize the KV cache in the worker(s).
401
@@ -1304,16 +1320,40 @@ class LLMEngine:
1320
"Pipeline parallelism is only supported through AsyncLLMEngine "
1321
"as performance will be severely degraded otherwise.")
1322
1307
- if self.scheduler_config.num_scheduler_steps > 1:
1308
- raise NotImplementedError(
1309
- "Multiple scheduler steps (multi-step) are only supported "
1310
- "through AsyncLLMEngine. ")
1311
- seq_group_metadata_list, scheduler_outputs = self.scheduler[
1312
- 0].schedule()
1323
+ # These are cached outputs from previous iterations. None if on first
1324
+ # iteration
1325
+ cached_outputs = self.cached_scheduler_outputs[0]
1326
+ seq_group_metadata_list = cached_outputs.seq_group_metadata_list
1327
+ scheduler_outputs = cached_outputs.scheduler_outputs
1328
+
1329
+ # Skip the scheduler if there are any remaining steps in the seq groups.
1330
+ # This ensures that the scheduler is only called again when the current
1331
+ # batch has completed.
1332
+ if not self._has_remaining_steps(seq_group_metadata_list):
1333
+ seq_group_metadata_list, scheduler_outputs = self.scheduler[
1334
+ 0].schedule()
1335
+
1336
+ if (self.scheduler_config.is_multi_step
1337
+ and scheduler_outputs.num_lookahead_slots > 0):
1338
+ # cache the scheduler outputs for the next iteration if we have
1339
+ # lookahead slots
1340
+ self._cache_scheduler_outputs_for_multi_step(
1341
+ 0, seq_group_metadata_list, scheduler_outputs)
1342
+
1343
+ assert seq_group_metadata_list is not None
1344
+ assert scheduler_outputs is not None
1345
1346
if not scheduler_outputs.is_empty():
1347
finished_requests_ids = self.scheduler[
1348
0].get_and_reset_finished_requests_ids()
1349
+
1350
+ # Check if we have a cached last_output from the previous iteration.
1351
+ # For supporting PP this is probably the best way to pass the
1352
+ # sampled_token_ids, as a separate broadcast over all the PP stages
1353
+ # will cause one virtual engine's microbatch to block the pipeline.
1354
+ last_sampled_token_ids = \
1355
+ self._get_last_sampled_token_ids(0)
1356
+
1357
execute_model_req = ExecuteModelRequest(
1358
seq_group_metadata_list=seq_group_metadata_list,
1359
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
@@ -1321,15 +1361,36 @@ class LLMEngine:
1361
blocks_to_copy=scheduler_outputs.blocks_to_copy,
1362
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
1363
running_queue_size=scheduler_outputs.running_queue_size,
1324
- finished_requests_ids=finished_requests_ids)
1364
+ finished_requests_ids=finished_requests_ids,
1365
+ # We use ExecuteModelRequest to pass the last sampled_token_ids
1366
+ # to each of the non-last PP stages for in-place prepare_input.
1367
+ last_sampled_token_ids=last_sampled_token_ids)
1368
+
1369
output = self.model_executor.execute_model(
1370
execute_model_req=execute_model_req)
1371
+
1372
+ # we need to do this here so that last step's sampled_token_ids can
1373
+ # be passed to the next iteration for PP.
1374
+ if self.scheduler_config.is_multi_step:
1375
+ self._update_cached_scheduler_output(0, output)
1376
else:
1377
output = []
1378
1330
- request_outputs = self._process_model_outputs(
1331
- output, scheduler_outputs.scheduled_seq_groups,
1332
- scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
1379
+ # Finish the current step for all the sequence groups.
1380
+ if self.scheduler_config.is_multi_step:
1381
+ for seq_group in seq_group_metadata_list:
1382
+ seq_group.finish_step()
1383
+
1384
+ if not self._has_remaining_steps(seq_group_metadata_list):
1385
+ # clear the cache if we have finished all the steps
1386
+ if self.scheduler_config.is_multi_step:
1387
+ self.cached_scheduler_outputs[0] = SchedulerOutputState()
1388
+ request_outputs = self._process_model_outputs(
1389
+ output, scheduler_outputs.scheduled_seq_groups,
1390
+ scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
1391
+
1392
+ else:
1393
+ request_outputs = []
1394
1395
# Log stats.
1396
self.do_log_stats(scheduler_outputs, output)
@@ -1347,6 +1408,60 @@ class LLMEngine:
1408
1409
return request_outputs
1410
1411
+ def _has_remaining_steps(
1412
+ self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
1413
+ ) -> bool:
1414
+ if (not self.scheduler_config.is_multi_step
1415
+ or not seq_group_metadata_list):
1416
+ return False
1417
+
1418
+ # TODO(will) this is a sanity check for nowto make sure that all the
1419
+ # seqs are on the same steps. Eventually we will want to do some sort of
1420
+ # dynamic scheduling when doing multi-step decoding.
1421
+ ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps
1422
+ if any([
1423
+ seq_group.state.remaining_steps != ref_remaining_steps
1424
+ for seq_group in seq_group_metadata_list[1:]
1425
+ ]):
1426
+ raise AssertionError(("All running sequence groups should "
1427
+ "have the same remaining steps."))
1428
+
1429
+ return ref_remaining_steps > 0
1430
+
1431
+ def _cache_scheduler_outputs_for_multi_step(
1432
+ self, virtual_engine: int,
1433
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
1434
+ scheduler_outputs: SchedulerOutputs) -> None:
1435
+ self.cached_scheduler_outputs[
1436
+ virtual_engine].seq_group_metadata_list = seq_group_metadata_list
1437
+ self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
1438
+ scheduler_outputs
1439
+ self.cached_scheduler_outputs[virtual_engine].last_output = None
1440
+
1441
+ def _update_cached_scheduler_output(
1442
+ self, virtual_engine: int,
1443
+ output: List[Optional[SamplerOutput]]) -> None:
1444
+ if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0
1445
+ and output[0] is not None):
1446
+ last_output = output[-1]
1447
+ assert last_output is not None
1448
+ assert last_output.sampled_token_ids_cpu is not None
1449
+ assert last_output.sampled_token_ids is None
1450
+ assert last_output.sampled_token_probs is None
1451
+ self.cached_scheduler_outputs[
1452
+ virtual_engine].last_output = last_output
1453
+
1454
+ def _get_last_sampled_token_ids(
1455
+ self, virtual_engine: int) -> Optional[torch.Tensor]:
1456
+ cached_last_output = self.cached_scheduler_outputs[
1457
+ virtual_engine].last_output
1458
+ if (self.scheduler_config.is_multi_step
1459
+ and self.parallel_config.pipeline_parallel_size > 1
1460
+ and cached_last_output is not None
1461
+ and cached_last_output.sampled_token_ids_cpu is not None):
1462
+ return cached_last_output.sampled_token_ids_cpu
1463
+ return None
1464
+
1465
def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
1466
if logger_name in self.stat_loggers:
1467
raise KeyError(f"Logger with name {logger_name} already exists.")