@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f92038b99
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add comments on swap space (#154)

Woosuk Kwon committed Jun 18, 2023 at 11:39 UTC 3f92038b990b26fdb9e6a9bccab0e3ec0cdc6aea
2 files changed +8 -2
benchmarks/benchmark_serving.py
+2 -1
@@ -3,7 +3,8 @@
3 On the server side, run one of the following commands:
4 (vLLM backend)
5 python -m vllm.entrypoints.api_server \
6 - --disable-log-requests --model <your_model>
6 + --model <your_model> --swap-space 16 \
7 + --disable-log-requests
8
9 (TGI backend)
10 ./launch_hf_server.sh <your_model>
vllm/core/scheduler.py
+6 -1
@@ -409,7 +409,12 @@ class Scheduler:
409 seq_group: SequenceGroup,
410 blocks_to_swap_out: Dict[int, int],
411 ) -> None:
412 - assert self.block_manager.can_swap_out(seq_group)
412 + if not self.block_manager.can_swap_out(seq_group):
413 + # FIXME(woosuk): Abort the sequence group instead of aborting the
414 + # entire engine.
415 + raise RuntimeError(
416 + "Aborted due to the lack of CPU swap space. Please increase "
417 + "the swap space to avoid this error.")
418 mapping = self.block_manager.swap_out(seq_group)
419 blocks_to_swap_out.update(mapping)
420 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):