A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add comments on swap space (#154)
Woosuk Kwon committed
Jun 18, 2023 at 11:39 UTC
3f92038b990b26fdb9e6a9bccab0e3ec0cdc6aea
2 files changed
+8
-2
benchmarks/benchmark_serving.py
+2
-1
@@ -3,7 +3,8 @@
3
On the server side, run one of the following commands:
4
(vLLM backend)
5
python -m vllm.entrypoints.api_server \
6
- --disable-log-requests --model <your_model>
6
+ --model <your_model> --swap-space 16 \
7
+ --disable-log-requests
8
9
(TGI backend)
10
./launch_hf_server.sh <your_model>
vllm/core/scheduler.py
+6
-1
@@ -409,7 +409,12 @@ class Scheduler:
409
seq_group: SequenceGroup,
410
blocks_to_swap_out: Dict[int, int],
411
) -> None:
412
- assert self.block_manager.can_swap_out(seq_group)
412
+ if not self.block_manager.can_swap_out(seq_group):
413
+ # FIXME(woosuk): Abort the sequence group instead of aborting the
414
+ # entire engine.
415
+ raise RuntimeError(
416
+ "Aborted due to the lack of CPU swap space. Please increase "
417
+ "the swap space to avoid this error.")
418
mapping = self.block_manager.swap_out(seq_group)
419
blocks_to_swap_out.update(mapping)
420
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):