A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Doc] fix doc string typo in block_manager `swap_out` function (#10212)
Yangcheng Li committed
Nov 12, 2024 at 00:53 UTC
36fc439de00a11d82d75d1e571cc4360fab11cdb
1 file changed
+3
-3
vllm/core/block_manager.py
+3
-3
@@ -393,7 +393,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
393
with num_lookahead_slots.
394
395
Args:
396
- seq_group (SequenceGroup): The sequence group to swap in.
396
+ seq_group (SequenceGroup): The sequence group to swap out.
397
num_lookahead_slots (int): Number of lookahead slots used in
398
speculative decoding, default to 0.
399
@@ -409,7 +409,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
409
swapping out the given sequence_group with num_lookahead_slots.
410
411
Args:
412
- sequence_group (SequenceGroup): The sequence group to swap in.
412
+ sequence_group (SequenceGroup): The sequence group to swap out.
413
414
Returns:
415
List[Tuple[int, int]]: The mapping of swapping block from
@@ -459,7 +459,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
459
on to the 'device'.
460
461
Args:
462
- sequence_group (SequenceGroup): The sequence group to swap in.
462
+ sequence_group (SequenceGroup): The sequence group to swap in/out.
463
device (Device): device to swap the 'seq_group' on.
464
status (SequenceStatus): The status of sequence which is needed
465
for action. RUNNING for swap out and SWAPPED for swap in