A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
fix: fix some args desc (#2487)
zspo committed
Jan 19, 2024 at 01:41 UTC
4df417d0593b56d189e8aed8b9c85a4476cfc630
2 files changed
+1
-1
vllm/prefix.py
-1
@@ -11,7 +11,6 @@ class Prefix:
11
prefix caching in the future.
12
13
Args:
14
- prefix_id: The id of the prefix in the prefix pool.
14
token_ids: The token ids of the prefix.
15
block_size: The block size of the executed model.
16
"""
vllm/sequence.py
+1
@@ -229,6 +229,7 @@ class SequenceGroup:
229
seqs: The list of sequences.
230
sampling_params: The sampling parameters used to generate the outputs.
231
arrival_time: The arrival time of the request.
232
+ prefix: The prefix of the prompt of the sequence group.
233
"""
234
235
def __init__(