A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] fix docstrings (#4191)
Co-authored-by: Zhong Wang <wangzhong@infini-ai.com>
Uranus committed
Apr 19, 2024 at 16:18 UTC
8f20fc04bf7384089395caa021766cd352d0cf0b
1 file changed
+3
-6
vllm/sequence.py
+3
-6
@@ -160,7 +160,7 @@ class SequenceData:
160
self._stage = SequenceStage.PREFILL
161
162
def get_num_uncomputed_tokens(self) -> int:
163
- """Return the number of prefil tokens that are not computed."""
163
+ """Return the number of prefill tokens that are not computed."""
164
# we use `get_len()` which includes prompt_len + output_len instead
165
# of prompt_len here. This is because during recompute we need to
166
# prefill for both prompt and output.
@@ -345,12 +345,9 @@ class Sequence:
345
def get_num_new_tokens(self) -> int:
346
"""Get the number of new tokens to be computed.
347
348
- Args:
349
- remainig_token_budget: The remaining token budgets.
348
Returns:
351
- The new number of tokens to be computed. I.e., 1 for decode, prompt
352
- size for prefill. If there's not enough remainig_token_budget, it
353
- can return the chunked number of new tokens.
349
+ The new number of tokens to be computed. I.e., 1 for decode, or
350
+ the remaining prompt size for prefill.
351
"""
352
if self.data.stage == SequenceStage.DECODE:
353
return 1