@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f20fc04bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] fix docstrings (#4191)

Co-authored-by: Zhong Wang <wangzhong@infini-ai.com>

Uranus committed Apr 19, 2024 at 16:18 UTC 8f20fc04bf7384089395caa021766cd352d0cf0b
1 file changed +3 -6
vllm/sequence.py
+3 -6
@@ -160,7 +160,7 @@ class SequenceData:
160 self._stage = SequenceStage.PREFILL
161
162 def get_num_uncomputed_tokens(self) -> int:
163 - """Return the number of prefil tokens that are not computed."""
163 + """Return the number of prefill tokens that are not computed."""
164 # we use `get_len()` which includes prompt_len + output_len instead
165 # of prompt_len here. This is because during recompute we need to
166 # prefill for both prompt and output.
@@ -345,12 +345,9 @@ class Sequence:
345 def get_num_new_tokens(self) -> int:
346 """Get the number of new tokens to be computed.
347
348 - Args:
349 - remainig_token_budget: The remaining token budgets.
348 Returns:
351 - The new number of tokens to be computed. I.e., 1 for decode, prompt
352 - size for prefill. If there's not enough remainig_token_budget, it
353 - can return the chunked number of new tokens.
349 + The new number of tokens to be computed. I.e., 1 for decode, or
350 + the remaining prompt size for prefill.
351 """
352 if self.data.stage == SequenceStage.DECODE:
353 return 1