A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Minor] Fix type annotations (#1238)
Woosuk Kwon committed
Oct 2, 2023 at 15:28 UTC
84e4e37d1427bd254bea6ff366026773d36f3982
2 files changed
+3
-4
vllm/model_executor/layers/sampler.py
+1
-1
@@ -290,7 +290,7 @@ def _get_topk_logprobs(
290
def _build_sequence_outputs(
291
parent_ids: List[int],
292
next_token_ids: List[int],
293
- selected_token_logprobs: torch.Tensor,
293
+ selected_token_logprobs: List[float],
294
parent_seq_ids: List[int],
295
parent_logprobs: torch.Tensor,
296
num_output_logprobs: Optional[int],
vllm/sampling_params.py
+2
-3
@@ -61,7 +61,6 @@ class SamplingParams:
61
max_tokens: Maximum number of tokens to generate per output sequence.
62
logprobs: Number of log probabilities to return per output token.
63
skip_special_tokens: Whether to skip special tokens in the output.
64
- Defaults to true.
64
"""
65
66
def __init__(
@@ -76,8 +75,8 @@ class SamplingParams:
75
use_beam_search: bool = False,
76
length_penalty: float = 1.0,
77
early_stopping: Union[bool, str] = False,
79
- stop: Union[None, str, List[str]] = None,
80
- stop_token_ids: List[int] = None,
78
+ stop: Optional[Union[str, List[str]]] = None,
79
+ stop_token_ids: Optional[List[int]] = None,
80
ignore_eos: bool = False,
81
max_tokens: int = 16,
82
logprobs: Optional[int] = None,