@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 84e4e37d14
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Minor] Fix type annotations (#1238)

Woosuk Kwon committed Oct 2, 2023 at 15:28 UTC 84e4e37d1427bd254bea6ff366026773d36f3982
2 files changed +3 -4
vllm/model_executor/layers/sampler.py
+1 -1
@@ -290,7 +290,7 @@ def _get_topk_logprobs(
290 def _build_sequence_outputs(
291 parent_ids: List[int],
292 next_token_ids: List[int],
293 - selected_token_logprobs: torch.Tensor,
293 + selected_token_logprobs: List[float],
294 parent_seq_ids: List[int],
295 parent_logprobs: torch.Tensor,
296 num_output_logprobs: Optional[int],
vllm/sampling_params.py
+2 -3
@@ -61,7 +61,6 @@ class SamplingParams:
61 max_tokens: Maximum number of tokens to generate per output sequence.
62 logprobs: Number of log probabilities to return per output token.
63 skip_special_tokens: Whether to skip special tokens in the output.
64 - Defaults to true.
64 """
65
66 def __init__(
@@ -76,8 +75,8 @@ class SamplingParams:
75 use_beam_search: bool = False,
76 length_penalty: float = 1.0,
77 early_stopping: Union[bool, str] = False,
79 - stop: Union[None, str, List[str]] = None,
80 - stop_token_ids: List[int] = None,
78 + stop: Optional[Union[str, List[str]]] = None,
79 + stop_token_ids: Optional[List[int]] = None,
80 ignore_eos: bool = False,
81 max_tokens: int = 16,
82 logprobs: Optional[int] = None,