@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9140561059
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Fix typo and remove unused code (#2305)

Roy committed Jan 3, 2024 at 11:23 UTC 9140561059904f133a3c146f04afe13d1cdd5a6c
2 files changed +1 -22
vllm/model_executor/layers/sampler.py
-21
@@ -112,27 +112,6 @@ def _prune_hidden_states(
112 sampling_metadata.selected_token_indices)
113
114
115 -def _get_prompt_and_output_tokens(
116 - sampling_metadata: SamplingMetadata,
117 -) -> Tuple[List[List[int]], List[List[int]]]:
118 - prompt_tokens: List[List[int]] = []
119 - output_tokens: List[List[int]] = []
120 - for i, seq_group in enumerate(sampling_metadata.seq_groups):
121 - seq_ids, sampling_params = seq_group
122 - if (i < sampling_metadata.num_prompts
123 - and sampling_params.prompt_logprobs is not None):
124 - # NOTE: prompt token positions do not need output tokens to
125 - # compute penalties.
126 - prompt_len = sampling_metadata.prompt_lens[i]
127 - prompt_tokens.extend([] for _ in range(prompt_len - 1))
128 - output_tokens.extend([] for _ in range(prompt_len - 1))
129 - for seq_id in seq_ids:
130 - seq_data = sampling_metadata.seq_data[seq_id]
131 - prompt_tokens.append(seq_data.prompt_token_ids)
132 - output_tokens.append(seq_data.output_token_ids)
133 - return prompt_tokens, output_tokens
134 -
135 -
115 def _get_bin_counts_and_mask(
116 tokens: torch.Tensor,
117 vocab_size: int,
vllm/sampling_params.py
+1 -1
@@ -100,7 +100,7 @@ class SamplingParams:
100 temperature: float = 1.0,
101 top_p: float = 1.0,
102 top_k: int = -1,
103 - min_p: int = 0.0,
103 + min_p: float = 0.0,
104 use_beam_search: bool = False,
105 length_penalty: float = 1.0,
106 early_stopping: Union[bool, str] = False,