A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] Fix typo and remove unused code (#2305)
Roy committed
Jan 3, 2024 at 11:23 UTC
9140561059904f133a3c146f04afe13d1cdd5a6c
2 files changed
+1
-22
vllm/model_executor/layers/sampler.py
-21
@@ -112,27 +112,6 @@ def _prune_hidden_states(
112
sampling_metadata.selected_token_indices)
113
114
115
-def _get_prompt_and_output_tokens(
116
- sampling_metadata: SamplingMetadata,
117
-) -> Tuple[List[List[int]], List[List[int]]]:
118
- prompt_tokens: List[List[int]] = []
119
- output_tokens: List[List[int]] = []
120
- for i, seq_group in enumerate(sampling_metadata.seq_groups):
121
- seq_ids, sampling_params = seq_group
122
- if (i < sampling_metadata.num_prompts
123
- and sampling_params.prompt_logprobs is not None):
124
- # NOTE: prompt token positions do not need output tokens to
125
- # compute penalties.
126
- prompt_len = sampling_metadata.prompt_lens[i]
127
- prompt_tokens.extend([] for _ in range(prompt_len - 1))
128
- output_tokens.extend([] for _ in range(prompt_len - 1))
129
- for seq_id in seq_ids:
130
- seq_data = sampling_metadata.seq_data[seq_id]
131
- prompt_tokens.append(seq_data.prompt_token_ids)
132
- output_tokens.append(seq_data.output_token_ids)
133
- return prompt_tokens, output_tokens
134
-
135
-
115
def _get_bin_counts_and_mask(
116
tokens: torch.Tensor,
117
vocab_size: int,
vllm/sampling_params.py
+1
-1
@@ -100,7 +100,7 @@ class SamplingParams:
100
temperature: float = 1.0,
101
top_p: float = 1.0,
102
top_k: int = -1,
103
- min_p: int = 0.0,
103
+ min_p: float = 0.0,
104
use_beam_search: bool = False,
105
length_penalty: float = 1.0,
106
early_stopping: Union[bool, str] = False,