@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81661da7b2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[BugFix] Fix `min_tokens` when `eos_token_id` is None (#4389)

Co-authored-by: DefTruth <31974251+deftruth@users.noreply.github.com>

Nick Hill committed Apr 27, 2024 at 09:52 UTC 81661da7b2d446cff7065fd6b34f1b7051098d24
4 files changed +14 -18
tests/samplers/test_sampler.py
+3 -6
@@ -207,7 +207,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
207 def create_sampling_params(min_tokens,
208 eos_token_id=0,
209 *,
210 - stop_token_ids: Optional[List[str]] = None,
210 + stop_token_ids: Optional[List[int]] = None,
211 prompt_logprobs: Optional[int] = None):
212 sampling_params = SamplingParams(
213 min_tokens=min_tokens,
@@ -216,7 +216,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
216 # requesting prompt_logprobs changes the structure of `logits`
217 prompt_logprobs=prompt_logprobs,
218 )
219 - sampling_params.eos_token_id = eos_token_id
219 + sampling_params.all_stop_token_ids.add(eos_token_id)
220 return sampling_params
221
222 def create_sequence_data(num_input=3, num_generated=0):
@@ -461,10 +461,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
461 for logits_idx, (should_penalize, sampling_params) in enumerate(
462 zip(expected_penalization, sampling_params_per_row)):
463
464 - tokens_to_check = [sampling_params.eos_token_id]
465 - if sampling_params.stop_token_ids:
466 - tokens_to_check.extend(sampling_params.stop_token_ids)
467 - tokens_to_check = set(tokens_to_check)
464 + tokens_to_check = sampling_params.all_stop_token_ids
465
466 if should_penalize:
467 for token_id in tokens_to_check:
vllm/engine/llm_engine.py
+3 -2
@@ -431,9 +431,10 @@ class LLMEngine:
431 # Defensive copy of SamplingParams, which are used by the sampler,
432 # this doesn't deep-copy LogitsProcessor objects
433 sampling_params = sampling_params.clone()
434 - # inject the eos token id into the sampling_params to support min_tokens
434 + # Add the eos token id into the sampling_params to support min_tokens
435 # processing
436 - sampling_params.eos_token_id = seq.eos_token_id
436 + if seq.eos_token_id is not None:
437 + sampling_params.all_stop_token_ids.add(seq.eos_token_id)
438 sampling_params.update_from_generation_config(
439 self.generation_config_fields)
440
vllm/model_executor/layers/sampler.py
+6 -8
@@ -169,19 +169,17 @@ def _apply_min_tokens_penalty(
169
170 start_idx = sample_indices[0]
171 min_tokens = sampling_params.min_tokens
172 - if min_tokens > 0:
172 + token_ids_to_penalize = sampling_params.all_stop_token_ids
173 + if min_tokens > 0 and token_ids_to_penalize:
174 seqs_to_penalize = []
174 - for i, seq_id in enumerate(seq_ids):
175 + for j, seq_id in enumerate(seq_ids):
176 seq_data = seq_group.seq_data[seq_id]
177 if len(seq_data.output_token_ids) < min_tokens:
177 - seqs_to_penalize.append(i)
178 + seqs_to_penalize.append(j)
179
180 if seqs_to_penalize:
181 # convert to the index into logits
181 - seqs_to_penalize = [start_idx + i for i in seqs_to_penalize]
182 - # use set() to remove any duplicates
183 - token_ids_to_penalize = set(sampling_params.stop_token_ids +
184 - [sampling_params.eos_token_id])
182 + seqs_to_penalize = [start_idx + j for j in seqs_to_penalize]
183 # itertools.product pairs each seq index with every token id
184 logits_to_penalize.extend(
185 itertools.product(seqs_to_penalize, token_ids_to_penalize))
@@ -645,7 +643,7 @@ def _sample(
643 Returns:
644 (next_token_ids, parent_seq_ids) for each seq group in a batch.
645 If sampling is skipped, it returns ([], [])
648 - sampled_token_ids_tensor: A tensor of sampled token ids.
646 + sampled_token_ids_tensor: A tensor of sampled token ids.
647 """
648 return _sample_with_torch(
649 probs,
vllm/sampling_params.py
+2 -2
@@ -185,8 +185,8 @@ class SamplingParams:
185 self.top_k = -1
186 self.min_p = 0.0
187 self._verify_greedy_sampling()
188 - # injected by the engine
189 - self.eos_token_id = None
188 + # eos_token_id is added to this by the engine
189 + self.all_stop_token_ids = set(self.stop_token_ids)
190
191 def _verify_args(self) -> None:
192 if self.n < 1: