A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[BugFix] Fix `min_tokens` behaviour for multiple eos tokens (#5849)
Nick Hill committed
Jun 27, 2024 at 11:31 UTC
365791ff81181d0d08b719cc6ff976889f6e3288
2 files changed
+23
-13
vllm/engine/llm_engine.py
+2
-5
@@ -606,12 +606,9 @@ class LLMEngine:
606
# Defensive copy of SamplingParams, which are used by the sampler,
607
# this doesn't deep-copy LogitsProcessor objects
608
sampling_params = sampling_params.clone()
609
- # Add the eos token id into the sampling_params to support min_tokens
610
- # processing
611
- if seq.eos_token_id is not None:
612
- sampling_params.all_stop_token_ids.add(seq.eos_token_id)
609
+
610
sampling_params.update_from_generation_config(
614
- self.generation_config_fields)
611
+ self.generation_config_fields, seq.eos_token_id)
612
613
# Create the sequence group.
614
seq_group = SequenceGroup(
vllm/sampling_params.py
+21
-8
@@ -280,17 +280,30 @@ class SamplingParams:
280
f"Got {self.best_of}.")
281
282
def update_from_generation_config(
283
- self, generation_config: Dict[str, Any]) -> None:
283
+ self,
284
+ generation_config: Dict[str, Any],
285
+ model_eos_token_id: Optional[int] = None) -> None:
286
"""Update if there are non-default values from generation_config"""
287
+
288
+ if model_eos_token_id is not None:
289
+ # Add the eos token id into the sampling_params to support
290
+ # min_tokens processing.
291
+ self.all_stop_token_ids.add(model_eos_token_id)
292
+
293
# Update eos_token_id for generation
286
- if (not self.ignore_eos) and (eos_ids :=
287
- generation_config.get("eos_token_id")):
294
+ if (eos_ids := generation_config.get("eos_token_id")) is not None:
295
# it can be either int or list of int
289
- if isinstance(eos_ids, int):
290
- eos_ids = [eos_ids]
291
- original_stop_token_ids = set(self.stop_token_ids)
292
- original_stop_token_ids.update(eos_ids)
293
- self.stop_token_ids = list(original_stop_token_ids)
296
+ eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids)
297
+ if model_eos_token_id is not None:
298
+ # We don't need to include the primary eos_token_id in
299
+ # stop_token_ids since it's handled separately for stopping
300
+ # purposes.
301
+ eos_ids.discard(model_eos_token_id)
302
+ if eos_ids:
303
+ self.all_stop_token_ids.update(eos_ids)
304
+ if not self.ignore_eos:
305
+ eos_ids.update(self.stop_token_ids)
306
+ self.stop_token_ids = list(eos_ids)
307
308
@cached_property
309
def sampling_type(self) -> SamplingType: