@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd1a50a8bc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Minor] Make ignore_eos effective (#4468)

Li, Jiang committed May 1, 2024 at 07:33 UTC dd1a50a8bc520b0e52ce7914f0263ebd576c197f
1 file changed +2 -1
vllm/sampling_params.py
+2 -1
@@ -275,7 +275,8 @@ class SamplingParams:
275 self, generation_config: Dict[str, Any]) -> None:
276 """Update if there are non-default values from generation_config"""
277 # Update eos_token_id for generation
278 - if eos_ids := generation_config.get("eos_token_id"):
278 + if (not self.ignore_eos) and (eos_ids :=
279 + generation_config.get("eos_token_id")):
280 # it can be either int or list of int
281 if isinstance(eos_ids, int):
282 eos_ids = [eos_ids]