A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix][Minor] Make ignore_eos effective (#4468)
Li, Jiang committed
May 1, 2024 at 07:33 UTC
dd1a50a8bc520b0e52ce7914f0263ebd576c197f
1 file changed
+2
-1
vllm/sampling_params.py
+2
-1
@@ -275,7 +275,8 @@ class SamplingParams:
275
self, generation_config: Dict[str, Any]) -> None:
276
"""Update if there are non-default values from generation_config"""
277
# Update eos_token_id for generation
278
- if eos_ids := generation_config.get("eos_token_id"):
278
+ if (not self.ignore_eos) and (eos_ids :=
279
+ generation_config.get("eos_token_id")):
280
# it can be either int or list of int
281
if isinstance(eos_ids, int):
282
eos_ids = [eos_ids]