@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3209b49033
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Bugfix] fix crash if max_tokens=None (#2570)

Nikola Borisov committed Jan 23, 2024 at 22:38 UTC 3209b4903376fd723858b256dcfabb8420a0cc64
3 files changed +28 -2
tests/test_regression.py
+13
@@ -22,6 +22,19 @@ def test_duplicated_ignored_sequence_group():
22 assert len(prompts) == len(outputs)
23
24
25 +def test_max_tokens_none():
26 + sampling_params = SamplingParams(temperature=0.01,
27 + top_p=0.1,
28 + max_tokens=None)
29 + llm = LLM(model="facebook/opt-125m",
30 + max_num_batched_tokens=4096,
31 + tensor_parallel_size=1)
32 + prompts = ["Just say hello!"]
33 + outputs = llm.generate(prompts, sampling_params=sampling_params)
34 +
35 + assert len(prompts) == len(outputs)
36 +
37 +
38 if __name__ == "__main__":
39 import pytest
40 pytest.main([__file__])
tests/test_sampling_params.py new
+13
@@ -0,0 +1,13 @@
1 +"""Tests for the SamplingParams class.
2 +"""
3 +from vllm import SamplingParams
4 +
5 +
6 +def test_max_tokens_none():
7 + """max_tokens=None should be allowed"""
8 + SamplingParams(temperature=0.01, top_p=0.1, max_tokens=None)
9 +
10 +
11 +if __name__ == "__main__":
12 + import pytest
13 + pytest.main([__file__])
vllm/sampling_params.py
+2 -2
@@ -108,7 +108,7 @@ class SamplingParams:
108 stop_token_ids: Optional[List[int]] = None,
109 include_stop_str_in_output: bool = False,
110 ignore_eos: bool = False,
111 - max_tokens: int = 16,
111 + max_tokens: Optional[int] = 16,
112 logprobs: Optional[int] = None,
113 prompt_logprobs: Optional[int] = None,
114 skip_special_tokens: bool = True,
@@ -183,7 +183,7 @@ class SamplingParams:
183 if not 0.0 <= self.min_p <= 1.0:
184 raise ValueError("min_p must be in [0, 1], got "
185 f"{self.min_p}.")
186 - if self.max_tokens < 1:
186 + if self.max_tokens is not None and self.max_tokens < 1:
187 raise ValueError(
188 f"max_tokens must be at least 1, got {self.max_tokens}.")
189 if self.logprobs is not None and self.logprobs < 0: