A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Bugfix] fix crash if max_tokens=None (#2570)
Nikola Borisov committed
Jan 23, 2024 at 22:38 UTC
3209b4903376fd723858b256dcfabb8420a0cc64
3 files changed
+28
-2
tests/test_regression.py
+13
@@ -22,6 +22,19 @@ def test_duplicated_ignored_sequence_group():
22
assert len(prompts) == len(outputs)
23
24
25
+def test_max_tokens_none():
26
+ sampling_params = SamplingParams(temperature=0.01,
27
+ top_p=0.1,
28
+ max_tokens=None)
29
+ llm = LLM(model="facebook/opt-125m",
30
+ max_num_batched_tokens=4096,
31
+ tensor_parallel_size=1)
32
+ prompts = ["Just say hello!"]
33
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
34
+
35
+ assert len(prompts) == len(outputs)
36
+
37
+
38
if __name__ == "__main__":
39
import pytest
40
pytest.main([__file__])
tests/test_sampling_params.py
new
+13
@@ -0,0 +1,13 @@
1
+"""Tests for the SamplingParams class.
2
+"""
3
+from vllm import SamplingParams
4
+
5
+
6
+def test_max_tokens_none():
7
+ """max_tokens=None should be allowed"""
8
+ SamplingParams(temperature=0.01, top_p=0.1, max_tokens=None)
9
+
10
+
11
+if __name__ == "__main__":
12
+ import pytest
13
+ pytest.main([__file__])
vllm/sampling_params.py
+2
-2
@@ -108,7 +108,7 @@ class SamplingParams:
108
stop_token_ids: Optional[List[int]] = None,
109
include_stop_str_in_output: bool = False,
110
ignore_eos: bool = False,
111
- max_tokens: int = 16,
111
+ max_tokens: Optional[int] = 16,
112
logprobs: Optional[int] = None,
113
prompt_logprobs: Optional[int] = None,
114
skip_special_tokens: bool = True,
@@ -183,7 +183,7 @@ class SamplingParams:
183
if not 0.0 <= self.min_p <= 1.0:
184
raise ValueError("min_p must be in [0, 1], got "
185
f"{self.min_p}.")
186
- if self.max_tokens < 1:
186
+ if self.max_tokens is not None and self.max_tokens < 1:
187
raise ValueError(
188
f"max_tokens must be at least 1, got {self.max_tokens}.")
189
if self.logprobs is not None and self.logprobs < 0: