A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Test] Add ignore_eos test (#4519)
SangBin Cho committed
May 1, 2024 at 21:45 UTC
6f1df80436c46175e09f660a99075a5eba3a2273
1 file changed
+31
tests/samplers/test_ignore_eos.py
new
+31
@@ -0,0 +1,31 @@
1
+"""Make sure ignore_eos works.
2
+
3
+Run `pytest tests/samplers/test_ignore_eos.py`.
4
+"""
5
+
6
+import pytest
7
+
8
+from vllm import SamplingParams
9
+
10
+MODELS = ["facebook/opt-125m"]
11
+
12
+
13
+@pytest.mark.parametrize("model", MODELS)
14
+@pytest.mark.parametrize("dtype", ["half"])
15
+@pytest.mark.parametrize("max_tokens", [1024])
16
+def test_beam_search_single_input(
17
+ vllm_runner,
18
+ example_prompts,
19
+ model: str,
20
+ dtype: str,
21
+ max_tokens: int,
22
+) -> None:
23
+ example_prompts = "1 + 1 is"
24
+
25
+ vllm_model = vllm_runner(model, dtype=dtype)
26
+ sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
27
+ ignore_eos_output = vllm_model.model.generate(
28
+ example_prompts, sampling_params=sampling_params)
29
+ print(len(ignore_eos_output[0].outputs[0].token_ids))
30
+ assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) < 10
31
+ assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) >= 0