A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
Update test_ignore_eos (#4898)
Simon Mo committed
Jun 1, 2024 at 21:21 UTC
ed59a7ed23c6e91096ea82b03037e40b14b5375c
1 file changed
+11
-10
tests/samplers/test_ignore_eos.py
+11
-10
@@ -7,25 +7,26 @@ import pytest
7
8
from vllm import SamplingParams
9
10
-MODELS = ["facebook/opt-125m"]
10
+# We also test with llama because it has generation_config to specify EOS
11
+# (past regression).
12
+MODELS = ["facebook/opt-125m", "meta-llama/Llama-2-7b-hf"]
13
14
15
@pytest.mark.parametrize("model", MODELS)
16
@pytest.mark.parametrize("dtype", ["half"])
15
-@pytest.mark.parametrize("max_tokens", [1024])
16
-def test_beam_search_single_input(
17
+@pytest.mark.parametrize("max_tokens", [512])
18
+def test_ignore_eos(
19
vllm_runner,
20
example_prompts,
21
model: str,
22
dtype: str,
23
max_tokens: int,
24
) -> None:
23
- example_prompts = "1 + 1 is"
24
-
25
vllm_model = vllm_runner(model, dtype=dtype)
26
sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
27
- ignore_eos_output = vllm_model.model.generate(
28
- example_prompts, sampling_params=sampling_params)
29
- print(len(ignore_eos_output[0].outputs[0].token_ids))
30
- assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) < 10
31
- assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) >= 0
27
+
28
+ for prompt in example_prompts:
29
+ ignore_eos_output = vllm_model.model.generate(
30
+ prompt, sampling_params=sampling_params)
31
+ output_length = len(ignore_eos_output[0].outputs[0].token_ids)
32
+ assert output_length == max_tokens