@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ed59a7ed23
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

Update test_ignore_eos (#4898)

Simon Mo committed Jun 1, 2024 at 21:21 UTC ed59a7ed23c6e91096ea82b03037e40b14b5375c
1 file changed +11 -10
tests/samplers/test_ignore_eos.py
+11 -10
@@ -7,25 +7,26 @@ import pytest
7
8 from vllm import SamplingParams
9
10 -MODELS = ["facebook/opt-125m"]
10 +# We also test with llama because it has generation_config to specify EOS
11 +# (past regression).
12 +MODELS = ["facebook/opt-125m", "meta-llama/Llama-2-7b-hf"]
13
14
15 @pytest.mark.parametrize("model", MODELS)
16 @pytest.mark.parametrize("dtype", ["half"])
15 -@pytest.mark.parametrize("max_tokens", [1024])
16 -def test_beam_search_single_input(
17 +@pytest.mark.parametrize("max_tokens", [512])
18 +def test_ignore_eos(
19 vllm_runner,
20 example_prompts,
21 model: str,
22 dtype: str,
23 max_tokens: int,
24 ) -> None:
23 - example_prompts = "1 + 1 is"
24 -
25 vllm_model = vllm_runner(model, dtype=dtype)
26 sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
27 - ignore_eos_output = vllm_model.model.generate(
28 - example_prompts, sampling_params=sampling_params)
29 - print(len(ignore_eos_output[0].outputs[0].token_ids))
30 - assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) < 10
31 - assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) >= 0
27 +
28 + for prompt in example_prompts:
29 + ignore_eos_output = vllm_model.model.generate(
30 + prompt, sampling_params=sampling_params)
31 + output_length = len(ignore_eos_output[0].outputs[0].token_ids)
32 + assert output_length == max_tokens