@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6f1df80436
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Test] Add ignore_eos test (#4519)

SangBin Cho committed May 1, 2024 at 21:45 UTC 6f1df80436c46175e09f660a99075a5eba3a2273
1 file changed +31
tests/samplers/test_ignore_eos.py new
+31
@@ -0,0 +1,31 @@
1 +"""Make sure ignore_eos works.
2 +
3 +Run `pytest tests/samplers/test_ignore_eos.py`.
4 +"""
5 +
6 +import pytest
7 +
8 +from vllm import SamplingParams
9 +
10 +MODELS = ["facebook/opt-125m"]
11 +
12 +
13 +@pytest.mark.parametrize("model", MODELS)
14 +@pytest.mark.parametrize("dtype", ["half"])
15 +@pytest.mark.parametrize("max_tokens", [1024])
16 +def test_beam_search_single_input(
17 + vllm_runner,
18 + example_prompts,
19 + model: str,
20 + dtype: str,
21 + max_tokens: int,
22 +) -> None:
23 + example_prompts = "1 + 1 is"
24 +
25 + vllm_model = vllm_runner(model, dtype=dtype)
26 + sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
27 + ignore_eos_output = vllm_model.model.generate(
28 + example_prompts, sampling_params=sampling_params)
29 + print(len(ignore_eos_output[0].outputs[0].token_ids))
30 + assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) < 10
31 + assert max_tokens - len(ignore_eos_output[0].outputs[0].token_ids) >= 0