| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from typing import Any |
| 5 | |
| 6 | import pytest |
| 7 | |
| 8 | from vllm import LLM, SamplingParams |
| 9 | |
| 10 | MODEL = "meta-llama/llama-2-7b-hf" |
| 11 | MAX_TOKENS = 200 |
| 12 | |
| 13 | |
| 14 | def _test_stopping( |
| 15 | llm: LLM, |
| 16 | expected_output: str, |
| 17 | expected_reason: Any, |
| 18 | stop: list[str] | None = None, |
| 19 | stop_token_ids: list[int] | None = None, |
| 20 | include_in_output: bool = False, |
| 21 | ) -> None: |
| 22 | output = llm.generate( |
| 23 | "A story about vLLM:\n", |
| 24 | SamplingParams( |
| 25 | temperature=0.0, |
| 26 | max_tokens=MAX_TOKENS, |
| 27 | stop=stop, |
| 28 | stop_token_ids=stop_token_ids, |
| 29 | include_stop_str_in_output=include_in_output, |
| 30 | ), |
| 31 | )[0].outputs[0] |
| 32 | |
| 33 | assert output is not None |
| 34 | assert output.text == expected_output |
| 35 | assert output.stop_reason == expected_reason |
| 36 | |
| 37 | |
| 38 | def _stop_basic(llm): |
| 39 | _test_stopping( |
| 40 | llm, |
| 41 | stop=["."], |
| 42 | include_in_output=False, |
| 43 | expected_output="VLLM is a 100% volunteer organization", |
| 44 | expected_reason=".", |
| 45 | ) |
| 46 | |
| 47 | _test_stopping( |
| 48 | llm, |
| 49 | stop=["."], |
| 50 | include_in_output=True, |
| 51 | expected_output="VLLM is a 100% volunteer organization.", |
| 52 | expected_reason=".", |
| 53 | ) |
| 54 | |
| 55 | |
| 56 | def _stop_multi_tokens(llm): |
| 57 | _test_stopping( |
| 58 | llm, |
| 59 | stop=["group of peo", "short"], |
| 60 | include_in_output=False, |
| 61 | expected_output="VLLM is a 100% volunteer organization. We are a ", |
| 62 | expected_reason="group of peo", |
| 63 | ) |
| 64 | |
| 65 | _test_stopping( |
| 66 | llm, |
| 67 | stop=["group of peo", "short"], |
| 68 | include_in_output=True, |
| 69 | expected_output="VLLM is a 100% volunteer organization. We are a group of peo", |
| 70 | expected_reason="group of peo", |
| 71 | ) |
| 72 | |
| 73 | |
| 74 | def _stop_partial_token(llm): |
| 75 | _test_stopping( |
| 76 | llm, |
| 77 | stop=["gani"], |
| 78 | include_in_output=False, |
| 79 | expected_output="VLLM is a 100% volunteer or", |
| 80 | expected_reason="gani", |
| 81 | ) |
| 82 | |
| 83 | _test_stopping( |
| 84 | llm, |
| 85 | stop=["gani"], |
| 86 | include_in_output=True, |
| 87 | expected_output="VLLM is a 100% volunteer organi", |
| 88 | expected_reason="gani", |
| 89 | ) |
| 90 | |
| 91 | |
| 92 | def _stop_token_id(llm): |
| 93 | # token id 13013 => " organization" |
| 94 | |
| 95 | _test_stopping( |
| 96 | llm, |
| 97 | stop_token_ids=[13013], |
| 98 | include_in_output=False, |
| 99 | expected_output="VLLM is a 100% volunteer", |
| 100 | expected_reason=13013, |
| 101 | ) |
| 102 | |
| 103 | _test_stopping( |
| 104 | llm, |
| 105 | stop_token_ids=[13013], |
| 106 | include_in_output=True, |
| 107 | expected_output="VLLM is a 100% volunteer organization", |
| 108 | expected_reason=13013, |
| 109 | ) |
| 110 | |
| 111 | |
| 112 | @pytest.mark.skip_global_cleanup |
| 113 | def test_stop_strings(): |
| 114 | llm = LLM(MODEL, enforce_eager=True) |
| 115 | |
| 116 | _stop_basic(llm) |
| 117 | _stop_multi_tokens(llm) |
| 118 | _stop_partial_token(llm) |
| 119 | # FIXME: this does not respect include_in_output=False |
| 120 | # _stop_token_id(llm) |