| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from vllm.sampling_params import SamplingParams |
| 7 | from vllm.v1.engine import EngineCoreRequest |
| 8 | from vllm.v1.engine.detokenizer import BaseIncrementalDetokenizer |
| 9 | |
| 10 | |
| 11 | @pytest.fixture(params=[True, False]) |
| 12 | def include_stop_str_in_output(request): |
| 13 | return request.param |
| 14 | |
| 15 | |
| 16 | class _DummyDetokenizer(BaseIncrementalDetokenizer): |
| 17 | def __init__(self, request: EngineCoreRequest): |
| 18 | super().__init__(request) |
| 19 | |
| 20 | def decode_next(self, next_token_id: int) -> str: |
| 21 | # Map token id to single ASCII character for deterministic testing. |
| 22 | return chr(next_token_id) |
| 23 | |
| 24 | |
| 25 | def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0): |
| 26 | params = SamplingParams( |
| 27 | stop=stop, |
| 28 | include_stop_str_in_output=include_stop_str_in_output, |
| 29 | min_tokens=min_tokens, |
| 30 | ) |
| 31 | # Keep other fields minimal for unit test purposes. |
| 32 | req = EngineCoreRequest( |
| 33 | request_id="test", |
| 34 | prompt_token_ids=[], |
| 35 | mm_features=None, |
| 36 | sampling_params=params, |
| 37 | pooling_params=None, |
| 38 | arrival_time=0.0, |
| 39 | lora_request=None, |
| 40 | cache_salt=None, |
| 41 | data_parallel_rank=None, |
| 42 | ) |
| 43 | return req |
| 44 | |
| 45 | |
| 46 | def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: bool): |
| 47 | """This test verifies that the detokenizer correctly handles the case where |
| 48 | the generated token sequence contains both: |
| 49 | - a stop token |
| 50 | - an <eos> token |
| 51 | |
| 52 | The detokenizer should respect the stop string and truncate the output |
| 53 | accordingly. |
| 54 | |
| 55 | Imagine the following sequence: |
| 56 | - "abcdeZ" is generated, where "Z" is the <eos> token. |
| 57 | - "cd" is the stop string. |
| 58 | |
| 59 | If include_stop_str_in_output=False, the detokenizer should truncate the |
| 60 | output to "ab" because the stop string "cd" is excluded. |
| 61 | If include_stop_str_in_output=True, the detokenizer should include the stop |
| 62 | string "cd" in the output, resulting in "abcd". |
| 63 | |
| 64 | |
| 65 | This verifies the behavioral change introduced in BaseIncrementalDetokenizer |
| 66 | where stop-string evaluation occurs before the early-return on |
| 67 | stop_terminated. |
| 68 | """ |
| 69 | # Generate text "abcdeZ" and tokenize it. |
| 70 | generated_text = "abcde" |
| 71 | eos_token = "Z" |
| 72 | stop_string = "cd" |
| 73 | generated_text = generated_text + eos_token |
| 74 | token_ids = [ord(c) for c in generated_text] |
| 75 | |
| 76 | # Create a request with the stop string and initialize the detokenizer. |
| 77 | req = _make_request( |
| 78 | stop=[stop_string], include_stop_str_in_output=include_stop_str_in_output |
| 79 | ) |
| 80 | detok = _DummyDetokenizer(req) |
| 81 | |
| 82 | # Simulate that the last token ('Z') is a stop token (stop_terminated=True). |
| 83 | result = detok.update(new_token_ids=token_ids, stop_terminated=True) |
| 84 | |
| 85 | # The update should not report a stop string |
| 86 | assert result == stop_string |
| 87 | |
| 88 | # Output text should reflect stop-string handling: |
| 89 | # - include_stop_str_in_output=False => exclude "cd" => "ab" |
| 90 | # - include_stop_str_in_output=True => include "cd" => "abcd" |
| 91 | expected_text = "abcd" if include_stop_str_in_output else "ab" |
| 92 | assert detok.output_text == expected_text |
| 93 | |
| 94 | # The skipped final token should still be recorded in token_ids. |
| 95 | assert detok.output_token_ids == token_ids |
| 96 | |
| 97 | # get_next_output_text should return the full text when finished=True. |
| 98 | # (Buffering only applies during streaming when finished=False.) |
| 99 | assert detok.get_next_output_text(finished=True, delta=False) == expected_text |