@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dfba529b40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Remove the last EOS token unless explicitly specified (#5077)

Junichi Sato committed May 29, 2024 at 09:15 UTC dfba529b4024fba9ce1346467318b35e8f2fa9d9
2 files changed +91
tests/engine/output_processor/test_stop_checker.py new
+86
@@ -0,0 +1,86 @@
1 +from unittest.mock import MagicMock
2 +
3 +import pytest
4 +from transformers import PreTrainedTokenizer
5 +
6 +from vllm.engine.output_processor.stop_checker import StopChecker
7 +from vllm.sampling_params import SamplingParams
8 +from vllm.sequence import Logprob, Sequence, SequenceStatus
9 +
10 +
11 +def sequence_with_eos(text: str, eos_token: str,
12 + eos_token_id: int) -> Sequence:
13 + """
14 + Create a Sequence that ends with an EOS token.
15 + """
16 + seq = Sequence(
17 + seq_id=0,
18 + prompt="",
19 + prompt_token_ids=[],
20 + block_size=16,
21 + eos_token_id=eos_token_id,
22 + )
23 + seq.output_text = text + eos_token
24 +
25 + offset = eos_token_id + 1
26 + for i in range(offset, len(text) + offset):
27 + seq.append_token_id(token_id=i, logprobs={i: Logprob(0.0)})
28 + seq.append_token_id(token_id=eos_token_id,
29 + logprobs={eos_token_id: Logprob(0.0)})
30 +
31 + seq.status = SequenceStatus.RUNNING
32 +
33 + return seq
34 +
35 +
36 +@pytest.mark.parametrize(["text_wo_eos", "eos_token", "eos_token_id"], [
37 + ("This text ends with EOS token", "</s>", 2),
38 +])
39 +@pytest.mark.parametrize("ignore_eos", [True, False, None])
40 +@pytest.mark.parametrize("include_stop_str_in_output", [True, False, None])
41 +@pytest.mark.skip_global_cleanup
42 +def test_stop_on_eos_token(text_wo_eos: str, eos_token: str, eos_token_id: int,
43 + ignore_eos: bool, include_stop_str_in_output: bool):
44 + """
45 + Test the behavior of the StopChecker's maybe_stop_sequence method
46 + when an EOS token is encountered.
47 +
48 + This test covers:
49 + - When the EOS token should stop the sequence and be removed from the output
50 + - When the EOS token should stop the sequence and be included in the output
51 + - When the EOS token should be ignored, and the sequence continues
52 + """
53 +
54 + tokenizer = MagicMock(spec=PreTrainedTokenizer)
55 + get_tokenizer_for_seq = MagicMock(return_value=tokenizer)
56 + stop_checker = StopChecker(max_model_len=1024,
57 + get_tokenizer_for_seq=get_tokenizer_for_seq)
58 +
59 + seq = sequence_with_eos(
60 + text=text_wo_eos,
61 + eos_token=eos_token,
62 + eos_token_id=eos_token_id,
63 + )
64 + new_char_count = len(eos_token)
65 +
66 + # Note that `stop` and `stop_token_ids` are not specified
67 + sampling_params = SamplingParams(
68 + min_tokens=1,
69 + ignore_eos=ignore_eos,
70 + include_stop_str_in_output=include_stop_str_in_output)
71 +
72 + stop_checker.maybe_stop_sequence(
73 + seq=seq,
74 + new_char_count=new_char_count,
75 + sampling_params=sampling_params,
76 + )
77 +
78 + if ignore_eos:
79 + assert seq.status == SequenceStatus.RUNNING
80 + assert seq.output_text == text_wo_eos + eos_token
81 + elif include_stop_str_in_output:
82 + assert seq.status == SequenceStatus.FINISHED_STOPPED
83 + assert seq.output_text == text_wo_eos + eos_token
84 + else:
85 + assert seq.status == SequenceStatus.FINISHED_STOPPED
86 + assert seq.output_text == text_wo_eos
vllm/engine/output_processor/stop_checker.py
+5
@@ -48,6 +48,11 @@ class StopChecker:
48 # Check if the sequence has generated the EOS token.
49 if ((not sampling_params.ignore_eos)
50 and seq.get_last_token_id() == seq.eos_token_id):
51 + # Remove the last EOS token unless explicitly specified
52 + # This prevents unintended exposure of the EOS token
53 + if new_char_count and (
54 + not sampling_params.include_stop_str_in_output):
55 + seq.output_text = seq.output_text[:-new_char_count]
56 seq.status = SequenceStatus.FINISHED_STOPPED
57 return
58