@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e46a60aa4c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[BugFix] Fix handling of stop strings and stop token ids (#3672)

Nick Hill committed Apr 11, 2024 at 23:34 UTC e46a60aa4c90cf3dfd9b90782f2eeabbda935eef
8 files changed +202 -37
tests/conftest.py
+1 -1
@@ -401,7 +401,7 @@ class VllmRunner:
401 cleanup()
402
403
404 -@pytest.fixture
404 +@pytest.fixture(scope="session")
405 def vllm_runner():
406 return VllmRunner
407
tests/engine/test_stop_reason.py renamed
+1 -1
@@ -3,7 +3,7 @@
3 2. One of the provided stop tokens
4 3. The EOS token
5
6 -Run `pytest tests/samplers/test_stop_reason.py`.
6 +Run `pytest tests/engine/test_stop_reason.py`.
7 """
8
9 import pytest
tests/engine/test_stop_strings.py new
+111
@@ -0,0 +1,111 @@
1 +from typing import Any, List, Optional
2 +
3 +import pytest
4 +
5 +from vllm import CompletionOutput, LLMEngine, SamplingParams
6 +
7 +MODEL = "meta-llama/llama-2-7b-hf"
8 +MAX_TOKENS = 200
9 +
10 +
11 +@pytest.fixture(scope="session")
12 +def vllm_model(vllm_runner):
13 + return vllm_runner(MODEL)
14 +
15 +
16 +@pytest.mark.skip_global_cleanup
17 +def test_stop_basic(vllm_model):
18 + _test_stopping(vllm_model.model.llm_engine,
19 + stop=["."],
20 + include_in_output=False,
21 + expected_output="VLLM is a 100% volunteer organization",
22 + expected_reason=".")
23 +
24 + _test_stopping(vllm_model.model.llm_engine,
25 + stop=["."],
26 + include_in_output=True,
27 + expected_output="VLLM is a 100% volunteer organization.",
28 + expected_reason=".")
29 +
30 +
31 +@pytest.mark.skip_global_cleanup
32 +def test_stop_multi_tokens(vllm_model):
33 + _test_stopping(
34 + vllm_model.model.llm_engine,
35 + stop=["group of peo", "short"],
36 + include_in_output=False,
37 + expected_output="VLLM is a 100% volunteer organization. We are a ",
38 + expected_reason="group of peo")
39 +
40 + _test_stopping(
41 + vllm_model.model.llm_engine,
42 + stop=["group of peo", "short"],
43 + include_in_output=True,
44 + expected_output=
45 + "VLLM is a 100% volunteer organization. We are a group of peo",
46 + expected_reason="group of peo")
47 +
48 +
49 +@pytest.mark.skip_global_cleanup
50 +def test_stop_partial_token(vllm_model):
51 + _test_stopping(vllm_model.model.llm_engine,
52 + stop=["gani"],
53 + include_in_output=False,
54 + expected_output="VLLM is a 100% volunteer or",
55 + expected_reason="gani")
56 +
57 + _test_stopping(vllm_model.model.llm_engine,
58 + stop=["gani"],
59 + include_in_output=True,
60 + expected_output="VLLM is a 100% volunteer organi",
61 + expected_reason="gani")
62 +
63 +
64 +@pytest.mark.skip_global_cleanup
65 +def test_stop_token_id(vllm_model):
66 + # token id 13013 => " organization"
67 +
68 + _test_stopping(vllm_model.model.llm_engine,
69 + stop_token_ids=[13013],
70 + include_in_output=False,
71 + expected_output="VLLM is a 100% volunteer",
72 + expected_reason=13013)
73 +
74 + _test_stopping(vllm_model.model.llm_engine,
75 + stop_token_ids=[13013],
76 + include_in_output=True,
77 + expected_output="VLLM is a 100% volunteer organization",
78 + expected_reason=13013)
79 +
80 +
81 +def _test_stopping(llm_engine: LLMEngine,
82 + expected_output: str,
83 + expected_reason: Any,
84 + stop: Optional[List[str]] = None,
85 + stop_token_ids: Optional[List[int]] = None,
86 + include_in_output: bool = False) -> None:
87 + llm_engine.add_request(
88 + "id", "A story about vLLM:\n",
89 + SamplingParams(
90 + temperature=0.0,
91 + max_tokens=MAX_TOKENS,
92 + stop=stop,
93 + stop_token_ids=stop_token_ids,
94 + include_stop_str_in_output=include_in_output,
95 + ), None)
96 +
97 + output: Optional[CompletionOutput] = None
98 + output_text = ""
99 + stop_reason = None
100 + while llm_engine.has_unfinished_requests():
101 + (request_output, ) = llm_engine.step()
102 + (output, ) = request_output.outputs
103 +
104 + # Ensure we don't backtrack
105 + assert output.text.startswith(output_text)
106 + output_text = output.text
107 + stop_reason = output.stop_reason
108 +
109 + assert output is not None
110 + assert output_text == expected_output
111 + assert stop_reason == expected_reason
vllm/engine/llm_engine.py
+65 -33
@@ -501,9 +501,11 @@ class LLMEngine:
501
502 for seq, _ in child_seqs:
503 if seq_group.sampling_params.detokenize:
504 - self.detokenizer.decode_sequence_inplace(
504 + new_char_count = self.detokenizer.decode_sequence_inplace(
505 seq, seq_group.sampling_params)
506 - self._check_stop(seq, seq_group.sampling_params)
506 + else:
507 + new_char_count = 0
508 + self._check_stop(seq, new_char_count, seq_group.sampling_params)
509
510 # Non-beam search case
511 if not seq_group.sampling_params.use_beam_search:
@@ -798,56 +800,86 @@ class LLMEngine:
800 time_e2e_requests=time_e2e_requests,
801 )
802
801 - def _check_stop(self, seq: Sequence,
803 + def _check_stop(self, seq: Sequence, new_char_count: int,
804 sampling_params: SamplingParams) -> None:
803 - """Stop the finished sequences."""
804 - # Check if the sequence has reached max_model_len.
805 - if seq.get_len() > self.scheduler_config.max_model_len:
806 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
807 - return
805 + """Stop the finished sequences.
806
809 - # Check if the sequence has reached max_tokens.
810 - if seq.get_output_len() == sampling_params.max_tokens:
811 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
812 - return
807 + new_char_count is the number of chars added to the
808 + sequence's output text for the newly generated token
809 + """
810
811 # Check if the minimum number of tokens has been generated yet;
812 # skip the stop string/token checks if not
813 if seq.get_output_len() < sampling_params.min_tokens:
814 return
815
819 - if sampling_params.detokenize:
820 - for stop_str in sampling_params.stop:
821 - if seq.output_text.endswith(stop_str):
822 - self._finalize_sequence(seq, sampling_params, stop_str)
823 - seq.status = SequenceStatus.FINISHED_STOPPED
824 - seq.stop_reason = stop_str
825 - return
816 + # Check if the sequence has generated the EOS token.
817 + if ((not sampling_params.ignore_eos)
818 + and seq.get_last_token_id() == seq.eos_token_id):
819 + seq.status = SequenceStatus.FINISHED_STOPPED
820 + return
821 +
822 + # Check if a stop token was encountered.
823 + # This assumes a single token produced per step.
824 last_token_id = seq.get_last_token_id()
825 if last_token_id in sampling_params.stop_token_ids:
828 - stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
829 - last_token_id)
830 - self._finalize_sequence(seq, sampling_params, stop_str)
826 + if new_char_count and (
827 + not sampling_params.include_stop_str_in_output):
828 + # Remove last token
829 + seq.output_text = seq.output_text[:-new_char_count]
830 seq.status = SequenceStatus.FINISHED_STOPPED
831 seq.stop_reason = last_token_id
832 return
833
835 - # Check if the sequence has generated the EOS token.
836 - if ((not sampling_params.ignore_eos)
837 - and seq.get_last_token_id() == seq.eos_token_id):
834 + # Check if any stop strings are matched.
835 + stop_str = self._check_stop_strings(seq, new_char_count,
836 + sampling_params)
837 + if stop_str is not None:
838 seq.status = SequenceStatus.FINISHED_STOPPED
839 + seq.stop_reason = stop_str
840 return
841
841 - def _finalize_sequence(self, seq: Sequence,
842 - sampling_params: SamplingParams,
843 - stop_string: str) -> None:
844 - if sampling_params.include_stop_str_in_output:
842 + # Check if the sequence has reached max_model_len.
843 + if seq.get_len() > self.scheduler_config.max_model_len:
844 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
845 return
846
847 - if stop_string and seq.output_text.endswith(stop_string):
848 - # Truncate the output text so that the stop string is
849 - # not included in the output.
850 - seq.output_text = seq.output_text[:-len(stop_string)]
847 + # Check if the sequence has reached max_tokens.
848 + if seq.get_output_len() == sampling_params.max_tokens:
849 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
850 + return
851 +
852 + @staticmethod
853 + def _check_stop_strings(seq: Sequence, new_char_count: int,
854 + sampling_params: SamplingParams) -> Optional[str]:
855 + """Check if any stop strings are matched and truncate sequence
856 + output text accordingly.
857 +
858 + Returns the stop string if matched or else None.
859 + """
860 + if not new_char_count:
861 + return None
862 +
863 + for stop_str in sampling_params.stop:
864 + stop_string_len = len(stop_str)
865 + # Avoid searching already-searched text.
866 + stop_index = seq.output_text.find(
867 + stop_str, -new_char_count - stop_string_len)
868 + if stop_index == -1:
869 + continue
870 +
871 + if sampling_params.include_stop_str_in_output:
872 + # Truncate to end of stop string.
873 + stop_index += stop_string_len
874 + if stop_index >= len(seq.output_text):
875 + # No truncation required.
876 + return stop_str
877 +
878 + # Truncate the output text to either the beginning
879 + # or end of the stop string.
880 + seq.output_text = seq.output_text[:stop_index]
881 + return stop_str
882 + return None
883
884 def add_lora(self, lora_request: LoRARequest) -> bool:
885 return self.model_executor.add_lora(lora_request)
vllm/outputs.py
+3 -1
@@ -112,8 +112,10 @@ class RequestOutput:
112 # always has the logprobs of the sampled tokens even if the
113 # logprobs are not requested.
114 include_logprobs = seq_group.sampling_params.logprobs is not None
115 + text_buffer_length = seq_group.sampling_params.output_text_buffer_length
116 outputs = [
116 - CompletionOutput(seqs.index(seq), seq.output_text,
117 + CompletionOutput(seqs.index(seq),
118 + seq.get_output_text_to_return(text_buffer_length),
119 seq.get_output_token_ids(),
120 seq.get_cumulative_logprob(),
121 seq.output_logprobs if include_logprobs else None,
vllm/sampling_params.py
+9
@@ -166,6 +166,13 @@ class SamplingParams:
166 self.logits_processors = logits_processors
167 self.include_stop_str_in_output = include_stop_str_in_output
168 self.truncate_prompt_tokens = truncate_prompt_tokens
169 + # Number of characters to hold back for stop string evaluation
170 + # until sequence is finished.
171 + if self.stop and not include_stop_str_in_output:
172 + self.output_text_buffer_length = max(len(s) for s in self.stop) - 1
173 + else:
174 + self.output_text_buffer_length = 0
175 +
176 self._verify_args()
177 if self.use_beam_search:
178 self._verify_beam_search()
@@ -226,6 +233,8 @@ class SamplingParams:
233 and self.truncate_prompt_tokens < 1):
234 raise ValueError(f"truncate_prompt_tokens must be >= 1, "
235 f"got {self.truncate_prompt_tokens}")
236 + if any(not stop_str for stop_str in self.stop):
237 + raise ValueError("stop cannot contain an empty string.")
238 if self.stop and not self.detokenize:
239 raise ValueError(
240 "stop strings are only supported when detokenize is True. "
vllm/sequence.py
+6
@@ -235,6 +235,12 @@ class Sequence:
235 def lora_int_id(self) -> int:
236 return self.lora_request.lora_int_id if self.lora_request else 0
237
238 + def get_output_text_to_return(self, buffer_length: int):
239 + # We return the full output text if the sequence is finished.
240 + truncate = buffer_length and not self.is_finished()
241 + return self.output_text[:-buffer_length] if truncate else (
242 + self.output_text)
243 +
244 def hash_of_block(self, logical_idx: int) -> int:
245 # TODO This can produce incorrect hash when block size > prompt size
246
vllm/transformers_utils/detokenizer.py
+6 -1
@@ -87,12 +87,15 @@ class Detokenizer:
87 prev_tokens.extend(next_iter_tokens)
88
89 def decode_sequence_inplace(self, seq: Sequence,
90 - prms: SamplingParams) -> None:
90 + prms: SamplingParams) -> int:
91 """Decodes the new token for a sequence. In-place operation.
92
93 Args:
94 seq: The sequence to decode.
95 prms: The sampling parameters used to generate the sequence.
96 +
97 + Returns:
98 + The number of characters added to the output text.
99 """
100 all_input_ids = seq.get_token_ids()
101 token_id_generated_this_iteration = all_input_ids[-1]
@@ -151,6 +154,8 @@ class Detokenizer:
154 seq.read_offset = read_offset
155 seq.output_text += new_decoded_token_text
156
157 + return len(new_decoded_token_text)
158 +
159
160 def _convert_tokens_to_string_with_added_encoders(
161 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],