A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc] Include matched stop string/token in responses (#2976)
Co-authored-by: Sahil Suneja <sahilsuneja@gmail.com>
Nick Hill committed
Mar 25, 2024 at 17:31 UTC
dfeb2ecc3abe5c1d1a3161b39d8cead5484591e9
7 files changed
+97
-7
tests/samplers/test_stop_reason.py
new
+59
@@ -0,0 +1,59 @@
1
+"""Test the different finish_reason="stop" situations during generation:
2
+ 1. One of the provided stop strings
3
+ 2. One of the provided stop tokens
4
+ 3. The EOS token
5
+
6
+Run `pytest tests/samplers/test_stop_reason.py`.
7
+"""
8
+
9
+import pytest
10
+import transformers
11
+
12
+from vllm import SamplingParams
13
+
14
+MODEL = "facebook/opt-350m"
15
+STOP_STR = "."
16
+SEED = 42
17
+MAX_TOKENS = 1024
18
+
19
+
20
+@pytest.fixture
21
+def vllm_model(vllm_runner):
22
+ vllm_model = vllm_runner(MODEL)
23
+ yield vllm_model
24
+ del vllm_model
25
+
26
+
27
+def test_stop_reason(vllm_model, example_prompts):
28
+ tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
29
+ stop_token_id = tokenizer.convert_tokens_to_ids(STOP_STR)
30
+ llm = vllm_model.model
31
+
32
+ # test stop token
33
+ outputs = llm.generate(example_prompts,
34
+ sampling_params=SamplingParams(
35
+ seed=SEED,
36
+ max_tokens=MAX_TOKENS,
37
+ stop_token_ids=[stop_token_id]))
38
+ for output in outputs:
39
+ output = output.outputs[0]
40
+ assert output.finish_reason == "stop"
41
+ assert output.stop_reason == stop_token_id
42
+
43
+ # test stop string
44
+ outputs = llm.generate(example_prompts,
45
+ sampling_params=SamplingParams(
46
+ seed=SEED, max_tokens=MAX_TOKENS, stop="."))
47
+ for output in outputs:
48
+ output = output.outputs[0]
49
+ assert output.finish_reason == "stop"
50
+ assert output.stop_reason == STOP_STR
51
+
52
+ # test EOS token
53
+ outputs = llm.generate(example_prompts,
54
+ sampling_params=SamplingParams(
55
+ seed=SEED, max_tokens=MAX_TOKENS))
56
+ for output in outputs:
57
+ output = output.outputs[0]
58
+ assert output.finish_reason == "length" or (
59
+ output.finish_reason == "stop" and output.stop_reason is None)
vllm/engine/llm_engine.py
+5
-2
@@ -740,12 +740,15 @@ class LLMEngine:
740
if seq.output_text.endswith(stop_str):
741
self._finalize_sequence(seq, sampling_params, stop_str)
742
seq.status = SequenceStatus.FINISHED_STOPPED
743
+ seq.stop_reason = stop_str
744
return
744
- if seq.get_last_token_id() in sampling_params.stop_token_ids:
745
+ last_token_id = seq.get_last_token_id()
746
+ if last_token_id in sampling_params.stop_token_ids:
747
stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
746
- seq.get_last_token_id())
748
+ last_token_id)
749
self._finalize_sequence(seq, sampling_params, stop_str)
750
seq.status = SequenceStatus.FINISHED_STOPPED
751
+ seq.stop_reason = last_token_id
752
return
753
754
# Check if the sequence has generated the EOS token.
vllm/entrypoints/openai/protocol.py
+16
@@ -338,6 +338,13 @@ class CompletionResponseChoice(BaseModel):
338
text: str
339
logprobs: Optional[LogProbs] = None
340
finish_reason: Optional[Literal["stop", "length"]] = None
341
+ stop_reason: Union[None, int, str] = Field(
342
+ default=None,
343
+ description=(
344
+ "The stop string or token id that caused the completion "
345
+ "to stop, None if the completion finished for some other reason "
346
+ "including encountering the EOS token"),
347
+ )
348
349
350
class CompletionResponse(BaseModel):
@@ -354,6 +361,13 @@ class CompletionResponseStreamChoice(BaseModel):
361
text: str
362
logprobs: Optional[LogProbs] = None
363
finish_reason: Optional[Literal["stop", "length"]] = None
364
+ stop_reason: Union[None, int, str] = Field(
365
+ default=None,
366
+ description=(
367
+ "The stop string or token id that caused the completion "
368
+ "to stop, None if the completion finished for some other reason "
369
+ "including encountering the EOS token"),
370
+ )
371
372
373
class CompletionStreamResponse(BaseModel):
@@ -375,6 +389,7 @@ class ChatCompletionResponseChoice(BaseModel):
389
message: ChatMessage
390
logprobs: Optional[LogProbs] = None
391
finish_reason: Optional[Literal["stop", "length"]] = None
392
+ stop_reason: Union[None, int, str] = None
393
394
395
class ChatCompletionResponse(BaseModel):
@@ -396,6 +411,7 @@ class ChatCompletionResponseStreamChoice(BaseModel):
411
delta: DeltaMessage
412
logprobs: Optional[LogProbs] = None
413
finish_reason: Optional[Literal["stop", "length"]] = None
414
+ stop_reason: Union[None, int, str] = None
415
416
417
class ChatCompletionStreamResponse(BaseModel):
vllm/entrypoints/openai/serving_chat.py
+3
-1
@@ -220,7 +220,8 @@ class OpenAIServingChat(OpenAIServing):
220
index=i,
221
delta=DeltaMessage(content=delta_text),
222
logprobs=logprobs,
223
- finish_reason=output.finish_reason)
223
+ finish_reason=output.finish_reason,
224
+ stop_reason=output.stop_reason)
225
chunk = ChatCompletionStreamResponse(
226
id=request_id,
227
object=chunk_object_type,
@@ -278,6 +279,7 @@ class OpenAIServingChat(OpenAIServing):
279
message=ChatMessage(role=role, content=output.text),
280
logprobs=logprobs,
281
finish_reason=output.finish_reason,
282
+ stop_reason=output.stop_reason,
283
)
284
choices.append(choice_data)
285
vllm/entrypoints/openai/serving_completion.py
+3
@@ -266,6 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
266
previous_texts[i] = output.text
267
previous_num_tokens[i] = len(output.token_ids)
268
finish_reason = output.finish_reason
269
+ stop_reason = output.stop_reason
270
if output.finish_reason is not None: # return final usage
271
prompt_tokens = len(res.prompt_token_ids)
272
completion_tokens = len(output.token_ids)
@@ -286,6 +287,7 @@ class OpenAIServingCompletion(OpenAIServing):
287
text=delta_text,
288
logprobs=logprobs,
289
finish_reason=finish_reason,
290
+ stop_reason=stop_reason,
291
)
292
],
293
usage=final_usage,
@@ -342,6 +344,7 @@ class OpenAIServingCompletion(OpenAIServing):
344
text=output_text,
345
logprobs=logprobs,
346
finish_reason=output.finish_reason,
347
+ stop_reason=output.stop_reason,
348
)
349
choices.append(choice_data)
350
vllm/outputs.py
+10
-4
@@ -1,5 +1,5 @@
1
import time
2
-from typing import List, Optional
2
+from typing import List, Optional, Union
3
4
from vllm.lora.request import LoRARequest
5
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -18,6 +18,9 @@ class CompletionOutput:
18
logprobs: The log probabilities of the top probability words at each
19
position if the logprobs are requested.
20
finish_reason: The reason why the sequence is finished.
21
+ stop_reason: The stop string or token id that caused the completion
22
+ to stop, None if the completion finished for some other reason
23
+ including encountering the EOS token.
24
lora_request: The LoRA request that was used to generate the output.
25
"""
26
@@ -29,6 +32,7 @@ class CompletionOutput:
32
cumulative_logprob: float,
33
logprobs: Optional[SampleLogprobs],
34
finish_reason: Optional[str] = None,
35
+ stop_reason: Union[int, str, None] = None,
36
lora_request: Optional[LoRARequest] = None,
37
) -> None:
38
self.index = index
@@ -37,6 +41,7 @@ class CompletionOutput:
41
self.cumulative_logprob = cumulative_logprob
42
self.logprobs = logprobs
43
self.finish_reason = finish_reason
44
+ self.stop_reason = stop_reason
45
self.lora_request = lora_request
46
47
def finished(self) -> bool:
@@ -48,7 +53,8 @@ class CompletionOutput:
53
f"token_ids={self.token_ids}, "
54
f"cumulative_logprob={self.cumulative_logprob}, "
55
f"logprobs={self.logprobs}, "
51
- f"finish_reason={self.finish_reason})")
56
+ f"finish_reason={self.finish_reason}, "
57
+ f"stop_reason={self.stop_reason})")
58
59
60
class RequestOutput:
@@ -111,8 +117,8 @@ class RequestOutput:
117
seq.get_output_token_ids(),
118
seq.get_cumulative_logprob(),
119
seq.output_logprobs if include_logprobs else None,
114
- SequenceStatus.get_finished_reason(seq.status))
115
- for seq in top_n_seqs
120
+ SequenceStatus.get_finished_reason(seq.status),
121
+ seq.stop_reason) for seq in top_n_seqs
122
]
123
124
# Every sequence in the sequence group should have the same prompt.
vllm/sequence.py
+1
@@ -183,6 +183,7 @@ class Sequence:
183
# Initialize the logical token blocks with the prompt token ids.
184
self._append_tokens_to_blocks(prompt_token_ids)
185
self.status = SequenceStatus.WAITING
186
+ self.stop_reason: Union[int, str, None] = None
187
188
# Used for incremental detokenization
189
self.prefix_offset = 0