@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dfeb2ecc3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] Include matched stop string/token in responses (#2976)

Co-authored-by: Sahil Suneja <sahilsuneja@gmail.com>

Nick Hill committed Mar 25, 2024 at 17:31 UTC dfeb2ecc3abe5c1d1a3161b39d8cead5484591e9
7 files changed +97 -7
tests/samplers/test_stop_reason.py new
+59
@@ -0,0 +1,59 @@
1 +"""Test the different finish_reason="stop" situations during generation:
2 + 1. One of the provided stop strings
3 + 2. One of the provided stop tokens
4 + 3. The EOS token
5 +
6 +Run `pytest tests/samplers/test_stop_reason.py`.
7 +"""
8 +
9 +import pytest
10 +import transformers
11 +
12 +from vllm import SamplingParams
13 +
14 +MODEL = "facebook/opt-350m"
15 +STOP_STR = "."
16 +SEED = 42
17 +MAX_TOKENS = 1024
18 +
19 +
20 +@pytest.fixture
21 +def vllm_model(vllm_runner):
22 + vllm_model = vllm_runner(MODEL)
23 + yield vllm_model
24 + del vllm_model
25 +
26 +
27 +def test_stop_reason(vllm_model, example_prompts):
28 + tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
29 + stop_token_id = tokenizer.convert_tokens_to_ids(STOP_STR)
30 + llm = vllm_model.model
31 +
32 + # test stop token
33 + outputs = llm.generate(example_prompts,
34 + sampling_params=SamplingParams(
35 + seed=SEED,
36 + max_tokens=MAX_TOKENS,
37 + stop_token_ids=[stop_token_id]))
38 + for output in outputs:
39 + output = output.outputs[0]
40 + assert output.finish_reason == "stop"
41 + assert output.stop_reason == stop_token_id
42 +
43 + # test stop string
44 + outputs = llm.generate(example_prompts,
45 + sampling_params=SamplingParams(
46 + seed=SEED, max_tokens=MAX_TOKENS, stop="."))
47 + for output in outputs:
48 + output = output.outputs[0]
49 + assert output.finish_reason == "stop"
50 + assert output.stop_reason == STOP_STR
51 +
52 + # test EOS token
53 + outputs = llm.generate(example_prompts,
54 + sampling_params=SamplingParams(
55 + seed=SEED, max_tokens=MAX_TOKENS))
56 + for output in outputs:
57 + output = output.outputs[0]
58 + assert output.finish_reason == "length" or (
59 + output.finish_reason == "stop" and output.stop_reason is None)
vllm/engine/llm_engine.py
+5 -2
@@ -740,12 +740,15 @@ class LLMEngine:
740 if seq.output_text.endswith(stop_str):
741 self._finalize_sequence(seq, sampling_params, stop_str)
742 seq.status = SequenceStatus.FINISHED_STOPPED
743 + seq.stop_reason = stop_str
744 return
744 - if seq.get_last_token_id() in sampling_params.stop_token_ids:
745 + last_token_id = seq.get_last_token_id()
746 + if last_token_id in sampling_params.stop_token_ids:
747 stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
746 - seq.get_last_token_id())
748 + last_token_id)
749 self._finalize_sequence(seq, sampling_params, stop_str)
750 seq.status = SequenceStatus.FINISHED_STOPPED
751 + seq.stop_reason = last_token_id
752 return
753
754 # Check if the sequence has generated the EOS token.
vllm/entrypoints/openai/protocol.py
+16
@@ -338,6 +338,13 @@ class CompletionResponseChoice(BaseModel):
338 text: str
339 logprobs: Optional[LogProbs] = None
340 finish_reason: Optional[Literal["stop", "length"]] = None
341 + stop_reason: Union[None, int, str] = Field(
342 + default=None,
343 + description=(
344 + "The stop string or token id that caused the completion "
345 + "to stop, None if the completion finished for some other reason "
346 + "including encountering the EOS token"),
347 + )
348
349
350 class CompletionResponse(BaseModel):
@@ -354,6 +361,13 @@ class CompletionResponseStreamChoice(BaseModel):
361 text: str
362 logprobs: Optional[LogProbs] = None
363 finish_reason: Optional[Literal["stop", "length"]] = None
364 + stop_reason: Union[None, int, str] = Field(
365 + default=None,
366 + description=(
367 + "The stop string or token id that caused the completion "
368 + "to stop, None if the completion finished for some other reason "
369 + "including encountering the EOS token"),
370 + )
371
372
373 class CompletionStreamResponse(BaseModel):
@@ -375,6 +389,7 @@ class ChatCompletionResponseChoice(BaseModel):
389 message: ChatMessage
390 logprobs: Optional[LogProbs] = None
391 finish_reason: Optional[Literal["stop", "length"]] = None
392 + stop_reason: Union[None, int, str] = None
393
394
395 class ChatCompletionResponse(BaseModel):
@@ -396,6 +411,7 @@ class ChatCompletionResponseStreamChoice(BaseModel):
411 delta: DeltaMessage
412 logprobs: Optional[LogProbs] = None
413 finish_reason: Optional[Literal["stop", "length"]] = None
414 + stop_reason: Union[None, int, str] = None
415
416
417 class ChatCompletionStreamResponse(BaseModel):
vllm/entrypoints/openai/serving_chat.py
+3 -1
@@ -220,7 +220,8 @@ class OpenAIServingChat(OpenAIServing):
220 index=i,
221 delta=DeltaMessage(content=delta_text),
222 logprobs=logprobs,
223 - finish_reason=output.finish_reason)
223 + finish_reason=output.finish_reason,
224 + stop_reason=output.stop_reason)
225 chunk = ChatCompletionStreamResponse(
226 id=request_id,
227 object=chunk_object_type,
@@ -278,6 +279,7 @@ class OpenAIServingChat(OpenAIServing):
279 message=ChatMessage(role=role, content=output.text),
280 logprobs=logprobs,
281 finish_reason=output.finish_reason,
282 + stop_reason=output.stop_reason,
283 )
284 choices.append(choice_data)
285
vllm/entrypoints/openai/serving_completion.py
+3
@@ -266,6 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
266 previous_texts[i] = output.text
267 previous_num_tokens[i] = len(output.token_ids)
268 finish_reason = output.finish_reason
269 + stop_reason = output.stop_reason
270 if output.finish_reason is not None: # return final usage
271 prompt_tokens = len(res.prompt_token_ids)
272 completion_tokens = len(output.token_ids)
@@ -286,6 +287,7 @@ class OpenAIServingCompletion(OpenAIServing):
287 text=delta_text,
288 logprobs=logprobs,
289 finish_reason=finish_reason,
290 + stop_reason=stop_reason,
291 )
292 ],
293 usage=final_usage,
@@ -342,6 +344,7 @@ class OpenAIServingCompletion(OpenAIServing):
344 text=output_text,
345 logprobs=logprobs,
346 finish_reason=output.finish_reason,
347 + stop_reason=output.stop_reason,
348 )
349 choices.append(choice_data)
350
vllm/outputs.py
+10 -4
@@ -1,5 +1,5 @@
1 import time
2 -from typing import List, Optional
2 +from typing import List, Optional, Union
3
4 from vllm.lora.request import LoRARequest
5 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -18,6 +18,9 @@ class CompletionOutput:
18 logprobs: The log probabilities of the top probability words at each
19 position if the logprobs are requested.
20 finish_reason: The reason why the sequence is finished.
21 + stop_reason: The stop string or token id that caused the completion
22 + to stop, None if the completion finished for some other reason
23 + including encountering the EOS token.
24 lora_request: The LoRA request that was used to generate the output.
25 """
26
@@ -29,6 +32,7 @@ class CompletionOutput:
32 cumulative_logprob: float,
33 logprobs: Optional[SampleLogprobs],
34 finish_reason: Optional[str] = None,
35 + stop_reason: Union[int, str, None] = None,
36 lora_request: Optional[LoRARequest] = None,
37 ) -> None:
38 self.index = index
@@ -37,6 +41,7 @@ class CompletionOutput:
41 self.cumulative_logprob = cumulative_logprob
42 self.logprobs = logprobs
43 self.finish_reason = finish_reason
44 + self.stop_reason = stop_reason
45 self.lora_request = lora_request
46
47 def finished(self) -> bool:
@@ -48,7 +53,8 @@ class CompletionOutput:
53 f"token_ids={self.token_ids}, "
54 f"cumulative_logprob={self.cumulative_logprob}, "
55 f"logprobs={self.logprobs}, "
51 - f"finish_reason={self.finish_reason})")
56 + f"finish_reason={self.finish_reason}, "
57 + f"stop_reason={self.stop_reason})")
58
59
60 class RequestOutput:
@@ -111,8 +117,8 @@ class RequestOutput:
117 seq.get_output_token_ids(),
118 seq.get_cumulative_logprob(),
119 seq.output_logprobs if include_logprobs else None,
114 - SequenceStatus.get_finished_reason(seq.status))
115 - for seq in top_n_seqs
120 + SequenceStatus.get_finished_reason(seq.status),
121 + seq.stop_reason) for seq in top_n_seqs
122 ]
123
124 # Every sequence in the sequence group should have the same prompt.
vllm/sequence.py
+1
@@ -183,6 +183,7 @@ class Sequence:
183 # Initialize the logical token blocks with the prompt token ids.
184 self._append_tokens_to_blocks(prompt_token_ids)
185 self.status = SequenceStatus.WAITING
186 + self.stop_reason: Union[int, str, None] = None
187
188 # Used for incremental detokenization
189 self.prefix_offset = 0