@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 70f3e8e3a1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add LogProbs for Chat Completions in OpenAI (#2918)

Jared Moore committed Feb 25, 2024 at 18:39 UTC 70f3e8e3a1ed081003c0a2b70de151bb144f98e0
3 files changed +57 -14
tests/entrypoints/test_openai_server.py
+13 -12
@@ -155,15 +155,18 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
155 }]
156
157 # test single completion
158 - chat_completion = await client.chat.completions.create(
159 - model=model_name,
160 - messages=messages,
161 - max_tokens=10,
162 - )
158 + chat_completion = await client.chat.completions.create(model=model_name,
159 + messages=messages,
160 + max_tokens=10,
161 + logprobs=True,
162 + top_logprobs=10)
163 assert chat_completion.id is not None
164 assert chat_completion.choices is not None and len(
165 chat_completion.choices) == 1
166 assert chat_completion.choices[0].message is not None
167 + assert chat_completion.choices[0].logprobs is not None
168 + assert chat_completion.choices[0].logprobs.top_logprobs is not None
169 + assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 10
170 message = chat_completion.choices[0].message
171 assert message.content is not None and len(message.content) >= 10
172 assert message.role == "assistant"
@@ -198,13 +201,11 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
201 single_output = single_completion.choices[0].text
202 single_usage = single_completion.usage
203
201 - stream = await client.completions.create(
202 - model=model_name,
203 - prompt=prompt,
204 - max_tokens=5,
205 - temperature=0.0,
206 - stream=True,
207 - )
204 + stream = await client.completions.create(model=model_name,
205 + prompt=prompt,
206 + max_tokens=5,
207 + temperature=0.0,
208 + stream=True)
209 chunks = []
210 async for chunk in stream:
211 chunks.append(chunk.choices[0].text)
vllm/entrypoints/openai/protocol.py
+8
@@ -63,6 +63,8 @@ class ChatCompletionRequest(BaseModel):
63 seed: Optional[int] = None
64 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
65 stream: Optional[bool] = False
66 + logprobs: Optional[bool] = False
67 + top_logprobs: Optional[int] = None
68 presence_penalty: Optional[float] = 0.0
69 frequency_penalty: Optional[float] = 0.0
70 logit_bias: Optional[Dict[str, float]] = None
@@ -84,6 +86,8 @@ class ChatCompletionRequest(BaseModel):
86 length_penalty: Optional[float] = 1.0
87
88 def to_sampling_params(self) -> SamplingParams:
89 + if self.logprobs and not self.top_logprobs:
90 + raise ValueError("Top logprobs must be set when logprobs is.")
91 return SamplingParams(
92 n=self.n,
93 presence_penalty=self.presence_penalty,
@@ -96,6 +100,8 @@ class ChatCompletionRequest(BaseModel):
100 stop=self.stop,
101 stop_token_ids=self.stop_token_ids,
102 max_tokens=self.max_tokens,
103 + logprobs=self.top_logprobs if self.logprobs else None,
104 + prompt_logprobs=self.top_logprobs if self.echo else None,
105 best_of=self.best_of,
106 top_k=self.top_k,
107 ignore_eos=self.ignore_eos,
@@ -216,6 +222,7 @@ class ChatMessage(BaseModel):
222 class ChatCompletionResponseChoice(BaseModel):
223 index: int
224 message: ChatMessage
225 + logprobs: Optional[LogProbs] = None
226 finish_reason: Optional[Literal["stop", "length"]] = None
227
228
@@ -236,6 +243,7 @@ class DeltaMessage(BaseModel):
243 class ChatCompletionResponseStreamChoice(BaseModel):
244 index: int
245 delta: DeltaMessage
246 + logprobs: Optional[LogProbs] = None
247 finish_reason: Optional[Literal["stop", "length"]] = None
248
249
vllm/entrypoints/openai/serving_chat.py
+36 -2
@@ -101,7 +101,10 @@ class OpenAIServingChat(OpenAIServing):
101 role = self.get_chat_request_role(request)
102 for i in range(request.n):
103 choice_data = ChatCompletionResponseStreamChoice(
104 - index=i, delta=DeltaMessage(role=role), finish_reason=None)
104 + index=i,
105 + delta=DeltaMessage(role=role),
106 + logprobs=None,
107 + finish_reason=None)
108 chunk = ChatCompletionStreamResponse(id=request_id,
109 object=chunk_object_type,
110 created=created_time,
@@ -118,6 +121,7 @@ class OpenAIServingChat(OpenAIServing):
121 "content") and request.messages[-1].get(
122 "role") == role:
123 last_msg_content = request.messages[-1]["content"]
124 +
125 if last_msg_content:
126 for i in range(request.n):
127 choice_data = ChatCompletionResponseStreamChoice(
@@ -129,6 +133,7 @@ class OpenAIServingChat(OpenAIServing):
133 object=chunk_object_type,
134 created=created_time,
135 choices=[choice_data],
136 + logprobs=None,
137 model=model_name)
138 data = chunk.model_dump_json(exclude_unset=True)
139 yield f"data: {data}\n\n"
@@ -145,15 +150,29 @@ class OpenAIServingChat(OpenAIServing):
150 if finish_reason_sent[i]:
151 continue
152
153 + delta_token_ids = output.token_ids[previous_num_tokens[i]:]
154 + top_logprobs = output.logprobs[
155 + previous_num_tokens[i]:] if output.logprobs else None
156 +
157 + if request.logprobs:
158 + logprobs = self._create_logprobs(
159 + token_ids=delta_token_ids,
160 + top_logprobs=top_logprobs,
161 + num_output_top_logprobs=request.logprobs,
162 + initial_text_offset=len(previous_texts[i]),
163 + )
164 + else:
165 + logprobs = None
166 +
167 delta_text = output.text[len(previous_texts[i]):]
168 previous_texts[i] = output.text
169 previous_num_tokens[i] = len(output.token_ids)
151 -
170 if output.finish_reason is None:
171 # Send token-by-token response for each request.n
172 choice_data = ChatCompletionResponseStreamChoice(
173 index=i,
174 delta=DeltaMessage(content=delta_text),
175 + logprobs=logprobs,
176 finish_reason=None)
177 chunk = ChatCompletionStreamResponse(
178 id=request_id,
@@ -174,6 +193,7 @@ class OpenAIServingChat(OpenAIServing):
193 choice_data = ChatCompletionResponseStreamChoice(
194 index=i,
195 delta=DeltaMessage(content=delta_text),
196 + logprobs=logprobs,
197 finish_reason=output.finish_reason)
198 chunk = ChatCompletionStreamResponse(
199 id=request_id,
@@ -208,11 +228,25 @@ class OpenAIServingChat(OpenAIServing):
228 assert final_res is not None
229
230 choices = []
231 +
232 role = self.get_chat_request_role(request)
233 for output in final_res.outputs:
234 + token_ids = output.token_ids
235 + top_logprobs = output.logprobs
236 +
237 + if request.logprobs:
238 + logprobs = self._create_logprobs(
239 + token_ids=token_ids,
240 + top_logprobs=top_logprobs,
241 + num_output_top_logprobs=request.logprobs,
242 + )
243 + else:
244 + logprobs = None
245 +
246 choice_data = ChatCompletionResponseChoice(
247 index=output.index,
248 message=ChatMessage(role=role, content=output.text),
249 + logprobs=logprobs,
250 finish_reason=output.finish_reason,
251 )
252 choices.append(choice_data)