A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add LogProbs for Chat Completions in OpenAI (#2918)
Jared Moore committed
Feb 25, 2024 at 18:39 UTC
70f3e8e3a1ed081003c0a2b70de151bb144f98e0
3 files changed
+57
-14
tests/entrypoints/test_openai_server.py
+13
-12
@@ -155,15 +155,18 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
155
}]
156
157
# test single completion
158
- chat_completion = await client.chat.completions.create(
159
- model=model_name,
160
- messages=messages,
161
- max_tokens=10,
162
- )
158
+ chat_completion = await client.chat.completions.create(model=model_name,
159
+ messages=messages,
160
+ max_tokens=10,
161
+ logprobs=True,
162
+ top_logprobs=10)
163
assert chat_completion.id is not None
164
assert chat_completion.choices is not None and len(
165
chat_completion.choices) == 1
166
assert chat_completion.choices[0].message is not None
167
+ assert chat_completion.choices[0].logprobs is not None
168
+ assert chat_completion.choices[0].logprobs.top_logprobs is not None
169
+ assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 10
170
message = chat_completion.choices[0].message
171
assert message.content is not None and len(message.content) >= 10
172
assert message.role == "assistant"
@@ -198,13 +201,11 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
201
single_output = single_completion.choices[0].text
202
single_usage = single_completion.usage
203
201
- stream = await client.completions.create(
202
- model=model_name,
203
- prompt=prompt,
204
- max_tokens=5,
205
- temperature=0.0,
206
- stream=True,
207
- )
204
+ stream = await client.completions.create(model=model_name,
205
+ prompt=prompt,
206
+ max_tokens=5,
207
+ temperature=0.0,
208
+ stream=True)
209
chunks = []
210
async for chunk in stream:
211
chunks.append(chunk.choices[0].text)
vllm/entrypoints/openai/protocol.py
+8
@@ -63,6 +63,8 @@ class ChatCompletionRequest(BaseModel):
63
seed: Optional[int] = None
64
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
65
stream: Optional[bool] = False
66
+ logprobs: Optional[bool] = False
67
+ top_logprobs: Optional[int] = None
68
presence_penalty: Optional[float] = 0.0
69
frequency_penalty: Optional[float] = 0.0
70
logit_bias: Optional[Dict[str, float]] = None
@@ -84,6 +86,8 @@ class ChatCompletionRequest(BaseModel):
86
length_penalty: Optional[float] = 1.0
87
88
def to_sampling_params(self) -> SamplingParams:
89
+ if self.logprobs and not self.top_logprobs:
90
+ raise ValueError("Top logprobs must be set when logprobs is.")
91
return SamplingParams(
92
n=self.n,
93
presence_penalty=self.presence_penalty,
@@ -96,6 +100,8 @@ class ChatCompletionRequest(BaseModel):
100
stop=self.stop,
101
stop_token_ids=self.stop_token_ids,
102
max_tokens=self.max_tokens,
103
+ logprobs=self.top_logprobs if self.logprobs else None,
104
+ prompt_logprobs=self.top_logprobs if self.echo else None,
105
best_of=self.best_of,
106
top_k=self.top_k,
107
ignore_eos=self.ignore_eos,
@@ -216,6 +222,7 @@ class ChatMessage(BaseModel):
222
class ChatCompletionResponseChoice(BaseModel):
223
index: int
224
message: ChatMessage
225
+ logprobs: Optional[LogProbs] = None
226
finish_reason: Optional[Literal["stop", "length"]] = None
227
228
@@ -236,6 +243,7 @@ class DeltaMessage(BaseModel):
243
class ChatCompletionResponseStreamChoice(BaseModel):
244
index: int
245
delta: DeltaMessage
246
+ logprobs: Optional[LogProbs] = None
247
finish_reason: Optional[Literal["stop", "length"]] = None
248
249
vllm/entrypoints/openai/serving_chat.py
+36
-2
@@ -101,7 +101,10 @@ class OpenAIServingChat(OpenAIServing):
101
role = self.get_chat_request_role(request)
102
for i in range(request.n):
103
choice_data = ChatCompletionResponseStreamChoice(
104
- index=i, delta=DeltaMessage(role=role), finish_reason=None)
104
+ index=i,
105
+ delta=DeltaMessage(role=role),
106
+ logprobs=None,
107
+ finish_reason=None)
108
chunk = ChatCompletionStreamResponse(id=request_id,
109
object=chunk_object_type,
110
created=created_time,
@@ -118,6 +121,7 @@ class OpenAIServingChat(OpenAIServing):
121
"content") and request.messages[-1].get(
122
"role") == role:
123
last_msg_content = request.messages[-1]["content"]
124
+
125
if last_msg_content:
126
for i in range(request.n):
127
choice_data = ChatCompletionResponseStreamChoice(
@@ -129,6 +133,7 @@ class OpenAIServingChat(OpenAIServing):
133
object=chunk_object_type,
134
created=created_time,
135
choices=[choice_data],
136
+ logprobs=None,
137
model=model_name)
138
data = chunk.model_dump_json(exclude_unset=True)
139
yield f"data: {data}\n\n"
@@ -145,15 +150,29 @@ class OpenAIServingChat(OpenAIServing):
150
if finish_reason_sent[i]:
151
continue
152
153
+ delta_token_ids = output.token_ids[previous_num_tokens[i]:]
154
+ top_logprobs = output.logprobs[
155
+ previous_num_tokens[i]:] if output.logprobs else None
156
+
157
+ if request.logprobs:
158
+ logprobs = self._create_logprobs(
159
+ token_ids=delta_token_ids,
160
+ top_logprobs=top_logprobs,
161
+ num_output_top_logprobs=request.logprobs,
162
+ initial_text_offset=len(previous_texts[i]),
163
+ )
164
+ else:
165
+ logprobs = None
166
+
167
delta_text = output.text[len(previous_texts[i]):]
168
previous_texts[i] = output.text
169
previous_num_tokens[i] = len(output.token_ids)
151
-
170
if output.finish_reason is None:
171
# Send token-by-token response for each request.n
172
choice_data = ChatCompletionResponseStreamChoice(
173
index=i,
174
delta=DeltaMessage(content=delta_text),
175
+ logprobs=logprobs,
176
finish_reason=None)
177
chunk = ChatCompletionStreamResponse(
178
id=request_id,
@@ -174,6 +193,7 @@ class OpenAIServingChat(OpenAIServing):
193
choice_data = ChatCompletionResponseStreamChoice(
194
index=i,
195
delta=DeltaMessage(content=delta_text),
196
+ logprobs=logprobs,
197
finish_reason=output.finish_reason)
198
chunk = ChatCompletionStreamResponse(
199
id=request_id,
@@ -208,11 +228,25 @@ class OpenAIServingChat(OpenAIServing):
228
assert final_res is not None
229
230
choices = []
231
+
232
role = self.get_chat_request_role(request)
233
for output in final_res.outputs:
234
+ token_ids = output.token_ids
235
+ top_logprobs = output.logprobs
236
+
237
+ if request.logprobs:
238
+ logprobs = self._create_logprobs(
239
+ token_ids=token_ids,
240
+ top_logprobs=top_logprobs,
241
+ num_output_top_logprobs=request.logprobs,
242
+ )
243
+ else:
244
+ logprobs = None
245
+
246
choice_data = ChatCompletionResponseChoice(
247
index=output.index,
248
message=ChatMessage(role=role, content=output.text),
249
+ logprobs=logprobs,
250
finish_reason=output.finish_reason,
251
)
252
choices.append(choice_data)