@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b4997e05c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Bugfix] API stream returning two stops (#3450)

Co-authored-by: Dylan Hawk <dylanwawk@gmail.com>

Dylan Hawk committed Mar 25, 2024 at 10:14 UTC 0b4997e05cf5aefcf648cae4d91c8e1d651008b9
2 files changed +25 -27
tests/entrypoints/test_openai_server.py
+12
@@ -322,9 +322,15 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
322 temperature=0.0,
323 stream=True)
324 chunks = []
325 + finish_reason_count = 0
326 async for chunk in stream:
327 chunks.append(chunk.choices[0].text)
328 + if chunk.choices[0].finish_reason is not None:
329 + finish_reason_count += 1
330 + # finish reason should only return in last block
331 + assert finish_reason_count == 1
332 assert chunk.choices[0].finish_reason == "length"
333 + assert chunk.choices[0].text
334 assert chunk.usage == single_usage
335 assert "".join(chunks) == single_output
336
@@ -363,13 +369,19 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
369 stream=True,
370 )
371 chunks = []
372 + finish_reason_count = 0
373 async for chunk in stream:
374 delta = chunk.choices[0].delta
375 if delta.role:
376 assert delta.role == "assistant"
377 if delta.content:
378 chunks.append(delta.content)
379 + if chunk.choices[0].finish_reason is not None:
380 + finish_reason_count += 1
381 + # finish reason should only return in last block
382 + assert finish_reason_count == 1
383 assert chunk.choices[0].finish_reason == stop_reason
384 + assert delta.content
385 assert "".join(chunks) == output
386
387
vllm/entrypoints/openai/serving_completion.py
+13 -27
@@ -266,6 +266,16 @@ class OpenAIServingCompletion(OpenAIServing):
266 previous_texts[i] = output.text
267 previous_num_tokens[i] = len(output.token_ids)
268 finish_reason = output.finish_reason
269 + if output.finish_reason is not None: # return final usage
270 + prompt_tokens = len(res.prompt_token_ids)
271 + completion_tokens = len(output.token_ids)
272 + final_usage = UsageInfo(
273 + prompt_tokens=prompt_tokens,
274 + completion_tokens=completion_tokens,
275 + total_tokens=prompt_tokens + completion_tokens,
276 + )
277 + else:
278 + final_usage = None
279 response_json = CompletionStreamResponse(
280 id=request_id,
281 created=created_time,
@@ -277,34 +287,10 @@ class OpenAIServingCompletion(OpenAIServing):
287 logprobs=logprobs,
288 finish_reason=finish_reason,
289 )
280 - ]).model_dump_json()
290 + ],
291 + usage=final_usage,
292 + ).model_dump_json(exclude_unset=True)
293 yield f"data: {response_json}\n\n"
282 -
283 - if output.finish_reason is not None: # return final usage
284 - logprobs = LogProbs(
285 - ) if request.logprobs is not None else None
286 - prompt_tokens = len(res.prompt_token_ids)
287 - completion_tokens = len(output.token_ids)
288 - final_usage = UsageInfo(
289 - prompt_tokens=prompt_tokens,
290 - completion_tokens=completion_tokens,
291 - total_tokens=prompt_tokens + completion_tokens,
292 - )
293 - response_json = CompletionStreamResponse(
294 - id=request_id,
295 - created=created_time,
296 - model=model_name,
297 - choices=[
298 - CompletionResponseStreamChoice(
299 - index=i,
300 - text="",
301 - logprobs=logprobs,
302 - finish_reason=output.finish_reason,
303 - )
304 - ],
305 - usage=final_usage,
306 - ).model_dump_json()
307 - yield f"data: {response_json}\n\n"
294 except ValueError as e:
295 # TODO: Use a vllm-specific Validation Error
296 data = self.create_streaming_error_response(str(e))