A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Bugfix] API stream returning two stops (#3450)
Co-authored-by: Dylan Hawk <dylanwawk@gmail.com>
Dylan Hawk committed
Mar 25, 2024 at 10:14 UTC
0b4997e05cf5aefcf648cae4d91c8e1d651008b9
2 files changed
+25
-27
tests/entrypoints/test_openai_server.py
+12
@@ -322,9 +322,15 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
322
temperature=0.0,
323
stream=True)
324
chunks = []
325
+ finish_reason_count = 0
326
async for chunk in stream:
327
chunks.append(chunk.choices[0].text)
328
+ if chunk.choices[0].finish_reason is not None:
329
+ finish_reason_count += 1
330
+ # finish reason should only return in last block
331
+ assert finish_reason_count == 1
332
assert chunk.choices[0].finish_reason == "length"
333
+ assert chunk.choices[0].text
334
assert chunk.usage == single_usage
335
assert "".join(chunks) == single_output
336
@@ -363,13 +369,19 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
369
stream=True,
370
)
371
chunks = []
372
+ finish_reason_count = 0
373
async for chunk in stream:
374
delta = chunk.choices[0].delta
375
if delta.role:
376
assert delta.role == "assistant"
377
if delta.content:
378
chunks.append(delta.content)
379
+ if chunk.choices[0].finish_reason is not None:
380
+ finish_reason_count += 1
381
+ # finish reason should only return in last block
382
+ assert finish_reason_count == 1
383
assert chunk.choices[0].finish_reason == stop_reason
384
+ assert delta.content
385
assert "".join(chunks) == output
386
387
vllm/entrypoints/openai/serving_completion.py
+13
-27
@@ -266,6 +266,16 @@ class OpenAIServingCompletion(OpenAIServing):
266
previous_texts[i] = output.text
267
previous_num_tokens[i] = len(output.token_ids)
268
finish_reason = output.finish_reason
269
+ if output.finish_reason is not None: # return final usage
270
+ prompt_tokens = len(res.prompt_token_ids)
271
+ completion_tokens = len(output.token_ids)
272
+ final_usage = UsageInfo(
273
+ prompt_tokens=prompt_tokens,
274
+ completion_tokens=completion_tokens,
275
+ total_tokens=prompt_tokens + completion_tokens,
276
+ )
277
+ else:
278
+ final_usage = None
279
response_json = CompletionStreamResponse(
280
id=request_id,
281
created=created_time,
@@ -277,34 +287,10 @@ class OpenAIServingCompletion(OpenAIServing):
287
logprobs=logprobs,
288
finish_reason=finish_reason,
289
)
280
- ]).model_dump_json()
290
+ ],
291
+ usage=final_usage,
292
+ ).model_dump_json(exclude_unset=True)
293
yield f"data: {response_json}\n\n"
282
-
283
- if output.finish_reason is not None: # return final usage
284
- logprobs = LogProbs(
285
- ) if request.logprobs is not None else None
286
- prompt_tokens = len(res.prompt_token_ids)
287
- completion_tokens = len(output.token_ids)
288
- final_usage = UsageInfo(
289
- prompt_tokens=prompt_tokens,
290
- completion_tokens=completion_tokens,
291
- total_tokens=prompt_tokens + completion_tokens,
292
- )
293
- response_json = CompletionStreamResponse(
294
- id=request_id,
295
- created=created_time,
296
- model=model_name,
297
- choices=[
298
- CompletionResponseStreamChoice(
299
- index=i,
300
- text="",
301
- logprobs=logprobs,
302
- finish_reason=output.finish_reason,
303
- )
304
- ],
305
- usage=final_usage,
306
- ).model_dump_json()
307
- yield f"data: {response_json}\n\n"
294
except ValueError as e:
295
# TODO: Use a vllm-specific Validation Error
296
data = self.create_streaming_error_response(str(e))