A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Frontend] OpenAI server: propagate usage accounting to FastAPI middleware layer (#8672)
Adam Tilghman committed
Sep 25, 2024 at 00:49 UTC
1ac3de09cd87290f7494ce6337623d6edd3f8667
3 files changed
+57
-11
vllm/entrypoints/openai/protocol.py
+5
@@ -107,6 +107,11 @@ class UsageInfo(OpenAIBaseModel):
107
completion_tokens: Optional[int] = 0
108
109
110
+class RequestResponseMetadata(BaseModel):
111
+ request_id: str
112
+ final_usage_info: Optional[UsageInfo] = None
113
+
114
+
115
class JsonSchemaResponseFormat(OpenAIBaseModel):
116
name: str
117
description: Optional[str] = None
vllm/entrypoints/openai/serving_chat.py
+23
-3
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (
22
ChatCompletionRequest, ChatCompletionResponse,
23
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
24
ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
25
- DeltaToolCall, ErrorResponse, FunctionCall, ToolCall, UsageInfo)
25
+ DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata,
26
+ ToolCall, UsageInfo)
27
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
28
LoRAModulePath,
29
OpenAIServing,
@@ -175,6 +176,11 @@ class OpenAIServingChat(OpenAIServing):
176
"--enable-auto-tool-choice and --tool-call-parser to be set")
177
178
request_id = f"chat-{random_uuid()}"
179
+
180
+ request_metadata = RequestResponseMetadata(request_id=request_id)
181
+ if raw_request:
182
+ raw_request.state.request_metadata = request_metadata
183
+
184
try:
185
guided_decode_logits_processor = (
186
await self._guided_decode_logits_processor(request, tokenizer))
@@ -241,11 +247,13 @@ class OpenAIServingChat(OpenAIServing):
247
# Streaming response
248
if request.stream:
249
return self.chat_completion_stream_generator(
244
- request, result_generator, request_id, conversation, tokenizer)
250
+ request, result_generator, request_id, conversation, tokenizer,
251
+ request_metadata)
252
253
try:
254
return await self.chat_completion_full_generator(
248
- request, result_generator, request_id, conversation, tokenizer)
255
+ request, result_generator, request_id, conversation, tokenizer,
256
+ request_metadata)
257
except ValueError as e:
258
# TODO: Use a vllm-specific Validation Error
259
return self.create_error_response(str(e))
@@ -262,6 +270,7 @@ class OpenAIServingChat(OpenAIServing):
270
request_id: str,
271
conversation: List[ConversationMessage],
272
tokenizer: AnyTokenizer,
273
+ request_metadata: RequestResponseMetadata,
274
) -> AsyncGenerator[str, None]:
275
model_name = self.base_model_paths[0].name
276
created_time = int(time.time())
@@ -580,6 +589,13 @@ class OpenAIServingChat(OpenAIServing):
589
exclude_unset=True, exclude_none=True))
590
yield f"data: {final_usage_data}\n\n"
591
592
+ # report to FastAPI middleware aggregate usage across all choices
593
+ num_completion_tokens = sum(previous_num_tokens)
594
+ request_metadata.final_usage_info = UsageInfo(
595
+ prompt_tokens=num_prompt_tokens,
596
+ completion_tokens=num_completion_tokens,
597
+ total_tokens=num_prompt_tokens + num_completion_tokens)
598
+
599
except ValueError as e:
600
# TODO: Use a vllm-specific Validation Error
601
logger.error("error in chat completion stream generator: %s", e)
@@ -595,6 +611,7 @@ class OpenAIServingChat(OpenAIServing):
611
request_id: str,
612
conversation: List[ConversationMessage],
613
tokenizer: AnyTokenizer,
614
+ request_metadata: RequestResponseMetadata,
615
) -> Union[ErrorResponse, ChatCompletionResponse]:
616
617
model_name = self.base_model_paths[0].name
@@ -714,6 +731,9 @@ class OpenAIServingChat(OpenAIServing):
731
completion_tokens=num_generated_tokens,
732
total_tokens=num_prompt_tokens + num_generated_tokens,
733
)
734
+
735
+ request_metadata.final_usage_info = usage
736
+
737
response = ChatCompletionResponse(
738
id=request_id,
739
created=created_time,
vllm/entrypoints/openai/serving_completion.py
+29
-8
@@ -18,7 +18,9 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
18
CompletionResponseChoice,
19
CompletionResponseStreamChoice,
20
CompletionStreamResponse,
21
- ErrorResponse, UsageInfo)
21
+ ErrorResponse,
22
+ RequestResponseMetadata,
23
+ UsageInfo)
24
# yapf: enable
25
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
26
LoRAModulePath,
@@ -94,6 +96,10 @@ class OpenAIServingCompletion(OpenAIServing):
96
request_id = f"cmpl-{random_uuid()}"
97
created_time = int(time.time())
98
99
+ request_metadata = RequestResponseMetadata(request_id=request_id)
100
+ if raw_request:
101
+ raw_request.state.request_metadata = request_metadata
102
+
103
# Schedule the request and get the result generator.
104
generators: List[AsyncGenerator[RequestOutput, None]] = []
105
try:
@@ -165,13 +171,15 @@ class OpenAIServingCompletion(OpenAIServing):
171
172
# Streaming response
173
if stream:
168
- return self.completion_stream_generator(request,
169
- result_generator,
170
- request_id,
171
- created_time,
172
- model_name,
173
- num_prompts=len(prompts),
174
- tokenizer=tokenizer)
174
+ return self.completion_stream_generator(
175
+ request,
176
+ result_generator,
177
+ request_id,
178
+ created_time,
179
+ model_name,
180
+ num_prompts=len(prompts),
181
+ tokenizer=tokenizer,
182
+ request_metadata=request_metadata)
183
184
# Non-streaming response
185
final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
@@ -198,6 +206,7 @@ class OpenAIServingCompletion(OpenAIServing):
206
created_time,
207
model_name,
208
tokenizer,
209
+ request_metadata,
210
)
211
except asyncio.CancelledError:
212
return self.create_error_response("Client disconnected")
@@ -227,6 +236,7 @@ class OpenAIServingCompletion(OpenAIServing):
236
model_name: str,
237
num_prompts: int,
238
tokenizer: AnyTokenizer,
239
+ request_metadata: RequestResponseMetadata,
240
) -> AsyncGenerator[str, None]:
241
num_choices = 1 if request.n is None else request.n
242
previous_text_lens = [0] * num_choices * num_prompts
@@ -346,6 +356,14 @@ class OpenAIServingCompletion(OpenAIServing):
356
exclude_unset=False, exclude_none=True))
357
yield f"data: {final_usage_data}\n\n"
358
359
+ # report to FastAPI middleware aggregate usage across all choices
360
+ total_prompt_tokens = sum(num_prompt_tokens)
361
+ total_completion_tokens = sum(previous_num_tokens)
362
+ request_metadata.final_usage_info = UsageInfo(
363
+ prompt_tokens=total_prompt_tokens,
364
+ completion_tokens=total_completion_tokens,
365
+ total_tokens=total_prompt_tokens + total_completion_tokens)
366
+
367
except ValueError as e:
368
# TODO: Use a vllm-specific Validation Error
369
data = self.create_streaming_error_response(str(e))
@@ -360,6 +378,7 @@ class OpenAIServingCompletion(OpenAIServing):
378
created_time: int,
379
model_name: str,
380
tokenizer: AnyTokenizer,
381
+ request_metadata: RequestResponseMetadata,
382
) -> CompletionResponse:
383
choices: List[CompletionResponseChoice] = []
384
num_prompt_tokens = 0
@@ -433,6 +452,8 @@ class OpenAIServingCompletion(OpenAIServing):
452
total_tokens=num_prompt_tokens + num_generated_tokens,
453
)
454
455
+ request_metadata.final_usage_info = usage
456
+
457
return CompletionResponse(
458
id=request_id,
459
created=created_time,