@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ac3de09cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Frontend] OpenAI server: propagate usage accounting to FastAPI middleware layer (#8672)

Adam Tilghman committed Sep 25, 2024 at 00:49 UTC 1ac3de09cd87290f7494ce6337623d6edd3f8667
3 files changed +57 -11
vllm/entrypoints/openai/protocol.py
+5
@@ -107,6 +107,11 @@ class UsageInfo(OpenAIBaseModel):
107 completion_tokens: Optional[int] = 0
108
109
110 +class RequestResponseMetadata(BaseModel):
111 + request_id: str
112 + final_usage_info: Optional[UsageInfo] = None
113 +
114 +
115 class JsonSchemaResponseFormat(OpenAIBaseModel):
116 name: str
117 description: Optional[str] = None
vllm/entrypoints/openai/serving_chat.py
+23 -3
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (
22 ChatCompletionRequest, ChatCompletionResponse,
23 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
24 ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
25 - DeltaToolCall, ErrorResponse, FunctionCall, ToolCall, UsageInfo)
25 + DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata,
26 + ToolCall, UsageInfo)
27 from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
28 LoRAModulePath,
29 OpenAIServing,
@@ -175,6 +176,11 @@ class OpenAIServingChat(OpenAIServing):
176 "--enable-auto-tool-choice and --tool-call-parser to be set")
177
178 request_id = f"chat-{random_uuid()}"
179 +
180 + request_metadata = RequestResponseMetadata(request_id=request_id)
181 + if raw_request:
182 + raw_request.state.request_metadata = request_metadata
183 +
184 try:
185 guided_decode_logits_processor = (
186 await self._guided_decode_logits_processor(request, tokenizer))
@@ -241,11 +247,13 @@ class OpenAIServingChat(OpenAIServing):
247 # Streaming response
248 if request.stream:
249 return self.chat_completion_stream_generator(
244 - request, result_generator, request_id, conversation, tokenizer)
250 + request, result_generator, request_id, conversation, tokenizer,
251 + request_metadata)
252
253 try:
254 return await self.chat_completion_full_generator(
248 - request, result_generator, request_id, conversation, tokenizer)
255 + request, result_generator, request_id, conversation, tokenizer,
256 + request_metadata)
257 except ValueError as e:
258 # TODO: Use a vllm-specific Validation Error
259 return self.create_error_response(str(e))
@@ -262,6 +270,7 @@ class OpenAIServingChat(OpenAIServing):
270 request_id: str,
271 conversation: List[ConversationMessage],
272 tokenizer: AnyTokenizer,
273 + request_metadata: RequestResponseMetadata,
274 ) -> AsyncGenerator[str, None]:
275 model_name = self.base_model_paths[0].name
276 created_time = int(time.time())
@@ -580,6 +589,13 @@ class OpenAIServingChat(OpenAIServing):
589 exclude_unset=True, exclude_none=True))
590 yield f"data: {final_usage_data}\n\n"
591
592 + # report to FastAPI middleware aggregate usage across all choices
593 + num_completion_tokens = sum(previous_num_tokens)
594 + request_metadata.final_usage_info = UsageInfo(
595 + prompt_tokens=num_prompt_tokens,
596 + completion_tokens=num_completion_tokens,
597 + total_tokens=num_prompt_tokens + num_completion_tokens)
598 +
599 except ValueError as e:
600 # TODO: Use a vllm-specific Validation Error
601 logger.error("error in chat completion stream generator: %s", e)
@@ -595,6 +611,7 @@ class OpenAIServingChat(OpenAIServing):
611 request_id: str,
612 conversation: List[ConversationMessage],
613 tokenizer: AnyTokenizer,
614 + request_metadata: RequestResponseMetadata,
615 ) -> Union[ErrorResponse, ChatCompletionResponse]:
616
617 model_name = self.base_model_paths[0].name
@@ -714,6 +731,9 @@ class OpenAIServingChat(OpenAIServing):
731 completion_tokens=num_generated_tokens,
732 total_tokens=num_prompt_tokens + num_generated_tokens,
733 )
734 +
735 + request_metadata.final_usage_info = usage
736 +
737 response = ChatCompletionResponse(
738 id=request_id,
739 created=created_time,
vllm/entrypoints/openai/serving_completion.py
+29 -8
@@ -18,7 +18,9 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
18 CompletionResponseChoice,
19 CompletionResponseStreamChoice,
20 CompletionStreamResponse,
21 - ErrorResponse, UsageInfo)
21 + ErrorResponse,
22 + RequestResponseMetadata,
23 + UsageInfo)
24 # yapf: enable
25 from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
26 LoRAModulePath,
@@ -94,6 +96,10 @@ class OpenAIServingCompletion(OpenAIServing):
96 request_id = f"cmpl-{random_uuid()}"
97 created_time = int(time.time())
98
99 + request_metadata = RequestResponseMetadata(request_id=request_id)
100 + if raw_request:
101 + raw_request.state.request_metadata = request_metadata
102 +
103 # Schedule the request and get the result generator.
104 generators: List[AsyncGenerator[RequestOutput, None]] = []
105 try:
@@ -165,13 +171,15 @@ class OpenAIServingCompletion(OpenAIServing):
171
172 # Streaming response
173 if stream:
168 - return self.completion_stream_generator(request,
169 - result_generator,
170 - request_id,
171 - created_time,
172 - model_name,
173 - num_prompts=len(prompts),
174 - tokenizer=tokenizer)
174 + return self.completion_stream_generator(
175 + request,
176 + result_generator,
177 + request_id,
178 + created_time,
179 + model_name,
180 + num_prompts=len(prompts),
181 + tokenizer=tokenizer,
182 + request_metadata=request_metadata)
183
184 # Non-streaming response
185 final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
@@ -198,6 +206,7 @@ class OpenAIServingCompletion(OpenAIServing):
206 created_time,
207 model_name,
208 tokenizer,
209 + request_metadata,
210 )
211 except asyncio.CancelledError:
212 return self.create_error_response("Client disconnected")
@@ -227,6 +236,7 @@ class OpenAIServingCompletion(OpenAIServing):
236 model_name: str,
237 num_prompts: int,
238 tokenizer: AnyTokenizer,
239 + request_metadata: RequestResponseMetadata,
240 ) -> AsyncGenerator[str, None]:
241 num_choices = 1 if request.n is None else request.n
242 previous_text_lens = [0] * num_choices * num_prompts
@@ -346,6 +356,14 @@ class OpenAIServingCompletion(OpenAIServing):
356 exclude_unset=False, exclude_none=True))
357 yield f"data: {final_usage_data}\n\n"
358
359 + # report to FastAPI middleware aggregate usage across all choices
360 + total_prompt_tokens = sum(num_prompt_tokens)
361 + total_completion_tokens = sum(previous_num_tokens)
362 + request_metadata.final_usage_info = UsageInfo(
363 + prompt_tokens=total_prompt_tokens,
364 + completion_tokens=total_completion_tokens,
365 + total_tokens=total_prompt_tokens + total_completion_tokens)
366 +
367 except ValueError as e:
368 # TODO: Use a vllm-specific Validation Error
369 data = self.create_streaming_error_response(str(e))
@@ -360,6 +378,7 @@ class OpenAIServingCompletion(OpenAIServing):
378 created_time: int,
379 model_name: str,
380 tokenizer: AnyTokenizer,
381 + request_metadata: RequestResponseMetadata,
382 ) -> CompletionResponse:
383 choices: List[CompletionResponseChoice] = []
384 num_prompt_tokens = 0
@@ -433,6 +452,8 @@ class OpenAIServingCompletion(OpenAIServing):
452 total_tokens=num_prompt_tokens + num_generated_tokens,
453 )
454
455 + request_metadata.final_usage_info = usage
456 +
457 return CompletionResponse(
458 id=request_id,
459 created=created_time,