A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] add input/output usage in responses api when harmony context is leveraged (#22667)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Aug 22, 2025 at 10:32 UTC
0ba1b54ac6958de7a02dfd39be7b59dd430be9ca
1 file changed
+26
-2
vllm/entrypoints/context.py
+26
-2
@@ -3,6 +3,7 @@
3
import json
4
import logging
5
from abc import ABC, abstractmethod
6
+from collections.abc import Sequence
7
from typing import TYPE_CHECKING, Union
8
9
from openai_harmony import Author, Message, Role, StreamState, TextContent
@@ -67,15 +68,27 @@ class HarmonyContext(ConversationContext):
68
69
self.parser = get_streamable_parser_for_assistant()
70
self.num_init_messages = len(messages)
70
- # TODO(woosuk): Implement the following fields.
71
self.num_prompt_tokens = 0
72
- self.num_cached_tokens = 0
72
self.num_output_tokens = 0
73
+ # TODO(woosuk): Implement the following fields.
74
+ self.num_cached_tokens = 0
75
self.num_reasoning_tokens = 0
76
77
+ def _update_num_prompt_tokens(self, output: RequestOutput):
78
+ if output.prompt_token_ids and len(output.prompt_token_ids) > 0:
79
+ # NOTE: with built-in tools, there might be multiple rounds in
80
+ # the conversation, with the full conversation being resent
81
+ # as new prompt each time. Hence the sum.
82
+ self.num_prompt_tokens += len(output.prompt_token_ids)
83
+
84
+ def _update_num_output_tokens(self, token_ids: Sequence[int]):
85
+ self.num_output_tokens += len(token_ids)
86
+
87
def append_output(self, output) -> None:
88
if isinstance(output, RequestOutput):
89
+ self._update_num_prompt_tokens(output)
90
output_token_ids = output.outputs[0].token_ids
91
+ self._update_num_output_tokens(output_token_ids)
92
self.parser = get_streamable_parser_for_assistant()
93
for token_id in output_token_ids:
94
self.parser.process(token_id)
@@ -158,6 +171,7 @@ class StreamingHarmonyContext(HarmonyContext):
171
self.parser = get_streamable_parser_for_assistant()
172
self.encoding = get_encoding()
173
self.last_tok = None
174
+ self.first_tok_of_message = True
175
176
@property
177
def messages(self) -> list:
@@ -165,8 +179,18 @@ class StreamingHarmonyContext(HarmonyContext):
179
180
def append_output(self, output) -> None:
181
if isinstance(output, RequestOutput):
182
+ # append_output is called for each output token in streaming case,
183
+ # so we only want to add the prompt tokens once for each message.
184
+ if self.first_tok_of_message:
185
+ self._update_num_prompt_tokens(output)
186
+ # Reset self.first_tok_of_message if needed:
187
+ # if the current token is the last one of the current message
188
+ # (finished=True), then the next token processed will mark the
189
+ # beginning of a new message
190
+ self.first_tok_of_message = output.finished
191
tok = output.outputs[0].token_ids[0]
192
self.parser.process(tok)
193
+ self._update_num_output_tokens(output.outputs[0].token_ids)
194
self.last_tok = tok
195
else:
196
# Handle the case of tool output in direct message format