@tduncan / Nvidia-Optimization-Stack-VLLM / commits / fbf2c5e8be

[Frontend] Add per-request metrics to Responses API (#55084)

Signed-off-by: wxin <wxin@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>

xinnywinne committed Sep 16, 2026 at 15:52 UTC fbf2c5e8be9754f31c4e0f549189fc9f3bdd213c

6 files changed +184 -14

docs/features/per_request_metrics.md

+12

@@ -120,6 +120,18 @@ the same `metrics` response field. As with `n > 1`, metrics are omitted for

120 requests with multiple prompts, because the timing data cannot be attributed to

121 a single prompt's generation.

122

123 +## Responses API

124 +

125 +Per-request metrics are available on the `/v1/responses` endpoint using the

126 +common `metrics` object described above. Non-streaming responses include it at

127 +the top level. Streaming responses include it in the final response carried by

128 +the `response.completed` event; intermediate events do not include metrics.

129 +

130 +Metrics are omitted for Responses requests that perform multiple

131 +model-generation turns, such as built-in tool-call workflows, because the

132 +response retains timing data for only one generation turn while token usage is

133 +accumulated across all turns.

134 +

135 ## Relationship to Prometheus Metrics

136

137 The `metrics` response field provides per-request values for a single request.

tests/entrypoints/openai/responses/test_serving_responses.py

+135 -12

@@ -48,6 +48,7 @@ from vllm.entrypoints.openai.responses.context import (

48 SimpleContext,

49 )

50 from vllm.entrypoints.openai.responses.protocol import (

51 + ResponseCompletedEvent,

52 ResponseCreatedEvent,

53 ResponseRawMessageAndToken,

54 ResponsesRequest,

@@ -71,6 +72,7 @@ from vllm.renderers.online_renderer import (

72 _extract_allowed_tools_from_mcp_requests,

73 )

74 from vllm.sampling_params import SamplingParams

75 +from vllm.v1.metrics.stats import RequestStateStats

77 pytestmark = pytest.mark.skip_global_cleanup

@@ -795,6 +797,7 @@ class TestInitializeToolSessions:

797 pass

798

799 assert serving_responses_instance.engine_client.generate.call_count == 2

800 + assert context.request_metrics_cover_all_generation_turns is False

801 followup_engine_input = (

802 serving_responses_instance.engine_client.generate.call_args_list[1].args[0]

803 )

@@ -1248,9 +1251,19 @@ class TestHarmonyPreambleStreaming:

1251 assert "response.output_item.done" in type_names

1252

1253

1251 -def _make_simple_context_with_output(text, token_ids, response_parser=None):

1252 - """Create a SimpleContext with a RequestOutput containing the given text."""

1253 - ctx = SimpleContext(response_parser=response_parser)

1254 +_PER_REQUEST_STATS = RequestStateStats(

1255 + queued_ts=1.0,

1256 + scheduled_ts=1.5,

1257 + first_token_ts=2.0,

1258 + last_token_ts=3.0,

1259 +)

1260 +

1261 +

1262 +def _make_request_output(

1263 + text,

1264 + token_ids,

1265 + metrics: RequestStateStats | None = None,

1266 +):

1267 completion = CompletionOutput(

1268 index=0,

1269 text=text,

@@ -1260,7 +1273,7 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):

1273 finish_reason=None,

1274 stop_reason=None,

1275 )

1263 - req_output = RequestOutput(

1276 + return RequestOutput(

1277 request_id="req",

1278 prompt="hi",

1279 prompt_token_ids=[7, 8],

@@ -1268,13 +1281,29 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None):

1281 outputs=[completion],

1282 finished=False,

1283 num_cached_tokens=0,

1284 + metrics=metrics,

1285 )

1286 +

1287 +

1288 +def _make_simple_context_with_output(

1289 + text,

1290 + token_ids,

1291 + response_parser=None,

1292 + metrics: RequestStateStats | None = None,

1293 +):

1294 + """Create a SimpleContext with a RequestOutput containing the given text."""

1295 + ctx = SimpleContext(response_parser=response_parser)

1296 + req_output = _make_request_output(text, token_ids, metrics)

1297 ctx.append_output(req_output)

1298 + ctx.request_metrics = metrics

1299 return ctx

1300

1301

1276 -def _make_serving_instance_with_reasoning():

1277 - """Create an OpenAIServingResponses with a mocked reasoning parser."""

1302 +def _make_serving_instance(

1303 + *,

1304 + reasoning_parser: str = "",

1305 + enable_per_request_metrics: bool = False,

1306 +) -> OpenAIServingResponses:

1307 engine_client = MagicMock()

1308 model_config = MagicMock()

1309 model_config.max_model_len = 100

@@ -1285,18 +1314,112 @@ def _make_serving_instance_with_reasoning():

1314 engine_client.input_processor = MagicMock()

1315 engine_client.renderer = MagicMock()

1316

1288 - models = MagicMock()

1289 -

1290 - serving = OpenAIServingResponses(

1317 + return OpenAIServingResponses(

1318 engine_client=engine_client,

1292 - models=models,

1319 + models=MagicMock(),

1320 online_renderer=MagicMock(),

1321 request_logger=None,

1322 chat_template=None,

1323 chat_template_content_format="auto",

1297 - reasoning_parser="qwen3",

1324 + reasoning_parser=reasoning_parser,

1325 + enable_per_request_metrics=enable_per_request_metrics,

1326 + )

1327 +

1328 +

1329 +async def _empty_context_generator():

1330 + if False:

1331 + yield

1332 +

1333 +

1334 +async def _make_full_metrics_response(

1335 + enable_per_request_metrics: bool,

1336 + request_metrics_cover_all_generation_turns: bool = True,

1337 +):

1338 + serving = _make_serving_instance(

1339 + enable_per_request_metrics=enable_per_request_metrics

1340 + )

1341 + request = ResponsesRequest(input="hi", tools=[], stream=False, store=False)

1342 + sampling_params = SamplingParams(max_tokens=16)

1343 + context = SimpleContext()

1344 + context.request_metrics_cover_all_generation_turns = (

1345 + request_metrics_cover_all_generation_turns

1346 + )

1347 +

1348 + async def generate(*args, **kwargs):

1349 + yield _make_request_output("hello", [10, 20], _PER_REQUEST_STATS)

1350 +

1351 + serving.engine_client.generate.side_effect = generate

1352 + result_generator = serving._generate_with_builtin_tools(

1353 + request_id=request.request_id,

1354 + engine_input=tokens_input([7, 8]),

1355 + sampling_params=sampling_params,

1356 + context=context,

1357 )

1299 - return serving

1358 + response = await serving.responses_full_generator(

1359 + request=request,

1360 + sampling_params=sampling_params,

1361 + result_generator=result_generator,

1362 + context=context,

1363 + model_name="test-model",

1364 + tokenizer=MagicMock(),

1365 + request_metadata=RequestResponseMetadata(request_id="req"),

1366 + )

1367 + assert isinstance(response, ResponsesResponse)

1368 + return response

1369 +

1370 +

1371 +@pytest.mark.asyncio

1372 +async def test_responses_per_request_metrics_follow_server_flag():

1373 + disabled_response = await _make_full_metrics_response(False)

1374 + assert disabled_response.metrics is None

1375 + assert "metrics" not in disabled_response.model_dump(mode="json")

1376 +

1377 + enabled_response = await _make_full_metrics_response(True)

1378 + assert enabled_response.metrics is not None

1379 + assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0)

1380 +

1381 +

1382 +@pytest.mark.asyncio

1383 +async def test_responses_streaming_metrics_only_on_completed_event():

1384 + serving = _make_serving_instance(enable_per_request_metrics=True)

1385 + request = ResponsesRequest(input="hi", tools=[], stream=True, store=False)

1386 + context = _make_simple_context_with_output(

1387 + "hello", [10, 20], metrics=_PER_REQUEST_STATS

1388 + )

1389 +

1390 + events = [

1391 + event

1392 + async for event in serving.responses_stream_generator(

1393 + request=request,

1394 + sampling_params=SamplingParams(max_tokens=16),

1395 + result_generator=_empty_context_generator(),

1396 + context=context,

1397 + model_name="test-model",

1398 + tokenizer=MagicMock(),

1399 + request_metadata=RequestResponseMetadata(request_id="req"),

1400 + )

1401 + ]

1402 +

1403 + for event in events[:-1]:

1404 + assert "metrics" not in event.response.model_dump(mode="json")

1405 + assert isinstance(events[-1], ResponseCompletedEvent)

1406 + assert events[-1].response.metrics is not None

1407 + assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0)

1408 +

1409 +

1410 +@pytest.mark.asyncio

1411 +async def test_responses_metrics_suppressed_for_multiple_generation_turns():

1412 + response = await _make_full_metrics_response(

1413 + True, request_metrics_cover_all_generation_turns=False

1414 + )

1415 +

1416 + assert response.metrics is None

1417 + assert "metrics" not in response.model_dump(mode="json")

1418 +

1419 +

1420 +def _make_serving_instance_with_reasoning():

1421 + """Create an OpenAIServingResponses with a mocked reasoning parser."""

1422 + return _make_serving_instance(reasoning_parser="qwen3")

1423

1424

1425 def _identity_increment(event):

vllm/entrypoints/generate/api_router.py

@@ -116,6 +116,7 @@ async def init_generate_state(

116 reasoning_parser=args.structured_outputs_config.reasoning_parser,

117 enable_prompt_tokens_details=args.enable_prompt_tokens_details,

118 enable_force_include_usage=args.enable_force_include_usage,

119 + enable_per_request_metrics=args.enable_per_request_metrics,

120 enable_log_outputs=args.enable_log_outputs,

121 default_chat_template_kwargs=default_chat_template_kwargs,

122 )

vllm/entrypoints/openai/responses/context.py

@@ -45,6 +45,8 @@ from vllm.utils import random_uuid

45 if TYPE_CHECKING:

46 from mcp.client import ClientSession

48 + from vllm.v1.metrics.stats import RequestStateStats

49 +

50 logger = logging.getLogger(__name__)

52 # This is currently needed as the tool type doesn't 1:1 match the

@@ -102,6 +104,11 @@ class TurnMetrics:

104

105 class ConversationContext(ABC):

106 response_parser: Parser | None = None

107 + request_metrics: "RequestStateStats | None" = None

108 + # Built-in tools can trigger additional model generations. In that case,

109 + # the stored engine timestamps cover only one turn, while token usage is

110 + # accumulated across all turns.

111 + request_metrics_cover_all_generation_turns: bool = True

112

113 @abstractmethod

114 def append_output(self, output: RequestOutput) -> None:

vllm/entrypoints/openai/responses/protocol.py

+12 -1

@@ -60,7 +60,11 @@ from vllm.entrypoints.chat_utils import (

60 ChatCompletionMessageParam,

61 ChatTemplateContentFormatOption,

62 )

63 -from vllm.entrypoints.generate.base.protocol import StopParam, validate_cache_salt

63 +from vllm.entrypoints.generate.base.protocol import (

64 + PerRequestMetrics,

65 + StopParam,

66 + validate_cache_salt,

67 +)

68 from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel

69 from vllm.exceptions import VLLMValidationError

70 from vllm.logger import init_logger

@@ -684,6 +688,11 @@ class ResponsesResponse(OpenAIBaseModel):

688 usage: ResponseUsage | None = None

689 user: str | None = None

690

691 + # vLLM-specific per-request metrics. Omitted unless enabled server-side.

692 + metrics: PerRequestMetrics | None = Field(

693 + default=None, exclude_if=lambda value: value is None

694 + )

695 +

696 presence_penalty: float | None = Field(

697 default=None,

698 ge=-2.0,

@@ -752,6 +761,7 @@ class ResponsesResponse(OpenAIBaseModel):

761 output: list[ResponseOutputItem],

762 status: ResponseStatus,

763 usage: ResponseUsage | None = None,

764 + metrics: PerRequestMetrics | None = None,

765 input_messages: ResponseInputOutputMessage | None = None,

766 output_messages: ResponseInputOutputMessage | None = None,

767 kv_transfer_params: dict[str, Any] | None = None,

@@ -795,6 +805,7 @@ class ResponsesResponse(OpenAIBaseModel):

805 truncation=request.truncation,

806 user=request.user,

807 usage=usage,

808 + metrics=metrics,

809 kv_transfer_params=kv_transfer_params,

810 ec_transfer_params=ec_transfer_params,

811 )

vllm/entrypoints/openai/responses/serving.py

+17 -1

@@ -30,7 +30,10 @@ from vllm.entrypoints.generate.base.protocol import (

30 DeltaMessage,

31 RequestResponseMetadata,

32 )

33 -from vllm.entrypoints.generate.base.serving import GenerateBaseServing

33 +from vllm.entrypoints.generate.base.serving import (

34 + GenerateBaseServing,

35 + build_per_request_timing_metrics,

36 +)

37 from vllm.entrypoints.mcp.tool_server import ToolServer

38 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

39 from vllm.entrypoints.openai.responses.context import (

@@ -109,6 +112,7 @@ class OpenAIServingResponses(GenerateBaseServing):

112 tool_server: ToolServer | None = None,

113 enable_prompt_tokens_details: bool = False,

114 enable_force_include_usage: bool = False,

115 + enable_per_request_metrics: bool = False,

116 enable_log_outputs: bool = False,

117 default_chat_template_kwargs: dict[str, Any] | None = None,

118 ) -> None:

@@ -136,6 +140,7 @@ class OpenAIServingResponses(GenerateBaseServing):

140 )

141 self.enable_prompt_tokens_details = enable_prompt_tokens_details

142 self.enable_force_include_usage = enable_force_include_usage

143 + self.enable_per_request_metrics = enable_per_request_metrics

144

145 self.default_sampling_params = self.model_config.get_diff_sampling_param()

146 mc = self.model_config

@@ -645,6 +650,7 @@ class OpenAIServingResponses(GenerateBaseServing):

650 )

651

652 async for res in generator:

653 + context.request_metrics = res.metrics

654 context.append_output(res)

655 # NOTE(woosuk): The stop condition is handled by the engine.

656 yield context

@@ -654,6 +660,7 @@ class OpenAIServingResponses(GenerateBaseServing):

660 break

661

662 # Call the tool and update the context with the result.

663 + context.request_metrics_cover_all_generation_turns = False

664 tool_output = await context.call_tool()

665 context.append_tool_output(tool_output)

666

@@ -857,6 +864,14 @@ class OpenAIServingResponses(GenerateBaseServing):

864 ],

865 ),

866 )

867 + per_request_metrics = None

868 + if (

869 + self.enable_per_request_metrics

870 + and context.request_metrics_cover_all_generation_turns

871 + ):

872 + per_request_metrics = build_per_request_timing_metrics(

873 + context.request_metrics, num_generated_tokens

874 + )

875 response = ResponsesResponse.from_request(

876 request,

877 sampling_params,

@@ -867,6 +882,7 @@ class OpenAIServingResponses(GenerateBaseServing):

882 output=output,

883 status=status,

884 usage=usage,

885 + metrics=per_request_metrics,

886 kv_transfer_params=context.kv_transfer_params,

887 ec_transfer_params=context.ec_transfer_params,

888 )