@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8e8e0b6af1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

feat: Add --enable-log-outputs flag for logging model generations (#20707)

Signed-off-by: Adrian Garcia <adrian.garcia@inceptionai.ai>

Adrián García García committed Aug 7, 2025 at 10:10 UTC 8e8e0b6af189d262bcfdaef6c0cfb94772e86b0b
5 files changed +412 -26
tests/test_logger.py
+248 -4
@@ -10,11 +10,12 @@ from dataclasses import dataclass
10 from json.decoder import JSONDecodeError
11 from tempfile import NamedTemporaryFile
12 from typing import Any
13 -from unittest.mock import patch
13 +from unittest.mock import MagicMock, patch
14 from uuid import uuid4
15
16 import pytest
17
18 +from vllm.entrypoints.logger import RequestLogger
19 from vllm.logger import (_DATE_FORMAT, _FORMAT, _configure_vllm_root_logger,
20 enable_trace_function_call, init_logger)
21 from vllm.logging_utils import NewLineFormatter
@@ -228,9 +229,10 @@ def test_prepare_object_to_dump():
229 list_obj = [1, 2, 3]
230 assert prepare_object_to_dump(list_obj) == '[1, 2, 3]'
231
231 - dict_obj = {'a': 1, 'b': 'b'}
232 + dict_obj = {"a": 1, "b": "b"}
233 assert prepare_object_to_dump(dict_obj) in [
233 - "{a: 1, b: 'b'}", "{b: 'b', a: 1}"
234 + "{a: 1, b: 'b'}",
235 + "{b: 'b', a: 1}",
236 ]
237
238 set_obj = {1, 2, 3}
@@ -252,4 +254,246 @@ def test_prepare_object_to_dump():
254 b: str
255
256 assert (prepare_object_to_dump(CustomClass(
255 - 1, 'b')) == "CustomClass(a=1, b='b')")
257 + 1, "b")) == "CustomClass(a=1, b='b')")
258 +
259 +
260 +def test_request_logger_log_outputs():
261 + """Test the new log_outputs functionality."""
262 + # Create a mock logger to capture log calls
263 + mock_logger = MagicMock()
264 +
265 + with patch("vllm.entrypoints.logger.logger", mock_logger):
266 + request_logger = RequestLogger(max_log_len=None)
267 +
268 + # Test basic output logging
269 + request_logger.log_outputs(
270 + request_id="test-123",
271 + outputs="Hello, world!",
272 + output_token_ids=[1, 2, 3, 4],
273 + finish_reason="stop",
274 + is_streaming=False,
275 + delta=False,
276 + )
277 +
278 + mock_logger.info.assert_called_once()
279 + call_args = mock_logger.info.call_args.args
280 + assert "Generated response %s%s" in call_args[0]
281 + assert call_args[1] == "test-123"
282 + assert call_args[3] == "Hello, world!"
283 + assert call_args[4] == [1, 2, 3, 4]
284 + assert call_args[5] == "stop"
285 +
286 +
287 +def test_request_logger_log_outputs_streaming_delta():
288 + """Test log_outputs with streaming delta mode."""
289 + mock_logger = MagicMock()
290 +
291 + with patch("vllm.entrypoints.logger.logger", mock_logger):
292 + request_logger = RequestLogger(max_log_len=None)
293 +
294 + # Test streaming delta logging
295 + request_logger.log_outputs(
296 + request_id="test-456",
297 + outputs="Hello",
298 + output_token_ids=[1],
299 + finish_reason=None,
300 + is_streaming=True,
301 + delta=True,
302 + )
303 +
304 + mock_logger.info.assert_called_once()
305 + call_args = mock_logger.info.call_args.args
306 + assert "Generated response %s%s" in call_args[0]
307 + assert call_args[1] == "test-456"
308 + assert call_args[2] == " (streaming delta)"
309 + assert call_args[3] == "Hello"
310 + assert call_args[4] == [1]
311 + assert call_args[5] is None
312 +
313 +
314 +def test_request_logger_log_outputs_streaming_complete():
315 + """Test log_outputs with streaming complete mode."""
316 + mock_logger = MagicMock()
317 +
318 + with patch("vllm.entrypoints.logger.logger", mock_logger):
319 + request_logger = RequestLogger(max_log_len=None)
320 +
321 + # Test streaming complete logging
322 + request_logger.log_outputs(
323 + request_id="test-789",
324 + outputs="Complete response",
325 + output_token_ids=[1, 2, 3],
326 + finish_reason="length",
327 + is_streaming=True,
328 + delta=False,
329 + )
330 +
331 + mock_logger.info.assert_called_once()
332 + call_args = mock_logger.info.call_args.args
333 + assert "Generated response %s%s" in call_args[0]
334 + assert call_args[1] == "test-789"
335 + assert call_args[2] == " (streaming complete)"
336 + assert call_args[3] == "Complete response"
337 + assert call_args[4] == [1, 2, 3]
338 + assert call_args[5] == "length"
339 +
340 +
341 +def test_request_logger_log_outputs_with_truncation():
342 + """Test log_outputs respects max_log_len setting."""
343 + mock_logger = MagicMock()
344 +
345 + with patch("vllm.entrypoints.logger.logger", mock_logger):
346 + # Set max_log_len to 10
347 + request_logger = RequestLogger(max_log_len=10)
348 +
349 + # Test output truncation
350 + long_output = "This is a very long output that should be truncated"
351 + long_token_ids = list(range(20)) # 20 tokens
352 +
353 + request_logger.log_outputs(
354 + request_id="test-truncate",
355 + outputs=long_output,
356 + output_token_ids=long_token_ids,
357 + finish_reason="stop",
358 + is_streaming=False,
359 + delta=False,
360 + )
361 +
362 + mock_logger.info.assert_called_once()
363 + call_args = mock_logger.info.call_args
364 +
365 + # Check that output was truncated to first 10 characters
366 + logged_output = call_args[0][3]
367 + assert logged_output == "This is a "
368 + assert len(logged_output) == 10
369 +
370 + # Check that token IDs were truncated to first 10 tokens
371 + logged_token_ids = call_args[0][4]
372 + assert logged_token_ids == list(range(10))
373 + assert len(logged_token_ids) == 10
374 +
375 +
376 +def test_request_logger_log_outputs_none_values():
377 + """Test log_outputs handles None values correctly."""
378 + mock_logger = MagicMock()
379 +
380 + with patch("vllm.entrypoints.logger.logger", mock_logger):
381 + request_logger = RequestLogger(max_log_len=None)
382 +
383 + # Test with None output_token_ids
384 + request_logger.log_outputs(
385 + request_id="test-none",
386 + outputs="Test output",
387 + output_token_ids=None,
388 + finish_reason="stop",
389 + is_streaming=False,
390 + delta=False,
391 + )
392 +
393 + mock_logger.info.assert_called_once()
394 + call_args = mock_logger.info.call_args.args
395 + assert "Generated response %s%s" in call_args[0]
396 + assert call_args[1] == "test-none"
397 + assert call_args[3] == "Test output"
398 + assert call_args[4] is None
399 + assert call_args[5] == "stop"
400 +
401 +
402 +def test_request_logger_log_outputs_empty_output():
403 + """Test log_outputs handles empty output correctly."""
404 + mock_logger = MagicMock()
405 +
406 + with patch("vllm.entrypoints.logger.logger", mock_logger):
407 + request_logger = RequestLogger(max_log_len=5)
408 +
409 + # Test with empty output
410 + request_logger.log_outputs(
411 + request_id="test-empty",
412 + outputs="",
413 + output_token_ids=[],
414 + finish_reason="stop",
415 + is_streaming=False,
416 + delta=False,
417 + )
418 +
419 + mock_logger.info.assert_called_once()
420 + call_args = mock_logger.info.call_args.args
421 + assert "Generated response %s%s" in call_args[0]
422 + assert call_args[1] == "test-empty"
423 + assert call_args[3] == ""
424 + assert call_args[4] == []
425 + assert call_args[5] == "stop"
426 +
427 +
428 +def test_request_logger_log_outputs_integration():
429 + """Test that log_outputs can be called alongside log_inputs."""
430 + mock_logger = MagicMock()
431 +
432 + with patch("vllm.entrypoints.logger.logger", mock_logger):
433 + request_logger = RequestLogger(max_log_len=None)
434 +
435 + # Test that both methods can be called without interference
436 + request_logger.log_inputs(
437 + request_id="test-integration",
438 + prompt="Test prompt",
439 + prompt_token_ids=[1, 2, 3],
440 + prompt_embeds=None,
441 + params=None,
442 + lora_request=None,
443 + )
444 +
445 + request_logger.log_outputs(
446 + request_id="test-integration",
447 + outputs="Test output",
448 + output_token_ids=[4, 5, 6],
449 + finish_reason="stop",
450 + is_streaming=False,
451 + delta=False,
452 + )
453 +
454 + # Should have been called twice - once for inputs, once for outputs
455 + assert mock_logger.info.call_count == 2
456 +
457 + # Check that the calls were made with correct patterns
458 + input_call = mock_logger.info.call_args_list[0][0]
459 + output_call = mock_logger.info.call_args_list[1][0]
460 +
461 + assert "Received request %s" in input_call[0]
462 + assert input_call[1] == "test-integration"
463 +
464 + assert "Generated response %s%s" in output_call[0]
465 + assert output_call[1] == "test-integration"
466 +
467 +
468 +def test_streaming_complete_logs_full_text_content():
469 + """Test that streaming complete logging includes
470 + full accumulated text, not just token count."""
471 + mock_logger = MagicMock()
472 +
473 + with patch("vllm.entrypoints.logger.logger", mock_logger):
474 + request_logger = RequestLogger(max_log_len=None)
475 +
476 + # Test with actual content instead of token count format
477 + full_response = "This is a complete response from streaming"
478 + request_logger.log_outputs(
479 + request_id="test-streaming-full-text",
480 + outputs=full_response,
481 + output_token_ids=None,
482 + finish_reason="streaming_complete",
483 + is_streaming=True,
484 + delta=False,
485 + )
486 +
487 + mock_logger.info.assert_called_once()
488 + call_args = mock_logger.info.call_args.args
489 +
490 + # Verify the logged output is the full text, not a token count format
491 + logged_output = call_args[3]
492 + assert logged_output == full_response
493 + assert "tokens>" not in logged_output
494 + assert "streaming_complete" not in logged_output
495 +
496 + # Verify other parameters
497 + assert call_args[1] == "test-streaming-full-text"
498 + assert call_args[2] == " (streaming complete)"
499 + assert call_args[5] == "streaming_complete"
vllm/entrypoints/logger.py
+34 -2
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from collections.abc import Sequence
5 from typing import Optional, Union
6
7 import torch
@@ -16,8 +17,6 @@ logger = init_logger(__name__)
17 class RequestLogger:
18
19 def __init__(self, *, max_log_len: Optional[int]) -> None:
19 - super().__init__()
20 -
20 self.max_log_len = max_log_len
21
22 def log_inputs(
@@ -45,3 +44,36 @@ class RequestLogger:
44 "lora_request: %s.", request_id, prompt, params, prompt_token_ids,
45 prompt_embeds.shape if prompt_embeds is not None else None,
46 lora_request)
47 +
48 + def log_outputs(
49 + self,
50 + request_id: str,
51 + outputs: str,
52 + output_token_ids: Optional[Sequence[int]],
53 + finish_reason: Optional[str] = None,
54 + is_streaming: bool = False,
55 + delta: bool = False,
56 + ) -> None:
57 + max_log_len = self.max_log_len
58 + if max_log_len is not None:
59 + if outputs is not None:
60 + outputs = outputs[:max_log_len]
61 +
62 + if output_token_ids is not None:
63 + # Convert to list and apply truncation
64 + output_token_ids = list(output_token_ids)[:max_log_len]
65 +
66 + stream_info = ""
67 + if is_streaming:
68 + stream_info = (" (streaming delta)"
69 + if delta else " (streaming complete)")
70 +
71 + logger.info(
72 + "Generated response %s%s: output: %r, "
73 + "output_token_ids: %s, finish_reason: %s",
74 + request_id,
75 + stream_info,
76 + outputs,
77 + output_token_ids,
78 + finish_reason,
79 + )
vllm/entrypoints/openai/cli_args.py
+6 -3
@@ -44,10 +44,10 @@ class LoRAParserAction(argparse.Action):
44
45 lora_list: list[LoRAModulePath] = []
46 for item in values:
47 - if item in [None, '']: # Skip if item is None or empty string
47 + if item in [None, ""]: # Skip if item is None or empty string
48 continue
49 - if '=' in item and ',' not in item: # Old format: name=path
50 - name, path = item.split('=')
49 + if "=" in item and "," not in item: # Old format: name=path
50 + name, path = item.split("=")
51 lora_list.append(LoRAModulePath(name, path))
52 else: # Assume JSON format
53 try:
@@ -167,6 +167,9 @@ schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
167 enable_tokenizer_info_endpoint: bool = False
168 """Enable the /get_tokenizer_info endpoint. May expose chat
169 templates and other tokenizer configuration."""
170 + enable_log_outputs: bool = False
171 + """If set to True, enable logging of model outputs (generations)
172 + in addition to the input logging that is enabled by default."""
173
174 @staticmethod
175 def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
vllm/entrypoints/openai/serving_chat.py
+104 -17
@@ -73,6 +73,7 @@ class OpenAIServingChat(OpenAIServing):
73 tool_parser: Optional[str] = None,
74 enable_prompt_tokens_details: bool = False,
75 enable_force_include_usage: bool = False,
76 + enable_log_outputs: bool = False,
77 ) -> None:
78 super().__init__(engine_client=engine_client,
79 model_config=model_config,
@@ -84,6 +85,7 @@ class OpenAIServingChat(OpenAIServing):
85 self.response_role = response_role
86 self.chat_template = chat_template
87 self.chat_template_content_format: Final = chat_template_content_format
88 + self.enable_log_outputs = enable_log_outputs
89
90 # set up tool use
91 self.enable_auto_tools: bool = enable_auto_tools
@@ -489,20 +491,21 @@ class OpenAIServingChat(OpenAIServing):
491 all_previous_token_ids: Optional[list[list[int]]]
492 function_name_returned = [False] * num_choices
493
494 + # Always track previous_texts for comprehensive output logging
495 + previous_texts = [""] * num_choices
496 +
497 # Only one of these will be used, thus previous_texts and
498 # all_previous_token_ids will not be used twice in the same iteration.
499 if tool_choice_auto or self.reasoning_parser:
500 # These are only required in "auto" tool choice case
496 - previous_texts = [""] * num_choices
501 all_previous_token_ids = [[]] * num_choices
502 # For reasoning parser and tool call all enabled
503 added_content_delta_arr = [False] * num_choices
504 reasoning_end_arr = [False] * num_choices
505 elif request.tool_choice == "required":
502 - previous_texts = [""] * num_choices
506 all_previous_token_ids = None
507 else:
505 - previous_texts, all_previous_token_ids = None, None
508 + all_previous_token_ids = None
509
510 try:
511 if self.reasoning_parser:
@@ -844,6 +847,7 @@ class OpenAIServingChat(OpenAIServing):
847 current_token_ids=current_token_ids,
848 delta_token_ids=output.token_ids,
849 request=request))
850 +
851 # when only reasoning
852 elif self.reasoning_parser:
853 delta_message = (reasoning_parser.
@@ -865,6 +869,10 @@ class OpenAIServingChat(OpenAIServing):
869 assert all_previous_token_ids is not None
870 previous_texts[i] = current_text
871 all_previous_token_ids[i] = current_token_ids
872 + else:
873 + # Update for comprehensive logging even in simple case
874 + assert previous_texts is not None
875 + previous_texts[i] += delta_text
876
877 # set the previous values for the next iteration
878 previous_num_tokens[i] += len(output.token_ids)
@@ -876,6 +884,27 @@ class OpenAIServingChat(OpenAIServing):
884 if delta_message is None:
885 continue
886
887 + # Log streaming delta if output logging is enabled
888 + if self.enable_log_outputs and self.request_logger:
889 + delta_content = ""
890 + if delta_message.content:
891 + delta_content = delta_message.content
892 + elif delta_message.tool_calls:
893 + delta_content = "".join(
894 + tc.function.arguments
895 + for tc in delta_message.tool_calls
896 + if tc.function and tc.function.arguments)
897 +
898 + if delta_content:
899 + self.request_logger.log_outputs(
900 + request_id=request_id,
901 + outputs=delta_content,
902 + output_token_ids=list(output.token_ids),
903 + finish_reason=output.finish_reason,
904 + is_streaming=True,
905 + delta=True,
906 + )
907 +
908 if output.finish_reason is None:
909 # Send token-by-token response for each request.n
910 choice_data = ChatCompletionResponseStreamChoice(
@@ -994,7 +1023,27 @@ class OpenAIServingChat(OpenAIServing):
1023 request_metadata.final_usage_info = UsageInfo(
1024 prompt_tokens=num_prompt_tokens,
1025 completion_tokens=num_completion_tokens,
997 - total_tokens=num_prompt_tokens + num_completion_tokens)
1026 + total_tokens=num_prompt_tokens + num_completion_tokens,
1027 + )
1028 +
1029 + # Log complete streaming response if output logging is enabled
1030 + if self.enable_log_outputs and self.request_logger:
1031 + # Log the complete response for each choice
1032 + for i in range(num_choices):
1033 + full_text = (
1034 + previous_texts[i]
1035 + if previous_texts and i < len(previous_texts) else
1036 + f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1037 + )
1038 + self.request_logger.log_outputs(
1039 + request_id=request_id,
1040 + outputs=full_text,
1041 + output_token_ids=
1042 + None, # Consider also logging all token IDs
1043 + finish_reason="streaming_complete",
1044 + is_streaming=True,
1045 + delta=False,
1046 + )
1047
1048 except Exception as e:
1049 # TODO: Use a vllm-specific Validation Error
@@ -1121,8 +1170,10 @@ class OpenAIServingChat(OpenAIServing):
1170 tool_calls=[
1171 tool_call_class(function=FunctionCall(
1172 name=request.tool_choice.function.name,
1124 - arguments=content))
1125 - ])
1173 + arguments=content,
1174 + ))
1175 + ],
1176 + )
1177
1178 elif request.tool_choice and request.tool_choice == "required":
1179 tool_call_class = MistralToolCall if isinstance(
@@ -1209,12 +1260,13 @@ class OpenAIServingChat(OpenAIServing):
1260 finish_reason="tool_calls" if auto_tools_called else
1261 output.finish_reason if output.finish_reason else "stop",
1262 stop_reason=output.stop_reason)
1263 +
1264 choices.append(choice_data)
1265
1266 if request.echo:
1267 last_msg_content: Union[str, list[dict[str, str]]] = ""
1216 - if conversation and "content" in conversation[-1] and conversation[
1217 - -1].get("role") == role:
1268 + if (conversation and "content" in conversation[-1]
1269 + and conversation[-1].get("role") == role):
1270 last_msg_content = conversation[-1]["content"] or ""
1271 if isinstance(last_msg_content, list):
1272 last_msg_content = "\n".join(msg['text']
@@ -1251,6 +1303,40 @@ class OpenAIServingChat(OpenAIServing):
1303 kv_transfer_params=final_res.kv_transfer_params,
1304 )
1305
1306 + # Log complete response if output logging is enabled
1307 + if self.enable_log_outputs and self.request_logger:
1308 + for choice in choices:
1309 + output_text = ""
1310 + if choice.message.content:
1311 + output_text = choice.message.content
1312 + elif choice.message.tool_calls:
1313 + # For tool calls, log the function name and arguments
1314 + tool_call_descriptions = []
1315 + for tool_call in choice.message.tool_calls:
1316 + if hasattr(tool_call.function, "name") and hasattr(
1317 + tool_call.function, "arguments"):
1318 + tool_call_descriptions.append(
1319 + f"{tool_call.function.name}({tool_call.function.arguments})"
1320 + )
1321 + tool_calls_str = ", ".join(tool_call_descriptions)
1322 + output_text = f"[tool_calls: {tool_calls_str}]"
1323 +
1324 + if output_text:
1325 + # Get the corresponding output token IDs
1326 + output_token_ids = None
1327 + if choice.index < len(final_res.outputs):
1328 + output_token_ids = final_res.outputs[
1329 + choice.index].token_ids
1330 +
1331 + self.request_logger.log_outputs(
1332 + request_id=request_id,
1333 + outputs=output_text,
1334 + output_token_ids=output_token_ids,
1335 + finish_reason=choice.finish_reason,
1336 + is_streaming=False,
1337 + delta=False,
1338 + )
1339 +
1340 return response
1341
1342 def _get_top_logprobs(
@@ -1258,15 +1344,16 @@ class OpenAIServingChat(OpenAIServing):
1344 tokenizer: AnyTokenizer,
1345 should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1346 return [
1261 - ChatCompletionLogProb(token=(token := self._get_decoded_token(
1262 - p[1],
1263 - p[0],
1264 - tokenizer,
1265 - return_as_token_id=should_return_as_token_id)),
1266 - logprob=max(p[1].logprob, -9999.0),
1267 - bytes=list(
1268 - token.encode("utf-8", errors="replace")))
1269 - for i, p in enumerate(logprobs.items())
1347 + ChatCompletionLogProb(
1348 + token=(token := self._get_decoded_token(
1349 + p[1],
1350 + p[0],
1351 + tokenizer,
1352 + return_as_token_id=should_return_as_token_id,
1353 + )),
1354 + logprob=max(p[1].logprob, -9999.0),
1355 + bytes=list(token.encode("utf-8", errors="replace")),
1356 + ) for i, p in enumerate(logprobs.items())
1357 if top_logprobs and i < top_logprobs
1358 ]
1359
vllm/entrypoints/openai/serving_responses.py
+20
@@ -65,6 +65,7 @@ class OpenAIServingResponses(OpenAIServing):
65 tool_server: Optional[ToolServer] = None,
66 enable_prompt_tokens_details: bool = False,
67 enable_force_include_usage: bool = False,
68 + enable_log_outputs: bool = False,
69 ) -> None:
70 super().__init__(
71 engine_client=engine_client,
@@ -77,6 +78,7 @@ class OpenAIServingResponses(OpenAIServing):
78
79 self.chat_template = chat_template
80 self.chat_template_content_format: Final = chat_template_content_format
81 + self.enable_log_outputs = enable_log_outputs
82
83 self.reasoning_parser: Optional[Callable[[AnyTokenizer],
84 ReasoningParser]] = None
@@ -428,6 +430,24 @@ class OpenAIServingResponses(OpenAIServing):
430 usage=usage,
431 )
432
433 + # Log complete response if output logging is enabled
434 + if self.enable_log_outputs and self.request_logger:
435 + output_text = ""
436 + if content:
437 + output_text = content
438 + elif reasoning_content:
439 + output_text = f"[reasoning: {reasoning_content}]"
440 +
441 + if output_text:
442 + self.request_logger.log_outputs(
443 + request_id=request.request_id,
444 + outputs=output_text,
445 + output_token_ids=final_output.token_ids,
446 + finish_reason=final_output.finish_reason,
447 + is_streaming=False,
448 + delta=False,
449 + )
450 +
451 if request.store:
452 async with self.response_store_lock:
453 stored_response = self.response_store.get(response.id)