A high-throughput and memory-efficient inference and serving engine for LLMs
[Feature][Frontend] Report multimodal token counts in usage.prompt_tokens_details (#45458)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
Ting SUN committed
Jun 15, 2026 at 13:20 UTC
48df95c43e05347070b6a99d39c053acfdeb8900
3 files changed
+93
-14
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+37
-1
@@ -23,7 +23,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
23
ChatCompletionRequest,
24
ChatCompletionResponse,
25
)
26
-from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
26
+from vllm.entrypoints.openai.chat_completion.serving import (
27
+ OpenAIServingChat,
28
+ _get_mm_token_counts,
29
+ _make_prompt_tokens_details,
30
+)
31
from vllm.entrypoints.openai.engine.protocol import (
32
ErrorResponse,
33
RequestResponseMetadata,
@@ -37,6 +41,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
41
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
42
from vllm.exceptions import VLLMValidationError
43
from vllm.inputs import TokensPrompt
44
+from vllm.multimodal.inputs import PlaceholderRange
45
from vllm.outputs import CompletionOutput, RequestOutput
46
from vllm.parser import HarmonyParser
47
from vllm.renderers.hf import HfRenderer
@@ -635,6 +640,37 @@ def test_async_serving_chat_init():
640
assert serving_completion.chat_template == CHAT_TEMPLATE
641
642
643
+def test_mm_prompt_tokens_details():
644
+ # Text-only input has no multimodal placeholders.
645
+ assert _get_mm_token_counts({"type": "tokens"}) == {}
646
+
647
+ # Per-modality counts sum each modality's placeholder ranges.
648
+ counts = _get_mm_token_counts(
649
+ {
650
+ "mm_placeholders": {
651
+ "image": [
652
+ PlaceholderRange(offset=0, length=576),
653
+ PlaceholderRange(offset=600, length=24),
654
+ ],
655
+ "video": [PlaceholderRange(offset=700, length=1200)],
656
+ }
657
+ }
658
+ )
659
+ assert counts == {"image": 600, "video": 1200}
660
+
661
+ # Gated off, or nothing to report -> no details.
662
+ assert _make_prompt_tokens_details(False, 5, counts) is None
663
+ assert _make_prompt_tokens_details(True, None, None) is None
664
+
665
+ # Zero cached_tokens is still reported (not None), matching the cached-only
666
+ # behavior; multimodal counts ride alongside even when cached_tokens is None.
667
+ assert _make_prompt_tokens_details(True, 0, None).cached_tokens == 0
668
+ details = _make_prompt_tokens_details(True, None, counts)
669
+ assert details.cached_tokens is None
670
+ assert details.multimodal_tokens == {"image": 600, "video": 1200}
671
+ assert _make_prompt_tokens_details(True, 3, counts).cached_tokens == 3
672
+
673
+
674
@pytest.mark.asyncio
675
async def test_serving_chat_returns_correct_model_name():
676
mock_engine = MagicMock(spec=AsyncLLM)
vllm/entrypoints/openai/chat_completion/serving.py
+51
-13
@@ -7,7 +7,7 @@ import time
7
from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
from http import HTTPStatus
10
-from typing import TYPE_CHECKING, Any, Final
10
+from typing import TYPE_CHECKING, Any, Final, cast
11
12
import numpy as np
13
import pybase64 as base64
@@ -54,7 +54,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
54
from vllm.entrypoints.serve.utils.tool_calls_utils import (
55
maybe_filter_parallel_tool_calls,
56
)
57
-from vllm.inputs import EngineInput
57
+from vllm.inputs import EngineInput, MultiModalPlaceholders
58
from vllm.logger import init_logger
59
from vllm.logprobs import Logprob
60
from vllm.outputs import RequestOutput
@@ -73,6 +73,39 @@ if TYPE_CHECKING:
73
logger = init_logger(__name__)
74
75
76
+def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
77
+ """Sum per-modality placeholder tokens from ``mm_placeholders``.
78
+
79
+ Keyed by modality name; ``PlaceholderRange.length`` is the placeholder's
80
+ prompt token span, so each sum matches the placeholder tokens already
81
+ counted in ``usage.prompt_tokens``.
82
+ """
83
+ mm_placeholders = cast(
84
+ "MultiModalPlaceholders | None", engine_input.get("mm_placeholders")
85
+ )
86
+ return {
87
+ modality: sum(p.length for p in ranges)
88
+ for modality, ranges in (mm_placeholders or {}).items()
89
+ if ranges
90
+ }
91
+
92
+
93
+def _make_prompt_tokens_details(
94
+ enable_prompt_tokens_details: bool,
95
+ num_cached_tokens: int | None,
96
+ mm_token_counts: dict[str, int] | None,
97
+) -> PromptTokenUsageInfo | None:
98
+ """Build ``prompt_tokens_details`` from cached + multimodal token counts."""
99
+ if not enable_prompt_tokens_details:
100
+ return None
101
+ if num_cached_tokens is None and not mm_token_counts:
102
+ return None
103
+ return PromptTokenUsageInfo(
104
+ cached_tokens=num_cached_tokens,
105
+ multimodal_tokens=mm_token_counts or None,
106
+ )
107
+
108
+
109
class OpenAIServingChat(OpenAIServing):
110
def __init__(
111
self,
@@ -264,8 +297,10 @@ class OpenAIServingChat(OpenAIServing):
297
# Schedule the request and get the result generator.
298
max_model_len = self.model_config.max_model_len
299
generators: list[AsyncGenerator[RequestOutput, None]] = []
300
+ mm_token_counts: dict[str, int] | None = None
301
for i, engine_input in enumerate(engine_inputs):
302
prompt_token_ids = self._extract_prompt_components(engine_input).token_ids
303
+ mm_token_counts = _get_mm_token_counts(engine_input)
304
305
# If we are creating sub requests for multiple prompts, ensure that they
306
# have unique request ids.
@@ -362,6 +397,7 @@ class OpenAIServingChat(OpenAIServing):
397
tokenizer,
398
request_metadata,
399
chat_template_kwargs=chat_template_kwargs,
400
+ mm_token_counts=mm_token_counts,
401
)
402
403
return await self.chat_completion_full_generator(
@@ -373,6 +409,7 @@ class OpenAIServingChat(OpenAIServing):
409
tokenizer,
410
request_metadata,
411
chat_template_kwargs=chat_template_kwargs,
412
+ mm_token_counts=mm_token_counts,
413
)
414
415
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -390,6 +427,7 @@ class OpenAIServingChat(OpenAIServing):
427
tokenizer: TokenizerLike,
428
request_metadata: RequestResponseMetadata,
429
chat_template_kwargs: dict[str, Any] | None = None,
430
+ mm_token_counts: dict[str, int] | None = None,
431
) -> AsyncGenerator[str, None]:
432
created_time = int(time.time())
433
chunk_object_type: Final = "chat.completion.chunk"
@@ -733,10 +771,11 @@ class OpenAIServingChat(OpenAIServing):
771
completion_tokens=completion_tokens,
772
total_tokens=num_prompt_tokens + completion_tokens,
773
)
736
- if self.enable_prompt_tokens_details and num_cached_tokens is not None:
737
- final_usage.prompt_tokens_details = PromptTokenUsageInfo(
738
- cached_tokens=num_cached_tokens
739
- )
774
+ final_usage.prompt_tokens_details = _make_prompt_tokens_details(
775
+ self.enable_prompt_tokens_details,
776
+ num_cached_tokens,
777
+ mm_token_counts,
778
+ )
779
780
final_usage_chunk = ChatCompletionStreamResponse(
781
id=request_id,
@@ -797,6 +836,7 @@ class OpenAIServingChat(OpenAIServing):
836
tokenizer: TokenizerLike,
837
request_metadata: RequestResponseMetadata,
838
chat_template_kwargs: dict[str, Any] | None = None,
839
+ mm_token_counts: dict[str, int] | None = None,
840
) -> ErrorResponse | ChatCompletionResponse:
841
created_time = int(time.time())
842
final_res: RequestOutput | None = None
@@ -1024,13 +1064,11 @@ class OpenAIServingChat(OpenAIServing):
1064
completion_tokens=num_generated_tokens,
1065
total_tokens=num_prompt_tokens + num_generated_tokens,
1066
)
1027
- if (
1028
- self.enable_prompt_tokens_details
1029
- and final_res.num_cached_tokens is not None
1030
- ):
1031
- usage.prompt_tokens_details = PromptTokenUsageInfo(
1032
- cached_tokens=final_res.num_cached_tokens
1033
- )
1067
+ usage.prompt_tokens_details = _make_prompt_tokens_details(
1068
+ self.enable_prompt_tokens_details,
1069
+ final_res.num_cached_tokens,
1070
+ mm_token_counts,
1071
+ )
1072
1073
request_metadata.final_usage_info = usage
1074
vllm/entrypoints/openai/engine/protocol.py
+5
@@ -101,6 +101,11 @@ class ModelList(OpenAIBaseModel):
101
102
class PromptTokenUsageInfo(OpenAIBaseModel):
103
cached_tokens: int | None = None
104
+ multimodal_tokens: dict[str, int] | None = None
105
+ """Prompt tokens contributed by each input modality, keyed by modality name
106
+ (e.g. `image`, `audio`, `video`). A breakdown of the multimodal
107
+ placeholder tokens already counted in `prompt_tokens`; `None` when the
108
+ request has no multimodal input."""
109
110
111
class UsageInfo(OpenAIBaseModel):