A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][FE]: Always include usage with `--enable-force-include-usage ` (#20983)
Signed-off-by: Max Wittig <max.wittig@siemens.com> Signed-off-by: Antoine Auger <antoineauger@users.noreply.github.com> Co-authored-by: Antoine Auger <antoineauger@users.noreply.github.com>
Max Wittig committed
Oct 14, 2025 at 09:17 UTC
fd85c9f4263de8cf9bc9f51bef9471344436614c
11 files changed
+172
-30
pyproject.toml
+1
@@ -107,6 +107,7 @@ markers = [
107
"distributed: run this test only in distributed GPU tests",
108
"skip_v1: do not run this test with v1",
109
"optional: optional tests that are automatically skipped, include --optional to run them",
110
+ "extra_server_args: extra arguments to pass to the server fixture",
111
]
112
113
[tool.ty.src]
tests/entrypoints/openai/test_enable_force_include_usage.py
new
+126
@@ -0,0 +1,126 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import openai
4
+import pytest
5
+import pytest_asyncio
6
+
7
+from ...utils import RemoteOpenAIServer
8
+
9
+
10
+@pytest.fixture(scope="module")
11
+def chat_server_with_force_include_usage(request): # noqa: F811
12
+ args = [
13
+ # use half precision for speed and memory savings in CI environment
14
+ "--dtype",
15
+ "bfloat16",
16
+ "--max-model-len",
17
+ "128",
18
+ "--enforce-eager",
19
+ "--max-num-seqs",
20
+ "1",
21
+ "--enable-force-include-usage",
22
+ "--port",
23
+ "55857",
24
+ "--gpu-memory-utilization",
25
+ "0.2",
26
+ ]
27
+
28
+ with RemoteOpenAIServer("Qwen/Qwen3-0.6B", args, auto_port=False) as remote_server:
29
+ yield remote_server
30
+
31
+
32
+@pytest_asyncio.fixture
33
+async def chat_client_with_force_include_usage(chat_server_with_force_include_usage):
34
+ async with chat_server_with_force_include_usage.get_async_client() as async_client:
35
+ yield async_client
36
+
37
+
38
+@pytest.mark.asyncio
39
+async def test_chat_with_enable_force_include_usage(
40
+ chat_client_with_force_include_usage: openai.AsyncOpenAI,
41
+):
42
+ messages = [
43
+ {"role": "system", "content": "You are a helpful assistant."},
44
+ {"role": "user", "content": "What is the capital of France?"},
45
+ ]
46
+
47
+ stream = await chat_client_with_force_include_usage.chat.completions.create(
48
+ model="Qwen/Qwen3-0.6B",
49
+ messages=messages,
50
+ max_completion_tokens=10,
51
+ extra_body=dict(min_tokens=10),
52
+ temperature=0.0,
53
+ stream=True,
54
+ )
55
+ last_completion_tokens = 0
56
+ async for chunk in stream:
57
+ if not len(chunk.choices):
58
+ assert chunk.usage.prompt_tokens >= 0
59
+ assert (
60
+ last_completion_tokens == 0
61
+ or chunk.usage.completion_tokens > last_completion_tokens
62
+ or (
63
+ not chunk.choices
64
+ and chunk.usage.completion_tokens == last_completion_tokens
65
+ )
66
+ )
67
+ assert chunk.usage.total_tokens == (
68
+ chunk.usage.prompt_tokens + chunk.usage.completion_tokens
69
+ )
70
+ else:
71
+ assert chunk.usage is None
72
+
73
+
74
+@pytest.fixture(scope="module")
75
+def transcription_server_with_force_include_usage():
76
+ args = [
77
+ # use half precision for speed and memory savings in CI environment
78
+ "--dtype",
79
+ "bfloat16",
80
+ "--max-num-seqs",
81
+ "1",
82
+ "--enforce-eager",
83
+ "--enable-force-include-usage",
84
+ "--gpu-memory-utilization",
85
+ "0.2",
86
+ ]
87
+
88
+ with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
89
+ yield remote_server
90
+
91
+
92
+@pytest_asyncio.fixture
93
+async def transcription_client_with_force_include_usage(
94
+ transcription_server_with_force_include_usage,
95
+):
96
+ async with (
97
+ transcription_server_with_force_include_usage.get_async_client() as async_client
98
+ ):
99
+ yield async_client
100
+
101
+
102
+@pytest.mark.asyncio
103
+async def test_transcription_with_enable_force_include_usage(
104
+ transcription_client_with_force_include_usage, winning_call
105
+):
106
+ res = (
107
+ await transcription_client_with_force_include_usage.audio.transcriptions.create(
108
+ model="openai/whisper-large-v3-turbo",
109
+ file=winning_call,
110
+ language="en",
111
+ temperature=0.0,
112
+ stream=True,
113
+ timeout=30,
114
+ )
115
+ )
116
+
117
+ async for chunk in res:
118
+ if not len(chunk.choices):
119
+ # final usage sent
120
+ usage = chunk.usage
121
+ assert isinstance(usage, dict)
122
+ assert usage["prompt_tokens"] > 0
123
+ assert usage["completion_tokens"] > 0
124
+ assert usage["total_tokens"] > 0
125
+ else:
126
+ assert not hasattr(chunk, "usage")
vllm/entrypoints/openai/api_server.py
+2
@@ -1808,6 +1808,7 @@ async def init_app_state(
1808
state.openai_serving_models,
1809
request_logger=request_logger,
1810
log_error_stack=args.log_error_stack,
1811
+ enable_force_include_usage=args.enable_force_include_usage,
1812
)
1813
if "transcription" in supported_tasks
1814
else None
@@ -1818,6 +1819,7 @@ async def init_app_state(
1819
state.openai_serving_models,
1820
request_logger=request_logger,
1821
log_error_stack=args.log_error_stack,
1822
+ enable_force_include_usage=args.enable_force_include_usage,
1823
)
1824
if "transcription" in supported_tasks
1825
else None
vllm/entrypoints/openai/run_batch.py
+8
@@ -104,6 +104,13 @@ def make_arg_parser(parser: FlexibleArgumentParser):
104
default=False,
105
help="If set to True, enable prompt_tokens_details in usage.",
106
)
107
+ parser.add_argument(
108
+ "--enable-force-include-usage",
109
+ action="store_true",
110
+ default=False,
111
+ help="If set to True, include usage on every request "
112
+ "(even when stream_options is not specified)",
113
+ )
114
115
return parser
116
@@ -361,6 +368,7 @@ async def run_batch(
368
chat_template=None,
369
chat_template_content_format="auto",
370
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
371
+ enable_force_include_usage=args.enable_force_include_usage,
372
)
373
if "generate" in supported_tasks
374
else None
vllm/entrypoints/openai/serving_chat.py
+4
-11
@@ -58,7 +58,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_l
58
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
59
from vllm.entrypoints.openai.tool_parsers import ToolParser
60
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
61
-from vllm.entrypoints.utils import get_max_tokens
61
+from vllm.entrypoints.utils import get_max_tokens, should_include_usage
62
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
63
from vllm.logger import init_logger
64
from vllm.logprobs import Logprob
@@ -101,7 +101,6 @@ class OpenAIServingChat(OpenAIServing):
101
models=models,
102
request_logger=request_logger,
103
return_tokens_as_token_ids=return_tokens_as_token_ids,
104
- enable_force_include_usage=enable_force_include_usage,
104
log_error_stack=log_error_stack,
105
)
106
@@ -352,7 +351,6 @@ class OpenAIServingChat(OpenAIServing):
351
conversation,
352
tokenizer,
353
request_metadata,
355
- enable_force_include_usage=self.enable_force_include_usage,
354
)
355
356
try:
@@ -518,7 +516,6 @@ class OpenAIServingChat(OpenAIServing):
516
conversation: list[ConversationMessage],
517
tokenizer: AnyTokenizer,
518
request_metadata: RequestResponseMetadata,
521
- enable_force_include_usage: bool,
519
) -> AsyncGenerator[str, None]:
520
created_time = int(time.time())
521
chunk_object_type: Final = "chat.completion.chunk"
@@ -596,13 +593,9 @@ class OpenAIServingChat(OpenAIServing):
593
return
594
595
stream_options = request.stream_options
599
- if stream_options:
600
- include_usage = stream_options.include_usage or enable_force_include_usage
601
- include_continuous_usage = (
602
- include_usage and stream_options.continuous_usage_stats
603
- )
604
- else:
605
- include_usage, include_continuous_usage = False, False
596
+ include_usage, include_continuous_usage = should_include_usage(
597
+ stream_options, self.enable_force_include_usage
598
+ )
599
600
try:
601
async for res in result_generator:
vllm/entrypoints/openai/serving_completion.py
+5
-11
@@ -27,7 +27,7 @@ from vllm.entrypoints.openai.protocol import (
27
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
28
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
29
from vllm.entrypoints.renderer import RenderConfig
30
-from vllm.entrypoints.utils import get_max_tokens
30
+from vllm.entrypoints.utils import get_max_tokens, should_include_usage
31
from vllm.inputs.data import EmbedsPrompt, TokensPrompt, is_embeds_prompt
32
from vllm.logger import init_logger
33
from vllm.logprobs import Logprob
@@ -56,11 +56,11 @@ class OpenAIServingCompletion(OpenAIServing):
56
models=models,
57
request_logger=request_logger,
58
return_tokens_as_token_ids=return_tokens_as_token_ids,
59
- enable_force_include_usage=enable_force_include_usage,
59
log_error_stack=log_error_stack,
60
)
61
self.enable_prompt_tokens_details = enable_prompt_tokens_details
62
self.default_sampling_params = self.model_config.get_diff_sampling_param()
63
+ self.enable_force_include_usage = enable_force_include_usage
64
if self.default_sampling_params:
65
source = self.model_config.generation_config
66
source = "model" if source == "auto" else source
@@ -256,7 +256,6 @@ class OpenAIServingCompletion(OpenAIServing):
256
num_prompts=num_prompts,
257
tokenizer=tokenizer,
258
request_metadata=request_metadata,
259
- enable_force_include_usage=self.enable_force_include_usage,
259
)
260
261
# Non-streaming response
@@ -320,7 +319,6 @@ class OpenAIServingCompletion(OpenAIServing):
319
num_prompts: int,
320
tokenizer: AnyTokenizer,
321
request_metadata: RequestResponseMetadata,
323
- enable_force_include_usage: bool,
322
) -> AsyncGenerator[str, None]:
323
num_choices = 1 if request.n is None else request.n
324
previous_text_lens = [0] * num_choices * num_prompts
@@ -331,13 +329,9 @@ class OpenAIServingCompletion(OpenAIServing):
329
first_iteration = True
330
331
stream_options = request.stream_options
334
- if stream_options:
335
- include_usage = stream_options.include_usage or enable_force_include_usage
336
- include_continuous_usage = (
337
- include_usage and stream_options.continuous_usage_stats
338
- )
339
- else:
340
- include_usage, include_continuous_usage = False, False
332
+ include_usage, include_continuous_usage = should_include_usage(
333
+ stream_options, self.enable_force_include_usage
334
+ )
335
336
try:
337
async for prompt_idx, res in result_generator:
vllm/entrypoints/openai/serving_engine.py
-3
@@ -249,7 +249,6 @@ class OpenAIServing:
249
*,
250
request_logger: RequestLogger | None,
251
return_tokens_as_token_ids: bool = False,
252
- enable_force_include_usage: bool = False,
252
log_error_stack: bool = False,
253
):
254
super().__init__()
@@ -260,8 +259,6 @@ class OpenAIServing:
259
260
self.request_logger = request_logger
261
self.return_tokens_as_token_ids = return_tokens_as_token_ids
263
- self.enable_force_include_usage = enable_force_include_usage
264
-
262
self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
263
self._apply_mistral_chat_template_async = make_async(
264
apply_mistral_chat_template, executor=self._tokenizer_executor
vllm/entrypoints/openai/serving_responses.py
-1
@@ -127,7 +127,6 @@ class OpenAIServingResponses(OpenAIServing):
127
models=models,
128
request_logger=request_logger,
129
return_tokens_as_token_ids=return_tokens_as_token_ids,
130
- enable_force_include_usage=enable_force_include_usage,
130
log_error_stack=log_error_stack,
131
)
132
vllm/entrypoints/openai/serving_transcription.py
+4
@@ -37,6 +37,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
37
request_logger: RequestLogger | None,
38
return_tokens_as_token_ids: bool = False,
39
log_error_stack: bool = False,
40
+ enable_force_include_usage: bool = False,
41
):
42
super().__init__(
43
engine_client=engine_client,
@@ -45,6 +46,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
46
return_tokens_as_token_ids=return_tokens_as_token_ids,
47
task_type="transcribe",
48
log_error_stack=log_error_stack,
49
+ enable_force_include_usage=enable_force_include_usage,
50
)
51
52
async def create_transcription(
@@ -96,6 +98,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
98
request_logger: RequestLogger | None,
99
return_tokens_as_token_ids: bool = False,
100
log_error_stack: bool = False,
101
+ enable_force_include_usage: bool = False,
102
):
103
super().__init__(
104
engine_client=engine_client,
@@ -104,6 +107,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
107
return_tokens_as_token_ids=return_tokens_as_token_ids,
108
task_type="translate",
109
log_error_stack=log_error_stack,
110
+ enable_force_include_usage=enable_force_include_usage,
111
)
112
113
async def create_translation(
vllm/entrypoints/openai/speech_to_text.py
+4
-3
@@ -58,6 +58,7 @@ class OpenAISpeechToText(OpenAIServing):
58
return_tokens_as_token_ids: bool = False,
59
task_type: Literal["transcribe", "translate"] = "transcribe",
60
log_error_stack: bool = False,
61
+ enable_force_include_usage: bool = False,
62
):
63
super().__init__(
64
engine_client=engine_client,
@@ -74,6 +75,8 @@ class OpenAISpeechToText(OpenAIServing):
75
self.model_config, task_type
76
)
77
78
+ self.enable_force_include_usage = enable_force_include_usage
79
+
80
self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
81
82
if self.default_sampling_params:
@@ -261,9 +264,7 @@ class OpenAISpeechToText(OpenAIServing):
264
completion_tokens = 0
265
num_prompt_tokens = 0
266
264
- include_usage = (
265
- request.stream_include_usage if request.stream_include_usage else False
266
- )
267
+ include_usage = self.enable_force_include_usage or request.stream_include_usage
268
include_continuous_usage = (
269
request.stream_continuous_usage_stats
270
if include_usage and request.stream_continuous_usage_stats
vllm/entrypoints/utils.py
+18
-1
@@ -14,7 +14,11 @@ from starlette.background import BackgroundTask, BackgroundTasks
14
15
from vllm.engine.arg_utils import EngineArgs
16
from vllm.entrypoints.openai.cli_args import make_arg_parser
17
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, CompletionRequest
17
+from vllm.entrypoints.openai.protocol import (
18
+ ChatCompletionRequest,
19
+ CompletionRequest,
20
+ StreamOptions,
21
+)
22
from vllm.logger import init_logger
23
from vllm.platforms import current_platform
24
from vllm.utils import FlexibleArgumentParser
@@ -237,3 +241,16 @@ def log_non_default_args(args: Namespace | EngineArgs):
241
)
242
243
logger.info("non-default args: %s", non_default_args)
244
+
245
+
246
+def should_include_usage(
247
+ stream_options: StreamOptions | None, enable_force_include_usage: bool
248
+) -> tuple[bool, bool]:
249
+ if stream_options:
250
+ include_usage = stream_options.include_usage or enable_force_include_usage
251
+ include_continuous_usage = include_usage and bool(
252
+ stream_options.continuous_usage_stats
253
+ )
254
+ else:
255
+ include_usage, include_continuous_usage = enable_force_include_usage, False
256
+ return include_usage, include_continuous_usage