A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
[Bugfix] Force continuous usage stats when CLI override is enabled (#37923)
Signed-off-by: Your Name <you@example.com> Co-authored-by: Your Name <you@example.com> Co-authored-by: OpenCode <noreply@openai.com>
Dhruv Singal committed
Mar 24, 2026 at 10:29 UTC
4df5fa74396bef3995a6192573f75d58a2976ebc
3 files changed
+41
-17
tests/entrypoints/openai/chat_completion/test_enable_force_include_usage.py
+12
-14
@@ -54,21 +54,19 @@ async def test_chat_with_enable_force_include_usage(
54
)
55
last_completion_tokens = 0
56
async for chunk in stream:
57
- if not len(chunk.choices):
58
- assert chunk.usage.prompt_tokens >= 0
59
- assert (
60
- last_completion_tokens == 0
61
- or chunk.usage.completion_tokens > last_completion_tokens
62
- or (
63
- not chunk.choices
64
- and chunk.usage.completion_tokens == last_completion_tokens
65
- )
66
- )
67
- assert chunk.usage.total_tokens == (
68
- chunk.usage.prompt_tokens + chunk.usage.completion_tokens
57
+ assert chunk.usage.prompt_tokens >= 0
58
+ assert (
59
+ last_completion_tokens == 0
60
+ or chunk.usage.completion_tokens > last_completion_tokens
61
+ or (
62
+ not chunk.choices
63
+ and chunk.usage.completion_tokens == last_completion_tokens
64
)
70
- else:
71
- assert chunk.usage is None
65
+ )
66
+ assert chunk.usage.total_tokens == (
67
+ chunk.usage.prompt_tokens + chunk.usage.completion_tokens
68
+ )
69
+ last_completion_tokens = chunk.usage.completion_tokens
70
71
72
@pytest.fixture(scope="module")
tests/entrypoints/test_utils.py
+25
-1
@@ -3,7 +3,12 @@
3
4
import pytest
5
6
-from vllm.entrypoints.utils import get_max_tokens, sanitize_message
6
+from vllm.entrypoints.openai.engine.protocol import StreamOptions
7
+from vllm.entrypoints.utils import (
8
+ get_max_tokens,
9
+ sanitize_message,
10
+ should_include_usage,
11
+)
12
13
14
def test_sanitize_message():
@@ -13,6 +18,25 @@ def test_sanitize_message():
18
)
19
20
21
+@pytest.mark.parametrize(
22
+ ("stream_options", "expected"),
23
+ [
24
+ (None, (True, True)),
25
+ (StreamOptions(include_usage=False), (True, True)),
26
+ (
27
+ StreamOptions(include_usage=False, continuous_usage_stats=False),
28
+ (True, True),
29
+ ),
30
+ (
31
+ StreamOptions(include_usage=True, continuous_usage_stats=False),
32
+ (True, True),
33
+ ),
34
+ ],
35
+)
36
+def test_should_include_usage_force_enables_continuous_usage(stream_options, expected):
37
+ assert should_include_usage(stream_options, True) == expected
38
+
39
+
40
class TestGetMaxTokens:
41
"""Tests for get_max_tokens() to ensure generation_config's max_tokens
42
acts as a default when from model author, and as a ceiling when
vllm/entrypoints/utils.py
+4
-2
@@ -236,13 +236,15 @@ def log_non_default_args(args: Namespace | EngineArgs):
236
def should_include_usage(
237
stream_options: "StreamOptions | None", enable_force_include_usage: bool
238
) -> tuple[bool, bool]:
239
+ if enable_force_include_usage:
240
+ return True, True
241
if stream_options:
240
- include_usage = stream_options.include_usage or enable_force_include_usage
242
+ include_usage = bool(stream_options.include_usage)
243
include_continuous_usage = include_usage and bool(
244
stream_options.continuous_usage_stats
245
)
246
else:
245
- include_usage, include_continuous_usage = enable_force_include_usage, False
247
+ include_usage, include_continuous_usage = False, False
248
return include_usage, include_continuous_usage
249
250