A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 15:00 UTC
Add system_fingerprint field to OpenAI-compatible API responses (#40537)
Co-authored-by: Claude <noreply@anthropic.com>
Simon Mo committed
Apr 27, 2026 at 01:17 UTC
ebf862c351dc4bcaf65de34c3caebe6df6e9e214
10 files changed
+225
-2
tests/entrypoints/openai/test_fingerprint.py
new
+76
@@ -0,0 +1,76 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Unit tests for ``system_fingerprint`` construction."""
4
+
5
+from types import SimpleNamespace
6
+
7
+import pytest
8
+
9
+from vllm.entrypoints.openai import fingerprint as fp
10
+
11
+
12
+def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
13
+ c = SimpleNamespace(
14
+ parallel_config=SimpleNamespace(
15
+ tensor_parallel_size=tp,
16
+ pipeline_parallel_size=pp,
17
+ data_parallel_size=dp,
18
+ enable_expert_parallel=ep,
19
+ )
20
+ )
21
+ c.compute_hash = lambda: digest # type: ignore[attr-defined]
22
+ return c
23
+
24
+
25
+@pytest.fixture(autouse=True)
26
+def _reset():
27
+ fp.set_default_fingerprint_mode("full")
28
+ yield
29
+ fp.set_default_fingerprint_mode("full")
30
+
31
+
32
+def test_four_modes_produce_expected_shapes():
33
+ from vllm import __version__ as v
34
+
35
+ cfg = _cfg(tp=8, ep=True)
36
+
37
+ assert fp.build_system_fingerprint(cfg, "full") == (f"vllm-{v}-tp8-ep-a3b21f94")
38
+ assert fp.build_system_fingerprint(cfg, "hash") == f"vllm-{v}-a3b21f94"
39
+ assert fp.build_system_fingerprint(cfg, "custom", "my-fp") == "my-fp"
40
+ assert fp.build_system_fingerprint(cfg, "none") is None
41
+
42
+
43
+def test_full_mode_emits_only_non_trivial_parallelism():
44
+ from vllm import __version__ as v
45
+
46
+ # Single-GPU: nothing between version and hash.
47
+ assert fp.build_system_fingerprint(_cfg(), "full") == f"vllm-{v}-a3b21f94"
48
+ # All parallelism axes.
49
+ assert (
50
+ fp.build_system_fingerprint(_cfg(tp=8, pp=2, dp=4, ep=True), "full")
51
+ == f"vllm-{v}-tp8-pp2-dp4-ep-a3b21f94"
52
+ )
53
+
54
+
55
+def test_get_respects_set_default():
56
+ cfg = _cfg(tp=8)
57
+ full = fp.get_system_fingerprint(cfg)
58
+ assert full == fp.get_system_fingerprint(cfg)
59
+
60
+ fp.set_default_fingerprint_mode("hash")
61
+ hashed = fp.get_system_fingerprint(cfg)
62
+ assert hashed != full
63
+ assert "tp8" not in hashed
64
+
65
+ fp.set_default_fingerprint_mode("custom", "deploy-42")
66
+ assert fp.get_system_fingerprint(cfg) == "deploy-42"
67
+
68
+ fp.set_default_fingerprint_mode("none")
69
+ assert fp.get_system_fingerprint(cfg) is None
70
+
71
+
72
+def test_compute_hash_failure_does_not_raise():
73
+ cfg = _cfg()
74
+ cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom"))
75
+ assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash")
76
+ assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash")
vllm/entrypoints/openai/chat_completion/batch_serving.py
+1
@@ -317,4 +317,5 @@ class OpenAIServingChatBatch(OpenAIServingChat):
317
model=model_name,
318
choices=choices,
319
usage=usage,
320
+ system_fingerprint=self.system_fingerprint,
321
)
vllm/entrypoints/openai/chat_completion/protocol.py
+3
@@ -129,6 +129,9 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
129
model: str
130
choices: list[ChatCompletionResponseStreamChoice]
131
usage: UsageInfo | None = Field(default=None)
132
+ # Set only on the final chunk of a stream to mirror non-streaming responses
133
+ # without the per-chunk serialization overhead.
134
+ system_fingerprint: str | None = None
135
# not part of the OpenAI spec but for tracing the tokens
136
prompt_token_ids: list[int] | None = None
137
vllm/entrypoints/openai/chat_completion/serving.py
+12
@@ -1195,6 +1195,16 @@ class OpenAIServingChat(OpenAIServing):
1195
choices=[choice_data],
1196
model=model_name,
1197
)
1198
+ # Stamp the fingerprint on terminal chunks only (those with
1199
+ # finish_reason set). When ``include_usage`` is on, the
1200
+ # trailing usage chunk below overrides this as the true
1201
+ # final message.
1202
+ if (
1203
+ not include_usage
1204
+ and self.system_fingerprint is not None
1205
+ and choice_data.finish_reason is not None
1206
+ ):
1207
+ chunk.system_fingerprint = self.system_fingerprint
1208
1209
# handle usage stats if requested & if continuous
1210
if include_continuous_usage:
@@ -1229,6 +1239,7 @@ class OpenAIServingChat(OpenAIServing):
1239
choices=[],
1240
model=model_name,
1241
usage=final_usage,
1242
+ system_fingerprint=self.system_fingerprint,
1243
)
1244
final_usage_data = final_usage_chunk.model_dump_json(
1245
exclude_unset=True, exclude_none=True
@@ -1637,6 +1648,7 @@ class OpenAIServingChat(OpenAIServing):
1648
model=model_name,
1649
choices=choices,
1650
usage=usage,
1651
+ system_fingerprint=self.system_fingerprint,
1652
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1653
prompt_token_ids=(
1654
final_res.prompt_token_ids if request.return_token_ids else None
vllm/entrypoints/openai/cli_args.py
+13
-1
@@ -153,9 +153,21 @@ class BaseFrontendArgs:
153
"""If set to True, log the stack trace of error responses"""
154
tokens_only: bool = False
155
"""
156
- If set to True, only enable the Tokens In<>Out endpoint.
156
+ If set to True, only enable the Tokens In<>Out endpoint.
157
This is intended for use in a Disaggregated Everything setup.
158
"""
159
+ fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
160
+ """Controls the ``system_fingerprint`` field on responses.
161
+
162
+ - ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
163
+ server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
164
+ 8-char config hash.
165
+ - ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
166
+ - ``custom``: emits the literal string from ``--fingerprint-value``.
167
+ - ``none``: the field is omitted (serialized as ``null``).
168
+ """
169
+ fingerprint_value: str | None = None
170
+ """Literal fingerprint string used when ``--fingerprint-mode=custom``."""
171
172
@classmethod
173
def _customize_cli_kwargs(
vllm/entrypoints/openai/completion/protocol.py
+3
@@ -512,3 +512,6 @@ class CompletionStreamResponse(OpenAIBaseModel):
512
model: str
513
choices: list[CompletionResponseStreamChoice]
514
usage: UsageInfo | None = Field(default=None)
515
+ # Set only on the final chunk of a stream to mirror non-streaming responses
516
+ # without the per-chunk serialization overhead.
517
+ system_fingerprint: str | None = None
vllm/entrypoints/openai/completion/serving.py
+12
-1
@@ -383,6 +383,7 @@ class OpenAIServingCompletion(OpenAIServing):
383
384
chunk = CompletionStreamResponse(
385
id=request_id,
386
+ object="text_completion",
387
created=created_time,
388
model=model_name,
389
choices=[
@@ -401,6 +402,14 @@ class OpenAIServingCompletion(OpenAIServing):
402
)
403
],
404
)
405
+ # Stamp on terminal chunk only when no trailing usage chunk
406
+ # will follow (that one is the true final message).
407
+ if (
408
+ not include_usage
409
+ and self.system_fingerprint is not None
410
+ and finish_reason is not None
411
+ ):
412
+ chunk.system_fingerprint = self.system_fingerprint
413
if include_continuous_usage:
414
prompt_tokens = num_prompt_tokens[prompt_idx]
415
completion_tokens = previous_num_tokens[i]
@@ -410,7 +419,7 @@ class OpenAIServingCompletion(OpenAIServing):
419
total_tokens=prompt_tokens + completion_tokens,
420
)
421
413
- response_json = chunk.model_dump_json(exclude_unset=False)
422
+ response_json = chunk.model_dump_json(exclude_unset=True)
423
yield f"data: {response_json}\n\n"
424
425
total_prompt_tokens = sum(num_prompt_tokens)
@@ -433,6 +442,7 @@ class OpenAIServingCompletion(OpenAIServing):
442
model=model_name,
443
choices=[],
444
usage=final_usage_info,
445
+ system_fingerprint=self.system_fingerprint,
446
)
447
final_usage_data = final_usage_chunk.model_dump_json(
448
exclude_unset=False, exclude_none=True
@@ -562,6 +572,7 @@ class OpenAIServingCompletion(OpenAIServing):
572
model=model_name,
573
choices=choices,
574
usage=usage,
575
+ system_fingerprint=self.system_fingerprint,
576
kv_transfer_params=kv_transfer_params,
577
)
578
vllm/entrypoints/openai/engine/serving.py
+13
@@ -157,6 +157,19 @@ class OpenAIServing:
157
self.renderer = engine_client.renderer
158
self.input_processor = engine_client.input_processor
159
160
+ # Computed once at startup (cached by ``vllm_config`` identity) and
161
+ # stamped on non-streaming responses. Streaming chunks deliberately
162
+ # omit it to avoid per-chunk overhead.
163
+ from vllm.entrypoints.openai.fingerprint import get_system_fingerprint
164
+
165
+ try:
166
+ self.system_fingerprint: str | None = get_system_fingerprint(
167
+ engine_client.vllm_config
168
+ )
169
+ except Exception:
170
+ # Never fail server startup over the fingerprint.
171
+ self.system_fingerprint = None
172
+
173
async def beam_search(
174
self,
175
prompt: EngineInput,
vllm/entrypoints/openai/fingerprint.py
new
+84
@@ -0,0 +1,84 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Build the ``system_fingerprint`` string returned by the OpenAI-compatible
4
+server.
5
+
6
+Four modes, configured via ``--fingerprint-mode``:
7
+
8
+* ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>`` — encodes
9
+ server version, any non-trivial parallelism degree (tp/pp/dp/ep), and an
10
+ 8-char prefix of ``vllm_config.compute_hash()`` (covers model identity,
11
+ quant config, speculative, attention backend, etc.).
12
+* ``hash``: ``vllm-<version>-<hash8>`` — parallelism stripped.
13
+* ``custom``: user-provided literal via ``--fingerprint-value``.
14
+* ``none``: the field is omitted (serialized as ``null``).
15
+
16
+``get_system_fingerprint`` is only called at serving-class init (a handful
17
+of times per server); each subclass caches the returned string on
18
+``self.system_fingerprint``, so per-request cost is one attribute read.
19
+"""
20
+
21
+from __future__ import annotations
22
+
23
+from typing import Any, Literal
24
+
25
+FingerprintMode = Literal["full", "hash", "custom", "none"]
26
+
27
+_DEFAULT_MODE: FingerprintMode = "full"
28
+_CUSTOM_VALUE: str | None = None
29
+
30
+
31
+def set_default_fingerprint_mode(
32
+ mode: FingerprintMode,
33
+ custom_value: str | None = None,
34
+) -> None:
35
+ """Configure the fingerprint mode for subsequent ``get_system_fingerprint``
36
+ calls. Called once at server startup."""
37
+ global _DEFAULT_MODE, _CUSTOM_VALUE
38
+ _DEFAULT_MODE = mode
39
+ _CUSTOM_VALUE = custom_value
40
+
41
+
42
+def get_system_fingerprint(vllm_config: Any) -> str | None:
43
+ """Return the fingerprint for ``vllm_config`` using the mode configured by
44
+ ``set_default_fingerprint_mode``."""
45
+ return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE)
46
+
47
+
48
+def build_system_fingerprint(
49
+ vllm_config: Any,
50
+ mode: FingerprintMode = "full",
51
+ custom_value: str | None = None,
52
+) -> str | None:
53
+ if mode == "none":
54
+ return None
55
+ if mode == "custom":
56
+ return custom_value
57
+
58
+ from vllm import __version__ as vllm_version
59
+
60
+ try:
61
+ hash8 = vllm_config.compute_hash()[:8]
62
+ except Exception:
63
+ hash8 = "nohash"
64
+
65
+ if mode == "hash":
66
+ return f"vllm-{vllm_version}-{hash8}"
67
+
68
+ # mode == "full"
69
+ parts: list[str] = [f"vllm-{vllm_version}"]
70
+ pc = getattr(vllm_config, "parallel_config", None)
71
+ if pc is not None:
72
+ tp = getattr(pc, "tensor_parallel_size", 1)
73
+ if tp > 1:
74
+ parts.append(f"tp{tp}")
75
+ pp = getattr(pc, "pipeline_parallel_size", 1)
76
+ if pp > 1:
77
+ parts.append(f"pp{pp}")
78
+ dp = getattr(pc, "data_parallel_size", 1)
79
+ if dp > 1:
80
+ parts.append(f"dp{dp}")
81
+ if getattr(pc, "enable_expert_parallel", False):
82
+ parts.append("ep")
83
+ parts.append(hash8)
84
+ return "-".join(parts)
vllm/entrypoints/openai/generate/api_router.py
+8
@@ -61,9 +61,17 @@ async def init_generate_state(
61
)
62
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
63
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
64
+ from vllm.entrypoints.openai.fingerprint import set_default_fingerprint_mode
65
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
66
from vllm.entrypoints.serve.disagg.serving import ServingTokens
67
68
+ # Applied before any serving class is constructed so that each one picks
69
+ # up the chosen mode on its first cache miss.
70
+ set_default_fingerprint_mode(
71
+ getattr(args, "fingerprint_mode", "full"),
72
+ getattr(args, "fingerprint_value", None),
73
+ )
74
+
75
if args.tool_server == "demo":
76
tool_server: ToolServer | None = DemoToolServer()
77
assert isinstance(tool_server, DemoToolServer)