A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 16:30 UTC
[Spec Decode][UX] Add acceptance stats to `vllm bench serve` report (#31739)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
Matthew Bonanni committed
Jan 6, 2026 at 16:21 UTC
d49899732edd3e6c011ec9f922601d919250a4d2
1 file changed
+155
vllm/benchmarks/serve.py
+155
@@ -86,6 +86,76 @@ async def get_first_model_from_server(
86
) from e
87
88
89
+@dataclass
90
+class SpecDecodeMetrics:
91
+ """Speculative decoding metrics from the server's Prometheus endpoint."""
92
+
93
+ num_drafts: int
94
+ num_draft_tokens: int
95
+ num_accepted_tokens: int
96
+ accepted_per_pos: dict[int, int]
97
+
98
+
99
+async def fetch_spec_decode_metrics(
100
+ base_url: str, session: aiohttp.ClientSession
101
+) -> SpecDecodeMetrics | None:
102
+ """Fetch speculative decoding metrics from the server's Prometheus endpoint.
103
+
104
+ Returns None if speculative decoding is not enabled or metrics are not available.
105
+ """
106
+ metrics_url = f"{base_url}/metrics"
107
+ try:
108
+ async with session.get(metrics_url) as response:
109
+ if response.status != 200:
110
+ return None
111
+ text = await response.text()
112
+
113
+ num_drafts = 0
114
+ num_draft_tokens = 0
115
+ num_accepted_tokens = 0
116
+ accepted_per_pos: dict[int, int] = {}
117
+ found_spec_decode = False
118
+
119
+ for line in text.split("\n"):
120
+ line = line.strip()
121
+ if not line or line.startswith("#"):
122
+ continue
123
+
124
+ if line.startswith("vllm:spec_decode"):
125
+ found_spec_decode = True
126
+ parts = line.split()
127
+ if parts:
128
+ with contextlib.suppress(ValueError):
129
+ if "num_drafts" in line:
130
+ num_drafts += int(float(parts[-1]))
131
+ elif "num_draft_tokens" in line:
132
+ num_draft_tokens += int(float(parts[-1]))
133
+ elif "num_accepted_tokens_per_pos" in line:
134
+ pos_label = 'position="'
135
+ if pos_label in line:
136
+ start = line.index(pos_label) + len(pos_label)
137
+ end = line.index('"', start)
138
+ pos = int(line[start:end])
139
+ val = int(float(parts[-1]))
140
+ accepted_per_pos[pos] = (
141
+ accepted_per_pos.get(pos, 0) + val
142
+ )
143
+ elif "num_accepted_tokens" in line:
144
+ num_accepted_tokens += int(float(parts[-1]))
145
+
146
+ if not found_spec_decode:
147
+ return None
148
+
149
+ return SpecDecodeMetrics(
150
+ num_drafts=num_drafts,
151
+ num_draft_tokens=num_draft_tokens,
152
+ num_accepted_tokens=num_accepted_tokens,
153
+ accepted_per_pos=accepted_per_pos,
154
+ )
155
+ except (aiohttp.ClientError, asyncio.TimeoutError):
156
+ return None
157
+
158
+
159
class TaskType(Enum):
160
GENERATION = "generation"
161
POOLING = "pooling"
@@ -685,6 +755,8 @@ async def benchmark(
755
print(f"Burstiness factor: {burstiness} ({distribution})")
756
print(f"Maximum request concurrency: {max_concurrency}")
757
758
+ spec_decode_metrics_before = await fetch_spec_decode_metrics(base_url, session)
759
+
760
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
761
762
semaphore = (
@@ -768,6 +840,48 @@ async def benchmark(
840
841
benchmark_duration = time.perf_counter() - benchmark_start_time
842
843
+ spec_decode_metrics_after = await fetch_spec_decode_metrics(base_url, session)
844
+ spec_decode_stats: dict[str, Any] | None = None
845
+ if spec_decode_metrics_before is not None and spec_decode_metrics_after is not None:
846
+ delta_drafts = (
847
+ spec_decode_metrics_after.num_drafts - spec_decode_metrics_before.num_drafts
848
+ )
849
+ delta_draft_tokens = (
850
+ spec_decode_metrics_after.num_draft_tokens
851
+ - spec_decode_metrics_before.num_draft_tokens
852
+ )
853
+ delta_accepted = (
854
+ spec_decode_metrics_after.num_accepted_tokens
855
+ - spec_decode_metrics_before.num_accepted_tokens
856
+ )
857
+ per_pos_rates: list[float] = []
858
+ if delta_drafts > 0:
859
+ positions = sorted(
860
+ set(spec_decode_metrics_before.accepted_per_pos.keys())
861
+ | set(spec_decode_metrics_after.accepted_per_pos.keys())
862
+ )
863
+ for pos in positions:
864
+ before_val = spec_decode_metrics_before.accepted_per_pos.get(pos, 0)
865
+ after_val = spec_decode_metrics_after.accepted_per_pos.get(
866
+ pos, before_val
867
+ )
868
+ delta_pos = after_val - before_val
869
+ per_pos_rates.append(delta_pos / delta_drafts)
870
+
871
+ if delta_draft_tokens > 0:
872
+ acceptance_rate = (delta_accepted / delta_draft_tokens) * 100
873
+ acceptance_length = (
874
+ 1 + delta_accepted / delta_drafts if delta_drafts > 0 else 0.0
875
+ )
876
+ spec_decode_stats = {
877
+ "num_drafts": delta_drafts,
878
+ "draft_tokens": delta_draft_tokens,
879
+ "accepted_tokens": delta_accepted,
880
+ "acceptance_rate": acceptance_rate,
881
+ "acceptance_length": acceptance_length,
882
+ "per_position_acceptance_rates": per_pos_rates,
883
+ }
884
+
885
if task_type == TaskType.GENERATION:
886
metrics, actual_output_lens = calculate_metrics(
887
input_requests=input_requests,
@@ -863,6 +977,18 @@ async def benchmark(
977
if rps_change_events:
978
result["rps_change_events"] = rps_change_events
979
980
+ if spec_decode_stats is not None:
981
+ result["spec_decode_acceptance_rate"] = spec_decode_stats["acceptance_rate"]
982
+ result["spec_decode_acceptance_length"] = spec_decode_stats["acceptance_length"]
983
+ result["spec_decode_num_drafts"] = int(spec_decode_stats["num_drafts"])
984
+ result["spec_decode_draft_tokens"] = int(spec_decode_stats["draft_tokens"])
985
+ result["spec_decode_accepted_tokens"] = int(
986
+ spec_decode_stats["accepted_tokens"]
987
+ )
988
+ result["spec_decode_per_position_acceptance_rates"] = spec_decode_stats.get(
989
+ "per_position_acceptance_rates", []
990
+ )
991
+
992
def process_one_metric(
993
# E.g., "ttft"
994
metric_attribute_name: str,
@@ -908,6 +1034,35 @@ async def benchmark(
1034
process_one_metric("itl", "ITL", "Inter-token Latency")
1035
process_one_metric("e2el", "E2EL", "End-to-end Latency")
1036
1037
+ if spec_decode_stats is not None:
1038
+ print("{s:{c}^{n}}".format(s="Speculative Decoding", n=50, c="-"))
1039
+ print(
1040
+ "{:<40} {:<10.2f}".format(
1041
+ "Acceptance rate (%):", spec_decode_stats["acceptance_rate"]
1042
+ )
1043
+ )
1044
+ print(
1045
+ "{:<40} {:<10.2f}".format(
1046
+ "Acceptance length:", spec_decode_stats["acceptance_length"]
1047
+ )
1048
+ )
1049
+ print("{:<40} {:<10}".format("Drafts:", int(spec_decode_stats["num_drafts"])))
1050
+ print(
1051
+ "{:<40} {:<10}".format(
1052
+ "Draft tokens:", int(spec_decode_stats["draft_tokens"])
1053
+ )
1054
+ )
1055
+ print(
1056
+ "{:<40} {:<10}".format(
1057
+ "Accepted tokens:", int(spec_decode_stats["accepted_tokens"])
1058
+ )
1059
+ )
1060
+ per_pos = spec_decode_stats.get("per_position_acceptance_rates", [])
1061
+ if per_pos:
1062
+ print("Per-position acceptance (%):")
1063
+ for i, rate in enumerate(per_pos):
1064
+ print("{:<40} {:<10.2f}".format(f" Position {i}:", rate * 100))
1065
+
1066
print("=" * 50)
1067
1068
if profile: