A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Update Spec Decode metrics to include drafted and accepted token throughput (#24127)
Signed-off-by: Andrew Xia <axia@meta.com>
Andrew Xia committed
Sep 11, 2025 at 12:58 UTC
79ac59f32e5e04b1da606ce4bdad14cadb0075d6
1 file changed
+16
-3
vllm/v1/spec_decode/metrics.py
+16
-3
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import time
5
from dataclasses import dataclass, field
6
from typing import Optional
7
@@ -58,6 +59,7 @@ class SpecDecodingLogging:
59
self.num_draft_tokens: list[int] = []
60
self.num_accepted_tokens: list[int] = []
61
self.accepted_tokens_per_pos_lists: list[list[int]] = []
62
+ self.last_log_time = time.monotonic()
63
64
def observe(self, spec_decoding_stats: SpecDecodingStats):
65
self.num_drafts.append(spec_decoding_stats.num_drafts)
@@ -73,6 +75,13 @@ class SpecDecodingLogging:
75
num_drafts = np.sum(self.num_drafts)
76
num_draft_tokens = np.sum(self.num_draft_tokens)
77
num_accepted_tokens = np.sum(self.num_accepted_tokens)
78
+ draft_throughput = 0
79
+ accepted_throughput = 0
80
+
81
+ elapsed_time = time.monotonic() - self.last_log_time
82
+ if elapsed_time > 0:
83
+ draft_throughput = num_draft_tokens / elapsed_time
84
+ accepted_throughput = num_accepted_tokens / elapsed_time
85
86
draft_acceptance_rate = (num_accepted_tokens / num_draft_tokens *
87
100 if num_draft_tokens > 0 else float("nan"))
@@ -86,16 +95,20 @@ class SpecDecodingLogging:
95
96
log_fn(
97
"SpecDecoding metrics: "
89
- "Draft acceptance rate: %.1f%%, "
98
"Mean acceptance length: %.2f, "
99
+ "Accepted throughput: %.2f tokens/s, "
100
+ "Drafted throughput: %.2f tokens/s, "
101
"Accepted: %d tokens, "
102
"Drafted: %d tokens, "
93
- "Per-position acceptance rate: %s",
94
- draft_acceptance_rate,
103
+ "Per-position acceptance rate: %s, "
104
+ "Avg Draft acceptance rate: %.1f%%",
105
mean_acceptance_length,
106
+ accepted_throughput,
107
+ draft_throughput,
108
num_accepted_tokens,
109
num_draft_tokens,
110
rates_str,
111
+ draft_acceptance_rate,
112
)
113
self.reset()
114