A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[V1][Spec Decoding] Include bonus tokens in mean acceptance length (#17908)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
May 9, 2025 at 21:32 UTC
7e3571134fa8385f4795c7a1c2a40f3b2859a22c
2 files changed
+9
-5
examples/offline_inference/eagle.py
+2
-2
@@ -118,8 +118,8 @@ def main():
118
acceptance_counts[step] += count
119
120
print("-" * 50)
121
- print(f"mean acceptance length: \
122
- {sum(acceptance_counts) / acceptance_counts[0]:.2f}")
121
+ print(f"mean acceptance length (including bonus tokens): \
122
+ {1 + (sum(acceptance_counts) / acceptance_counts[0]):.2f}")
123
print("-" * 50)
124
125
# print acceptance at each token position
vllm/v1/spec_decode/metrics.py
+7
-3
@@ -73,7 +73,9 @@ class SpecDecodingLogging:
73
74
draft_acceptance_rate = (num_accepted_tokens / num_draft_tokens *
75
100 if num_draft_tokens > 0 else float("nan"))
76
- mean_acceptance_length = (num_accepted_tokens / num_drafts)
76
+
77
+ # Conventionally, mean acceptance length includes the bonus token
78
+ mean_acceptance_length = 1 + (num_accepted_tokens / num_drafts)
79
80
pos_matrix = np.array(self.accepted_tokens_per_pos_lists)
81
acceptance_rates = np.sum(pos_matrix, axis=0) / num_drafts
@@ -103,10 +105,12 @@ class SpecDecodingProm:
105
rate(vllm:spec_decode_num_accepted_tokens_total[$interval]) /
106
rate(vllm:spec_decode_num_draft_tokens_total[$interval])
107
106
- The mean acceptance length can be calculated using:
108
+ The mean acceptance length (conventionally including bonus tokens)
109
+ can be calculated using:
110
111
+ 1 + (
112
rate(vllm:spec_decode_num_accepted_tokens_total[$interval]) /
109
- rate(vllm:spec_decode_num_drafts[$interval])
113
+ rate(vllm:spec_decode_num_drafts[$interval]))
114
115
A per-position acceptance rate vector can be computed using
116