A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 22:45 UTC
[BugFix][Spec Decoding] Fix negative accepted tokens metric crash (#33729)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Feb 3, 2026 at 15:34 UTC
52ee21021a87735d46c4245c60bc0be42dd58c73
2 files changed
+61
-1
tests/v1/core/test_scheduler.py
+60
@@ -885,6 +885,66 @@ def test_schedule_spec_decoding_stats(spec_tokens, output_tokens, expected):
885
assert stats.num_accepted_tokens_per_pos == expected[3]
886
887
888
+def test_spec_decoding_stats_empty_output():
889
+ """Test that spec decoding stats handle empty output tokens gracefully.
890
+
891
+ This is a regression test for a bug where empty sampled_token_ids
892
+ would cause num_accepted = len([]) - 1 = -1, leading to a
893
+ ValueError when incrementing a Prometheus counter with a negative value.
894
+ """
895
+ num_spec_tokens = 3
896
+ scheduler = create_scheduler(num_speculative_tokens=num_spec_tokens)
897
+ requests = create_requests(num_requests=1, num_tokens=1)
898
+ request = requests[0]
899
+ req_id = request.request_id
900
+
901
+ scheduler.add_request(request)
902
+
903
+ # Initial schedule (prefill)
904
+ output = scheduler.schedule()
905
+ assert len(output.scheduled_new_reqs) == 1
906
+
907
+ # Complete the prefill with a sampled token
908
+ model_runner_output = ModelRunnerOutput(
909
+ req_ids=[req_id],
910
+ req_id_to_index={req_id: 0},
911
+ sampled_token_ids=[[0]],
912
+ logprobs=None,
913
+ prompt_logprobs_dict={},
914
+ pooler_output=[],
915
+ )
916
+ scheduler.update_from_output(output, model_runner_output)
917
+
918
+ # Add draft tokens for speculation
919
+ draft_token_ids = DraftTokenIds([req_id], [[1, 2, 3]])
920
+ scheduler.update_draft_token_ids(draft_token_ids)
921
+
922
+ # Schedule the speculated tokens for validation
923
+ output = scheduler.schedule()
924
+ assert req_id in output.scheduled_spec_decode_tokens
925
+ assert len(output.scheduled_spec_decode_tokens[req_id]) == 3
926
+
927
+ # Simulate empty output tokens (e.g., due to request abortion or error)
928
+ # This would previously cause num_accepted = -1 and crash
929
+ model_runner_output = ModelRunnerOutput(
930
+ req_ids=[req_id],
931
+ req_id_to_index={req_id: 0},
932
+ sampled_token_ids=[[]], # Empty output tokens
933
+ logprobs=None,
934
+ prompt_logprobs_dict={},
935
+ pooler_output=[],
936
+ )
937
+
938
+ # This should not raise an error
939
+ engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
940
+
941
+ # Spec decoding stats should be None since no tokens were generated
942
+ scheduler_stats = (
943
+ engine_core_outputs[0].scheduler_stats if engine_core_outputs else None
944
+ )
945
+ assert scheduler_stats is None or scheduler_stats.spec_decoding_stats is None
946
+
947
+
948
def _assert_right_scheduler_output(
949
output: SchedulerOutput,
950
num_requests: int,
vllm/v1/core/sched/scheduler.py
+1
-1
@@ -1276,7 +1276,7 @@ class Scheduler(SchedulerInterface):
1276
scheduled_spec_token_ids = (
1277
scheduler_output.scheduled_spec_decode_tokens.get(req_id)
1278
)
1279
- if scheduled_spec_token_ids:
1279
+ if scheduled_spec_token_ids and generated_token_ids:
1280
num_draft_tokens = len(scheduled_spec_token_ids)
1281
num_accepted = len(generated_token_ids) - 1
1282
num_rejected = num_draft_tokens - num_accepted