A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Respect min_tokens in scheduler stop check (#26317)
Signed-off-by: Elaine Zhao <elaineyz@amazon.com>
Elaine Zhao committed
Oct 8, 2025 at 14:08 UTC
f08919b7d17f6adf4eea26ca410c4a3de41a71da
2 files changed
+95
tests/v1/core/test_scheduler.py
+90
@@ -497,6 +497,96 @@ def test_stop_via_update_from_output():
497
assert list(requests[0].output_token_ids) == [EOS_TOKEN_ID, 10, 11]
498
499
500
+def test_check_stop_min_tokens():
501
+ """Test that requests don't stop when min_tokens requirement isn't met."""
502
+ from vllm.v1.core.sched.utils import check_stop
503
+
504
+ # Test case 1: num_output_tokens < min_tokens
505
+ # Should return False (don't stop)
506
+ sampling_params = SamplingParams(
507
+ ignore_eos=False,
508
+ max_tokens=20,
509
+ min_tokens=5,
510
+ )
511
+ request = Request(
512
+ request_id="0",
513
+ prompt_token_ids=[0, 1, 2],
514
+ sampling_params=sampling_params,
515
+ pooling_params=None,
516
+ eos_token_id=EOS_TOKEN_ID,
517
+ )
518
+ # Simulate having generated 3 output tokens (less than min_tokens=5)
519
+ request.append_output_token_ids([10, 11, EOS_TOKEN_ID]) # EOS token present
520
+
521
+ result = check_stop(request, max_model_len=100)
522
+ assert result is False, "Should not stop when num_output_tokens<min_tokens"
523
+
524
+ # Test case 2: num_output_tokens >= min_tokens
525
+ # Should follow normal stopping logic (stop on EOS)
526
+ request.append_output_token_ids(
527
+ [
528
+ 10,
529
+ 11,
530
+ 12,
531
+ 13,
532
+ 14,
533
+ EOS_TOKEN_ID,
534
+ ]
535
+ ) # 6 tokens > min_tokens
536
+
537
+ result = check_stop(request, max_model_len=100)
538
+ assert result is True, "Should stop on EOS when min_tokens met"
539
+ assert request.status == RequestStatus.FINISHED_STOPPED
540
+
541
+ # Test case 3: min_tokens = 0, should follow normal stopping logic
542
+ sampling_params_no_min = SamplingParams(
543
+ ignore_eos=False,
544
+ max_tokens=20,
545
+ min_tokens=0,
546
+ )
547
+ request_no_min = Request(
548
+ request_id="1",
549
+ prompt_token_ids=[0, 1, 2],
550
+ sampling_params=sampling_params_no_min,
551
+ pooling_params=None,
552
+ eos_token_id=EOS_TOKEN_ID,
553
+ )
554
+ request_no_min.append_output_token_ids([10, EOS_TOKEN_ID])
555
+
556
+ result = check_stop(request_no_min, max_model_len=100)
557
+ assert result is True, "Should stop on EOS when min_tokens=0"
558
+ assert request_no_min.status == RequestStatus.FINISHED_STOPPED
559
+
560
+ # Test case 4: min_tokens > 0 with stop token (not EOS)
561
+ sampling_params_stop = SamplingParams(
562
+ ignore_eos=False,
563
+ max_tokens=20,
564
+ min_tokens=5,
565
+ stop_token_ids=[42],
566
+ )
567
+ request_stop = Request(
568
+ request_id="2",
569
+ prompt_token_ids=[0, 1, 2],
570
+ sampling_params=sampling_params_stop,
571
+ pooling_params=None,
572
+ eos_token_id=EOS_TOKEN_ID,
573
+ )
574
+ # Only 3 output tokens, less than min_tokens=5, but has stop token
575
+ request_stop.append_output_token_ids([10, 11, 42])
576
+ result = check_stop(request_stop, max_model_len=100)
577
+ assert result is False, "Should not stop when num_output_tokens<min_tokens"
578
+
579
+ # Test case 5: min_tokens met, should stop on stop token
580
+ request_stop.append_output_token_ids(
581
+ [10, 11, 12, 13, 14, 42]
582
+ ) # 6 tokens >= min_tokens=5
583
+
584
+ result = check_stop(request_stop, max_model_len=100)
585
+ assert result is True, "Should stop on stop token when min_tokens met"
586
+ assert request_stop.status == RequestStatus.FINISHED_STOPPED
587
+ assert request_stop.stop_reason == 42
588
+
589
+
590
@pytest.mark.parametrize(
591
"enable_prefix_caching, prompt_logprobs",
592
[
vllm/v1/core/sched/utils.py
+5
@@ -58,6 +58,11 @@ def check_stop(
58
59
sampling_params = request.sampling_params
60
assert sampling_params is not None
61
+
62
+ min_tokens = sampling_params.min_tokens
63
+ if request.num_output_tokens < min_tokens:
64
+ return False
65
+
66
last_token_id = request.output_token_ids[-1]
67
if not sampling_params.ignore_eos and last_token_id == request.eos_token_id:
68
request.status = RequestStatus.FINISHED_STOPPED