| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Comprehensive end-to-end tests for `min_tokens` in the V1 engine. |
| 4 | |
| 5 | Addresses #21950: verify and add CI coverage. |
| 6 | |
| 7 | Covers: |
| 8 | 1) Basic functionality |
| 9 | 2) Stop strings with `min_tokens` (bug #21987; fix in PR #22014) |
| 10 | 3) EOS behavior with `min_tokens` (potential logits-processor bug) |
| 11 | 4) Edge cases (min_tokens == max_tokens, min_tokens == 0) |
| 12 | 5) Multiple stop conditions |
| 13 | """ |
| 14 | |
| 15 | import json |
| 16 | |
| 17 | import pytest |
| 18 | |
| 19 | from vllm import LLM, SamplingParams |
| 20 | from vllm.exceptions import VLLMValidationError |
| 21 | from vllm.outputs import RequestOutput |
| 22 | from vllm.sampling_params import StructuredOutputsParams |
| 23 | |
| 24 | # Test configuration |
| 25 | TEST_MODEL = "facebook/opt-125m" # Small model for fast CI execution |
| 26 | GREEDY = 0.0 # Deterministic generation for consistent testing |
| 27 | |
| 28 | |
| 29 | class MinTokensTestCase: |
| 30 | """Data class for min_tokens test scenarios.""" |
| 31 | |
| 32 | def __init__( |
| 33 | self, |
| 34 | name: str, |
| 35 | min_tokens: int, |
| 36 | max_tokens: int, |
| 37 | stop: str | list[str] | None = None, |
| 38 | expected_min_len: int | None = None, |
| 39 | expected_exact_len: int | None = None, |
| 40 | ): |
| 41 | self.name = name |
| 42 | self.min_tokens = min_tokens |
| 43 | self.max_tokens = max_tokens |
| 44 | self.stop = stop |
| 45 | self.expected_min_len = expected_min_len or min_tokens |
| 46 | self.expected_exact_len = expected_exact_len |
| 47 | |
| 48 | def __str__(self): |
| 49 | return ( |
| 50 | f"{self.name}: min={self.min_tokens}, " |
| 51 | f"max={self.max_tokens}, stop={self.stop}" |
| 52 | ) |
| 53 | |
| 54 | |
| 55 | # Test scenarios covering all critical cases |
| 56 | MIN_TOKENS_TEST_CASES = [ |
| 57 | # === BASIC FUNCTIONALITY (should work) === |
| 58 | MinTokensTestCase( |
| 59 | name="basic_min_tokens_no_stop", |
| 60 | min_tokens=8, |
| 61 | max_tokens=20, |
| 62 | stop=None, |
| 63 | expected_min_len=8, |
| 64 | ), |
| 65 | MinTokensTestCase( |
| 66 | name="min_tokens_zero", |
| 67 | min_tokens=0, |
| 68 | max_tokens=10, |
| 69 | stop=None, |
| 70 | expected_min_len=0, |
| 71 | ), |
| 72 | MinTokensTestCase( |
| 73 | name="min_equals_max_no_stop", |
| 74 | min_tokens=15, |
| 75 | max_tokens=15, |
| 76 | stop=None, |
| 77 | expected_exact_len=15, |
| 78 | ), |
| 79 | # === STOP STRINGS WITH MIN_TOKENS === |
| 80 | # These tests expose the detokenizer bug where stop strings |
| 81 | # bypass min_tokens |
| 82 | # Using mathematically guaranteed approach with wide stop nets |
| 83 | pytest.param( |
| 84 | MinTokensTestCase( |
| 85 | name="min_tokens_with_comprehensive_stops", |
| 86 | min_tokens=5, |
| 87 | max_tokens=20, |
| 88 | stop=[ |
| 89 | "a", |
| 90 | "e", |
| 91 | "i", |
| 92 | "o", |
| 93 | "u", |
| 94 | "t", |
| 95 | "n", |
| 96 | "s", |
| 97 | "r", |
| 98 | "l", |
| 99 | " ", |
| 100 | ], |
| 101 | expected_min_len=5, |
| 102 | ), |
| 103 | marks=pytest.mark.xfail( |
| 104 | reason=( |
| 105 | "Known bug #21987: stop strings bypass min_tokens (fixed by PR #22014)" |
| 106 | ), |
| 107 | strict=False, |
| 108 | ), |
| 109 | id="min_tokens_with_comprehensive_stops", |
| 110 | ), |
| 111 | pytest.param( |
| 112 | MinTokensTestCase( |
| 113 | name="min_tokens_with_simple_char_stop", |
| 114 | min_tokens=3, |
| 115 | max_tokens=15, |
| 116 | stop=["e", "a", " "], |
| 117 | expected_min_len=3, |
| 118 | ), |
| 119 | marks=pytest.mark.xfail( |
| 120 | reason=( |
| 121 | "Known bug #21987: stop strings bypass min_tokens (fixed by PR #22014)" |
| 122 | ), |
| 123 | strict=False, |
| 124 | ), |
| 125 | id="min_tokens_with_simple_char_stop", |
| 126 | ), |
| 127 | # === EOS TOKEN WITH MIN_TOKENS (potential LogitsProcessor bug) === |
| 128 | # These test the MinTokensLogitsProcessor handling of EOS tokens |
| 129 | pytest.param( |
| 130 | MinTokensTestCase( |
| 131 | name="min_equals_max_eos_only", |
| 132 | min_tokens=20, |
| 133 | max_tokens=20, |
| 134 | stop=None, # Relies on default EOS token behavior |
| 135 | expected_exact_len=20, |
| 136 | ), |
| 137 | marks=pytest.mark.xfail( |
| 138 | reason=("Potential logits-processor bug: EOS tokens may bypass min_tokens"), |
| 139 | strict=False, |
| 140 | ), |
| 141 | id="min_equals_max_eos_only", |
| 142 | ), |
| 143 | # === EDGE CASES === |
| 144 | MinTokensTestCase( |
| 145 | name="large_min_tokens", |
| 146 | min_tokens=50, |
| 147 | max_tokens=60, |
| 148 | stop=None, |
| 149 | expected_min_len=50, |
| 150 | ), |
| 151 | MinTokensTestCase( |
| 152 | name="min_tokens_with_empty_stop_list", |
| 153 | min_tokens=5, |
| 154 | max_tokens=15, |
| 155 | stop=[], # Empty stop list |
| 156 | expected_min_len=5, |
| 157 | ), |
| 158 | ] |
| 159 | |
| 160 | |
| 161 | @pytest.fixture(scope="module") |
| 162 | def llm_v1(): |
| 163 | """Create V1 LLM instance for testing.""" |
| 164 | llm = LLM( |
| 165 | model=TEST_MODEL, |
| 166 | tensor_parallel_size=1, |
| 167 | max_model_len=1024, # Small context for fast testing |
| 168 | enforce_eager=True, # Avoid graph compilation overhead |
| 169 | ) |
| 170 | return llm |
| 171 | |
| 172 | |
| 173 | def test_min_tokens_with_terminal_structured_output(llm_v1: LLM): |
| 174 | """A grammar whose only legal continuation is EOS must stop under |
| 175 | min_tokens instead of sampling outside the grammar.""" |
| 176 | params = SamplingParams( |
| 177 | temperature=GREEDY, |
| 178 | min_tokens=20, |
| 179 | max_tokens=40, |
| 180 | structured_outputs=StructuredOutputsParams( |
| 181 | json=json.dumps({"type": "boolean"}) |
| 182 | ), |
| 183 | ) |
| 184 | |
| 185 | output = llm_v1.generate("Answer:", params)[0].outputs[0] |
| 186 | eos_token_id = llm_v1.get_tokenizer().eos_token_id |
| 187 | |
| 188 | assert output.finish_reason == "stop" |
| 189 | assert isinstance(json.loads(output.text), bool) |
| 190 | assert output.token_ids[-1] == eos_token_id |
| 191 | assert len(output.token_ids) < params.min_tokens |
| 192 | |
| 193 | |
| 194 | def get_token_count(output: RequestOutput) -> int: |
| 195 | """Extract token count from LLM output.""" |
| 196 | if not output.outputs: |
| 197 | return 0 |
| 198 | return len(output.outputs[0].token_ids) |
| 199 | |
| 200 | |
| 201 | def assert_min_tokens_satisfied( |
| 202 | output: RequestOutput, test_case: MinTokensTestCase |
| 203 | ) -> None: |
| 204 | """Assert that min_tokens requirement is satisfied.""" |
| 205 | token_count = get_token_count(output) |
| 206 | stop_reason = output.outputs[0].stop_reason if output.outputs else "no output" |
| 207 | |
| 208 | if test_case.expected_exact_len is not None: |
| 209 | # Exact length requirement |
| 210 | assert token_count == test_case.expected_exact_len, ( |
| 211 | f"Expected exactly {test_case.expected_exact_len} tokens, " |
| 212 | f"got {token_count} tokens. " |
| 213 | f"Stop reason: {stop_reason}" |
| 214 | ) |
| 215 | else: |
| 216 | # Minimum length requirement |
| 217 | assert token_count >= (test_case.expected_min_len or 0), ( |
| 218 | f"Expected at least {test_case.expected_min_len} tokens, " |
| 219 | f"got {token_count} tokens. " |
| 220 | f"Stop reason: {stop_reason}" |
| 221 | ) |
| 222 | |
| 223 | |
| 224 | @pytest.mark.parametrize( |
| 225 | "test_case", |
| 226 | MIN_TOKENS_TEST_CASES, |
| 227 | ids=lambda tc: tc.name, |
| 228 | ) |
| 229 | def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase): |
| 230 | """Comprehensive test for min_tokens functionality in V1 engine. |
| 231 | |
| 232 | This test covers all critical scenarios for min_tokens: |
| 233 | - Basic functionality (should work) |
| 234 | - Stop strings with min_tokens (known bug) |
| 235 | - EOS tokens with min_tokens (potential bug) |
| 236 | - Edge cases |
| 237 | |
| 238 | Args: |
| 239 | llm_v1: V1 LLM instance |
| 240 | test_case: Test scenario parameters |
| 241 | |
| 242 | """ |
| 243 | # Known failing cases are handled via param-level xfail marks above. |
| 244 | |
| 245 | # Create sampling parameters |
| 246 | sampling_params = SamplingParams( |
| 247 | min_tokens=test_case.min_tokens, |
| 248 | max_tokens=test_case.max_tokens, |
| 249 | stop=test_case.stop, |
| 250 | temperature=GREEDY, |
| 251 | include_stop_str_in_output=True, # Include stop strings for debugging |
| 252 | ) |
| 253 | |
| 254 | # Use simple prompt. Comprehensive stop lists should catch any generation |
| 255 | prompt = "Hello" |
| 256 | |
| 257 | # Generate output |
| 258 | outputs = llm_v1.generate([prompt], sampling_params) |
| 259 | |
| 260 | assert len(outputs) == 1, "Expected exactly one output" |
| 261 | output = outputs[0] |
| 262 | |
| 263 | # Debug information |
| 264 | token_count = get_token_count(output) |
| 265 | generated_text = output.outputs[0].text if output.outputs else "" |
| 266 | stop_reason = output.outputs[0].stop_reason if output.outputs else "unknown" |
| 267 | |
| 268 | print(f"\nTest: {test_case.name}") |
| 269 | print(f"Generated {token_count} tokens") |
| 270 | print(f"Stop reason: {stop_reason}") |
| 271 | print(f"Generated text: {repr(generated_text)}") |
| 272 | print(f"Expected min: {test_case.expected_min_len}") |
| 273 | if test_case.expected_exact_len: |
| 274 | print(f"Expected exact: {test_case.expected_exact_len}") |
| 275 | |
| 276 | # Validate min_tokens requirement |
| 277 | assert_min_tokens_satisfied(output, test_case) |
| 278 | |
| 279 | |
| 280 | def test_min_tokens_basic_functionality(llm_v1: LLM): |
| 281 | """Test basic min_tokens functionality without stop conditions. |
| 282 | |
| 283 | This is a baseline test that should always pass and validates |
| 284 | that min_tokens works correctly in the simple case. |
| 285 | """ |
| 286 | sampling_params = SamplingParams(min_tokens=10, max_tokens=20, temperature=GREEDY) |
| 287 | |
| 288 | prompt = "Once upon a time" |
| 289 | outputs = llm_v1.generate([prompt], sampling_params) |
| 290 | |
| 291 | assert len(outputs) == 1 |
| 292 | token_count = get_token_count(outputs[0]) |
| 293 | |
| 294 | assert token_count >= 10, f"Expected at least 10 tokens, got {token_count}" |
| 295 | assert token_count <= 20, f"Expected at most 20 tokens, got {token_count}" |
| 296 | |
| 297 | |
| 298 | @pytest.mark.xfail( |
| 299 | reason=("Known bug #21987: stop strings bypass min_tokens (fixed by PR #22014)"), |
| 300 | strict=False, |
| 301 | ) |
| 302 | def test_min_tokens_stop_strings_bug(llm_v1: LLM): |
| 303 | """Test the specific bug where stop strings bypass min_tokens. |
| 304 | |
| 305 | This test specifically reproduces the bug Calvin is fixing in PR #22014. |
| 306 | It should fail until that fix is merged. |
| 307 | |
| 308 | Strategy: Use guaranteed stop characters that will appear |
| 309 | in any generated text. |
| 310 | """ |
| 311 | # If the bug is fixed upstream, this test will XPASS |
| 312 | |
| 313 | sampling_params = SamplingParams( |
| 314 | min_tokens=15, |
| 315 | max_tokens=50, |
| 316 | # Common letter; likely appears early |
| 317 | stop=["e"], |
| 318 | temperature=GREEDY, |
| 319 | include_stop_str_in_output=True, |
| 320 | ) |
| 321 | |
| 322 | # Simple prompt that will generate text containing "e" |
| 323 | prompt = "The quick brown fox" |
| 324 | outputs = llm_v1.generate([prompt], sampling_params) |
| 325 | |
| 326 | assert len(outputs) == 1 |
| 327 | token_count = get_token_count(outputs[0]) |
| 328 | generated_text = outputs[0].outputs[0].text if outputs[0].outputs else "" |
| 329 | |
| 330 | # Debug info to understand what happened |
| 331 | print(f"Generated text: {repr(generated_text)}") |
| 332 | print(f"Token count: {token_count}") |
| 333 | print(f"Contains 'e': {'e' in generated_text}") |
| 334 | |
| 335 | # This assertion should fail due to the bug - if stop string is found early, |
| 336 | # the model should still continue generating until min_tokens is reached |
| 337 | stop_reason = ( |
| 338 | outputs[0].outputs[0].stop_reason if outputs[0].outputs else "no output" |
| 339 | ) |
| 340 | assert token_count >= 15, ( |
| 341 | "Bug confirmed: " |
| 342 | f"{token_count} tokens < min_tokens=15. " |
| 343 | f"Reason: {stop_reason}. " |
| 344 | f"Text: {repr(generated_text)}" |
| 345 | ) |
| 346 | |
| 347 | |
| 348 | @pytest.mark.xfail( |
| 349 | reason=("Known bug #21987: stop strings bypass min_tokens (fixed by PR #22014)"), |
| 350 | strict=False, |
| 351 | ) |
| 352 | def test_min_tokens_stop_strings_guaranteed_early_trigger(llm_v1: LLM): |
| 353 | """Guaranteed test for stop strings bypassing min_tokens bug. |
| 354 | |
| 355 | Strategy: Use very low temperature and multiple common stop strings |
| 356 | to virtually guarantee early detection, combined with long min_tokens |
| 357 | to ensure the bug is exposed regardless of model behavior. |
| 358 | """ |
| 359 | # If the bug is fixed upstream, this test will XPASS |
| 360 | |
| 361 | sampling_params = SamplingParams( |
| 362 | min_tokens=50, # Set high min_tokens to ensure bug detection |
| 363 | max_tokens=200, |
| 364 | # Use multiple very common patterns - at least one will appear |
| 365 | stop=["e", "a", "i", "o", "u", " ", "t", "n", "s", "r"], |
| 366 | temperature=GREEDY, |
| 367 | include_stop_str_in_output=True, |
| 368 | ) |
| 369 | |
| 370 | # Simple prompt that will generate some text |
| 371 | prompt = "The cat" |
| 372 | outputs = llm_v1.generate([prompt], sampling_params) |
| 373 | |
| 374 | assert len(outputs) == 1 |
| 375 | token_count = get_token_count(outputs[0]) |
| 376 | generated_text = outputs[0].outputs[0].text if outputs[0].outputs else "" |
| 377 | stop_reason = outputs[0].outputs[0].stop_reason if outputs[0].outputs else "unknown" |
| 378 | |
| 379 | print(f"Generated text: {repr(generated_text)}") |
| 380 | print(f"Token count: {token_count}") |
| 381 | print(f"Stop reason: {stop_reason}") |
| 382 | |
| 383 | # With the bug, this will fail because ANY of the common characters |
| 384 | # will trigger early termination before min_tokens=50 is reached |
| 385 | # It's virtually impossible to generate 50 tokens without hitting |
| 386 | # at least one of: e, a, i, o, u, space, t, n, s, r |
| 387 | finish_reason = ( |
| 388 | outputs[0].outputs[0].finish_reason if outputs[0].outputs else "unknown" |
| 389 | ) |
| 390 | |
| 391 | print(f"Finish reason: {finish_reason}") |
| 392 | |
| 393 | if finish_reason == "stop": |
| 394 | assert token_count >= 50, ( |
| 395 | "Bug confirmed: " |
| 396 | f"{token_count} tokens < min_tokens=50. " |
| 397 | f"Reason: {finish_reason}. " |
| 398 | f"Text: {repr(generated_text)}" |
| 399 | ) |
| 400 | |
| 401 | |
| 402 | @pytest.mark.xfail( |
| 403 | reason=("Potential logits-processor bug: EOS tokens may bypass min_tokens"), |
| 404 | strict=False, |
| 405 | ) |
| 406 | def test_min_tokens_eos_behavior(llm_v1: LLM): |
| 407 | """Verify EOS handling with and without min_tokens. |
| 408 | |
| 409 | - Without min_tokens: expect early EOS -> finish_reason == "stop", |
| 410 | stop_reason is None, and generated tokens < max_tokens (25). |
| 411 | - With min_tokens: EOS should be blocked until min_tokens is reached |
| 412 | (finish_reason == "length"); verify that eos_token_id does not appear |
| 413 | in generated token_ids. |
| 414 | """ |
| 415 | # tokenizer + eos id |
| 416 | tokenizer = llm_v1.get_tokenizer() |
| 417 | eos_token_id = tokenizer.eos_token_id |
| 418 | |
| 419 | prompt = "Give a file extension." |
| 420 | max_toks = 32 |
| 421 | |
| 422 | # Case 1: WITHOUT min_tokens |
| 423 | sp_no_min = SamplingParams( |
| 424 | max_tokens=max_toks, |
| 425 | temperature=GREEDY, |
| 426 | ) |
| 427 | out_no_min = llm_v1.generate([prompt], sp_no_min) |
| 428 | assert len(out_no_min) == 1 |
| 429 | choice_no_min = out_no_min[0].outputs[0] |
| 430 | |
| 431 | ids_no_min = choice_no_min.token_ids or [] |
| 432 | finish_no_min = choice_no_min.finish_reason |
| 433 | stop_no_min = choice_no_min.stop_reason |
| 434 | |
| 435 | print( |
| 436 | "[no-min] tokens=", |
| 437 | len(ids_no_min), |
| 438 | " finish=", |
| 439 | finish_no_min, |
| 440 | " stop_reason=", |
| 441 | stop_no_min, |
| 442 | ) |
| 443 | |
| 444 | assert finish_no_min == "stop", ( |
| 445 | f"Expected finish_reason 'stop' without min_tokens, got {finish_no_min}" |
| 446 | ) |
| 447 | assert stop_no_min is None, ( |
| 448 | "For EOS-based stop (no user stop strings), stop_reason should be None." |
| 449 | ) |
| 450 | assert len(ids_no_min) < max_toks, ( |
| 451 | f"Expected early EOS with < {max_toks} tokens, got {len(ids_no_min)}" |
| 452 | ) |
| 453 | |
| 454 | # Case 2: WITH min_tokens |
| 455 | sp_with_min = SamplingParams( |
| 456 | min_tokens=max_toks, |
| 457 | max_tokens=max_toks, |
| 458 | temperature=GREEDY, |
| 459 | ) |
| 460 | out_with_min = llm_v1.generate([prompt], sp_with_min) |
| 461 | assert len(out_with_min) == 1 |
| 462 | choice_with_min = out_with_min[0].outputs[0] |
| 463 | |
| 464 | ids_with_min = choice_with_min.token_ids or [] |
| 465 | finish_with_min = choice_with_min.finish_reason |
| 466 | stop_with_min = choice_with_min.stop_reason |
| 467 | |
| 468 | print( |
| 469 | "[with-min] tokens=", |
| 470 | len(ids_with_min), |
| 471 | " finish=", |
| 472 | finish_with_min, |
| 473 | " stop_reason=", |
| 474 | stop_with_min, |
| 475 | ) |
| 476 | |
| 477 | # Exact length reached; EOS should have been blocked |
| 478 | assert len(ids_with_min) == max_toks, ( |
| 479 | f"Expected exactly {max_toks} tokens with min_tokens; got {len(ids_with_min)}" |
| 480 | ) |
| 481 | assert finish_with_min == "length", ( |
| 482 | f"Expected finish_reason 'length'; got {finish_with_min}" |
| 483 | ) |
| 484 | assert eos_token_id not in ids_with_min, ( |
| 485 | "EOS token id should not appear when min_tokens prevents early EOS." |
| 486 | ) |
| 487 | |
| 488 | |
| 489 | def test_min_tokens_validation(): |
| 490 | """Test that SamplingParams correctly validates min_tokens parameters. |
| 491 | |
| 492 | This tests the parameter validation logic in SamplingParams. |
| 493 | """ |
| 494 | # Valid cases |
| 495 | SamplingParams(min_tokens=0, max_tokens=10) |
| 496 | SamplingParams(min_tokens=5, max_tokens=10) |
| 497 | SamplingParams(min_tokens=10, max_tokens=10) |
| 498 | |
| 499 | # Invalid cases |
| 500 | with pytest.raises( |
| 501 | VLLMValidationError, |
| 502 | match="min_tokens must be greater than or equal to 0", |
| 503 | ): |
| 504 | SamplingParams(min_tokens=-1, max_tokens=10) |
| 505 | |
| 506 | with pytest.raises( |
| 507 | VLLMValidationError, |
| 508 | match="min_tokens must be less than or equal to max_tokens", |
| 509 | ): |
| 510 | SamplingParams(min_tokens=15, max_tokens=10) |
| 511 | |
| 512 | |
| 513 | if __name__ == "__main__": |
| 514 | """ |
| 515 | Run tests locally for development. |
| 516 | |
| 517 | Usage: |
| 518 | cd vllm/ |
| 519 | python -m pytest tests/v1/e2e/general/test_min_tokens.py -v |
| 520 | """ |
| 521 | pytest.main([__file__, "-v"]) |