111692bb8c · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -0,0 +1,479 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +Comprehensive end-to-end tests for `min_tokens` in the V1 engine.

5 +

6 +Addresses #21950: verify and add CI coverage.

7 +

8 +Covers:

9 +1) Basic functionality

10 +2) Stop strings with `min_tokens` (bug #21987; fix in PR #22014)

11 +3) EOS behavior with `min_tokens` (potential logits-processor bug)

12 +4) Edge cases (min_tokens == max_tokens, min_tokens == 0)

13 +5) Multiple stop conditions

14 +"""

15 +

16 +import os

17 +from typing import Optional, Union

18 +

19 +import pytest

20 +

21 +from vllm import LLM, SamplingParams

22 +from vllm.outputs import RequestOutput

23 +

24 +# Test configuration

25 +TEST_MODEL = "facebook/opt-125m" # Small model for fast CI execution

26 +GREEDY = 0.0 # Deterministic generation for consistent testing

27 +

28 +

29 +class MinTokensTestCase:

30 + """Data class for min_tokens test scenarios"""

31 +

32 + def __init__(

33 + self,

34 + name: str,

35 + min_tokens: int,

36 + max_tokens: int,

37 + stop: Optional[Union[str, list[str]]] = None,

38 + expected_min_len: Optional[int] = None,

39 + expected_exact_len: Optional[int] = None,

40 + ):

41 + self.name = name

42 + self.min_tokens = min_tokens

43 + self.max_tokens = max_tokens

44 + self.stop = stop

45 + self.expected_min_len = expected_min_len or min_tokens

46 + self.expected_exact_len = expected_exact_len

47 +

48 + def __str__(self):

49 + return (f"{self.name}: min={self.min_tokens}, "

50 + f"max={self.max_tokens}, stop={self.stop}")

51 +

52 +

53 +# Test scenarios covering all critical cases

54 +MIN_TOKENS_TEST_CASES = [

55 + # === BASIC FUNCTIONALITY (should work) ===

56 + MinTokensTestCase(name="basic_min_tokens_no_stop",

57 + min_tokens=8,

58 + max_tokens=20,

59 + stop=None,

60 + expected_min_len=8),

61 + MinTokensTestCase(name="min_tokens_zero",

62 + min_tokens=0,

63 + max_tokens=10,

64 + stop=None,

65 + expected_min_len=0),

66 + MinTokensTestCase(name="min_equals_max_no_stop",

67 + min_tokens=15,

68 + max_tokens=15,

69 + stop=None,

70 + expected_exact_len=15),

71 +

72 + # === STOP STRINGS WITH MIN_TOKENS ===

73 + # These tests expose the detokenizer bug where stop strings

74 + # bypass min_tokens

75 + # Using mathematically guaranteed approach with wide stop nets

76 + pytest.param(

77 + MinTokensTestCase(

78 + name="min_tokens_with_comprehensive_stops",

79 + min_tokens=5,

80 + max_tokens=20,

81 + stop=[

82 + "a",

83 + "e",

84 + "i",

85 + "o",

86 + "u",

87 + "t",

88 + "n",

89 + "s",

90 + "r",

91 + "l",

92 + " ",

93 + ],

94 + expected_min_len=5,

95 + ),

96 + marks=pytest.mark.xfail(

97 + reason=("Known bug #21987: stop strings bypass min_tokens "

98 + "(fixed by PR #22014)"),

99 + strict=False),

100 + id="min_tokens_with_comprehensive_stops",

101 + ),

102 + pytest.param(

103 + MinTokensTestCase(

104 + name="min_tokens_with_simple_char_stop",

105 + min_tokens=3,

106 + max_tokens=15,

107 + stop=["e", "a", " "],

108 + expected_min_len=3,

109 + ),

110 + marks=pytest.mark.xfail(

111 + reason=("Known bug #21987: stop strings bypass min_tokens "

112 + "(fixed by PR #22014)"),

113 + strict=False),

114 + id="min_tokens_with_simple_char_stop",

115 + ),

116 +

117 + # === EOS TOKEN WITH MIN_TOKENS (potential LogitsProcessor bug) ===

118 + # These test the MinTokensLogitsProcessor handling of EOS tokens

119 + pytest.param(

120 + MinTokensTestCase(

121 + name="min_equals_max_eos_only",

122 + min_tokens=20,

123 + max_tokens=20,

124 + stop=None, # Relies on default EOS token behavior

125 + expected_exact_len=20,

126 + ),

127 + marks=pytest.mark.xfail(

128 + reason=

129 + ("Potential logits-processor bug: EOS tokens may bypass min_tokens"

130 + ),

131 + strict=False,

132 + ),

133 + id="min_equals_max_eos_only",

134 + ),

135 +

136 + # === EDGE CASES ===

137 + MinTokensTestCase(name="large_min_tokens",

138 + min_tokens=50,

139 + max_tokens=60,

140 + stop=None,

141 + expected_min_len=50),

142 + MinTokensTestCase(

143 + name="min_tokens_with_empty_stop_list",

144 + min_tokens=5,

145 + max_tokens=15,

146 + stop=[], # Empty stop list

147 + expected_min_len=5),

148 +]

149 +

150 +

151 +@pytest.fixture(scope="module")

152 +def llm_v1():

153 + """Create V1 LLM instance for testing"""

154 + # Ensure V1 engine is used

155 + os.environ["VLLM_USE_V1"] = "1"

156 +

157 + llm = LLM(

158 + model=TEST_MODEL,

159 + tensor_parallel_size=1,

160 + max_model_len=1024, # Small context for fast testing

161 + enforce_eager=True, # Avoid graph compilation overhead

162 + )

163 + return llm

164 +

165 +

166 +def get_token_count(output: RequestOutput) -> int:

167 + """Extract token count from LLM output"""

168 + if not output.outputs:

169 + return 0

170 + return len(output.outputs[0].token_ids)

171 +

172 +

173 +def assert_min_tokens_satisfied(output: RequestOutput,

174 + test_case: MinTokensTestCase) -> None:

175 + """Assert that min_tokens requirement is satisfied"""

176 + token_count = get_token_count(output)

177 + stop_reason = (output.outputs[0].stop_reason

178 + if output.outputs else "no output")

179 +

180 + if test_case.expected_exact_len is not None:

181 + # Exact length requirement

182 + assert token_count == test_case.expected_exact_len, (

183 + f"Expected exactly {test_case.expected_exact_len} tokens, "

184 + f"got {token_count} tokens. "

185 + f"Stop reason: {stop_reason}")

186 + else:

187 + # Minimum length requirement

188 + assert token_count >= (test_case.expected_min_len or 0), (

189 + f"Expected at least {test_case.expected_min_len} tokens, "

190 + f"got {token_count} tokens. "

191 + f"Stop reason: {stop_reason}")

192 +

193 +

194 +@pytest.mark.parametrize(

195 + "test_case",

196 + MIN_TOKENS_TEST_CASES,

197 + ids=lambda tc: tc.name,

198 +)

199 +def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):

200 + """

201 + Comprehensive test for min_tokens functionality in V1 engine.

202 +

203 + This test covers all critical scenarios for min_tokens:

204 + - Basic functionality (should work)

205 + - Stop strings with min_tokens (known bug)

206 + - EOS tokens with min_tokens (potential bug)

207 + - Edge cases

208 +

209 + Args:

210 + llm_v1: V1 LLM instance

211 + test_case: Test scenario parameters

212 + """

213 + # Known failing cases are handled via param-level xfail marks above.

214 +

215 + # Create sampling parameters

216 + sampling_params = SamplingParams(

217 + min_tokens=test_case.min_tokens,

218 + max_tokens=test_case.max_tokens,

219 + stop=test_case.stop,

220 + temperature=GREEDY,

221 + include_stop_str_in_output=True # Include stop strings for debugging

222 + )

223 +

224 + # Use simple prompt. Comprehensive stop lists should catch any generation

225 + prompt = "Hello"

226 +

227 + # Generate output

228 + outputs = llm_v1.generate([prompt], sampling_params)

229 +

230 + assert len(outputs) == 1, "Expected exactly one output"

231 + output = outputs[0]

232 +

233 + # Debug information

234 + token_count = get_token_count(output)

235 + generated_text = output.outputs[0].text if output.outputs else ""

236 + stop_reason = output.outputs[0].stop_reason if output.outputs else "unknown"

237 +

238 + print(f"\nTest: {test_case.name}")

239 + print(f"Generated {token_count} tokens")

240 + print(f"Stop reason: {stop_reason}")

241 + print(f"Generated text: {repr(generated_text)}")

242 + print(f"Expected min: {test_case.expected_min_len}")

243 + if test_case.expected_exact_len:

244 + print(f"Expected exact: {test_case.expected_exact_len}")

245 +

246 + # Validate min_tokens requirement

247 + assert_min_tokens_satisfied(output, test_case)

248 +

249 +

250 +def test_min_tokens_basic_functionality(llm_v1: LLM):

251 + """

252 + Test basic min_tokens functionality without stop conditions.

253 +

254 + This is a baseline test that should always pass and validates

255 + that min_tokens works correctly in the simple case.

256 + """

257 + sampling_params = SamplingParams(min_tokens=10,

258 + max_tokens=20,

259 + temperature=GREEDY)

260 +

261 + prompt = "Once upon a time"

262 + outputs = llm_v1.generate([prompt], sampling_params)

263 +

264 + assert len(outputs) == 1

265 + token_count = get_token_count(outputs[0])

266 +

267 + assert token_count >= 10, f"Expected at least 10 tokens, got {token_count}"

268 + assert token_count <= 20, f"Expected at most 20 tokens, got {token_count}"

269 +

270 +

271 +@pytest.mark.xfail(

272 + reason=("Known bug #21987: stop strings bypass min_tokens "

273 + "(fixed by PR #22014)"),

274 + strict=False,

275 +)

276 +def test_min_tokens_stop_strings_bug(llm_v1: LLM):

277 + """

278 + Test the specific bug where stop strings bypass min_tokens.

279 +

280 + This test specifically reproduces the bug Calvin is fixing in PR #22014.

281 + It should fail until that fix is merged.

282 +

283 + Strategy: Use guaranteed stop characters that will appear

284 + in any generated text.

285 + """

286 + # If the bug is fixed upstream, this test will XPASS

287 +

288 + sampling_params = SamplingParams(

289 + min_tokens=15,

290 + max_tokens=50,

291 + # Common letter; likely appears early

292 + stop=["e"],

293 + temperature=GREEDY,

294 + include_stop_str_in_output=True)

295 +

296 + # Simple prompt that will generate text containing "e"

297 + prompt = "The quick brown fox"

298 + outputs = llm_v1.generate([prompt], sampling_params)

299 +

300 + assert len(outputs) == 1

301 + token_count = get_token_count(outputs[0])

302 + generated_text = outputs[0].outputs[0].text if outputs[0].outputs else ""

303 +

304 + # Debug info to understand what happened

305 + print(f"Generated text: {repr(generated_text)}")

306 + print(f"Token count: {token_count}")

307 + print(f"Contains 'e': {'e' in generated_text}")

308 +

309 + # This assertion should fail due to the bug - if stop string is found early,

310 + # the model should still continue generating until min_tokens is reached

311 + stop_reason = (outputs[0].outputs[0].stop_reason

312 + if outputs[0].outputs else "no output")

313 + assert token_count >= 15, ("Bug confirmed: "

314 + f"{token_count} tokens < min_tokens=15. "

315 + f"Reason: {stop_reason}. "

316 + f"Text: {repr(generated_text)}")

317 +

318 +

319 +@pytest.mark.xfail(

320 + reason=("Known bug #21987: stop strings bypass min_tokens "

321 + "(fixed by PR #22014)"),

322 + strict=False,

323 +)

324 +def test_min_tokens_stop_strings_guaranteed_early_trigger(llm_v1: LLM):

325 + """

326 + Guaranteed test for stop strings bypassing min_tokens bug.

327 +

328 + Strategy: Use very low temperature and multiple common stop strings

329 + to virtually guarantee early detection, combined with long min_tokens

330 + to ensure the bug is exposed regardless of model behavior.

331 + """

332 + # If the bug is fixed upstream, this test will XPASS

333 +

334 + sampling_params = SamplingParams(

335 + min_tokens=50, # Set high min_tokens to ensure bug detection

336 + max_tokens=200,

337 + # Use multiple very common patterns - at least one will appear

338 + stop=["e", "a", "i", "o", "u", " ", "t", "n", "s", "r"],

339 + temperature=GREEDY,

340 + include_stop_str_in_output=True)

341 +

342 + # Simple prompt that will generate some text

343 + prompt = "The cat"

344 + outputs = llm_v1.generate([prompt], sampling_params)

345 +

346 + assert len(outputs) == 1

347 + token_count = get_token_count(outputs[0])

348 + generated_text = outputs[0].outputs[0].text if outputs[0].outputs else ""

349 + stop_reason = (outputs[0].outputs[0].stop_reason

350 + if outputs[0].outputs else "unknown")

351 +

352 + print(f"Generated text: {repr(generated_text)}")

353 + print(f"Token count: {token_count}")

354 + print(f"Stop reason: {stop_reason}")

355 +

356 + # With the bug, this will fail because ANY of the common characters

357 + # will trigger early termination before min_tokens=50 is reached

358 + # It's virtually impossible to generate 50 tokens without hitting

359 + # at least one of: e, a, i, o, u, space, t, n, s, r

360 + finish_reason = (outputs[0].outputs[0].finish_reason

361 + if outputs[0].outputs else "unknown")

362 +

363 + print(f"Finish reason: {finish_reason}")

364 +

365 + if finish_reason == "stop":

366 + assert token_count >= 50, ("Bug confirmed: "

367 + f"{token_count} tokens < min_tokens=50. "

368 + f"Reason: {finish_reason}. "

369 + f"Text: {repr(generated_text)}")

370 +

371 +

372 +@pytest.mark.xfail(

373 + reason=(

374 + "Potential logits-processor bug: EOS tokens may bypass min_tokens"),

375 + strict=False,

376 +)

377 +def test_min_tokens_eos_behavior(llm_v1: LLM):

378 + """

379 + Verify EOS handling with and without min_tokens.

380 +

381 + - Without min_tokens: expect early EOS -> finish_reason == "stop",

382 + stop_reason is None, and generated tokens < max_tokens (25).

383 + - With min_tokens: EOS should be blocked until min_tokens is reached

384 + (finish_reason == "length"); verify that eos_token_id does not appear

385 + in generated token_ids.

386 + """

387 + # tokenizer + eos id

388 + tokenizer = llm_v1.get_tokenizer()

389 + eos_token_id = tokenizer.eos_token_id

390 +

391 + prompt = "Give a file extension."

392 + max_toks = 32

393 +

394 + # Case 1: WITHOUT min_tokens

395 + sp_no_min = SamplingParams(

396 + max_tokens=max_toks,

397 + temperature=GREEDY,

398 + )

399 + out_no_min = llm_v1.generate([prompt], sp_no_min)

400 + assert len(out_no_min) == 1

401 + choice_no_min = out_no_min[0].outputs[0]

402 +

403 + ids_no_min = choice_no_min.token_ids or []

404 + finish_no_min = choice_no_min.finish_reason

405 + stop_no_min = choice_no_min.stop_reason

406 +

407 + print("[no-min] tokens=", len(ids_no_min), " finish=", finish_no_min,

408 + " stop_reason=", stop_no_min)

409 +

410 + assert finish_no_min == "stop", (

411 + f"Expected finish_reason 'stop' without min_tokens, got {finish_no_min}"

412 + )

413 + assert stop_no_min is None, (

414 + "For EOS-based stop (no user stop strings), stop_reason should be None."

415 + )

416 + assert len(ids_no_min) < max_toks, (

417 + f"Expected early EOS with < {max_toks} tokens, got {len(ids_no_min)}")

418 +

419 + # Case 2: WITH min_tokens

420 + sp_with_min = SamplingParams(

421 + min_tokens=max_toks,

422 + max_tokens=max_toks,

423 + temperature=GREEDY,

424 + )

425 + out_with_min = llm_v1.generate([prompt], sp_with_min)

426 + assert len(out_with_min) == 1

427 + choice_with_min = out_with_min[0].outputs[0]

428 +

429 + ids_with_min = choice_with_min.token_ids or []

430 + finish_with_min = choice_with_min.finish_reason

431 + stop_with_min = choice_with_min.stop_reason

432 +

433 + print("[with-min] tokens=", len(ids_with_min), " finish=", finish_with_min,

434 + " stop_reason=", stop_with_min)

435 +

436 + # Exact length reached; EOS should have been blocked

437 + assert len(ids_with_min) == max_toks, (

438 + f"Expected exactly {max_toks} tokens with min_tokens; "

439 + f"got {len(ids_with_min)}")

440 + assert finish_with_min == "length", (

441 + f"Expected finish_reason 'length'; got {finish_with_min}")

442 + assert eos_token_id not in ids_with_min, (

443 + "EOS token id should not appear when min_tokens prevents early EOS.")

444 +

445 +

446 +def test_min_tokens_validation():

447 + """

448 + Test that SamplingParams correctly validates min_tokens parameters.

449 +

450 + This tests the parameter validation logic in SamplingParams.

451 + """

452 + # Valid cases

453 + SamplingParams(min_tokens=0, max_tokens=10)

454 + SamplingParams(min_tokens=5, max_tokens=10)

455 + SamplingParams(min_tokens=10, max_tokens=10)

456 +

457 + # Invalid cases

458 + with pytest.raises(

459 + ValueError,

460 + match="min_tokens must be greater than or equal to 0",

461 + ):

462 + SamplingParams(min_tokens=-1, max_tokens=10)

463 +

464 + with pytest.raises(

465 + ValueError,

466 + match="min_tokens must be less than or equal to max_tokens",

467 + ):

468 + SamplingParams(min_tokens=15, max_tokens=10)

469 +

470 +

471 +if __name__ == "__main__":

472 + """

473 + Run tests locally for development.

474 +

475 + Usage:

476 + cd vllm/

477 + VLLM_USE_V1=1 python -m pytest tests/v1/e2e/test_min_tokens.py -v

478 + """

479 + pytest.main([__file__, "-v"])

[CI] Add end-to-end V1 min_tokens test coverage (#22495)