A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Test] Make `test_failure` more stable for batch invariance (#27054)
Wentao Ye committed
Oct 17, 2025 at 16:59 UTC
f50cc221ea7861686fec5ae9e949018c9628f328
1 file changed
+20
-5
tests/v1/generation/test_batch_invariance.py
+20
-5
@@ -59,12 +59,15 @@ def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
59
# Pick a random template
60
base_prompt = random.choice(prompt_templates)
61
62
- # Add some padding to vary the length if needed
63
- if min_words > 50:
62
+ if max_words < min_words:
63
+ max_words = min_words
64
+ target_words = random.randint(min_words, max_words)
65
+
66
+ if target_words > 50:
67
# For longer prompts, repeat context
68
padding_text = (
69
" This is an interesting topic that deserves more explanation. "
67
- * (min_words // 50)
70
+ * (target_words // 50)
71
)
72
base_prompt = base_prompt + padding_text
73
@@ -516,8 +519,20 @@ def test_logprobs_WITHOUT_batch_invariance_should_FAIL(backend):
519
dtype="bfloat16",
520
)
521
519
- # Use more realistic prompts for better token generation
520
- prompts = [_random_prompt(10, 50) for i in range(32)]
522
+ # build ragged prompts to change shapes significantly across BS=1 vs BS=N
523
+ long_min = int(os.getenv("VLLM_MIN_PROMPT", "768"))
524
+ long_max = int(os.getenv("VLLM_MAX_PROMPT", "2048"))
525
+ prompts: list[str] = []
526
+ options = [
527
+ (max(long_min, 1536), max(long_max, 3072)), # very long
528
+ (max(1024, long_min), max(2048, long_max)), # long
529
+ (256, 512), # mid
530
+ (10, 20), # short
531
+ ]
532
+
533
+ for _ in range(32):
534
+ lo, hi = random.choice(options)
535
+ prompts.append(_random_prompt(lo, hi))
536
537
sp = SamplingParams(
538
temperature=0.6,