| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Batch-sharded sampling must be a drop-in for replicated sampling under |
| 4 | speculative decoding. |
| 5 | |
| 6 | The non-spec arm of this claim lives in tests/v1/e2e/general/ |
| 7 | test_sharded_sampling.py. Spec decode is the harder case: draft tokens give |
| 8 | each request a different number of logits rows, so the shard plan's |
| 9 | per-request arithmetic and the rejection sampler are both exercised. |
| 10 | """ |
| 11 | |
| 12 | import pytest |
| 13 | import torch |
| 14 | |
| 15 | from tests.utils import wait_for_memory_to_settle |
| 16 | from vllm import LLM, SamplingParams |
| 17 | from vllm.distributed import cleanup_dist_env_and_memory |
| 18 | |
| 19 | from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling |
| 20 | |
| 21 | |
| 22 | def _is_sharded_sampling_active(worker) -> bool: |
| 23 | return worker.model_runner.batch_sharder is not None |
| 24 | |
| 25 | |
| 26 | def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch): |
| 27 | """Batch-sharded sampling must be a bit-exact drop-in for replicated |
| 28 | sampling under MTP spec decoding: the collectives move the same logits |
| 29 | bytes, Gumbel keys derive from (request slot, position, seed), and slot |
| 30 | assignment is rank-deterministic. Both runs here are spec decode with |
| 31 | identical math, so outputs must match exactly.""" |
| 32 | tp_size = 2 |
| 33 | _skip_if_insufficient_gpus_for_tp(tp_size) |
| 34 | model_name = "Qwen/Qwen3.5-0.8B-Base" |
| 35 | test_prompts = get_test_prompts(mm_enabled=False, num_prompts=20) |
| 36 | |
| 37 | with monkeypatch.context() as m: |
| 38 | # Batch-sharded sampling only exists in the V2 model runner. |
| 39 | m.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") |
| 40 | # Required for the collective_rpc mode check below. |
| 41 | m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") |
| 42 | # Queue every request before the first step, so both boots schedule |
| 43 | # the same batches -- see tests/v1/e2e/general/test_sharded_sampling.py. |
| 44 | m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") |
| 45 | |
| 46 | def run(disable_sharding: bool): |
| 47 | llm = LLM( |
| 48 | model=model_name, |
| 49 | tensor_parallel_size=tp_size, |
| 50 | enable_batch_sharded_sampling=not disable_sharding, |
| 51 | speculative_config={ |
| 52 | "method": "mtp", |
| 53 | "num_speculative_tokens": 1, |
| 54 | "max_model_len": 2048, |
| 55 | }, |
| 56 | max_model_len=2048, |
| 57 | limit_mm_per_prompt={"image": 0, "video": 0}, |
| 58 | # More requests than ranks, so round-robin slot ownership |
| 59 | # splits the batch across both ranks. |
| 60 | max_num_seqs=len(test_prompts), |
| 61 | # Timing-based kernel selection is the dominant source of |
| 62 | # cross-boot numeric noise; disable for a tight comparison. |
| 63 | kernel_config={"enable_flashinfer_autotune": False}, |
| 64 | ) |
| 65 | try: |
| 66 | # Guard against a vacuous comparison: verify every worker is |
| 67 | # in the intended sampling mode. |
| 68 | modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active) |
| 69 | assert all(mode == (not disable_sharding) for mode in modes) |
| 70 | greedy = llm.chat(test_prompts, greedy_sampling()) |
| 71 | seeded = llm.chat( |
| 72 | test_prompts, |
| 73 | SamplingParams(temperature=1.0, seed=33, max_tokens=10), |
| 74 | ) |
| 75 | return greedy, seeded |
| 76 | finally: |
| 77 | del llm |
| 78 | torch.accelerator.empty_cache() |
| 79 | cleanup_dist_env_and_memory() |
| 80 | wait_for_memory_to_settle() |
| 81 | |
| 82 | ref_greedy, ref_seeded = run(disable_sharding=True) |
| 83 | shard_greedy, shard_seeded = run(disable_sharding=False) |
| 84 | |
| 85 | # A sharding bug shows up as wholesale divergence, not isolated flips. |
| 86 | for name, ref_outputs, shard_outputs in ( |
| 87 | ("greedy", ref_greedy, shard_greedy), |
| 88 | ("seeded", ref_seeded, shard_seeded), |
| 89 | ): |
| 90 | divergent = [] |
| 91 | for i, (ref, out) in enumerate(zip(ref_outputs, shard_outputs)): |
| 92 | if list(ref.outputs[0].token_ids) != list(out.outputs[0].token_ids): |
| 93 | divergent.append(i) |
| 94 | print( |
| 95 | f"{name} prompt {i} diverged:\n" |
| 96 | f" replicated: {ref.outputs[0].text!r}\n" |
| 97 | f" sharded: {out.outputs[0].text!r}" |
| 98 | ) |
| 99 | assert not divergent, ( |
| 100 | f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged " |
| 101 | f"at {divergent}; the sampling modes did not agree token for " |
| 102 | "token with in-process scheduling" |
| 103 | ) |