| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import numpy as np |
| 5 | import pytest |
| 6 | import torch |
| 7 | |
| 8 | pytest.importorskip("triton") |
| 9 | if not torch.cuda.is_available(): |
| 10 | pytest.skip( |
| 11 | "CUDA required for Model Runner V2 logit bias tests", |
| 12 | allow_module_level=True, |
| 13 | ) |
| 14 | |
| 15 | from vllm.sampling_params import SamplingParams, StructuredOutputsParams |
| 16 | from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState |
| 17 | from vllm.v1.worker.gpu.sample.logits_processor import ( |
| 18 | LogitsContext, |
| 19 | LogitsProcRequestState, |
| 20 | ) |
| 21 | from vllm.v1.worker.gpu.states import RequestState |
| 22 | |
| 23 | DEVICE = torch.device("cuda") |
| 24 | VOCAB_SIZE = 128 |
| 25 | STOP_TOKEN = 3 |
| 26 | OTHER_TOKEN = 4 |
| 27 | PROMPT_LEN = 2 |
| 28 | MIN_TOKENS = 10 |
| 29 | # min_len is PROMPT_LEN + MIN_TOKENS, so this position is below the minimum |
| 30 | # and the kernel masks stop tokens. |
| 31 | POS = 5 |
| 32 | |
| 33 | |
| 34 | def _params(structured: bool) -> SamplingParams: |
| 35 | return SamplingParams( |
| 36 | min_tokens=MIN_TOKENS, |
| 37 | stop_token_ids=[STOP_TOKEN], |
| 38 | structured_outputs=( |
| 39 | StructuredOutputsParams(json={"type": "boolean"}) if structured else None |
| 40 | ), |
| 41 | ) |
| 42 | |
| 43 | |
| 44 | def _only_stop_token_left(num_rows: int) -> torch.Tensor: |
| 45 | """Logits as a terminal grammar leaves them: every token masked but one.""" |
| 46 | logits = torch.full((num_rows, VOCAB_SIZE), -float("inf"), device=DEVICE) |
| 47 | logits[:, STOP_TOKEN] = 1.0 |
| 48 | return logits |
| 49 | |
| 50 | |
| 51 | def _apply(logits: torch.Tensor, structured: list[bool]) -> torch.Tensor: |
| 52 | """Run the fused bias kernel with one logits row per request.""" |
| 53 | req_states = RequestState( |
| 54 | max_num_reqs=4, |
| 55 | max_model_len=64, |
| 56 | max_num_batched_tokens=16, |
| 57 | num_speculative_steps=1, |
| 58 | vocab_size=VOCAB_SIZE, |
| 59 | device=DEVICE, |
| 60 | ) |
| 61 | # add_request() reads prompt_len per slot; every slot here uses the same one. |
| 62 | req_states.prompt_len.np[:] = PROMPT_LEN |
| 63 | |
| 64 | state = LogitBiasState(None, LogitsProcRequestState.from_request_state(req_states)) |
| 65 | for req_idx, is_structured in enumerate(structured): |
| 66 | state.add_request(req_idx, _params(is_structured)) |
| 67 | state.apply_staged_writes() |
| 68 | |
| 69 | num_reqs = len(structured) |
| 70 | idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) |
| 71 | state.apply( |
| 72 | logits, |
| 73 | LogitsContext( |
| 74 | expanded_idx_mapping=idx_mapping, |
| 75 | idx_mapping=idx_mapping, |
| 76 | idx_mapping_np=np.arange(num_reqs, dtype=np.intp), |
| 77 | expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), |
| 78 | input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), |
| 79 | pos=torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE), |
| 80 | seq_lens_upper_bound_np=np.full(num_reqs, POS + 1, dtype=np.int64), |
| 81 | ), |
| 82 | ) |
| 83 | return logits.cpu() |
| 84 | |
| 85 | |
| 86 | def test_v2_min_tokens_restores_stop_token_when_row_would_be_empty(): |
| 87 | """Structured output plus min_tokens must not leave an all -inf row.""" |
| 88 | out = _apply(_only_stop_token_left(1), structured=[True]) |
| 89 | assert out[0, STOP_TOKEN] == 1.0 |
| 90 | |
| 91 | |
| 92 | def test_v2_min_tokens_keeps_empty_row_without_structured_output(): |
| 93 | """Plain min_tokens requests keep the pre-existing masking behavior.""" |
| 94 | out = _apply(_only_stop_token_left(1), structured=[False]) |
| 95 | assert torch.isneginf(out).all() |
| 96 | |
| 97 | |
| 98 | def test_v2_min_tokens_keeps_masking_when_other_candidates_remain(): |
| 99 | """The fallback must not weaken min_tokens while a non-stop token is legal.""" |
| 100 | logits = _only_stop_token_left(1) |
| 101 | logits[0, OTHER_TOKEN] = 2.0 |
| 102 | |
| 103 | out = _apply(logits, structured=[True]) |
| 104 | |
| 105 | assert torch.isneginf(out[0, STOP_TOKEN]) |
| 106 | assert out[0, OTHER_TOKEN] == 2.0 |
| 107 | |
| 108 | |
| 109 | def test_v2_min_tokens_mixed_batch_gates_restore_per_request(): |
| 110 | """CHECK_ALL_MASKED_ROWS is a batch-wide constexpr, so a mixed batch must |
| 111 | still consult the per-request flag.""" |
| 112 | out = _apply(_only_stop_token_left(2), structured=[True, False]) |
| 113 | |
| 114 | assert out[0, STOP_TOKEN] == 1.0 |
| 115 | assert torch.isneginf(out[1]).all() |