| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Bounds and alignment checks for request-controlled inputs. |
| 4 | |
| 5 | Covers prompt-mask alignment, stop-string caps, bad-words dedup/tokenization |
| 6 | limits, stop-token-id dedup, beam-width/sequence caps, and the DeepSeek |
| 7 | history-scan bound. |
| 8 | """ |
| 9 | |
| 10 | import os |
| 11 | import subprocess |
| 12 | import sys |
| 13 | from collections.abc import Callable |
| 14 | from typing import Protocol |
| 15 | from unittest.mock import Mock |
| 16 | |
| 17 | import pytest |
| 18 | import torch |
| 19 | from pydantic import ValidationError |
| 20 | |
| 21 | import vllm.envs as envs |
| 22 | from vllm import SamplingParams |
| 23 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 24 | BatchChatCompletionRequest, |
| 25 | ChatCompletionRequest, |
| 26 | ) |
| 27 | from vllm.entrypoints.openai.completion.protocol import CompletionRequest |
| 28 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 29 | from vllm.exceptions import VLLMValidationError |
| 30 | from vllm.inputs.engine import embeds_input |
| 31 | from vllm.sampling_params import BeamSearchParams |
| 32 | from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding |
| 33 | from vllm.v1.engine.input_processor import InputProcessor |
| 34 | |
| 35 | pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] |
| 36 | |
| 37 | |
| 38 | @pytest.fixture |
| 39 | def prompt_embeds_processor(): |
| 40 | model_config = Mock( |
| 41 | max_model_len=16, |
| 42 | max_logprobs=20, |
| 43 | logits_processors=None, |
| 44 | is_diffusion=False, |
| 45 | return_sampling_mask=False, |
| 46 | supports_multimodal_inputs=False, |
| 47 | ) |
| 48 | model_config.get_vocab_size.return_value = 32 |
| 49 | model_config.try_get_generation_config.return_value = {} |
| 50 | config = Mock( |
| 51 | model_config=model_config, |
| 52 | speculative_config=None, |
| 53 | structured_outputs_config=None, |
| 54 | ) |
| 55 | renderer = Mock(tokenizer=None) |
| 56 | renderer.get_eos_token_id.return_value = None |
| 57 | return InputProcessor(config, renderer, mm_registry=Mock()) |
| 58 | |
| 59 | |
| 60 | @pytest.mark.parametrize("mask_len", [0, 1, 2, 4]) |
| 61 | def test_prompt_embeds_mask_length_rejected_before_engine_submission( |
| 62 | prompt_embeds_processor, mask_len |
| 63 | ): |
| 64 | """Reject short, long, and broadcastable masks before they reach a worker.""" |
| 65 | prompt = embeds_input( |
| 66 | torch.zeros(3, 4), |
| 67 | prompt_token_ids=[1, 2, 3], |
| 68 | is_token_ids=[False] * mask_len, |
| 69 | ) |
| 70 | |
| 71 | with pytest.raises(VLLMValidationError, match="prompt_is_token_ids") as exc_info: |
| 72 | prompt_embeds_processor.process_inputs( |
| 73 | "invalid", prompt, SamplingParams(max_tokens=1), ("generate",) |
| 74 | ) |
| 75 | |
| 76 | assert exc_info.value.parameter == "prompt_is_token_ids" |
| 77 | |
| 78 | |
| 79 | @pytest.mark.parametrize( |
| 80 | ("token_ids", "mask"), |
| 81 | [(None, None), ([1, 2, 3], None), ([1, 2, 3], [True, False, True])], |
| 82 | ids=["pure-embeds", "token-ids-without-mask", "mixed"], |
| 83 | ) |
| 84 | def test_prompt_embeds_valid_inputs_preserved(prompt_embeds_processor, token_ids, mask): |
| 85 | embeds = torch.zeros(3, 4) |
| 86 | prompt = embeds_input(embeds, prompt_token_ids=token_ids, is_token_ids=mask) |
| 87 | |
| 88 | request = prompt_embeds_processor.process_inputs( |
| 89 | "valid", prompt, SamplingParams(max_tokens=1), ("generate",) |
| 90 | ) |
| 91 | |
| 92 | assert request.prompt_embeds is embeds |
| 93 | assert request.prompt_token_ids == token_ids |
| 94 | assert request.prompt_is_token_ids == mask |
| 95 | |
| 96 | |
| 97 | @pytest.mark.parametrize("offset", [-1, 0, 2, 3, 4, 2**127]) |
| 98 | def test_routed_experts_offset_validated_before_engine_submission(offset): |
| 99 | """Reject invalid offsets even when routed-expert output is disabled.""" |
| 100 | processor = Mock(spec=InputProcessor) |
| 101 | processor.tokenizer = None |
| 102 | processor.generation_config_fields = {} |
| 103 | processor.renderer = Mock() |
| 104 | processor.renderer.get_eos_token_id.return_value = None |
| 105 | processor.vllm_config = Mock() |
| 106 | params = SamplingParams(routed_experts_prompt_start=offset) |
| 107 | prompt = {"type": "token", "prompt_token_ids": [1, 2, 3]} |
| 108 | |
| 109 | if not 0 <= offset <= 3: |
| 110 | with pytest.raises(VLLMValidationError, match="routed_experts_prompt_start"): |
| 111 | InputProcessor.process_inputs( |
| 112 | processor, "invalid", prompt, params, ("generate",) |
| 113 | ) |
| 114 | params.routed_experts_prompt_start = 0 |
| 115 | |
| 116 | request = InputProcessor.process_inputs( |
| 117 | processor, "valid", prompt, params, ("generate",) |
| 118 | ) |
| 119 | assert ( |
| 120 | request.sampling_params.routed_experts_prompt_start |
| 121 | == params.routed_experts_prompt_start |
| 122 | ) |
| 123 | |
| 124 | |
| 125 | def _process_inputs_with_max_model_len( |
| 126 | params: SamplingParams, max_model_len: int = 2048 |
| 127 | ): |
| 128 | processor = Mock(spec=InputProcessor) |
| 129 | processor.tokenizer = None |
| 130 | processor.generation_config_fields = {} |
| 131 | processor.renderer = Mock() |
| 132 | processor.renderer.get_eos_token_id.return_value = None |
| 133 | processor.vllm_config = Mock() |
| 134 | processor.model_config = Mock() |
| 135 | processor.model_config.max_model_len = max_model_len |
| 136 | prompt = {"type": "token", "prompt_token_ids": [1, 2, 3]} |
| 137 | return InputProcessor.process_inputs( |
| 138 | processor, "req", prompt, params, ("generate",) |
| 139 | ) |
| 140 | |
| 141 | |
| 142 | def test_unset_max_tokens_rejects_oversized_min_tokens(): |
| 143 | """Reject min_tokens that only becomes invalid after max_tokens is filled.""" |
| 144 | params = SamplingParams(max_tokens=None, min_tokens=2147483648) |
| 145 | with pytest.raises( |
| 146 | VLLMValidationError, |
| 147 | match="min_tokens must be less than or equal to max_tokens", |
| 148 | ): |
| 149 | _process_inputs_with_max_model_len(params) |
| 150 | |
| 151 | |
| 152 | def test_unset_max_tokens_accepts_min_tokens_within_filled_max(): |
| 153 | params = SamplingParams(max_tokens=None, min_tokens=10) |
| 154 | request = _process_inputs_with_max_model_len(params) |
| 155 | assert request.sampling_params.max_tokens == 2045 |
| 156 | assert request.sampling_params.min_tokens == 10 |
| 157 | |
| 158 | |
| 159 | # --- Stop strings: public requests cap the number of stop strings --------- |
| 160 | |
| 161 | |
| 162 | class _StopRequest(Protocol): |
| 163 | stop: str | list[str] | None |
| 164 | |
| 165 | |
| 166 | def _completion_request(stop: list[str]) -> _StopRequest: |
| 167 | return CompletionRequest(model="test-model", prompt="hello", stop=stop) |
| 168 | |
| 169 | |
| 170 | def _chat_request(stop: list[str]) -> _StopRequest: |
| 171 | return ChatCompletionRequest( |
| 172 | model="test-model", |
| 173 | messages=[{"role": "user", "content": "hello"}], |
| 174 | stop=stop, |
| 175 | ) |
| 176 | |
| 177 | |
| 178 | def _batch_chat_request(stop: list[str]) -> _StopRequest: |
| 179 | return BatchChatCompletionRequest( |
| 180 | model="test-model", |
| 181 | messages=[[{"role": "user", "content": "hello"}]], |
| 182 | stop=stop, |
| 183 | ) |
| 184 | |
| 185 | |
| 186 | def _responses_request(stop: list[str]) -> _StopRequest: |
| 187 | return ResponsesRequest(model="test-model", input="hello", stop=stop) |
| 188 | |
| 189 | |
| 190 | REQUEST_BUILDERS: list[Callable[[list[str]], _StopRequest]] = [ |
| 191 | _completion_request, |
| 192 | _chat_request, |
| 193 | _batch_chat_request, |
| 194 | _responses_request, |
| 195 | ] |
| 196 | |
| 197 | |
| 198 | @pytest.mark.parametrize("build_request", REQUEST_BUILDERS) |
| 199 | def test_public_requests_accept_four_stop_strings( |
| 200 | build_request: Callable[[list[str]], _StopRequest], |
| 201 | ): |
| 202 | stop = ["one", "two", "three", "four"] |
| 203 | |
| 204 | request = build_request(stop) |
| 205 | |
| 206 | assert request.stop == stop |
| 207 | |
| 208 | |
| 209 | @pytest.mark.parametrize("build_request", REQUEST_BUILDERS) |
| 210 | def test_public_requests_reject_more_than_four_stop_strings( |
| 211 | build_request: Callable[[list[str]], _StopRequest], |
| 212 | ): |
| 213 | with pytest.raises(ValidationError, match="at most 4"): |
| 214 | build_request(["one", "two", "three", "four", "five"]) |
| 215 | |
| 216 | |
| 217 | def test_stop_string_limit_can_be_overridden(): |
| 218 | env = os.environ.copy() |
| 219 | env["VLLM_MAX_STOP_STRINGS"] = "1" |
| 220 | code = """ |
| 221 | from pydantic import ValidationError |
| 222 | from vllm.entrypoints.openai.completion.protocol import CompletionRequest |
| 223 | |
| 224 | try: |
| 225 | CompletionRequest( |
| 226 | model="test-model", |
| 227 | prompt="hello", |
| 228 | stop=["one", "two"], |
| 229 | ) |
| 230 | except ValidationError as error: |
| 231 | assert "at most 1" in str(error) |
| 232 | else: |
| 233 | raise AssertionError("configured stop-string limit was not enforced") |
| 234 | """ |
| 235 | |
| 236 | subprocess.run([sys.executable, "-c", code], check=True, env=env) |
| 237 | |
| 238 | |
| 239 | # --- Stop token ids: duplicates are deduplicated in order ------------------ |
| 240 | |
| 241 | |
| 242 | def test_duplicate_stop_token_ids_are_deduplicated_in_order(): |
| 243 | params = SamplingParams(stop_token_ids=[42, 7, 42, 9, 7]) |
| 244 | |
| 245 | assert params.stop_token_ids == [42, 7, 9] |
| 246 | assert params.all_stop_token_ids == {7, 9, 42} |
| 247 | |
| 248 | |
| 249 | # --- Bad words: dedup, and the tokenization pass is bounded ---------------- |
| 250 | |
| 251 | |
| 252 | class MockTokenizer: |
| 253 | max_token_id = 1024 |
| 254 | |
| 255 | def __init__(self): |
| 256 | self.calls = 0 |
| 257 | |
| 258 | def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: |
| 259 | self.calls += 1 |
| 260 | return [2] if text.startswith(" ") else [1] |
| 261 | |
| 262 | |
| 263 | def test_duplicate_bad_words_are_deduplicated_in_order(): |
| 264 | params = SamplingParams(bad_words=["bad", "worse", "bad", "worst"]) |
| 265 | |
| 266 | assert params.bad_words == ["bad", "worse", "worst"] |
| 267 | |
| 268 | |
| 269 | def test_bad_word_tokenization_stops_at_worker_limit(): |
| 270 | params = SamplingParams(bad_words=[f"word-{i}" for i in range(65)]) |
| 271 | tokenizer = MockTokenizer() |
| 272 | |
| 273 | with pytest.raises(VLLMValidationError, match="Too many bad words"): |
| 274 | params.update_from_tokenizer(tokenizer) |
| 275 | |
| 276 | assert tokenizer.calls == 129 |
| 277 | |
| 278 | |
| 279 | def test_bad_word_tokenization_limit_can_be_overridden(monkeypatch): |
| 280 | monkeypatch.setenv("VLLM_MAX_NUM_BAD_WORDS", "2") |
| 281 | params = SamplingParams(bad_words=["bad", "worse"]) |
| 282 | tokenizer = MockTokenizer() |
| 283 | |
| 284 | with pytest.raises(VLLMValidationError, match="The max number is 2"): |
| 285 | params.update_from_tokenizer(tokenizer) |
| 286 | |
| 287 | assert tokenizer.calls == 3 |
| 288 | |
| 289 | |
| 290 | class EmptyBaseEncodingTokenizer: |
| 291 | max_token_id = 1024 |
| 292 | |
| 293 | def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: |
| 294 | return [216] if text.startswith(" ") else [] |
| 295 | |
| 296 | |
| 297 | def test_bad_word_rejects_empty_base_tokenization(): |
| 298 | params = SamplingParams(bad_words=["\x16"]) |
| 299 | |
| 300 | with pytest.raises( |
| 301 | VLLMValidationError, |
| 302 | match="must tokenize to at least one token", |
| 303 | ) as exc_info: |
| 304 | params.update_from_tokenizer(EmptyBaseEncodingTokenizer()) |
| 305 | |
| 306 | assert exc_info.value.parameter == "bad_words" |
| 307 | |
| 308 | |
| 309 | class EmptyPrefixedEncodingTokenizer: |
| 310 | max_token_id = 1024 |
| 311 | |
| 312 | def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: |
| 313 | return [] if text.startswith(" ") else [321] |
| 314 | |
| 315 | |
| 316 | def test_bad_word_skips_empty_optional_prefixed_tokenization(): |
| 317 | params = SamplingParams(bad_words=["word"]) |
| 318 | |
| 319 | params.update_from_tokenizer(EmptyPrefixedEncodingTokenizer()) |
| 320 | |
| 321 | assert params.bad_words_token_ids == [[321]] |
| 322 | |
| 323 | |
| 324 | # --- Beam search: beam width / n honor the sequence cap -------------------- |
| 325 | |
| 326 | |
| 327 | def _set_max_n(monkeypatch: pytest.MonkeyPatch, value: int) -> None: |
| 328 | monkeypatch.setenv("VLLM_MAX_N_SEQUENCES", str(value)) |
| 329 | if hasattr(envs.__getattr__, "cache_clear"): |
| 330 | envs.__getattr__.cache_clear() |
| 331 | |
| 332 | |
| 333 | def test_direct_beam_width_rejects_values_over_sequence_cap( |
| 334 | monkeypatch: pytest.MonkeyPatch, |
| 335 | ): |
| 336 | _set_max_n(monkeypatch, 4) |
| 337 | |
| 338 | with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"): |
| 339 | BeamSearchParams(beam_width=5, max_tokens=1) |
| 340 | |
| 341 | |
| 342 | def test_chat_beam_conversion_rejects_n_before_stream_state_allocation( |
| 343 | monkeypatch: pytest.MonkeyPatch, |
| 344 | ): |
| 345 | _set_max_n(monkeypatch, 4) |
| 346 | request = ChatCompletionRequest( |
| 347 | model="test-model", |
| 348 | messages=[{"role": "user", "content": "test"}], |
| 349 | n=5, |
| 350 | stream=True, |
| 351 | use_beam_search=True, |
| 352 | max_tokens=1, |
| 353 | ) |
| 354 | |
| 355 | with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"): |
| 356 | request.to_beam_search_params(max_tokens=1, default_sampling_params={}) |
| 357 | |
| 358 | |
| 359 | def test_chat_beam_conversion_accepts_n_at_sequence_cap( |
| 360 | monkeypatch: pytest.MonkeyPatch, |
| 361 | ): |
| 362 | _set_max_n(monkeypatch, 4) |
| 363 | request = ChatCompletionRequest( |
| 364 | model="test-model", |
| 365 | messages=[{"role": "user", "content": "test"}], |
| 366 | n=4, |
| 367 | stream=True, |
| 368 | use_beam_search=True, |
| 369 | max_tokens=1, |
| 370 | ) |
| 371 | |
| 372 | params = request.to_beam_search_params(max_tokens=1, default_sampling_params={}) |
| 373 | |
| 374 | assert params.beam_width == 4 |
| 375 | |
| 376 | |
| 377 | # --- DeepSeek encoders: the last-user scan runs once per conversation ------ |
| 378 | |
| 379 | |
| 380 | ENCODING_MODULES = [deepseek_v32_encoding, deepseek_v4_encoding] |
| 381 | |
| 382 | |
| 383 | @pytest.mark.parametrize( |
| 384 | "encoding_module", |
| 385 | ENCODING_MODULES, |
| 386 | ids=["deepseek_v32", "deepseek_v4"], |
| 387 | ) |
| 388 | def test_encode_messages_scans_last_user_once_per_conversation( |
| 389 | monkeypatch: pytest.MonkeyPatch, |
| 390 | encoding_module, |
| 391 | ): |
| 392 | calls = 0 |
| 393 | original_find_last_user_index = encoding_module.find_last_user_index |
| 394 | |
| 395 | def counted_find_last_user_index(messages): |
| 396 | nonlocal calls |
| 397 | calls += 1 |
| 398 | return original_find_last_user_index(messages) |
| 399 | |
| 400 | monkeypatch.setattr( |
| 401 | encoding_module, |
| 402 | "find_last_user_index", |
| 403 | counted_find_last_user_index, |
| 404 | ) |
| 405 | |
| 406 | messages = [{"role": "user", "content": "Hello"}] |
| 407 | messages.extend({"role": "assistant", "content": "Hi"} for _ in range(8)) |
| 408 | |
| 409 | encoding_module.encode_messages(messages, thinking_mode="chat") |
| 410 | |
| 411 | assert calls == 1 |
| 412 | |
| 413 | |
| 414 | @pytest.mark.parametrize( |
| 415 | "encoding_module", |
| 416 | ENCODING_MODULES, |
| 417 | ids=["deepseek_v32", "deepseek_v4"], |
| 418 | ) |
| 419 | def test_encode_messages_preserves_small_chat_prompt(encoding_module): |
| 420 | prompt = encoding_module.encode_messages( |
| 421 | [ |
| 422 | {"role": "user", "content": "Hello"}, |
| 423 | {"role": "assistant", "content": "Hi"}, |
| 424 | {"role": "assistant", "content": "Again"}, |
| 425 | ], |
| 426 | thinking_mode="chat", |
| 427 | ) |
| 428 | |
| 429 | assert prompt == ( |
| 430 | "<｜begin▁of▁sentence｜><｜User｜>Hello<｜Assistant｜></think>" |
| 431 | "Hi<｜end▁of▁sentence｜>Again<｜end▁of▁sentence｜>" |
| 432 | ) |
| 433 | |
| 434 | |
| 435 | @pytest.mark.parametrize( |
| 436 | "encoding_module", |
| 437 | ENCODING_MODULES, |
| 438 | ids=["deepseek_v32", "deepseek_v4"], |
| 439 | ) |
| 440 | def test_encode_messages_unknown_role_raises_value_error(encoding_module): |
| 441 | # An invalid role (e.g. uppercase "SYSTEM") is a client error and must be |
| 442 | # raised as ValueError so the OpenAI serving layer maps it to HTTP 400 |
| 443 | # instead of NotImplementedError, which would map to HTTP 501. |
| 444 | with pytest.raises(ValueError, match="Invalid role: SYSTEM"): |
| 445 | encoding_module.encode_messages( |
| 446 | [{"role": "SYSTEM", "content": "Hello"}], |
| 447 | thinking_mode="chat", |
| 448 | ) |