31f2e70cd3 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/usage/security.md

+13

@@ -289,6 +289,19 @@ To mitigate this, vLLM enforces a configurable upper bound on the `n` parameter

289 - **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads.

290 - **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse.

291

292 +### Per-request multimodal arguments

293 +

294 +API server endpoints reject non-empty per-request `mm_processor_kwargs` and

295 +`media_io_kwargs` by default. These arguments can change image, video, or audio

296 +loading, sizing, sampling, and preprocessing behavior, causing excessive CPU,

297 +GPU, or memory use when controlled by an untrusted client. Server-level

298 +`--mm-processor-kwargs` and `--media-io-kwargs` remain available for deployment

299 +configuration.

300 +

301 +Only deployments whose API clients are trusted should start the server with

302 +`--trust-request-mm-kwargs` to restore per-request overrides. Do not enable

303 +this option on an endpoint exposed to untrusted clients.

304 +

305 ## Tool Server and MCP Security

306

307 vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures.

rust/src/cmd/src/cli/unsupported.rs

+11

@@ -409,6 +409,17 @@ pub struct ServerUnsupportedArgs {

409 )]

410 pub trust_request_chat_template: Option<Unsupported>,

411

412 + /// Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`

413 + /// and `media_io_kwargs`). If False, the server rejects non-empty values

414 + /// because they can change multimodal preprocessing resource usage.

415 + #[arg(

416 + long,

417 + visible_alias = "no-trust-request-mm-kwargs",

418 + default_missing_value = "true",

419 + num_args = 0..=1

420 + )]

421 + pub trust_request_mm_kwargs: Option<Unsupported>,

422 +

423 /// The role name to return if `request.add_generation_prompt=true`.

424 #[arg(long)]

425 pub response_role: Option<Unsupported>,

tests/engine/test_arg_utils.py

+12

@@ -480,6 +480,18 @@ def test_compilation_config():

480 )

481

482

483 +def test_trust_request_mm_kwargs_cli():

484 + from vllm.entrypoints.launchers.cli_args import FrontendArgs

485 +

486 + parser = FrontendArgs.add_cli_args(FlexibleArgumentParser())

487 +

488 + args = parser.parse_args([])

489 + assert not args.trust_request_mm_kwargs

490 +

491 + args = parser.parse_args(["--trust-request-mm-kwargs"])

492 + assert args.trust_request_mm_kwargs

493 +

494 +

495 def test_attention_config():

496 from vllm.v1.attention.backends.registry import AttentionBackendEnum

497

tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py

+2 -4

@@ -26,13 +26,11 @@ def server():

26 "--enforce-eager",

27 "--limit-mm-per-prompt",

28 json.dumps({"audio": 3, "video": 3}),

29 + "--trust-request-mm-kwargs",

30 *ROCM_EXTRA_ARGS,

31 ]

32

32 - with RemoteOpenAIServer(

33 - MODEL_NAME,

34 - args,

35 - ) as remote_server:

33 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

34 yield remote_server

35

36

tests/entrypoints/multimodal/openai/chat_completion/test_video.py

+1

@@ -37,6 +37,7 @@ def server():

37 json.dumps({"video": MAXIMUM_VIDEOS}),

38 "--media-io-kwargs",

39 json.dumps({"video": {"num_frames": 32}}),

40 + "--trust-request-mm-kwargs",

41 ]

42

43 # ROCm: Increase timeouts to handle potential network delays and slower

tests/entrypoints/openai/chat_completion/test_chat_error.py

+75

@@ -475,6 +475,81 @@ def test_json_schema_response_format_missing_schema():

475 )

476

477

478 +@pytest.mark.asyncio

479 +async def test_online_renderer_rejects_mm_processor_kwargs_by_default():

480 + model_config = MockModelConfig()

481 + model_config.multimodal_config = MultiModalConfig()

482 + online_renderer = OnlineRenderer(

483 + model_config=model_config,

484 + renderer=MagicMock(),

485 + request_logger=None,

486 + chat_template=None,

487 + chat_template_content_format="auto",

488 + )

489 +

490 + request = ChatCompletionRequest(

491 + model=MODEL_NAME,

492 + messages=[{"role": "user", "content": "hello"}],

493 + mm_processor_kwargs={

494 + "patch_size": 1,

495 + "vision_min_num_patches": 3_000_000_000,

496 + },

497 + )

498 +

499 + with pytest.raises(

500 + VLLMValidationError,

501 + match="Per-request mm_processor_kwargs are disabled",

502 + ):

503 + await online_renderer.preprocess_chat(

504 + request,

505 + request.messages,

506 + default_template=None,

507 + default_template_content_format="auto",

508 + default_template_kwargs=None,

509 + )

510 +

511 +

512 +@pytest.mark.asyncio

513 +async def test_online_renderer_rejects_media_io_kwargs_by_default():

514 + model_config = MockModelConfig()

515 + model_config.multimodal_config = MultiModalConfig()

516 + online_renderer = OnlineRenderer(

517 + model_config=model_config,

518 + renderer=MagicMock(),

519 + request_logger=None,

520 + chat_template=None,

521 + chat_template_content_format="auto",

522 + )

523 +

524 + request = ChatCompletionRequest(

525 + model=MODEL_NAME,

526 + messages=[{"role": "user", "content": "hello"}],

527 + media_io_kwargs={"video": {"num_frames": 1_000_000}},

528 + )

529 +

530 + with pytest.raises(

531 + VLLMValidationError,

532 + match="Per-request media_io_kwargs are disabled",

533 + ):

534 + await online_renderer.preprocess_chat(

535 + request,

536 + request.messages,

537 + default_template=None,

538 + default_template_content_format="auto",

539 + default_template_kwargs=None,

540 + )

541 +

542 +

543 +def test_trust_request_mm_kwargs_opt_in_allows_overrides():

544 + from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs

545 +

546 + validate_request_mm_kwargs(

547 + mm_processor_kwargs={"use_audio_in_video": True},

548 + media_io_kwargs={"video": {"num_frames": 4}},

549 + trust_request_mm_kwargs=True,

550 + )

551 +

552 +

553 @pytest.mark.parametrize("format_value", [None, {}])

554 def test_structural_tag_response_format_invalid(format_value):

555 """Malformed structural tags should be rejected during request validation."""

tests/entrypoints/openai/test_render_parity.py

+1

@@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:

233 enable_auto_tools=True,

234 tool_parser="openai",

235 exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,

236 + trust_request_mm_kwargs=True,

237 )

238

239

tests/entrypoints/pooling/embed/test_online_vision.py

+1

@@ -46,6 +46,7 @@ def server():

46 json.dumps({"image": MAXIMUM_IMAGES}),

47 "--chat-template",

48 str(vlm2vec_jinja_path),

49 + "--trust-request-mm-kwargs",

50 ]

51

52 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/entrypoints/pooling/scoring/test_late_interaction_online.py

+30

@@ -26,6 +26,36 @@ TEXTS_2 = [

26 ]

27

28

29 +@pytest.mark.asyncio

30 +async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():

31 + from unittest.mock import MagicMock

32 +

33 + from vllm.config.multimodal import MultiModalConfig

34 + from vllm.entrypoints.pooling.base.serving import PoolingBaseServing

35 + from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest

36 + from vllm.entrypoints.pooling.scoring.serving import ServingScores

37 + from vllm.exceptions import VLLMValidationError

38 +

39 + serving = MagicMock()

40 + serving.model_config = MagicMock(multimodal_config=MultiModalConfig())

41 + serving.trust_request_mm_kwargs = False

42 + serving.io_processor = MagicMock()

43 + serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)

44 + request = ScoreQueriesDocumentsRequest(

45 + model=MODEL_NAME,

46 + queries=TEXTS_1[0],

47 + documents=TEXTS_2[0],

48 + mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},

49 + )

50 +

51 + with pytest.raises(

52 + VLLMValidationError,

53 + match="Per-request mm_processor_kwargs are disabled",

54 + ):

55 + await ServingScores.flash_late_interaction(serving, request)

56 + serving._preprocessing.assert_not_called()

57 +

58 +

59 @pytest.fixture(scope="module", params=[True, False])

60 def server(request):

61 args = [

vllm/entrypoints/chat_utils.py

+1

@@ -1450,6 +1450,7 @@ class ChatTemplateConfig:

1450 chat_template: str | None = None

1451 chat_template_content_format: ChatTemplateContentFormatOption = "auto"

1452 trust_request_chat_template: bool = False

1453 + trust_request_mm_kwargs: bool = False

1454

1455

1456 def validate_chat_template(chat_template: Path | str | None):

vllm/entrypoints/generate/base/protocol.py

+28

@@ -49,6 +49,34 @@ def validate_cache_salt(cache_salt: object) -> None:

49 )

50

51

52 +def validate_request_mm_kwargs(

53 + *,

54 + mm_processor_kwargs: dict[str, Any] | None,

55 + media_io_kwargs: dict[str, dict[str, Any]] | None,

56 + trust_request_mm_kwargs: bool,

57 +) -> None:

58 + """Reject untrusted per-request multimodal kwarg overrides."""

59 + if trust_request_mm_kwargs:

60 + return

61 +

62 + if mm_processor_kwargs:

63 + raise VLLMValidationError(

64 + "Per-request mm_processor_kwargs are disabled by default because "

65 + "they can change multimodal preprocessing resource usage. Start "

66 + "the server with --trust-request-mm-kwargs only when clients "

67 + "are trusted.",

68 + parameter="mm_processor_kwargs",

69 + )

70 + if media_io_kwargs:

71 + raise VLLMValidationError(

72 + "Per-request media_io_kwargs are disabled by default because "

73 + "they can change multimodal media loading resource usage. Start "

74 + "the server with --trust-request-mm-kwargs only when clients "

75 + "are trusted.",

76 + parameter="media_io_kwargs",

77 + )

78 +

79 +

80 class SpeculativeDecodingMetrics(OpenAIBaseModel):

81 """Per-request speculative-decoding acceptance metrics.

82

vllm/entrypoints/launchers/api_server/app_state.py

+1

@@ -92,6 +92,7 @@ async def init_app_state(

92 chat_template=resolved_chat_template,

93 chat_template_content_format=args.chat_template_content_format,

94 trust_request_chat_template=args.trust_request_chat_template,

95 + trust_request_mm_kwargs=args.trust_request_mm_kwargs,

96 enable_auto_tools=args.enable_auto_tool_choice,

97 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

98 tool_parser=args.tool_call_parser,

vllm/entrypoints/launchers/cli_args.py

+5

@@ -93,6 +93,11 @@ class BaseFrontendArgs:

93 """Whether to trust the chat template provided in the request. If False,

94 the server will always use the chat template specified by `--chat-template`

95 or the ones from tokenizer."""

96 + trust_request_mm_kwargs: bool = False

97 + """Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`

98 + and `media_io_kwargs`). If False, the server rejects non-empty values

99 + because they can change multimodal preprocessing resource usage. Only

100 + enable this when API clients are trusted."""

101 default_chat_template_kwargs: dict[str, Any] | None = None

102 """Default keyword arguments to pass to the chat template renderer.

103 These will be merged with request-level chat_template_kwargs,

vllm/entrypoints/launchers/render/app_state.py

+1

@@ -57,6 +57,7 @@ async def init_render_app_state(

57 chat_template=resolved_chat_template,

58 chat_template_content_format=args.chat_template_content_format,

59 trust_request_chat_template=args.trust_request_chat_template,

60 + trust_request_mm_kwargs=args.trust_request_mm_kwargs,

61 enable_auto_tools=args.enable_auto_tool_choice,

62 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

63 tool_parser=args.tool_call_parser,

vllm/entrypoints/pooling/base/serving.py

+7

@@ -16,6 +16,7 @@ from vllm import PoolingRequestOutput, envs

16 from vllm.config import VllmConfig

17 from vllm.engine.protocol import EngineClient

18 from vllm.entrypoints.chat_utils import ChatTemplateConfig

19 +from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs

20 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

21 from vllm.entrypoints.serve.engine.serving import BaseServing

22 from vllm.entrypoints.serve.engine.typing import AnyRequest

@@ -60,6 +61,7 @@ class PoolingBaseServing(ABC, BaseServing):

61 self.return_tokens_as_token_ids = return_tokens_as_token_ids

62 self.log_error_stack = log_error_stack

63 self.chat_template_config = chat_template_config

64 + self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs

65

66 # Shared thread pool executor for preprocessing and postprocessing.

67 self._executor: Executor = self.renderer._executor

@@ -108,6 +110,11 @@ class PoolingBaseServing(ABC, BaseServing):

110 request: AnyPoolingRequest,

111 raw_request: Request | None = None,

112 ):

113 + validate_request_mm_kwargs(

114 + mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),

115 + media_io_kwargs=getattr(request, "media_io_kwargs", None),

116 + trust_request_mm_kwargs=self.trust_request_mm_kwargs,

117 + )

118 base_request_id = self._base_request_id(

119 raw_request, getattr(request, "request_id", None)

120 )

vllm/entrypoints/pooling/factories.py

+1

@@ -166,6 +166,7 @@ def init_pooling_state(

166 chat_template=resolved_chat_template,

167 chat_template_content_format=args.chat_template_content_format,

168 trust_request_chat_template=args.trust_request_chat_template,

169 + trust_request_mm_kwargs=args.trust_request_mm_kwargs,

170 )

171

172 state.serving_pooling = (

vllm/renderers/online_renderer.py

+13

@@ -17,6 +17,7 @@ from vllm.entrypoints.chat_utils import (

17 ChatTemplateContentFormatOption,

18 ConversationMessage,

19 )

20 +from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs

21 from vllm.entrypoints.openai.chat_completion.protocol import (

22 ChatCompletionNamedToolChoiceParam,

23 ChatCompletionRequest,

@@ -123,6 +124,7 @@ class OnlineRenderer:

124 chat_template: str | None,

125 chat_template_content_format: ChatTemplateContentFormatOption,

126 trust_request_chat_template: bool = False,

127 + trust_request_mm_kwargs: bool = False,

128 enable_auto_tools: bool = False,

129 exclude_tools_when_tool_choice_none: bool = False,

130 tool_parser: str | None = None,

@@ -155,6 +157,7 @@ class OnlineRenderer:

157 default_chat_template_kwargs or {}

158 )

159 self.trust_request_chat_template = trust_request_chat_template

160 + self.trust_request_mm_kwargs = trust_request_mm_kwargs

161

162 self.log_error_stack = log_error_stack

163 self.supports_browsing = False

@@ -674,6 +677,11 @@ class OnlineRenderer:

677 """Copied from GenerateBaseServing._preprocess_cmpl."""

678 renderer = self.renderer

679 model_config = self.model_config

680 + validate_request_mm_kwargs(

681 + mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),

682 + media_io_kwargs=getattr(request, "media_io_kwargs", None),

683 + trust_request_mm_kwargs=self.trust_request_mm_kwargs,

684 + )

685

686 parsed_prompts = [

687 (

@@ -710,6 +718,11 @@ class OnlineRenderer:

718 ) -> tuple[list[ConversationMessage], list[EngineInput]]:

719 """Copied from GenerateBaseServing._preprocess_chat."""

720 renderer = self.renderer

721 + validate_request_mm_kwargs(

722 + mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),

723 + media_io_kwargs=getattr(request, "media_io_kwargs", None),

724 + trust_request_mm_kwargs=self.trust_request_mm_kwargs,

725 + )

726 mm_config = self.model_config.multimodal_config

727

728 default_template_kwargs = merge_kwargs(

[Security] Gate per-request multimodal processor kwargs (#58830)