3958b96bf5 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/features/multimodal_inputs.md

+4

@@ -6,6 +6,10 @@ This page teaches you how to pass multi-modal inputs to [multi-modal models][sup

6 We are actively iterating on multi-modal support. See [this RFC](gh-issue:4194) for upcoming changes,

7 and [open an issue on GitHub](https://github.com/vllm-project/vllm/issues/new/choose) if you have any feedback or feature requests.

8

9 +!!! tip

10 + When serving multi-modal models, consider setting `--allowed-media-domains` to restrict domain that vLLM can access to prevent it from accessing arbitrary endpoints that can potentially be vulnerable to Server-Side Request Forgery (SSRF) attacks. You can provide a list of domains for this arg. For example: `--allowed-media-domains upload.wikimedia.org github.com www.bogotobogo.com`

11 + This restriction is especially important if you run vLLM in a containerized environment where the vLLM pods may have unrestricted access to internal networks.

12 +

13 ## Offline Inference

14

15 To input multi-modal data, follow this schema in [vllm.inputs.PromptType][]:

docs/usage/security.md

+6

@@ -60,6 +60,12 @@ Key points from the PyTorch security guide:

60 - Implement proper authentication and authorization for management interfaces

61 - Follow the principle of least privilege for all system components

62

63 +### 4. **Restrict Domains Access for Media URLs:**

64 +

65 +Restrict domains that vLLM can access for media URLs by setting

66 +`--allowed-media-domains` to prevent Server-Side Request Forgery (SSRF) attacks.

67 +(e.g. `--allowed-media-domains upload.wikimedia.org github.com www.bogotobogo.com`)

68 +

69 ## Security and Firewalls: Protecting Exposed vLLM Systems

70

71 While vLLM is designed to allow unsafe network services to be isolated to

tests/entrypoints/openai/test_lora_resolvers.py

+1

@@ -45,6 +45,7 @@ class MockModelConfig:

45 logits_processor_pattern: Optional[str] = None

46 diff_sampling_param: Optional[dict] = None

47 allowed_local_media_path: str = ""

48 + allowed_media_domains: Optional[list[str]] = None

49 encoder_config = None

50 generation_config: str = "auto"

51 skip_tokenizer_init: bool = False

tests/entrypoints/openai/test_serving_chat.py

+1

@@ -240,6 +240,7 @@ class MockModelConfig:

240 logits_processor_pattern = None

241 diff_sampling_param: Optional[dict] = None

242 allowed_local_media_path: str = ""

243 + allowed_media_domains: Optional[list[str]] = None

244 encoder_config = None

245 generation_config: str = "auto"

246 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

tests/multimodal/test_utils.py

+32 -1

@@ -66,7 +66,12 @@ async def test_fetch_image_http(image_url: str):

66 @pytest.mark.parametrize("suffix", get_supported_suffixes())

67 async def test_fetch_image_base64(url_images: dict[str, Image.Image],

68 raw_image_url: str, suffix: str):

69 - connector = MediaConnector()

69 + connector = MediaConnector(

70 + # Domain restriction should not apply to data URLs.

71 + allowed_media_domains=[

72 + "www.bogotobogo.com",

73 + "github.com",

74 + ])

75 url_image = url_images[raw_image_url]

76

77 try:

@@ -387,3 +392,29 @@ def test_argsort_mm_positions(case):

392 modality_idxs = argsort_mm_positions(mm_positions)

393

394 assert modality_idxs == expected_modality_idxs

395 +

396 +

397 +@pytest.mark.asyncio

398 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)

399 +@pytest.mark.parametrize("num_frames", [-1, 32, 1800])

400 +async def test_allowed_media_domains(video_url: str, num_frames: int):

401 + connector = MediaConnector(

402 + media_io_kwargs={"video": {

403 + "num_frames": num_frames,

404 + }},

405 + allowed_media_domains=[

406 + "www.bogotobogo.com",

407 + "github.com",

408 + ])

409 +

410 + video_sync, metadata_sync = connector.fetch_video(video_url)

411 + video_async, metadata_async = await connector.fetch_video_async(video_url)

412 + assert np.array_equal(video_sync, video_async)

413 + assert metadata_sync == metadata_async

414 +

415 + disallowed_url = "https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png"

416 + with pytest.raises(ValueError):

417 + _, _ = connector.fetch_video(disallowed_url)

418 +

419 + with pytest.raises(ValueError):

420 + _, _ = await connector.fetch_video_async(disallowed_url)

vllm/config/model.py

+3

@@ -137,6 +137,9 @@ class ModelConfig:

137 """Allowing API requests to read local images or videos from directories

138 specified by the server file system. This is a security risk. Should only

139 be enabled in trusted environments."""

140 + allowed_media_domains: Optional[list[str]] = None

141 + """If set, only media URLs that belong to this domain can be used for

142 + multi-modal inputs. """

143 revision: Optional[str] = None

144 """The specific model version to use. It can be a branch name, a tag name,

145 or a commit id. If unspecified, will use the default version."""

vllm/config/speculative.py

+2

@@ -288,6 +288,8 @@ class SpeculativeConfig:

288 trust_remote_code,

289 allowed_local_media_path=self.target_model_config.

290 allowed_local_media_path,

291 + allowed_media_domains=self.target_model_config.

292 + allowed_media_domains,

293 dtype=self.target_model_config.dtype,

294 seed=self.target_model_config.seed,

295 revision=self.revision,

vllm/engine/arg_utils.py

+5

@@ -297,6 +297,8 @@ class EngineArgs:

297 tokenizer_mode: TokenizerMode = ModelConfig.tokenizer_mode

298 trust_remote_code: bool = ModelConfig.trust_remote_code

299 allowed_local_media_path: str = ModelConfig.allowed_local_media_path

300 + allowed_media_domains: Optional[

301 + list[str]] = ModelConfig.allowed_media_domains

302 download_dir: Optional[str] = LoadConfig.download_dir

303 safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy

304 load_format: Union[str, LoadFormats] = LoadConfig.load_format

@@ -531,6 +533,8 @@ class EngineArgs:

533 **model_kwargs["hf_config_path"])

534 model_group.add_argument("--allowed-local-media-path",

535 **model_kwargs["allowed_local_media_path"])

536 + model_group.add_argument("--allowed-media-domains",

537 + **model_kwargs["allowed_media_domains"])

538 model_group.add_argument("--revision", **model_kwargs["revision"])

539 model_group.add_argument("--code-revision",

540 **model_kwargs["code_revision"])

@@ -997,6 +1001,7 @@ class EngineArgs:

1001 tokenizer_mode=self.tokenizer_mode,

1002 trust_remote_code=self.trust_remote_code,

1003 allowed_local_media_path=self.allowed_local_media_path,

1004 + allowed_media_domains=self.allowed_media_domains,

1005 dtype=self.dtype,

1006 seed=self.seed,

1007 revision=self.revision,

vllm/entrypoints/chat_utils.py

+6

@@ -632,6 +632,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

632 def allowed_local_media_path(self):

633 return self._model_config.allowed_local_media_path

634

635 + @property

636 + def allowed_media_domains(self):

637 + return self._model_config.allowed_media_domains

638 +

639 @property

640 def mm_registry(self):

641 return MULTIMODAL_REGISTRY

@@ -832,6 +836,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):

836 self._connector = MediaConnector(

837 media_io_kwargs=media_io_kwargs,

838 allowed_local_media_path=tracker.allowed_local_media_path,

839 + allowed_media_domains=tracker.allowed_media_domains,

840 )

841

842 def parse_image(

@@ -916,6 +921,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

921 self._connector = MediaConnector(

922 media_io_kwargs=media_io_kwargs,

923 allowed_local_media_path=tracker.allowed_local_media_path,

924 + allowed_media_domains=tracker.allowed_media_domains,

925 )

926

927 def parse_image(

vllm/entrypoints/llm.py

+4

@@ -86,6 +86,8 @@ class LLM:

86 or videos from directories specified by the server file system.

87 This is a security risk. Should only be enabled in trusted

88 environments.

89 + allowed_media_domains: If set, only media URLs that belong to this

90 + domain can be used for multi-modal inputs.

91 tensor_parallel_size: The number of GPUs to use for distributed

92 execution with tensor parallelism.

93 dtype: The data type for the model weights and activations. Currently,

@@ -169,6 +171,7 @@ class LLM:

171 skip_tokenizer_init: bool = False,

172 trust_remote_code: bool = False,

173 allowed_local_media_path: str = "",

174 + allowed_media_domains: Optional[list[str]] = None,

175 tensor_parallel_size: int = 1,

176 dtype: ModelDType = "auto",

177 quantization: Optional[QuantizationMethods] = None,

@@ -264,6 +267,7 @@ class LLM:

267 skip_tokenizer_init=skip_tokenizer_init,

268 trust_remote_code=trust_remote_code,

269 allowed_local_media_path=allowed_local_media_path,

270 + allowed_media_domains=allowed_media_domains,

271 tensor_parallel_size=tensor_parallel_size,

272 dtype=dtype,

273 quantization=quantization,

vllm/multimodal/utils.py

+16

@@ -50,6 +50,7 @@ class MediaConnector:

50 connection: HTTPConnection = global_http_connection,

51 *,

52 allowed_local_media_path: str = "",

53 + allowed_media_domains: Optional[list[str]] = None,

54 ) -> None:

55 """

56 Args:

@@ -82,6 +83,9 @@ class MediaConnector:

83 allowed_local_media_path_ = None

84

85 self.allowed_local_media_path = allowed_local_media_path_

86 + if allowed_media_domains is None:

87 + allowed_media_domains = []

88 + self.allowed_media_domains = allowed_media_domains

89

90 def _load_data_url(

91 self,

@@ -115,6 +119,14 @@ class MediaConnector:

119

120 return media_io.load_file(filepath)

121

122 + def _assert_url_in_allowed_media_domains(self, url_spec) -> None:

123 + if self.allowed_media_domains and url_spec.hostname not in \

124 + self.allowed_media_domains:

125 + raise ValueError(

126 + f"The URL must be from one of the allowed domains: "

127 + f"{self.allowed_media_domains}. Input URL domain: "

128 + f"{url_spec.hostname}")

129 +

130 def load_from_url(

131 self,

132 url: str,

@@ -125,6 +137,8 @@ class MediaConnector:

137 url_spec = urlparse(url)

138

139 if url_spec.scheme.startswith("http"):

140 + self._assert_url_in_allowed_media_domains(url_spec)

141 +

142 connection = self.connection

143 data = connection.get_bytes(url, timeout=fetch_timeout)

144

@@ -150,6 +164,8 @@ class MediaConnector:

164 loop = asyncio.get_running_loop()

165

166 if url_spec.scheme.startswith("http"):

167 + self._assert_url_in_allowed_media_domains(url_spec)

168 +

169 connection = self.connection

170 data = await connection.async_get_bytes(url, timeout=fetch_timeout)

171 future = loop.run_in_executor(global_thread_pool,

Add option to restrict media domains (#25783)