@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ac6b8f19b9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] Multi-Modality Support for Loading Local Image Files (#9915)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Chauncey committed Nov 4, 2024 at 23:34 UTC ac6b8f19b9007433b9cbf057c1d01ae9d4efdad5
6 files changed +132 -14
tests/multimodal/test_utils.py
+37 -2
@@ -1,11 +1,12 @@
1 import base64
2 import mimetypes
3 -from tempfile import NamedTemporaryFile
3 +import os
4 +from tempfile import NamedTemporaryFile, TemporaryDirectory
5 from typing import Dict, Tuple
6
7 import numpy as np
8 import pytest
8 -from PIL import Image
9 +from PIL import Image, ImageChops
10 from transformers import AutoConfig, AutoTokenizer
11
12 from vllm.multimodal.utils import (async_fetch_image, fetch_image,
@@ -84,6 +85,40 @@ async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
85 assert _image_equals(data_image_sync, data_image_async)
86
87
88 +@pytest.mark.asyncio
89 +@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
90 +async def test_fetch_image_local_files(image_url: str):
91 + with TemporaryDirectory() as temp_dir:
92 + origin_image = fetch_image(image_url)
93 + origin_image.save(os.path.join(temp_dir, os.path.basename(image_url)),
94 + quality=100,
95 + icc_profile=origin_image.info.get('icc_profile'))
96 +
97 + image_async = await async_fetch_image(
98 + f"file://{temp_dir}/{os.path.basename(image_url)}",
99 + allowed_local_media_path=temp_dir)
100 +
101 + image_sync = fetch_image(
102 + f"file://{temp_dir}/{os.path.basename(image_url)}",
103 + allowed_local_media_path=temp_dir)
104 + # Check that the images are equal
105 + assert not ImageChops.difference(image_sync, image_async).getbbox()
106 +
107 + with pytest.raises(ValueError):
108 + await async_fetch_image(
109 + f"file://{temp_dir}/../{os.path.basename(image_url)}",
110 + allowed_local_media_path=temp_dir)
111 + with pytest.raises(ValueError):
112 + await async_fetch_image(
113 + f"file://{temp_dir}/../{os.path.basename(image_url)}")
114 +
115 + with pytest.raises(ValueError):
116 + fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}",
117 + allowed_local_media_path=temp_dir)
118 + with pytest.raises(ValueError):
119 + fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}")
120 +
121 +
122 @pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"])
123 def test_repeat_and_pad_placeholder_tokens(model):
124 config = AutoConfig.from_pretrained(model)
vllm/config.py
+8
@@ -55,6 +55,10 @@ class ModelConfig:
55 "mistral" will always use the tokenizer from `mistral_common`.
56 trust_remote_code: Trust remote code (e.g., from HuggingFace) when
57 downloading the model and tokenizer.
58 + allowed_local_media_path: Allowing API requests to read local images or
59 + videos from directories specified by the server file system.
60 + This is a security risk. Should only be enabled in trusted
61 + environments.
62 dtype: Data type for model weights and activations. The "auto" option
63 will use FP16 precision for FP32 and FP16 models, and BF16 precision
64 for BF16 models.
@@ -134,6 +138,7 @@ class ModelConfig:
138 trust_remote_code: bool,
139 dtype: Union[str, torch.dtype],
140 seed: int,
141 + allowed_local_media_path: str = "",
142 revision: Optional[str] = None,
143 code_revision: Optional[str] = None,
144 rope_scaling: Optional[dict] = None,
@@ -164,6 +169,7 @@ class ModelConfig:
169 self.tokenizer = tokenizer
170 self.tokenizer_mode = tokenizer_mode
171 self.trust_remote_code = trust_remote_code
172 + self.allowed_local_media_path = allowed_local_media_path
173 self.seed = seed
174 self.revision = revision
175 self.code_revision = code_revision
@@ -1319,6 +1325,8 @@ class SpeculativeConfig:
1325 tokenizer=target_model_config.tokenizer,
1326 tokenizer_mode=target_model_config.tokenizer_mode,
1327 trust_remote_code=target_model_config.trust_remote_code,
1328 + allowed_local_media_path=target_model_config.
1329 + allowed_local_media_path,
1330 dtype=target_model_config.dtype,
1331 seed=target_model_config.seed,
1332 revision=draft_revision,
vllm/engine/arg_utils.py
+9
@@ -92,6 +92,7 @@ class EngineArgs:
92 tokenizer_mode: str = 'auto'
93 chat_template_text_format: str = 'string'
94 trust_remote_code: bool = False
95 + allowed_local_media_path: str = ""
96 download_dir: Optional[str] = None
97 load_format: str = 'auto'
98 config_format: ConfigFormat = ConfigFormat.AUTO
@@ -269,6 +270,13 @@ class EngineArgs:
270 parser.add_argument('--trust-remote-code',
271 action='store_true',
272 help='Trust remote code from huggingface.')
273 + parser.add_argument(
274 + '--allowed-local-media-path',
275 + type=str,
276 + help="Allowing API requests to read local images or videos"
277 + "from directories specified by the server file system."
278 + "This is a security risk."
279 + "Should only be enabled in trusted environments")
280 parser.add_argument('--download-dir',
281 type=nullable_str,
282 default=EngineArgs.download_dir,
@@ -920,6 +928,7 @@ class EngineArgs:
928 tokenizer_mode=self.tokenizer_mode,
929 chat_template_text_format=self.chat_template_text_format,
930 trust_remote_code=self.trust_remote_code,
931 + allowed_local_media_path=self.allowed_local_media_path,
932 dtype=self.dtype,
933 seed=self.seed,
934 revision=self.revision,
vllm/entrypoints/chat_utils.py
+7 -2
@@ -307,7 +307,9 @@ class MultiModalContentParser(BaseMultiModalContentParser):
307 self._tracker = tracker
308
309 def parse_image(self, image_url: str) -> None:
310 - image = get_and_parse_image(image_url)
310 + image = get_and_parse_image(image_url,
311 + allowed_local_media_path=self._tracker.
312 + _model_config.allowed_local_media_path)
313
314 placeholder = self._tracker.add("image", image)
315 self._add_placeholder(placeholder)
@@ -327,7 +329,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
329 self._tracker = tracker
330
331 def parse_image(self, image_url: str) -> None:
330 - image_coro = async_get_and_parse_image(image_url)
332 + image_coro = async_get_and_parse_image(
333 + image_url,
334 + allowed_local_media_path=self._tracker._model_config.
335 + allowed_local_media_path)
336
337 placeholder = self._tracker.add("image", image_coro)
338 self._add_placeholder(placeholder)
vllm/entrypoints/llm.py
+6
@@ -58,6 +58,10 @@ class LLM:
58 from the input.
59 trust_remote_code: Trust remote code (e.g., from HuggingFace) when
60 downloading the model and tokenizer.
61 + allowed_local_media_path: Allowing API requests to read local images
62 + or videos from directories specified by the server file system.
63 + This is a security risk. Should only be enabled in trusted
64 + environments.
65 tensor_parallel_size: The number of GPUs to use for distributed
66 execution with tensor parallelism.
67 dtype: The data type for the model weights and activations. Currently,
@@ -139,6 +143,7 @@ class LLM:
143 tokenizer_mode: str = "auto",
144 skip_tokenizer_init: bool = False,
145 trust_remote_code: bool = False,
146 + allowed_local_media_path: str = "",
147 tensor_parallel_size: int = 1,
148 dtype: str = "auto",
149 quantization: Optional[str] = None,
@@ -179,6 +184,7 @@ class LLM:
184 tokenizer_mode=tokenizer_mode,
185 skip_tokenizer_init=skip_tokenizer_init,
186 trust_remote_code=trust_remote_code,
187 + allowed_local_media_path=allowed_local_media_path,
188 tensor_parallel_size=tensor_parallel_size,
189 dtype=dtype,
190 quantization=quantization,
vllm/multimodal/utils.py
+65 -10
@@ -1,4 +1,5 @@
1 import base64
2 +import os
3 from functools import lru_cache
4 from io import BytesIO
5 from typing import Any, List, Optional, Tuple, TypeVar, Union
@@ -18,19 +19,60 @@ logger = init_logger(__name__)
19 cached_get_tokenizer = lru_cache(get_tokenizer)
20
21
21 -def _load_image_from_bytes(b: bytes):
22 +def _load_image_from_bytes(b: bytes) -> Image.Image:
23 image = Image.open(BytesIO(b))
24 image.load()
25 return image
26
27
27 -def _load_image_from_data_url(image_url: str):
28 +def _is_subpath(image_path: str, allowed_local_media_path: str) -> bool:
29 + # Get the common path
30 + common_path = os.path.commonpath([
31 + os.path.abspath(image_path),
32 + os.path.abspath(allowed_local_media_path)
33 + ])
34 + # Check if the common path is the same as allowed_local_media_path
35 + return common_path == os.path.abspath(allowed_local_media_path)
36 +
37 +
38 +def _load_image_from_file(image_url: str,
39 + allowed_local_media_path: str) -> Image.Image:
40 + if not allowed_local_media_path:
41 + raise ValueError("Invalid 'image_url': Cannot load local files without"
42 + "'--allowed-local-media-path'.")
43 + if allowed_local_media_path:
44 + if not os.path.exists(allowed_local_media_path):
45 + raise ValueError(
46 + "Invalid '--allowed-local-media-path': "
47 + f"The path {allowed_local_media_path} does not exist.")
48 + if not os.path.isdir(allowed_local_media_path):
49 + raise ValueError(
50 + "Invalid '--allowed-local-media-path': "
51 + f"The path {allowed_local_media_path} must be a directory.")
52 +
53 + # Only split once and assume the second part is the image path
54 + _, image_path = image_url.split("file://", 1)
55 + if not _is_subpath(image_path, allowed_local_media_path):
56 + raise ValueError(
57 + f"Invalid 'image_url': The file path {image_path} must"
58 + " be a subpath of '--allowed-local-media-path'"
59 + f" '{allowed_local_media_path}'.")
60 +
61 + image = Image.open(image_path)
62 + image.load()
63 + return image
64 +
65 +
66 +def _load_image_from_data_url(image_url: str) -> Image.Image:
67 # Only split once and assume the second part is the base64 encoded image
68 _, image_base64 = image_url.split(",", 1)
69 return load_image_from_base64(image_base64)
70
71
33 -def fetch_image(image_url: str, *, image_mode: str = "RGB") -> Image.Image:
72 +def fetch_image(image_url: str,
73 + *,
74 + image_mode: str = "RGB",
75 + allowed_local_media_path: str = "") -> Image.Image:
76 """
77 Load a PIL image from a HTTP or base64 data URL.
78
@@ -43,16 +85,19 @@ def fetch_image(image_url: str, *, image_mode: str = "RGB") -> Image.Image:
85
86 elif image_url.startswith('data:image'):
87 image = _load_image_from_data_url(image_url)
88 + elif image_url.startswith('file://'):
89 + image = _load_image_from_file(image_url, allowed_local_media_path)
90 else:
91 raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
48 - "with either 'data:image' or 'http'.")
92 + "with either 'data:image', 'file://' or 'http'.")
93
94 return image.convert(image_mode)
95
96
97 async def async_fetch_image(image_url: str,
98 *,
55 - image_mode: str = "RGB") -> Image.Image:
99 + image_mode: str = "RGB",
100 + allowed_local_media_path: str = "") -> Image.Image:
101 """
102 Asynchronously load a PIL image from a HTTP or base64 data URL.
103
@@ -65,9 +110,11 @@ async def async_fetch_image(image_url: str,
110
111 elif image_url.startswith('data:image'):
112 image = _load_image_from_data_url(image_url)
113 + elif image_url.startswith('file://'):
114 + image = _load_image_from_file(image_url, allowed_local_media_path)
115 else:
116 raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
70 - "with either 'data:image' or 'http'.")
117 + "with either 'data:image', 'file://' or 'http'.")
118
119 return image.convert(image_mode)
120
@@ -126,8 +173,12 @@ def get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
173 return {"audio": (audio, sr)}
174
175
129 -def get_and_parse_image(image_url: str) -> MultiModalDataDict:
130 - image = fetch_image(image_url)
176 +def get_and_parse_image(
177 + image_url: str,
178 + *,
179 + allowed_local_media_path: str = "") -> MultiModalDataDict:
180 + image = fetch_image(image_url,
181 + allowed_local_media_path=allowed_local_media_path)
182 return {"image": image}
183
184
@@ -136,8 +187,12 @@ async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
187 return {"audio": (audio, sr)}
188
189
139 -async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
140 - image = await async_fetch_image(image_url)
190 +async def async_get_and_parse_image(
191 + image_url: str,
192 + *,
193 + allowed_local_media_path: str = "") -> MultiModalDataDict:
194 + image = await async_fetch_image(
195 + image_url, allowed_local_media_path=allowed_local_media_path)
196 return {"image": image}
197
198