@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 755356b3d1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 02:45 UTC

feat: expose media_io_kwargs at runtime (#34778)

Signed-off-by: Alexandre Milesi <milesial@users.noreply.github.com>

milesial committed Mar 6, 2026 at 20:27 UTC 755356b3d18d8079b1b115dfd2111dc1accdb764
20 files changed +298 -16
tests/entrypoints/openai/test_video.py
+69
@@ -35,6 +35,8 @@ def server():
35 "--trust-remote-code",
36 "--limit-mm-per-prompt",
37 json.dumps({"video": MAXIMUM_VIDEOS}),
38 + "--media-io-kwargs",
39 + json.dumps({"video": {"num_frames": 32}}),
40 ]
41
42 # ROCm: Increase timeouts to handle potential network delays and slower
@@ -127,6 +129,73 @@ async def test_single_chat_session_video(
129 assert message.content is not None and len(message.content) >= 0
130
131
132 +@pytest.mark.asyncio
133 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
134 +@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
135 +async def test_request_media_io_kwargs_override_uses_fewer_video_frames(
136 + client: openai.AsyncOpenAI, model_name: str, video_url: str
137 +):
138 + messages = dummy_messages_from_video_url(video_url)
139 +
140 + default_resp = await client.chat.completions.create(
141 + model=model_name,
142 + messages=messages,
143 + max_completion_tokens=1,
144 + temperature=0.0,
145 + )
146 + override_resp = await client.chat.completions.create(
147 + model=model_name,
148 + messages=messages,
149 + max_completion_tokens=1,
150 + temperature=0.0,
151 + extra_body={
152 + "media_io_kwargs": {
153 + "video": {
154 + "num_frames": 4,
155 + }
156 + }
157 + },
158 + )
159 +
160 + assert default_resp.usage is not None
161 + assert override_resp.usage is not None
162 + assert override_resp.usage.prompt_tokens < default_resp.usage.prompt_tokens
163 +
164 +
165 +@pytest.mark.asyncio
166 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
167 +@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
168 +async def test_invalid_num_frames_request_recoverable(
169 + client: openai.AsyncOpenAI, model_name: str, video_url: str
170 +):
171 + messages = dummy_messages_from_video_url(video_url)
172 +
173 + with pytest.raises((openai.BadRequestError, openai.APIStatusError)):
174 + await client.chat.completions.create(
175 + model=model_name,
176 + messages=messages,
177 + max_completion_tokens=1,
178 + temperature=0.0,
179 + extra_body={
180 + "media_io_kwargs": {
181 + "video": {
182 + "num_frames": "invalid",
183 + }
184 + }
185 + },
186 + )
187 +
188 + # Server should still handle subsequent requests after the failed one.
189 + recovery_resp = await client.chat.completions.create(
190 + model=model_name,
191 + messages=messages,
192 + max_completion_tokens=1,
193 + temperature=0.0,
194 + )
195 + recovery_msg = recovery_resp.choices[0].message
196 + assert recovery_msg.content is not None and len(recovery_msg.content) >= 0
197 +
198 +
199 @pytest.mark.asyncio
200 @pytest.mark.parametrize("model_name", [MODEL_NAME])
201 @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
tests/entrypoints/pooling/embed/test_online_vision.py
+33
@@ -127,6 +127,39 @@ def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
127 assert output.usage.prompt_tokens == 767
128
129
130 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
131 +def test_chat_image_with_media_io_kwargs(server: RemoteOpenAIServer, model_name: str):
132 + rgba_image_url = (
133 + "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
134 + "/vision_model_images/RGBA_comp.png"
135 + )
136 + messages = [
137 + {
138 + "role": "user",
139 + "content": [
140 + {"type": "text", "text": "Represent the user's input."},
141 + {"type": "image_url", "image_url": {"url": rgba_image_url}},
142 + ],
143 + }
144 + ]
145 +
146 + response = requests.post(
147 + server.url_for("v1/embeddings"),
148 + json={
149 + "model": model_name,
150 + "messages": messages,
151 + "media_io_kwargs": {
152 + "image": {"rgba_background_color": [0, 0, 0]},
153 + },
154 + },
155 + )
156 + response.raise_for_status()
157 +
158 + output = EmbeddingResponse.model_validate(response.json())
159 + assert len(output.data) == 1
160 + assert len(output.data[0].embedding) == 3072
161 +
162 +
163 def get_hf_prompt_tokens(model_name, content, image_url):
164 processor = AutoProcessor.from_pretrained(
165 model_name, trust_remote_code=True, num_crops=4
vllm/entrypoints/chat_utils.py
+22 -9
@@ -462,10 +462,15 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
462 maximum per prompt.
463 """
464
465 - def __init__(self, model_config: ModelConfig):
465 + def __init__(
466 + self,
467 + model_config: ModelConfig,
468 + media_io_kwargs: dict[str, dict[str, Any]] | None = None,
469 + ):
470 super().__init__()
471
472 self._model_config = model_config
473 + self._media_io_kwargs = media_io_kwargs
474
475 self._items_by_modality = defaultdict[str, list[_T]](list)
476 # Track original modality for each vision_chunk item (image or video)
@@ -487,6 +492,14 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
492 model_cls = get_model_cls(self.model_config)
493 return cast(type[SupportsMultiModal], model_cls)
494
495 + @property
496 + def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None:
497 + return self._media_io_kwargs or (
498 + self._model_config.multimodal_config.media_io_kwargs
499 + if self._model_config.multimodal_config
500 + else None
501 + )
502 +
503 @property
504 def allowed_local_media_path(self):
505 return self._model_config.allowed_local_media_path
@@ -769,12 +782,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
782 super().__init__()
783
784 self._tracker = tracker
772 - multimodal_config = self._tracker.model_config.multimodal_config
773 - media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
785
786 self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
787 envs.VLLM_MEDIA_CONNECTOR,
777 - media_io_kwargs=media_io_kwargs,
788 + media_io_kwargs=tracker.media_io_kwargs,
789 allowed_local_media_path=tracker.allowed_local_media_path,
790 allowed_media_domains=tracker.allowed_media_domains,
791 )
@@ -881,11 +892,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
892 super().__init__()
893
894 self._tracker = tracker
884 - multimodal_config = self._tracker.model_config.multimodal_config
885 - media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
895 self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
896 envs.VLLM_MEDIA_CONNECTOR,
888 - media_io_kwargs=media_io_kwargs,
897 + media_io_kwargs=tracker.media_io_kwargs,
898 allowed_local_media_path=tracker.allowed_local_media_path,
899 allowed_media_domains=tracker.allowed_media_domains,
900 )
@@ -1530,13 +1539,14 @@ def parse_chat_messages(
1539 messages: list[ChatCompletionMessageParam],
1540 model_config: ModelConfig,
1541 content_format: ChatTemplateContentFormat,
1542 + media_io_kwargs: dict[str, dict[str, Any]] | None = None,
1543 ) -> tuple[
1544 list[ConversationMessage],
1545 MultiModalDataDict | None,
1546 MultiModalUUIDDict | None,
1547 ]:
1548 conversation: list[ConversationMessage] = []
1539 - mm_tracker = MultiModalItemTracker(model_config)
1549 + mm_tracker = MultiModalItemTracker(model_config, media_io_kwargs=media_io_kwargs)
1550
1551 for msg in messages:
1552 sub_messages = _parse_chat_message_content(
@@ -1563,13 +1573,16 @@ async def parse_chat_messages_async(
1573 messages: list[ChatCompletionMessageParam],
1574 model_config: ModelConfig,
1575 content_format: ChatTemplateContentFormat,
1576 + media_io_kwargs: dict[str, dict[str, Any]] | None = None,
1577 ) -> tuple[
1578 list[ConversationMessage],
1579 MultiModalDataDict | None,
1580 MultiModalUUIDDict | None,
1581 ]:
1582 conversation: list[ConversationMessage] = []
1572 - mm_tracker = AsyncMultiModalItemTracker(model_config)
1583 + mm_tracker = AsyncMultiModalItemTracker(
1584 + model_config, media_io_kwargs=media_io_kwargs
1585 + )
1586
1587 for msg in messages:
1588 sub_messages = _parse_chat_message_content(
vllm/entrypoints/openai/chat_completion/protocol.py
+8
@@ -268,6 +268,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
268 "Will be accessible by the chat template."
269 ),
270 )
271 + media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
272 + default=None,
273 + description=(
274 + "Additional kwargs to pass to the media IO connectors, "
275 + "keyed by modality. Merged with engine-level media_io_kwargs."
276 + ),
277 + )
278 mm_processor_kwargs: dict[str, Any] | None = Field(
279 default=None,
280 description=("Additional kwargs to pass to the HF processor."),
@@ -366,6 +373,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
373 reasoning_effort=self.reasoning_effort,
374 ),
375 ),
376 + media_io_kwargs=self.media_io_kwargs,
377 )
378
379 def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
vllm/entrypoints/openai/engine/serving.py
+6 -1
@@ -900,10 +900,15 @@ class OpenAIServing:
900 ),
901 )
902
903 + mm_config = self.model_config.multimodal_config
904 +
905 tok_params = request.build_tok_params(self.model_config)
906 chat_params = request.build_chat_params(
907 default_template, default_template_content_format
906 - ).with_defaults(default_template_kwargs)
908 + ).with_defaults(
909 + default_template_kwargs,
910 + default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
911 + )
912
913 (conversation,), (engine_prompt,) = await renderer.render_chat_async(
914 [messages],
vllm/entrypoints/openai/responses/protocol.py
+8
@@ -197,6 +197,13 @@ class ResponsesRequest(OpenAIBaseModel):
197 "through out the inference process and return in response."
198 ),
199 )
200 + media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
201 + default=None,
202 + description=(
203 + "Additional kwargs to pass to the media IO connectors, "
204 + "keyed by modality. Merged with engine-level media_io_kwargs."
205 + ),
206 + )
207 mm_processor_kwargs: dict[str, Any] | None = Field(
208 default=None,
209 description=("Additional kwargs to pass to the HF processor."),
@@ -276,6 +283,7 @@ class ResponsesRequest(OpenAIBaseModel):
283 reasoning_effort=None if reasoning is None else reasoning.effort,
284 ),
285 ),
286 + media_io_kwargs=self.media_io_kwargs,
287 )
288
289 def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
vllm/entrypoints/pooling/base/io_processor.py
+6 -1
@@ -123,10 +123,15 @@ class PoolingIOProcessor:
123 ),
124 )
125
126 + mm_config = self.model_config.multimodal_config
127 +
128 tok_params = request.build_tok_params(self.model_config)
129 chat_params = request.build_chat_params(
130 default_template, default_template_content_format
129 - ).with_defaults(default_template_kwargs)
131 + ).with_defaults(
132 + default_template_kwargs,
133 + default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
134 + )
135
136 (conversation,), (engine_prompt,) = renderer.render_chat(
137 [messages],
vllm/entrypoints/pooling/base/protocol.py
+8
@@ -124,6 +124,13 @@ class ChatRequestMixin(OpenAIBaseModel):
124 "Will be accessible by the chat template."
125 ),
126 )
127 + media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
128 + default=None,
129 + description=(
130 + "Additional kwargs to pass to the media IO connectors, "
131 + "keyed by modality. Merged with engine-level media_io_kwargs."
132 + ),
133 + )
134 # --8<-- [end:chat-extra-params]
135
136 @model_validator(mode="before")
@@ -151,6 +158,7 @@ class ChatRequestMixin(OpenAIBaseModel):
158 continue_final_message=self.continue_final_message,
159 ),
160 ),
161 + media_io_kwargs=self.media_io_kwargs,
162 )
163
164
vllm/entrypoints/serve/tokenize/protocol.py
+8
@@ -100,6 +100,13 @@ class TokenizeChatRequest(OpenAIBaseModel):
100 "Will be accessible by the chat template."
101 ),
102 )
103 + media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
104 + default=None,
105 + description=(
106 + "Additional kwargs to pass to the media IO connectors, "
107 + "keyed by modality. Merged with engine-level media_io_kwargs."
108 + ),
109 + )
110 mm_processor_kwargs: dict[str, Any] | None = Field(
111 default=None,
112 description="Additional kwargs to pass to the HF processor.",
@@ -134,6 +141,7 @@ class TokenizeChatRequest(OpenAIBaseModel):
141 continue_final_message=self.continue_final_message,
142 ),
143 ),
144 + media_io_kwargs=self.media_io_kwargs,
145 )
146
147 def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
vllm/multimodal/media/audio.py
+12 -1
@@ -83,11 +83,17 @@ def extract_audio_from_video_bytes(
83
84
85 class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
86 + """Configuration values can be user-provided either by --media-io-kwargs or
87 + by the runtime API field "media_io_kwargs". Ensure proper validation and
88 + error handling.
89 + """
90 +
91 def __init__(self, **kwargs) -> None:
92 super().__init__()
93
94 # `kwargs` contains custom arguments from
90 - # --media-io-kwargs for this modality.
95 + # --media-io-kwargs for this modality, merged with
96 + # per-request runtime media_io_kwargs via merge_kwargs().
97 # They can be passed to the underlying
98 # media loaders (e.g. custom implementations)
99 # for flexible control.
@@ -122,6 +128,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
128
129
130 class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
131 + """Configuration values can be user-provided either by --media-io-kwargs or
132 + by the runtime API field "media_io_kwargs". Ensure proper validation and
133 + error handling.
134 + """
135 +
136 def __init__(self) -> None:
137 super().__init__()
138
vllm/multimodal/media/base.py
+22
@@ -44,6 +44,28 @@ class MediaWithBytes(Generic[_T]):
44
45
46 class MediaIO(ABC, Generic[_T]):
47 + """Configuration values can be user-provided either by --media-io-kwargs or
48 + by the runtime API field "media_io_kwargs". Ensure proper validation and
49 + error handling.
50 + """
51 +
52 + @classmethod
53 + def merge_kwargs(
54 + cls,
55 + default_kwargs: dict[str, Any] | None,
56 + runtime_kwargs: dict[str, Any] | None,
57 + ) -> dict[str, Any]:
58 + """Merge config-level kwargs and request-level kwargs.
59 +
60 + By default this performs a shallow merge where runtime kwargs override
61 + keys in default kwargs. Subclasses may override to apply modality-
62 + specific behavior.
63 + """
64 + merged = dict(default_kwargs or {})
65 + if runtime_kwargs:
66 + merged.update(runtime_kwargs)
67 + return merged
68 +
69 @abstractmethod
70 def load_bytes(self, data: bytes) -> _T:
71 raise NotImplementedError
vllm/multimodal/media/connector.py
+34
@@ -32,9 +32,43 @@ atexit.register(global_thread_pool.shutdown)
32
33 MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
34
35 +MODALITY_IO_MAP: dict[str, type[MediaIO]] = {
36 + "audio": AudioMediaIO,
37 + "image": ImageMediaIO,
38 + "video": VideoMediaIO,
39 +}
40 +
41 +
42 +def merge_media_io_kwargs(
43 + defaults: dict[str, dict[str, Any]] | None,
44 + overrides: dict[str, dict[str, Any]] | None,
45 +) -> dict[str, dict[str, Any]] | None:
46 + """Merge config-level and per-request media_io_kwargs per modality.
47 +
48 + Each modality key is merged using the corresponding MediaIO subclass's
49 + ``merge_kwargs``, which may apply modality-specific logic (e.g.
50 + VideoMediaIO clears cross-dependent fps/num_frames fields).
51 + """
52 + if not defaults and not overrides:
53 + return None
54 + all_keys = set(defaults or {}) | set(overrides or {})
55 + merged = {}
56 + for key in all_keys:
57 + io_cls = MODALITY_IO_MAP.get(key, MediaIO)
58 + merged[key] = io_cls.merge_kwargs(
59 + (defaults or {}).get(key),
60 + (overrides or {}).get(key),
61 + )
62 + return merged or None
63 +
64
65 @MEDIA_CONNECTOR_REGISTRY.register("http")
66 class MediaConnector:
67 + """Configuration values can be user-provided either by --media-io-kwargs or
68 + by the runtime API field "media_io_kwargs". Ensure proper validation and
69 + error handling.
70 + """
71 +
72 def __init__(
73 self,
74 media_io_kwargs: dict[str, dict[str, Any]] | None = None,
vllm/multimodal/media/image.py
+14 -1
@@ -15,12 +15,18 @@ from .base import MediaIO, MediaWithBytes
15
16
17 class ImageMediaIO(MediaIO[Image.Image]):
18 + """Configuration values can be user-provided either by --media-io-kwargs or
19 + by the runtime API field "media_io_kwargs". Ensure proper validation and
20 + error handling.
21 + """
22 +
23 def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
24 super().__init__()
25
26 self.image_mode = image_mode
27 # `kwargs` contains custom arguments from
23 - # --media-io-kwargs for this modality.
28 + # --media-io-kwargs for this modality, merged with
29 + # per-request runtime media_io_kwargs via merge_kwargs().
30 # They can be passed to the underlying
31 # media loaders (e.g. custom implementations)
32 # for flexible control.
@@ -88,6 +94,13 @@ class ImageMediaIO(MediaIO[Image.Image]):
94
95
96 class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
97 + """Image embedding MediaIO implementation.
98 +
99 + Configuration values can be user-provided either by --media-io-kwargs or
100 + by the runtime API field "media_io_kwargs". Ensure proper validation and
101 + error handling.
102 + """
103 +
104 def __init__(self) -> None:
105 super().__init__()
106
vllm/multimodal/media/video.py
+24 -1
@@ -17,6 +17,28 @@ from .image import ImageMediaIO
17
18
19 class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
20 + """Configuration values can be user-provided either by --media-io-kwargs or
21 + by the runtime API field "media_io_kwargs". Ensure proper validation and
22 + error handling.
23 + """
24 +
25 + @classmethod
26 + def merge_kwargs(
27 + cls,
28 + default_kwargs: dict[str, Any] | None,
29 + runtime_kwargs: dict[str, Any] | None,
30 + ) -> dict[str, Any]:
31 + merged = super().merge_kwargs(default_kwargs, runtime_kwargs)
32 + # fps and num_frames interact with each other, so if either is
33 + # overridden at request time, wipe the other from defaults to
34 + # avoid unintuitive cross-field interactions.
35 + if runtime_kwargs:
36 + if "num_frames" in runtime_kwargs and "fps" not in runtime_kwargs:
37 + merged.pop("fps", None)
38 + elif "fps" in runtime_kwargs and "num_frames" not in runtime_kwargs:
39 + merged.pop("num_frames", None)
40 + return merged
41 +
42 def __init__(
43 self,
44 image_io: ImageMediaIO,
@@ -28,7 +50,8 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
50 self.image_io = image_io
51 self.num_frames = num_frames
52 # `kwargs` contains custom arguments from
31 - # --media-io-kwargs for this modality.
53 + # --media-io-kwargs for this modality, merged with
54 + # per-request runtime media_io_kwargs via merge_kwargs().
55 # They can be passed to the underlying
56 # media loaders (e.g. custom implementations)
57 # for flexible control.
vllm/renderers/deepseek_v32.py
+2
@@ -49,6 +49,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
49 messages,
50 self.model_config,
51 content_format="string",
52 + media_io_kwargs=params.media_io_kwargs,
53 )
54
55 prompt_raw = tokenizer.apply_chat_template(
@@ -75,6 +76,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
76 messages,
77 self.model_config,
78 content_format="string",
79 + media_io_kwargs=params.media_io_kwargs,
80 )
81
82 prompt_raw = tokenizer.apply_chat_template(
vllm/renderers/grok2.py
+2
@@ -49,6 +49,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):
49 messages,
50 self.model_config,
51 content_format="string",
52 + media_io_kwargs=params.media_io_kwargs,
53 )
54
55 prompt_raw = tokenizer.apply_chat_template(
@@ -75,6 +76,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):
76 messages,
77 self.model_config,
78 content_format="string",
79 + media_io_kwargs=params.media_io_kwargs,
80 )
81
82 prompt_raw = tokenizer.apply_chat_template(
vllm/renderers/hf.py
+2
@@ -635,6 +635,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
635 tokenizer=tokenizer,
636 model_config=model_config,
637 ),
638 + media_io_kwargs=params.media_io_kwargs,
639 )
640
641 prompt_raw = safe_apply_chat_template(
@@ -689,6 +690,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
690 tokenizer=tokenizer,
691 model_config=model_config,
692 ),
693 + media_io_kwargs=params.media_io_kwargs,
694 )
695
696 prompt_raw = safe_apply_chat_template(
vllm/renderers/mistral.py
+2
@@ -90,6 +90,7 @@ class MistralRenderer(BaseRenderer[MistralTokenizer]):
90 messages,
91 self.model_config,
92 content_format="string",
93 + media_io_kwargs=params.media_io_kwargs,
94 )
95
96 prompt_raw = safe_apply_chat_template(
@@ -116,6 +117,7 @@ class MistralRenderer(BaseRenderer[MistralTokenizer]):
117 messages,
118 self.model_config,
119 content_format="string",
120 + media_io_kwargs=params.media_io_kwargs,
121 )
122
123 prompt_raw = await self._apply_chat_template_async(
vllm/renderers/params.py
+14 -2
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, TypeVar
6 from vllm.exceptions import VLLMValidationError
7 from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
8 from vllm.logger import init_logger
9 +from vllm.multimodal.media.connector import merge_media_io_kwargs
10 from vllm.tokenizers import TokenizerLike
11 from vllm.utils.import_utils import LazyLoader
12
@@ -52,8 +53,15 @@ class ChatParams:
53 chat_template_kwargs: dict[str, Any] = field(default_factory=dict)
54 """The kwargs to pass to the chat template."""
55
55 - def with_defaults(self, default_chat_template_kwargs: dict[str, Any] | None):
56 - if not default_chat_template_kwargs:
56 + media_io_kwargs: dict[str, dict[str, Any]] | None = None
57 + """Per-modality kwargs for media I/O (loading/decoding images, videos, etc.)."""
58 +
59 + def with_defaults(
60 + self,
61 + default_chat_template_kwargs: dict[str, Any] | None = None,
62 + default_media_io_kwargs: dict[str, dict[str, Any]] | None = None,
63 + ):
64 + if not default_chat_template_kwargs and not default_media_io_kwargs:
65 return self
66
67 return ChatParams(
@@ -63,6 +71,10 @@ class ChatParams:
71 default_chat_template_kwargs,
72 self.chat_template_kwargs,
73 ),
74 + media_io_kwargs=merge_media_io_kwargs(
75 + default_media_io_kwargs,
76 + self.media_io_kwargs,
77 + ),
78 )
79
80 def get_apply_chat_template_kwargs(self) -> dict[str, Any]:
vllm/renderers/terratorch.py
+2
@@ -43,6 +43,7 @@ class TerratorchRenderer(BaseRenderer):
43 messages,
44 model_config,
45 content_format="string",
46 + media_io_kwargs=params.media_io_kwargs,
47 )
48
49 prompt = parse_dec_only_prompt([1]) # Dummy token IDs
@@ -64,6 +65,7 @@ class TerratorchRenderer(BaseRenderer):
65 messages,
66 model_config,
67 content_format="string",
68 + media_io_kwargs=params.media_io_kwargs,
69 )
70
71 prompt = parse_dec_only_prompt([1]) # Dummy token IDs