@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 671427efbf

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 21:15 UTC

[Model] Move `multimodal_cpu_fields` definition to field config (#30181)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 6, 2025 at 21:40 UTC 671427efbf57d4e40370a336cf1d3a4d1fd8eb91

15 files changed +141 -95

tests/distributed/test_shm_storage.py

+1 -1

@@ -28,7 +28,7 @@ def _dummy_elem(modality: str, key: str, size: int):

28 modality=modality,

29 key=key,

30 data=torch.empty((size,), dtype=torch.int8),

31 - field=MultiModalSharedField(1),

31 + field=MultiModalSharedField(batch_size=1),

32 )

tests/multimodal/test_cache.py

+1 -1

@@ -51,7 +51,7 @@ def _dummy_elem(

51 modality=modality,

52 key=key,

53 data=data,

54 - field=MultiModalSharedField(1),

54 + field=MultiModalSharedField(batch_size=1),

55 )

tests/v1/test_serial_utils.py

+15 -6

@@ -104,22 +104,31 @@ class MyRequest(msgspec.Struct):

104

105 def test_multimodal_kwargs():

106 e1 = MultiModalFieldElem(

107 - "audio", "a0", torch.zeros(1000, dtype=torch.bfloat16), MultiModalBatchedField()

107 + "audio",

108 + "a0",

109 + torch.zeros(1000, dtype=torch.bfloat16),

110 + MultiModalBatchedField(),

111 )

112 e2 = MultiModalFieldElem(

113 "video",

114 "v0",

115 [torch.zeros(1000, dtype=torch.int8) for _ in range(4)],

113 - MultiModalFlatField([[slice(1, 2, 3), slice(4, 5, 6)], [slice(None, 2)]], 0),

116 + MultiModalFlatField(

117 + slices=[[slice(1, 2, 3), slice(4, 5, 6)], [slice(None, 2)]],

118 + dim=0,

119 + ),

120 )

121 e3 = MultiModalFieldElem(

116 - "image", "i0", torch.zeros(1000, dtype=torch.int32), MultiModalSharedField(4)

122 + "image",

123 + "i0",

124 + torch.zeros(1000, dtype=torch.int32),

125 + MultiModalSharedField(batch_size=4),

126 )

127 e4 = MultiModalFieldElem(

128 "image",

129 "i1",

130 torch.zeros(1000, dtype=torch.int32),

122 - MultiModalFlatField([slice(1, 2, 3), slice(4, 5, 6)], 2),

131 + MultiModalFlatField(slices=[slice(1, 2, 3), slice(4, 5, 6)], dim=2),

132 )

133 audio = MultiModalKwargsItem.from_elems([e1])

134 video = MultiModalKwargsItem.from_elems([e2])

@@ -138,8 +147,8 @@ def test_multimodal_kwargs():

147

148 total_len = sum(memoryview(x).cast("B").nbytes for x in encoded)

149

141 - # expected total encoding length, should be 14306, +-20 for minor changes

142 - assert 14275 <= total_len <= 14325

150 + # expected total encoding length, should be 14395, +-20 for minor changes

151 + assert 14375 <= total_len <= 14425

152 decoded = decoder.decode(encoded).mm[0]

153 assert isinstance(decoded, MultiModalKwargsItems)

154

vllm/model_executor/models/glm4_1v.py

+11 -18

@@ -787,10 +787,10 @@ class Glm4vVisionTransformer(nn.Module):

787 def forward(

788 self,

789 x: torch.Tensor,

790 - grid_thw: list[list[int]],

790 + grid_thw: torch.Tensor | list[list[int]],

791 ) -> torch.Tensor:

792 - # Convert grid_thw to tensor (always expecting list format now)

793 - grid_thw = torch.tensor(grid_thw, device=x.device, dtype=torch.long)

792 + if isinstance(grid_thw, list):

793 + grid_thw = torch.tensor(grid_thw, dtype=torch.int32)

794

795 # patchify

796 x = x.to(device=self.device, dtype=self.dtype)

@@ -805,7 +805,8 @@ class Glm4vVisionTransformer(nn.Module):

805 cu_seqlens = torch.repeat_interleave(

806 grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]

807 ).cumsum(dim=0, dtype=torch.int32)

808 - cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)

808 + cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])

809 + cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)

810

811 # pre-compute max_seqlen for attn mask to reduce cuMemcpy operations

812 max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)

@@ -1548,7 +1549,6 @@ class Glm4vForConditionalGeneration(

1549 ) -> tuple[torch.Tensor, ...]:

1550 grid_thw = image_input["image_grid_thw"]

1551 assert grid_thw.ndim == 2

1551 - grid_thw_list = grid_thw.tolist()

1552

1553 if image_input["type"] == "image_embeds":

1554 image_embeds = image_input["image_embeds"].type(self.visual.dtype)

@@ -1559,12 +1559,10 @@ class Glm4vForConditionalGeneration(

1559 self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"

1560 )

1561 else:

1562 - image_embeds = self.visual(pixel_values, grid_thw=grid_thw.tolist())

1562 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw)

1563 +

1564 merge_size = self.visual.spatial_merge_size

1564 - sizes = (

1565 - torch.tensor(grid_thw_list, dtype=torch.long).prod(-1)

1566 - // (merge_size * merge_size)

1567 - ).tolist()

1565 + sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()

1566 return image_embeds.split(sizes)

1567

1568 def _process_video_input(

@@ -1572,7 +1570,6 @@ class Glm4vForConditionalGeneration(

1570 ) -> tuple[torch.Tensor, ...]:

1571 grid_thw = video_input["video_grid_thw"]

1572 assert grid_thw.ndim == 2

1575 - grid_thw_list = grid_thw.tolist()

1573

1574 if video_input["type"] == "video_embeds":

1575 video_embeds = video_input["video_embeds"].type(self.visual.dtype)

@@ -1588,15 +1585,11 @@ class Glm4vForConditionalGeneration(

1585 rope_type="rope_3d",

1586 )

1587 else:

1591 - video_embeds = self.visual(

1592 - pixel_values_videos, grid_thw=grid_thw.tolist()

1593 - )

1588 + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)

1589 +

1590 # Split concatenated embeddings for each video item.

1591 merge_size = self.visual.spatial_merge_size

1596 - sizes = (

1597 - torch.tensor(grid_thw_list, dtype=torch.long).prod(-1)

1598 - // (merge_size * merge_size)

1599 - ).tolist()

1592 + sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()

1593 return video_embeds.split(sizes)

1594

1595 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

vllm/model_executor/models/hunyuan_vision.py

+1 -3

@@ -563,7 +563,7 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):

563 return dict(

564 pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),

565 image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),

566 - image_grid_thw=MultiModalFieldConfig.batched("image"),

566 + image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),

567 )

568

569

@@ -786,8 +786,6 @@ class HunYuanVLForConditionalGeneration(

786 SupportsQuant,

787 SupportsXDRoPE,

788 ):

789 - multimodal_cpu_fields = {"image_grid_thw"}

790 -

789 # To ensure correct weight loading and mapping.

790 hf_to_vllm_mapper = WeightsMapper(

791 orig_to_new_prefix={

vllm/model_executor/models/interfaces.py

+11 -2

@@ -84,9 +84,9 @@ class SupportsMultiModal(Protocol):

84 `vllm.multimodal.utils.group_mm_kwargs_by_modality` to use.

85 """

87 - multimodal_cpu_fields: ClassVar[Set[str]] = frozenset()

87 + multimodal_cpu_fields: ClassVar[Set[str] | None] = None

88 """

89 - A set indicating CPU-only multimodal fields.

89 + [DEPRECATED] A set indicating CPU-only multimodal fields.

90 """

92 _processor_factory: ClassVar[_ProcessorFactories]

@@ -279,6 +279,15 @@ def supports_multimodal(

279 "please remove the override from your model."

280 )

281

282 + multimodal_cpu_fields = getattr(model, "multimodal_cpu_fields", None)

283 + if multimodal_cpu_fields is not None:

284 + raise ValueError(

285 + "`multimodal_cpu_fields` is no longer effective, "

286 + "please set `keep_on_cpu=True` in `MultiModalFieldConfig` "

287 + "(refer to https://github.com/vllm-project/vllm/pull/30181), "

288 + "and then remove the override from your model."

289 + )

290 +

291 return res

292

293

vllm/model_executor/models/opencua.py

-2

@@ -201,8 +201,6 @@ class OpenCUADummyInputsBuilder(Qwen2VLDummyInputsBuilder):

201 dummy_inputs=OpenCUADummyInputsBuilder,

202 )

203 class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):

204 - multimodal_cpu_fields = {"image_grid_thw"}

205 -

204 packed_modules_mapping = {

205 "qkv_proj": ["q_proj", "k_proj", "v_proj"],

206 "gate_up_proj": ["gate_proj", "up_proj"],

vllm/model_executor/models/qwen2_5_vl.py

-2

@@ -1039,8 +1039,6 @@ class Qwen2_5_VLForConditionalGeneration(

1039 SupportsMultiModalPruning,

1040 SupportsMRoPE,

1041 ):

1042 - multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}

1043 -

1042 packed_modules_mapping = {

1043 "qkv_proj": ["q_proj", "k_proj", "v_proj"],

1044 "gate_up_proj": ["gate_proj", "up_proj"],

vllm/model_executor/models/qwen2_vl.py

+6 -6

@@ -811,14 +811,14 @@ def _create_qwen2vl_field_factory(

811 image_embeds=MultiModalFieldConfig.flat_from_sizes(

812 "image", image_embed_grid_sizes

813 ),

814 - image_grid_thw=MultiModalFieldConfig.batched("image"),

814 + image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),

815 pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(

816 "video", video_grid_sizes

817 ),

818 video_embeds=MultiModalFieldConfig.flat_from_sizes(

819 "video", video_embed_grid_sizes

820 ),

821 - video_grid_thw=MultiModalFieldConfig.batched("video"),

821 + video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),

822 )

823

824 return _qwen2vl_field_config

@@ -1131,8 +1131,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])

1131 class Qwen2VLForConditionalGeneration(

1132 nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE

1133 ):

1134 - multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}

1135 -

1134 # To ensure correct weight loading and mapping.

1135 hf_to_vllm_mapper = WeightsMapper(

1136 orig_to_new_prefix={

@@ -1393,9 +1391,11 @@ class Qwen2VLForConditionalGeneration(

1391 else:

1392 pixel_values_videos = video_input["pixel_values_videos"]

1393 if self.use_data_parallel:

1396 - grid_thw_list = grid_thw.tolist()

1394 return run_dp_sharded_mrope_vision_model(

1398 - self.visual, pixel_values_videos, grid_thw_list, rope_type="rope_3d"

1395 + self.visual,

1396 + pixel_values_videos,

1397 + grid_thw.tolist(),

1398 + rope_type="rope_3d",

1399 )

1400 else:

1401 video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)

vllm/model_executor/models/qwen3_vl.py

+2 -4

@@ -984,14 +984,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

984 image_embeds=MultiModalFieldConfig.flat_from_sizes(

985 "image", image_grid_sizes

986 ),

987 - image_grid_thw=MultiModalFieldConfig.batched("image"),

987 + image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),

988 pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(

989 "video", video_grid_sizes

990 ),

991 video_embeds=MultiModalFieldConfig.flat_from_sizes(

992 "video", video_grid_sizes

993 ),

994 - video_grid_thw=MultiModalFieldConfig.batched("video"),

994 + video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),

995 )

996

997 def _get_prompt_updates(

@@ -1190,8 +1190,6 @@ class Qwen3VLForConditionalGeneration(

1190 SupportsMRoPE,

1191 SupportsEagle3,

1192 ):

1193 - multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}

1194 -

1193 packed_modules_mapping = {

1194 "qkv_proj": [

1195 "q_proj",

vllm/multimodal/inputs.py

+81 -35

@@ -3,7 +3,7 @@

4 from abc import ABC, abstractmethod

5 from collections import UserDict, defaultdict

6 -from collections.abc import Mapping, Sequence, Set

6 +from collections.abc import Mapping, Sequence

7 from dataclasses import dataclass

8 from functools import partial

9 from itertools import accumulate

@@ -223,6 +223,23 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:

223 return a == b

224

225

226 +def _nested_tensors_h2d(

227 + tensors: NestedTensors,

228 + device: torch.types.Device,

229 +) -> NestedTensors:

230 + if device is None:

231 + return tensors

232 +

233 + return json_map_leaves(

234 + (

235 + lambda x: x.to(device=device, non_blocking=True)

236 + if isinstance(x, torch.Tensor)

237 + else x

238 + ),

239 + tensors,

240 + )

241 +

242 +

243 BatchedTensorInputs: TypeAlias = dict[str, NestedTensors]

244 """

245 A dictionary containing nested tensors which have been batched via

@@ -334,7 +351,7 @@ class MultiModalFieldElem:

351 ) # noqa: E721

352

353

337 -@dataclass(frozen=True)

354 +@dataclass(frozen=True, kw_only=True)

355 class BaseMultiModalField(ABC):

356 """

357 Defines how to interpret tensor data belonging to a keyword argument in

@@ -342,6 +359,12 @@ class BaseMultiModalField(ABC):

359 multi-modal items, and vice versa.

360 """

361

362 + keep_on_cpu: bool = False

363 + """

364 + If `True`, then this field is excluded from being moved to the accelerator

365 + when `MultiModalKwargsItems.get_data()` is called to batch the data.

366 + """

367 +

368 def _field_factory(self, *, modality: str, key: str):

369 f = partial(

370 MultiModalFieldElem,

@@ -386,6 +409,7 @@ class BaseMultiModalField(ABC):

409 self,

410 elems: list[MultiModalFieldElem],

411 *,

412 + device: torch.types.Device = None,

413 pin_memory: bool = False,

414 ) -> NestedTensors:

415 """

@@ -399,11 +423,17 @@ class BaseMultiModalField(ABC):

423 if len(set(field_types)) > 1:

424 raise ValueError(f"Cannot merge different {field_types=}")

425

426 + if device is not None and self.keep_on_cpu:

427 + device = "cpu"

428 + if pin_memory and self.keep_on_cpu:

429 + pin_memory = False

430 +

431 batch = [elem.data for elem in elems]

403 - return self._reduce_data(batch, pin_memory=pin_memory)

432 + out = self._reduce_data(batch, pin_memory=pin_memory)

433 + return _nested_tensors_h2d(out, device=device)

434

435

406 -@dataclass(frozen=True)

436 +@dataclass(frozen=True, kw_only=True)

437 class MultiModalBatchedField(BaseMultiModalField):

438 """

439 Info:

@@ -445,7 +475,7 @@ class MultiModalBatchedField(BaseMultiModalField):

475 return batch

476

477

448 -@dataclass(frozen=True)

478 +@dataclass(frozen=True, kw_only=True)

479 class MultiModalFlatField(BaseMultiModalField):

480 """

481 Info:

@@ -505,7 +535,7 @@ class MultiModalFlatField(BaseMultiModalField):

535 return [e for elem in batch for e in elem]

536

537

508 -@dataclass(frozen=True)

538 +@dataclass(frozen=True, kw_only=True)

539 class MultiModalSharedField(BaseMultiModalField):

540 """

541 Info:

@@ -532,9 +562,10 @@ class MultiModalSharedField(BaseMultiModalField):

562 return batch[0]

563

564

565 +@dataclass(frozen=True)

566 class MultiModalFieldConfig:

567 @staticmethod

537 - def batched(modality: str):

568 + def batched(modality: str, *, keep_on_cpu: bool = False):

569 """

570 Defines a field where an element in the batch is obtained by

571 indexing into the first dimension of the underlying data.

@@ -542,6 +573,7 @@ class MultiModalFieldConfig:

573 Args:

574 modality: The modality of the multi-modal item that uses this

575 keyword argument.

576 + keep_on_cpu: Whether to keep this field on the CPU for the model inputs.

577

578 Example:

579

@@ -558,7 +590,7 @@ class MultiModalFieldConfig:

590 ```

591 """

592 return MultiModalFieldConfig(

561 - field=MultiModalBatchedField(),

593 + field=MultiModalBatchedField(keep_on_cpu=keep_on_cpu),

594 modality=modality,

595 )

596

@@ -567,6 +599,8 @@ class MultiModalFieldConfig:

599 modality: str,

600 slices: Sequence[slice] | Sequence[Sequence[slice]],

601 dim: int = 0,

602 + *,

603 + keep_on_cpu: bool = False,

604 ):

605 """

606 Defines a field where an element in the batch is obtained by

@@ -579,6 +613,7 @@ class MultiModalFieldConfig:

613 slices (dim>0) that is used to extract the data corresponding

614 to it.

615 dim: The dimension to extract data, default to 0.

616 + keep_on_cpu: Whether to keep this field on the CPU for the model inputs.

617

618 Example:

619

@@ -613,12 +648,22 @@ class MultiModalFieldConfig:

648 ```

649 """

650 return MultiModalFieldConfig(

616 - field=MultiModalFlatField(slices=slices, dim=dim),

651 + field=MultiModalFlatField(

652 + slices=slices,

653 + dim=dim,

654 + keep_on_cpu=keep_on_cpu,

655 + ),

656 modality=modality,

657 )

658

659 @staticmethod

621 - def flat_from_sizes(modality: str, size_per_item: "torch.Tensor", dim: int = 0):

660 + def flat_from_sizes(

661 + modality: str,

662 + size_per_item: "torch.Tensor",

663 + dim: int = 0,

664 + *,

665 + keep_on_cpu: bool = False,

666 + ):

667 """

668 Defines a field where an element in the batch is obtained by

669 slicing along the first dimension of the underlying data.

@@ -629,6 +674,7 @@ class MultiModalFieldConfig:

674 size_per_item: For each multi-modal item, the size of the slice

675 that is used to extract the data corresponding to it.

676 dim: The dimension to slice, default to 0.

677 + keep_on_cpu: Whether to keep this field on the CPU for the model inputs.

678

679 Example:

680

@@ -676,10 +722,20 @@ class MultiModalFieldConfig:

722 for i in range(len(size_per_item))

723 ]

724

679 - return MultiModalFieldConfig.flat(modality, slices, dim=dim)

725 + return MultiModalFieldConfig.flat(

726 + modality,

727 + slices,

728 + dim=dim,

729 + keep_on_cpu=keep_on_cpu,

730 + )

731

732 @staticmethod

682 - def shared(modality: str, batch_size: int):

733 + def shared(

734 + modality: str,

735 + batch_size: int,

736 + *,

737 + keep_on_cpu: bool = False,

738 + ):

739 """

740 Defines a field where an element in the batch is obtained by

741 taking the entirety of the underlying data.

@@ -690,6 +746,7 @@ class MultiModalFieldConfig:

746 modality: The modality of the multi-modal item that uses this

747 keyword argument.

748 batch_size: The number of multi-modal items which share this data.

749 + keep_on_cpu: Whether to keep this field on the CPU for the model inputs.

750

751 Example:

752

@@ -708,18 +765,15 @@ class MultiModalFieldConfig:

765 ```

766 """

767 return MultiModalFieldConfig(

711 - field=MultiModalSharedField(batch_size),

768 + field=MultiModalSharedField(

769 + batch_size=batch_size,

770 + keep_on_cpu=keep_on_cpu,

771 + ),

772 modality=modality,

773 )

774

715 - def __init__(self, field: BaseMultiModalField, modality: str) -> None:

716 - super().__init__()

717 -

718 - self.field = field

719 - self.modality = modality

720 -

721 - def __repr__(self) -> str:

722 - return f"MultiModalFieldConfig(field={self.field}, modality={self.modality})"

775 + field: BaseMultiModalField

776 + modality: str

777

778 def build_elems(

779 self,

@@ -744,7 +798,7 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):

798 modality=modality,

799 key="dummy",

800 data=torch.empty(nbytes, dtype=torch.uint8),

747 - field=MultiModalSharedField(1),

801 + field=MultiModalSharedField(batch_size=1),

802 )

803 return MultiModalKwargsItem.from_elems([mm_elem])

804

@@ -844,7 +898,6 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):

898 *,

899 device: torch.types.Device = None,

900 pin_memory: bool = False,

847 - cpu_fields: Set[str] = frozenset(),

901 ) -> BatchedTensorInputs:

902 """Construct a dictionary of keyword arguments to pass to the model."""

903 elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)

@@ -859,21 +912,14 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):

912 elems_by_key[key].append(elem)

913

914 data = {

862 - key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)

915 + key: elems[0].field.reduce_data(

916 + elems,

917 + device=device,

918 + pin_memory=pin_memory,

919 + )

920 for key, elems in elems_by_key.items()

921 }

922

866 - if device is not None:

867 - for k in data.keys() - cpu_fields:

868 - data[k] = json_map_leaves(

869 - (

870 - lambda x: x.to(device=device, non_blocking=True)

871 - if isinstance(x, torch.Tensor)

872 - else x

873 - ),

874 - data[k],

875 - )

876 -

923 return data

924

925

vllm/multimodal/utils.py

+6 -2

@@ -413,7 +413,7 @@ def group_mm_kwargs_by_modality(

413 device: torch.types.Device = None,

414 pin_memory: bool = False,

415 merge_by_field_config: bool | None = None,

416 - multimodal_cpu_fields: Set[str] = frozenset(),

416 + multimodal_cpu_fields: Set[str] | None = None,

417 ) -> Generator[tuple[str, int, BatchedTensorInputs], None, None]:

418 """Group consecutive `MultiModalKwargsItem`s from `mm_kwargs` with the same

419 modality together into the same `MultiModalKwargs` instance.

@@ -431,6 +431,11 @@ def group_mm_kwargs_by_modality(

431 "The `merge_by_field_config` argument of `group_mm_kwargs_by_modality` "

432 "is deprecated and will be removed in v0.13."

433 )

434 + if multimodal_cpu_fields is not None:

435 + logger.warning_once(

436 + "The `multimodal_cpu_fields` argument of `group_mm_kwargs_by_modality` "

437 + "is deprecated and will be removed in v0.13."

438 + )

439

440 from vllm.multimodal.inputs import MultiModalKwargsItems

441

@@ -440,7 +445,6 @@ def group_mm_kwargs_by_modality(

445 mm_kwargs_data = mm_kwargs_items.get_data(

446 device=device,

447 pin_memory=pin_memory,

443 - cpu_fields=multimodal_cpu_fields,

448 )

449

450 yield modality, len(items_lst), mm_kwargs_data

vllm/v1/serial_utils.py

+6 -5

@@ -269,10 +269,11 @@ class MsgpackEncoder:

269 name = MMF_CLASS_TO_FACTORY.get(field.__class__)

270 if not name:

271 raise TypeError(f"Unsupported field type: {field.__class__}")

272 +

273 # We just need to copy all of the field values in order

274 # which will be then used to reconstruct the field.

274 - field_values = (getattr(field, f.name) for f in dataclasses.fields(field))

275 - return name, *field_values

275 + factory_kw = {f.name: getattr(field, f.name) for f in dataclasses.fields(field)}

276 + return name, factory_kw

277

278

279 class MsgpackDecoder:

@@ -392,15 +393,15 @@ class MsgpackDecoder:

393 obj["data"] = self._decode_nested_tensors(obj["data"])

394

395 # Reconstruct the field processor using MultiModalFieldConfig

395 - factory_meth_name, *field_args = obj["field"]

396 + factory_meth_name, factory_kw = obj["field"]

397 factory_meth = getattr(MultiModalFieldConfig, factory_meth_name)

398

399 # Special case: decode the union "slices" field of

400 # MultiModalFlatField

401 if factory_meth_name == "flat":

401 - field_args[0] = self._decode_nested_slices(field_args[0])

402 + factory_kw["slices"] = self._decode_nested_slices(factory_kw["slices"])

403

403 - obj["field"] = factory_meth(None, *field_args).field

404 + obj["field"] = factory_meth("", **factory_kw).field

405 return MultiModalFieldElem(**obj)

406

407 def _decode_nested_tensors(self, obj: Any) -> NestedTensors:

vllm/v1/worker/gpu_model_runner.py

-5

@@ -1097,7 +1097,6 @@ class GPUModelRunner(

1097 device=self.device,

1098 pin_memory=self.pin_memory,

1099 merge_by_field_config=model.merge_by_field_config,

1100 - multimodal_cpu_fields=model.multimodal_cpu_fields,

1100 ):

1101 mm_kwargs_combined.update(mm_kwargs_group)

1102

@@ -2109,7 +2108,6 @@ class GPUModelRunner(

2108 mm_kwargs,

2109 device=self.device,

2110 pin_memory=self.pin_memory,

2112 - multimodal_cpu_fields=model.multimodal_cpu_fields,

2111 ):

2112 curr_group_outputs: list[torch.Tensor] = []

2113

@@ -2135,7 +2133,6 @@ class GPUModelRunner(

2133 [video_mm_kwargs_item],

2134 device=self.device,

2135 pin_memory=self.pin_memory,

2138 - multimodal_cpu_fields=model.multimodal_cpu_fields,

2136 )

2137 )

2138

@@ -3887,14 +3884,12 @@ class GPUModelRunner(

3884 dummy_mm_item = dummy_mm_data[modality][0]

3885 dummy_mm_items = [dummy_mm_item] * max_items_per_batch

3886

3890 - model = cast(SupportsMultiModal, self.model)

3887 return next(

3888 mm_kwargs_group

3889 for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(

3890 dummy_mm_items,

3891 device=self.device,

3892 pin_memory=self.pin_memory,

3897 - multimodal_cpu_fields=model.multimodal_cpu_fields,

3893 )

3894 )

3895

vllm/v1/worker/tpu_model_runner.py

-3

@@ -969,7 +969,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

969 mm_kwargs,

970 device=self.device,

971 pin_memory=self.pin_memory,

972 - multimodal_cpu_fields=model.multimodal_cpu_fields,

972 ):

973 # Run the encoder.

974 # `curr_group_outputs` is either of the following:

@@ -2050,14 +2049,12 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

2049 dummy_mm_item = dummy_mm_data[modality][0]

2050 dummy_mm_items = [dummy_mm_item] * max_items_per_batch

2051

2053 - model = cast(SupportsMultiModal, self.model)

2052 return next(

2053 grouped_mm_kwargs

2054 for _, _, grouped_mm_kwargs in group_mm_kwargs_by_modality(

2055 dummy_mm_items,

2056 device=self.device,

2057 pin_memory=self.pin_memory,

2060 - multimodal_cpu_fields=model.multimodal_cpu_fields,

2058 )

2059 )

2060