@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 29e5d10205

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 03:30 UTC

fix online fp8 for MiniCPM models (#39862)

Signed-off-by: Yan Ma <yan.ma@intel.com>

Yan Ma committed Apr 15, 2026 at 17:09 UTC 29e5d102050669d03992a2eb863ad364ea50fab2

1 file changed +27 -24

vllm/model_executor/models/minicpmv.py

+27 -24

@@ -1050,9 +1050,17 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

1050 quant_config=quant_config,

1051 prefix=maybe_prefix(prefix, "resampler"),

1052 )

1053 + self._resampler_moved = False

1054

1055 self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors

1056

1057 + def _ensure_resampler_device(self) -> None:

1058 + if self._resampler_moved:

1059 + return

1060 + # Only move device, DO NOT touch dtype (fp8 quant needs its own dtype)

1061 + self.resampler.to(current_platform.device_type)

1062 + self._resampler_moved = True

1063 +

1064 def _parse_and_validate_vision_input(

1065 self,

1066 modality: str,

@@ -1171,7 +1179,9 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

1179

1180 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

1181 loader = AutoWeightsLoader(self)

1174 - return loader.load_weights(weights)

1182 + loaded = loader.load_weights(weights)

1183 + self._ensure_resampler_device()

1184 + return loaded

1185

1186 def get_mm_mapping(self) -> MultiModelKeys:

1187 """

@@ -1276,9 +1286,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):

1286 prefix=prefix,

1287 )

1288

1279 - return resampler.to(

1280 - device=current_platform.device_type, dtype=torch.get_default_dtype()

1281 - )

1289 + return resampler.to(dtype=torch.get_default_dtype())

1290

1291 def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:

1292 pixel_values = data["pixel_values"]

@@ -1359,9 +1367,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):

1367 prefix=prefix,

1368 )

1369

1362 - return resampler.to(

1363 - device=current_platform.device_type, dtype=torch.get_default_dtype()

1364 - )

1370 + return resampler.to(dtype=torch.get_default_dtype())

1371

1372 def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:

1373 pixel_values = data["pixel_values"]

@@ -1452,11 +1458,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):

1458 quant_config=quant_config,

1459 prefix=prefix,

1460 )

1455 - target_device = current_platform.device_type

1456 - target_dtype = torch.get_default_dtype()

1457 - if any(p.is_meta for p in resampler.parameters()):

1458 - return resampler.to_empty(device=target_device).to(dtype=target_dtype)

1459 - return resampler.to(device=target_device, dtype=target_dtype)

1461 +

1462 + return resampler.to(dtype=torch.get_default_dtype())

1463

1464 def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:

1465 pixel_values = data["pixel_values"]

@@ -1491,7 +1494,9 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):

1494

1495 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

1496 loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])

1494 - return loader.load_weights(weights)

1497 + loaded = loader.load_weights(weights)

1498 + self._ensure_resampler_device()

1499 + return loaded

1500

1501

1502 class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):

@@ -1551,10 +1556,7 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):

1556 quant_config=quant_config,

1557 prefix=prefix,

1558 )

1554 -

1555 - return resampler.to(

1556 - device=current_platform.device_type, dtype=torch.get_default_dtype()

1557 - )

1559 + return resampler.to(dtype=torch.get_default_dtype())

1560

1561 def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:

1562 pixel_values = data["pixel_values"]

@@ -1589,7 +1591,9 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):

1591

1592 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

1593 loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])

1592 - return loader.load_weights(weights)

1594 + loaded = loader.load_weights(weights)

1595 + self._ensure_resampler_device()

1596 + return loaded

1597

1598

1599 class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):

@@ -1649,11 +1653,8 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):

1653 quant_config=quant_config,

1654 prefix=prefix,

1655 )

1652 - target_device = current_platform.device_type

1653 - target_dtype = torch.get_default_dtype()

1654 - if any(p.is_meta for p in resampler.parameters()):

1655 - return resampler.to_empty(device=target_device).to(dtype=target_dtype)

1656 - return resampler.to(device=target_device, dtype=target_dtype)

1656 +

1657 + return resampler.to(dtype=torch.get_default_dtype())

1658

1659 def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:

1660 pixel_values = data["pixel_values"]

@@ -1692,7 +1693,9 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):

1693

1694 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

1695 loader = AutoWeightsLoader(self, skip_prefixes=["apm.", "audio", "tts"])

1695 - return loader.load_weights(weights)

1696 + loaded = loader.load_weights(weights)

1697 + self._ensure_resampler_device()

1698 + return loaded

1699

1700

1701 _SUPPORT_VERSION = {