@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c0817e4d39

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 16:00 UTC

[Model] Add support for Cheers multimodal model (#38788)

Signed-off-by: bsliu <1187291748@qq.com> Signed-off-by: 吴炳贤 <wubingxian24@mails.ucas.ac.cn>

bsliu committed Apr 2, 2026 at 21:01 UTC c0817e4d39c78335952fb4b6bfae3cd3e45ac4c3

11 files changed +996

docs/models/supported_models.md

@@ -541,6 +541,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

541 | `BeeForConditionalGeneration` | Bee-8B | T + IE+ | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |

542 | `Blip2ForConditionalGeneration` | BLIP-2 | T + IE | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |

543 | `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ |

544 +| `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ |

545 | `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I+ | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ |

546 | `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I+ | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |

547 | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I+ | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |

examples/offline_inference/vision_language.py

+28

@@ -179,6 +179,33 @@ def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:

179 )

180

181

182 +# Cheers

183 +def run_cheers(questions: list[str], modality: str) -> ModelRequestData:

184 + assert modality == "image"

185 + model_name = "ai9stars/Cheers"

186 +

187 + engine_args = EngineArgs(

188 + model=model_name,

189 + trust_remote_code=True,

190 + max_model_len=4096,

191 + limit_mm_per_prompt={modality: 1},

192 + )

193 +

194 + prompts = [

195 + (

196 + f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

197 + f"<|im_start|>user\n<|image_pad|>{question}<|im_end|>\n"

198 + f"<|im_start|>assistant\n"

199 + )

200 + for question in questions

201 + ]

202 +

203 + return ModelRequestData(

204 + engine_args=engine_args,

205 + prompts=prompts,

206 + )

207 +

208 +

209 def run_command_a_vision(questions: list[str], modality: str) -> ModelRequestData:

210 assert modality == "image"

211

@@ -2140,6 +2167,7 @@ model_example_map = {

2167 "aria": run_aria,

2168 "aya_vision": run_aya_vision,

2169 "bagel": run_bagel,

2170 + "cheers": run_cheers,

2171 "bee": run_bee,

2172 "blip-2": run_blip2,

2173 "chameleon": run_chameleon,

tests/models/registry.py

@@ -766,6 +766,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {

766 extras={"6b": "Salesforce/blip2-opt-6.7b"},

767 ),

768 "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),

769 + "Cheers": _HfExamplesInfo(

770 + "ai9stars/Cheers",

771 + trust_remote_code=True,

772 + ),

773 + "CheersForConditionalGeneration": _HfExamplesInfo(

774 + "ai9stars/Cheers",

775 + trust_remote_code=True,

776 + ),

777 "Cohere2VisionForConditionalGeneration": _HfExamplesInfo(

778 "CohereLabs/command-a-vision-07-2025"

779 ),

vllm/config/model.py

@@ -1203,6 +1203,7 @@ class ModelConfig:

1203 "gemma3",

1204 "molmo2",

1205 "paligemma",

1206 + "umm",

1207 )

1208 if not hasattr(self.hf_config, "model_type"):

1209 return False

vllm/model_executor/models/cheers.py new

+753

@@ -0,0 +1,753 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Inference-only Cheers (UMM) model compatible with HuggingFace weights.

4 +

5 +Cheers is a unified multimodal model for image understanding and generation.

6 +For vLLM, we focus on the image understanding (vision-to-text) capabilities.

7 +The image generation part (gen_projector, hi_gate, etc.) is not supported,

8 +but the VAE encoder + decoder projector are required for image understanding.

9 +"""

10 +

11 +import math

12 +from collections.abc import Iterable, Mapping, Sequence

13 +from typing import Any, Literal, TypeAlias

14 +

15 +import torch

16 +import torch.nn as nn

17 +import torch.nn.functional as F

18 +from einops import rearrange

19 +from transformers import BatchFeature

20 +

21 +from vllm.config import VllmConfig

22 +from vllm.config.multimodal import BaseDummyOptions

23 +from vllm.inputs import MultiModalDataDict

24 +from vllm.logger import init_logger

25 +from vllm.model_executor.layers.quantization import QuantizationConfig

26 +from vllm.multimodal import MULTIMODAL_REGISTRY

27 +from vllm.multimodal.inputs import (

28 + MultiModalFieldConfig,

29 + MultiModalKwargsItems,

30 +)

31 +from vllm.multimodal.parse import MultiModalDataItems

32 +from vllm.multimodal.processing import (

33 + BaseDummyInputsBuilder,

34 + BaseMultiModalProcessor,

35 + BaseProcessingInfo,

36 + PromptReplacement,

37 +)

38 +from vllm.sequence import IntermediateTensors

39 +from vllm.transformers_utils.processors.cheers import CheersProcessor

40 +from vllm.utils.tensor_schema import TensorSchema

41 +

42 +from .interfaces import (

43 + MultiModalEmbeddings,

44 + SupportsLoRA,

45 + SupportsMultiModal,

46 + SupportsPP,

47 +)

48 +from .siglip import SiglipVisionModel

49 +from .utils import (

50 + AutoWeightsLoader,

51 + WeightsMapper,

52 + init_vllm_registered_model,

53 + maybe_prefix,

54 +)

55 +

56 +logger = init_logger(__name__)

57 +

58 +

59 +# ── VAE components (needed for image understanding pipeline) ────────

60 +

61 +

62 +def _swish(x: torch.Tensor) -> torch.Tensor:

63 + return x * torch.sigmoid(x)

64 +

65 +

66 +class _AttnBlock(nn.Module):

67 + def __init__(self, in_channels: int):

68 + super().__init__()

69 + self.norm = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)

70 + self.q = nn.Conv2d(in_channels, in_channels, 1)

71 + self.k = nn.Conv2d(in_channels, in_channels, 1)

72 + self.v = nn.Conv2d(in_channels, in_channels, 1)

73 + self.proj_out = nn.Conv2d(in_channels, in_channels, 1)

74 +

75 + def forward(self, x: torch.Tensor) -> torch.Tensor:

76 + h_ = self.norm(x)

77 + q = self.q(h_)

78 + k = self.k(h_)

79 + v = self.v(h_)

80 + b, c, h, w = q.shape

81 + q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous()

82 + k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous()

83 + v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous()

84 + h_ = F.scaled_dot_product_attention(q, k, v)

85 + h_ = rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)

86 + return x + self.proj_out(h_)

87 +

88 +

89 +class _ResnetBlock(nn.Module):

90 + def __init__(self, in_channels: int, out_channels: int):

91 + super().__init__()

92 + self.in_channels = in_channels

93 + self.out_channels = out_channels

94 + self.norm1 = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)

95 + self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, 1)

96 + self.norm2 = nn.GroupNorm(32, out_channels, eps=1e-6, affine=True)

97 + self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, 1)

98 + if in_channels != out_channels:

99 + self.nin_shortcut = nn.Conv2d(in_channels, out_channels, 1)

100 +

101 + def forward(self, x: torch.Tensor) -> torch.Tensor:

102 + h = _swish(self.norm1(x))

103 + h = self.conv1(h)

104 + h = _swish(self.norm2(h))

105 + h = self.conv2(h)

106 + if self.in_channels != self.out_channels:

107 + x = self.nin_shortcut(x)

108 + return x + h

109 +

110 +

111 +class _Downsample(nn.Module):

112 + def __init__(self, in_channels: int):

113 + super().__init__()

114 + self.conv = nn.Conv2d(in_channels, in_channels, 3, stride=2, padding=0)

115 +

116 + def forward(self, x: torch.Tensor) -> torch.Tensor:

117 + x = F.pad(x, (0, 1, 0, 1), mode="constant", value=0)

118 + return self.conv(x)

119 +

120 +

121 +class _Upsample(nn.Module):

122 + def __init__(self, in_channels: int):

123 + super().__init__()

124 + self.conv = nn.Conv2d(in_channels, in_channels, 3, 1, 1)

125 +

126 + def forward(self, x: torch.Tensor) -> torch.Tensor:

127 + x = F.interpolate(x, scale_factor=2.0, mode="nearest")

128 + return self.conv(x)

129 +

130 +

131 +_VAE_ENCODER_DEFAULTS = {

132 + "in_channels": 3,

133 + "ch": 128,

134 + "ch_mult": [1, 2, 4, 4],

135 + "num_res_blocks": 2,

136 + "z_channels": 32,

137 +}

138 +_VAE_DECODER_DEFAULTS = {

139 + "in_channels": 3,

140 + "out_ch": 3,

141 + "ch": 128,

142 + "ch_mult": [1, 2, 4, 4],

143 + "num_res_blocks": 2,

144 + "z_channels": 32,

145 +}

146 +

147 +

148 +def _cfg(config, key, defaults=None):

149 + """Access config attribute whether it's a dict or namespace object."""

150 + if isinstance(config, dict):

151 + if key in config:

152 + return config[key]

153 + if defaults and key in defaults:

154 + return defaults[key]

155 + raise KeyError(f"Key '{key}' not found in config dict: {list(config.keys())}")

156 + return getattr(config, key)

157 +

158 +

159 +class CheersVAEEncoder(nn.Module):

160 + """VAE encoder from the Cheers/UMM model."""

161 +

162 + def __init__(self, config):

163 + super().__init__()

164 + d = _VAE_ENCODER_DEFAULTS

165 + ch = _cfg(config, "ch", d)

166 + ch_mult = _cfg(config, "ch_mult", d)

167 + num_res_blocks = _cfg(config, "num_res_blocks", d)

168 + z_channels = _cfg(config, "z_channels", d)

169 + in_channels = _cfg(config, "in_channels", d)

170 + num_resolutions = len(ch_mult)

171 +

172 + self.quant_conv = nn.Conv2d(2 * z_channels, 2 * z_channels, 1)

173 + self.conv_in = nn.Conv2d(in_channels, ch, 3, 1, 1)

174 +

175 + in_ch_mult = (1,) + tuple(ch_mult)

176 + self.down = nn.ModuleList()

177 + block_in = ch

178 + for i_level in range(num_resolutions):

179 + block = nn.ModuleList()

180 + attn = nn.ModuleList()

181 + block_in = ch * in_ch_mult[i_level]

182 + block_out = ch * ch_mult[i_level]

183 + for _ in range(num_res_blocks):

184 + block.append(_ResnetBlock(block_in, block_out))

185 + block_in = block_out

186 + down = nn.Module()

187 + down.block = block

188 + down.attn = attn

189 + if i_level != num_resolutions - 1:

190 + down.downsample = _Downsample(block_in)

191 + self.down.append(down)

192 +

193 + self.mid = nn.Module()

194 + self.mid.block_1 = _ResnetBlock(block_in, block_in)

195 + self.mid.attn_1 = _AttnBlock(block_in)

196 + self.mid.block_2 = _ResnetBlock(block_in, block_in)

197 +

198 + self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)

199 + self.conv_out = nn.Conv2d(block_in, 2 * z_channels, 3, 1, 1)

200 + self._num_resolutions = num_resolutions

201 + self._num_res_blocks = num_res_blocks

202 +

203 + def forward(self, x: torch.Tensor) -> torch.Tensor:

204 + hs = [self.conv_in(x)]

205 + for i_level in range(self._num_resolutions):

206 + for i_block in range(self._num_res_blocks):

207 + h = self.down[i_level].block[i_block](hs[-1])

208 + if len(self.down[i_level].attn) > 0:

209 + h = self.down[i_level].attn[i_block](h)

210 + hs.append(h)

211 + if hasattr(self.down[i_level], "downsample"):

212 + hs.append(self.down[i_level].downsample(hs[-1]))

213 + h = hs[-1]

214 + h = self.mid.block_1(h)

215 + h = self.mid.attn_1(h)

216 + h = self.mid.block_2(h)

217 + h = _swish(self.norm_out(h))

218 + h = self.conv_out(h)

219 + h = self.quant_conv(h)

220 + return h

221 +

222 +

223 +class CheersVAEDecoder(nn.Module):

224 + """VAE decoder (used inside VAEDecoderProjector)."""

225 +

226 + def __init__(self, config):

227 + super().__init__()

228 + d = _VAE_DECODER_DEFAULTS

229 + ch = _cfg(config, "ch", d)

230 + ch_mult = _cfg(config, "ch_mult", d)

231 + num_res_blocks = _cfg(config, "num_res_blocks", d)

232 + z_channels = _cfg(config, "z_channels", d)

233 + out_ch = _cfg(config, "out_ch", d)

234 + num_resolutions = len(ch_mult)

235 +

236 + self.post_quant_conv = nn.Conv2d(z_channels, z_channels, 1)

237 + block_in = ch * ch_mult[num_resolutions - 1]

238 + self.conv_in = nn.Conv2d(z_channels, block_in, 3, 1, 1)

239 +

240 + self.mid = nn.Module()

241 + self.mid.block_1 = _ResnetBlock(block_in, block_in)

242 + self.mid.attn_1 = _AttnBlock(block_in)

243 + self.mid.block_2 = _ResnetBlock(block_in, block_in)

244 +

245 + self.up = nn.ModuleList()

246 + for i_level in reversed(range(num_resolutions)):

247 + block = nn.ModuleList()

248 + attn = nn.ModuleList()

249 + block_out = ch * ch_mult[i_level]

250 + for _ in range(num_res_blocks + 1):

251 + block.append(_ResnetBlock(block_in, block_out))

252 + block_in = block_out

253 + up = nn.Module()

254 + up.block = block

255 + up.attn = attn

256 + if i_level != 0:

257 + up.upsample = _Upsample(block_in)

258 + self.up.insert(0, up)

259 +

260 + self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)

261 + self.conv_out = nn.Conv2d(block_in, out_ch, 3, 1, 1)

262 + self._num_resolutions = num_resolutions

263 + self._num_res_blocks = num_res_blocks

264 +

265 + def forward(self, z: torch.Tensor) -> torch.Tensor:

266 + z = self.post_quant_conv(z)

267 + upscale_dtype = next(self.up.parameters()).dtype

268 + h = self.conv_in(z)

269 + h = self.mid.block_1(h)

270 + h = self.mid.attn_1(h)

271 + h = self.mid.block_2(h)

272 + h = h.to(upscale_dtype)

273 + for i_level in reversed(range(self._num_resolutions)):

274 + for i_block in range(self._num_res_blocks + 1):

275 + h = self.up[i_level].block[i_block](h)

276 + if len(self.up[i_level].attn) > 0:

277 + h = self.up[i_level].attn[i_block](h)

278 + if i_level != 0:

279 + h = self.up[i_level].upsample(h)

280 + h = _swish(self.norm_out(h))

281 + return self.conv_out(h)

282 +

283 +

284 +class CheersVAEModel(nn.Module):

285 + """VAE model with encoder only (for image understanding)."""

286 +

287 + def __init__(self, config):

288 + super().__init__()

289 + enc_cfg = _cfg(config, "vae_encoder_config")

290 + self.encoder = CheersVAEEncoder(enc_cfg)

291 + self.ps = [2, 2]

292 + z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)

293 + self.bn = nn.BatchNorm2d(

294 + math.prod(self.ps) * z_ch,

295 + eps=1e-4,

296 + momentum=0.1,

297 + affine=False,

298 + track_running_stats=True,

299 + )

300 +

301 + def encode(self, x: torch.Tensor) -> torch.Tensor:

302 + self.bn.eval()

303 + moments = self.encoder(x)

304 + mean = torch.chunk(moments, 2, dim=1)[0]

305 + z = rearrange(

306 + mean,

307 + "... c (i pi) (j pj) -> ... (c pi pj) i j",

308 + pi=self.ps[0],

309 + pj=self.ps[1],

310 + )

311 + return self.bn(z)

312 +

313 +

314 +class CheersVAEDecoderProjector(nn.Module):

315 + """VAE decoder projector that converts latent back to pixel-like space."""

316 +

317 + def __init__(self, config):

318 + super().__init__()

319 + dec_cfg = _cfg(config, "vae_decoder_config")

320 + enc_cfg = _cfg(config, "vae_encoder_config")

321 + self.decoder = CheersVAEDecoder(dec_cfg)

322 + self.ps = [2, 2]

323 + z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)

324 + self.bn = nn.BatchNorm2d(

325 + math.prod(self.ps) * z_ch,

326 + eps=1e-4,

327 + momentum=0.1,

328 + affine=False,

329 + track_running_stats=True,

330 + )

331 +

332 + def forward(self, z: torch.Tensor) -> torch.Tensor:

333 + self.bn.eval()

334 + s = torch.sqrt(self.bn.running_var.view(1, -1, 1, 1) + 1e-4)

335 + m = self.bn.running_mean.view(1, -1, 1, 1)

336 + z = z * s + m

337 + z = rearrange(

338 + z,

339 + "... (c pi pj) i j -> ... c (i pi) (j pj)",

340 + pi=self.ps[0],

341 + pj=self.ps[1],

342 + )

343 + return self.decoder(z)

344 +

345 +

346 +class CheersImagePixelInputs(TensorSchema):

347 + """

348 + Dimensions:

349 + - bn: Batch size * number of images

350 + - c: Number of channels (3)

351 + - h: Height of each image

352 + - w: Width of each image

353 + """

354 +

355 + type: Literal["pixel_values"]

356 + pixel_values: torch.Tensor # Shape: (bn, 3, h, w)

357 +

358 +

359 +CheersImageInputs: TypeAlias = CheersImagePixelInputs

360 +

361 +

362 +class CheersUndProjector(nn.Module):

363 + """Understanding projector that maps vision features to LLM dimension

364 + with 2x2 spatial compression (4x token reduction)."""

365 +

366 + def __init__(

367 + self,

368 + image_embed_dim: int,

369 + text_embed_dim: int,

370 + compression_factor: tuple[int, int] = (2, 2),

371 + quant_config: QuantizationConfig | None = None,

372 + prefix: str = "",

373 + ):

374 + super().__init__()

375 + self.image_embed_dim = image_embed_dim

376 + self.text_embed_dim = text_embed_dim

377 + self.compression_factor = compression_factor

378 + self.layernorm = nn.LayerNorm(image_embed_dim)

379 + hidden_size = image_embed_dim * (compression_factor[0] * compression_factor[1])

380 + self.mlp = nn.Sequential(

381 + nn.Linear(hidden_size, hidden_size),

382 + nn.GELU(),

383 + nn.Linear(hidden_size, text_embed_dim),

384 + )

385 +

386 + def forward(self, x: torch.Tensor) -> torch.Tensor:

387 + x = self.layernorm(x)

388 + height = width = int(x.size(1) ** 0.5)

389 + x = x.permute(0, 2, 1).unflatten(-1, (height, width))

390 + batch_size, dim, height, width = x.shape

391 + unfolded = x.unfold(

392 + 2, self.compression_factor[0], self.compression_factor[0]

393 + ).unfold(3, self.compression_factor[1], self.compression_factor[1])

394 + unfolded = unfolded.contiguous().view(

395 + batch_size,

396 + dim,

397 + -1,

398 + self.compression_factor[0] * self.compression_factor[1],

399 + )

400 + unfolded = (

401 + unfolded.permute(0, 2, 3, 1)

402 + .contiguous()

403 + .view(

404 + batch_size,

405 + -1,

406 + dim * self.compression_factor[0] * self.compression_factor[1],

407 + )

408 + )

409 + return self.mlp(unfolded)

410 +

411 +

412 +class CheersProcessingInfo(BaseProcessingInfo):

413 + """Processing information for Cheers model."""

414 +

415 + def get_hf_processor(self, **kwargs: object) -> CheersProcessor:

416 + from vllm.transformers_utils.processor import cached_get_image_processor

417 +

418 + image_processor = cached_get_image_processor(

419 + self.ctx.model_config.model,

420 + revision=self.ctx.model_config.revision,

421 + trust_remote_code=self.ctx.model_config.trust_remote_code,

422 + )

423 +

424 + tokenizer = self.get_tokenizer()

425 +

426 + return CheersProcessor(

427 + image_processor=image_processor,

428 + tokenizer=tokenizer,

429 + **kwargs,

430 + )

431 +

432 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:

433 + return {"image": None}

434 +

435 + def get_mm_max_tokens_per_item(

436 + self,

437 + seq_len: int,

438 + mm_counts: Mapping[str, int],

439 + ) -> Mapping[str, int]:

440 + hf_config = self.get_hf_config()

441 + vit_config = hf_config.vision_representation_config

442 + patch_size = vit_config.patch_size

443 + image_size = vit_config.image_size

444 + num_patches = (image_size // patch_size) ** 2

445 + # After 2x2 compression, tokens reduce by 4x

446 + num_tokens = num_patches // 4

447 + return {"image": num_tokens}

448 +

449 + def get_num_image_tokens(

450 + self,

451 + *,

452 + image_width: int,

453 + image_height: int,

454 + ) -> int:

455 + hf_config = self.get_hf_config()

456 + vit_config = hf_config.vision_representation_config

457 + patch_size = vit_config.patch_size

458 + image_size = vit_config.image_size

459 + num_patches = (image_size // patch_size) ** 2

460 + return num_patches // 4

461 +

462 +

463 +class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):

464 + """Build dummy inputs for Cheers model profiling."""

465 +

466 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

467 + num_images = mm_counts.get("image", 0)

468 + return "<|image_pad|>" * num_images

469 +

470 + def get_dummy_mm_data(

471 + self,

472 + seq_len: int,

473 + mm_counts: Mapping[str, int],

474 + mm_options: Mapping[str, BaseDummyOptions] | None = None,

475 + ) -> MultiModalDataDict:

476 + num_images = mm_counts.get("image", 0)

477 + hf_config = self.info.get_hf_config()

478 + vit_config = hf_config.vision_representation_config

479 + image_size = vit_config.image_size

480 + image_overrides = mm_options.get("image") if mm_options else None

481 +

482 + return {

483 + "image": self._get_dummy_images(

484 + width=image_size,

485 + height=image_size,

486 + num_images=num_images,

487 + overrides=image_overrides,

488 + ),

489 + }

490 +

491 +

492 +class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):

493 + """Multimodal processor for Cheers model."""

494 +

495 + def _call_hf_processor(

496 + self,

497 + prompt: str,

498 + mm_data: Mapping[str, object],

499 + mm_kwargs: Mapping[str, object],

500 + tok_kwargs: Mapping[str, object],

501 + ) -> BatchFeature:

502 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

503 +

504 + def _hf_processor_applies_updates(

505 + self,

506 + prompt_text: str,

507 + mm_items: MultiModalDataItems,

508 + hf_processor_mm_kwargs: Mapping[str, object],

509 + tokenization_kwargs: Mapping[str, object],

510 + ) -> bool:

511 + return False

512 +

513 + def _get_prompt_updates(

514 + self,

515 + mm_items: MultiModalDataItems,

516 + hf_processor_mm_kwargs: Mapping[str, Any],

517 + out_mm_kwargs: MultiModalKwargsItems,

518 + ) -> Sequence[PromptReplacement]:

519 + hf_config = self.info.get_hf_config()

520 + vit_config = hf_config.vision_representation_config

521 + patch_size = vit_config.patch_size

522 + image_size = vit_config.image_size

523 +

524 + tokenizer = self.info.get_tokenizer()

525 + image_token_id = tokenizer.get_vocab().get("<|image_pad|>")

526 + if image_token_id is None:

527 + raise ValueError(

528 + "Image token '<|image_pad|>' not found in tokenizer vocabulary"

529 + )

530 +

531 + def get_replacement_cheers(item_idx: int):

532 + num_patches = (image_size // patch_size) ** 2

533 + num_tokens = num_patches // 4

534 + return [image_token_id] * num_tokens

535 +

536 + return [

537 + PromptReplacement(

538 + modality="image",

539 + target=[image_token_id],

540 + replacement=get_replacement_cheers,

541 + )

542 + ]

543 +

544 + def _get_mm_fields_config(

545 + self,

546 + hf_inputs: Any,

547 + hf_processor_mm_kwargs: Mapping[str, object],

548 + ) -> Mapping[str, MultiModalFieldConfig]:

549 + return {

550 + "pixel_values": MultiModalFieldConfig.batched("image"),

551 + }

552 +

553 +

554 +@MULTIMODAL_REGISTRY.register_processor(

555 + CheersMultiModalProcessor,

556 + info=CheersProcessingInfo,

557 + dummy_inputs=CheersDummyInputsBuilder,

558 +)

559 +class CheersForConditionalGeneration(

560 + nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP

561 +):

562 + """

563 + Cheers: A unified multimodal model for image understanding and generation.

564 +

565 + For vLLM, we focus on the image understanding (vision-to-text) capabilities.

566 + The image generation part is not supported in vLLM.

567 + """

568 +

569 + requires_raw_input_tokens = True

570 +

571 + hf_to_vllm_mapper = WeightsMapper(

572 + orig_to_new_prefix={

573 + "model.language_model.": "language_model.model.",

574 + "model.vision_representation.": "vision_representation.vision_model.",

575 + "model.und_projector.": "und_projector.",

576 + "model.vae_model.": "vae_model.",

577 + "model.vae_decoder_projector.": "vae_decoder_projector.",

578 + "lm_head.": "language_model.lm_head.",

579 + }

580 + )

581 +

582 + @classmethod

583 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:

584 + if modality.startswith("image"):

585 + return "<|image_pad|>"

586 + raise ValueError("Only image modality is supported")

587 +

588 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

589 + super().__init__()

590 +

591 + config = vllm_config.model_config.hf_config

592 + quant_config = vllm_config.quant_config

593 + multimodal_config = vllm_config.model_config.multimodal_config

594 +

595 + if type(config).__name__ not in ("CheersConfig", "UMMConfig"):

596 + raise ValueError(

597 + f"Expected CheersConfig or UMMConfig, got {type(config).__name__}."

598 + )

599 +

600 + self.config = config

601 + self.multimodal_config = multimodal_config

602 +

603 + # The Cheers model's custom Qwen2Config defaults rope_theta to

604 + # 1_000_000, but this isn't stored in the JSON. vLLM's standard

605 + # Qwen2Config defaults to 10_000, causing a 100× mismatch.

606 + # We must patch BOTH the attribute AND rope_parameters (which

607 + # patch_rope_parameters may have already populated from the wrong

608 + # default before __init__ runs).

609 + _CHEERS_ROPE_THETA = 1_000_000.0

610 + tc = config.text_config

611 + old_theta = getattr(tc, "rope_theta", None)

612 + if old_theta != _CHEERS_ROPE_THETA:

613 + logger.info(

614 + "Overriding text_config.rope_theta from %s to %s",

615 + old_theta,

616 + _CHEERS_ROPE_THETA,

617 + )

618 + tc.rope_theta = _CHEERS_ROPE_THETA

619 + rp = getattr(tc, "rope_parameters", None)

620 + if rp is not None and rp.get("rope_theta") != _CHEERS_ROPE_THETA:

621 + logger.info(

622 + "Overriding rope_parameters.rope_theta from %s to %s",

623 + rp.get("rope_theta"),

624 + _CHEERS_ROPE_THETA,

625 + )

626 + rp["rope_theta"] = _CHEERS_ROPE_THETA

627 +

628 + with self._mark_language_model(vllm_config):

629 + self.language_model = init_vllm_registered_model(

630 + vllm_config=vllm_config,

631 + hf_config=config.text_config,

632 + prefix=maybe_prefix(prefix, "language_model"),

633 + architectures=["Qwen2ForCausalLM"],

634 + )

635 +

636 + vit_config = config.vision_representation_config

637 +

638 + with self._mark_tower_model(vllm_config, "image"):

639 + self.vae_model = CheersVAEModel(config)

640 + self.vae_decoder_projector = CheersVAEDecoderProjector(config)

641 +

642 + self.vision_representation = SiglipVisionModel(

643 + config=vit_config,

644 + quant_config=quant_config,

645 + prefix=maybe_prefix(prefix, "vision_representation"),

646 + )

647 +

648 + vit_hidden_size = vit_config.hidden_size

649 + llm_hidden_size = config.text_config.hidden_size

650 +

651 + self.und_projector = CheersUndProjector(

652 + image_embed_dim=vit_hidden_size,

653 + text_embed_dim=llm_hidden_size,

654 + compression_factor=(2, 2),

655 + quant_config=quant_config,

656 + prefix=maybe_prefix(prefix, "und_projector"),

657 + )

658 +

659 + self.make_empty_intermediate_tensors = (

660 + self.language_model.make_empty_intermediate_tensors

661 + )

662 +

663 + def _parse_and_validate_image_input(

664 + self, **kwargs: object

665 + ) -> CheersImageInputs | None:

666 + pixel_values = kwargs.pop("pixel_values", None)

667 + if pixel_values is None:

668 + return None

669 + return CheersImagePixelInputs(

670 + type="pixel_values",

671 + pixel_values=pixel_values,

672 + )

673 +

674 + def _process_image_input(

675 + self, image_input: CheersImageInputs

676 + ) -> tuple[torch.Tensor, ...]:

677 + """Process image inputs through VAE → SigLIP → projector pipeline.

678 +

679 + HF native path: pixel_values → VAE.encode(t=1.0) → vae_decoder_projector

680 + → SigLIP → und_projector → text-space embeddings

681 + """

682 + pixel_values = image_input["pixel_values"]

683 +

684 + if pixel_values.ndim == 5:

685 + batch_size, num_images, channels, height, width = pixel_values.shape

686 + pixel_values = pixel_values.reshape(

687 + batch_size * num_images, channels, height, width

688 + )

689 +

690 + with torch.no_grad():

691 + vae_dtype = next(self.vae_model.parameters()).dtype

692 + image_latent = self.vae_model.encode(pixel_values.to(dtype=vae_dtype))

693 + image_pixel_hat = self.vae_decoder_projector(image_latent)

694 +

695 + vision_features = self.vision_representation(image_pixel_hat)

696 + vision_embeds = self.und_projector(vision_features)

697 +

698 + return tuple(vision_embeds)

699 +

700 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

701 + image_input = self._parse_and_validate_image_input(**kwargs)

702 + if image_input is None:

703 + return []

704 + return self._process_image_input(image_input)

705 +

706 + def forward(

707 + self,

708 + input_ids: torch.Tensor | None,

709 + positions: torch.Tensor,

710 + intermediate_tensors: IntermediateTensors | None = None,

711 + inputs_embeds: torch.Tensor | None = None,

712 + **kwargs: object,

713 + ) -> torch.Tensor | IntermediateTensors:

714 + if intermediate_tensors is not None:

715 + inputs_embeds = None

716 +

717 + hidden_states = self.language_model.model(

718 + input_ids=input_ids,

719 + positions=positions,

720 + intermediate_tensors=intermediate_tensors,

721 + inputs_embeds=inputs_embeds,

722 + )

723 + return hidden_states

724 +

725 + def compute_logits(

726 + self,

727 + hidden_states: torch.Tensor,

728 + ) -> torch.Tensor | None:

729 + return self.language_model.compute_logits(hidden_states)

730 +

731 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

732 + """Load weights, keeping VAE encoder/decoder projector for understanding."""

733 + skip_prefixes = [

734 + "model.time_embed.",

735 + "model.gen_projector.",

736 + "model.hi_gate.",

737 + "model.hi_projector.",

738 + "model.vae_model.decoder.",

739 + ]

740 + skip_keywords = [

741 + "text_loss_fc",

742 + ]

743 +

744 + filtered_weights = []

745 + for name, tensor in weights:

746 + if any(name.startswith(p) for p in skip_prefixes):

747 + continue

748 + if any(kw in name for kw in skip_keywords):

749 + continue

750 + filtered_weights.append((name, tensor))

751 +

752 + loader = AutoWeightsLoader(self)

753 + return loader.load_weights(filtered_weights, mapper=self.hf_to_vllm_mapper)

vllm/model_executor/models/registry.py

@@ -350,6 +350,8 @@ _MULTIMODAL_MODELS = {

350 "chameleon",

351 "ChameleonForConditionalGeneration",

352 ),

353 + "Cheers": ("cheers", "CheersForConditionalGeneration"),

354 + "CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),

355 "Cohere2VisionForConditionalGeneration": (

356 "cohere2_vision",

357 "Cohere2VisionForConditionalGeneration",

vllm/transformers_utils/config.py

@@ -80,6 +80,7 @@ class LazyConfigDict(dict):

80 _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(

81 afmoe="AfmoeConfig",

82 bagel="BagelConfig",

83 + umm="CheersConfig",

84 chatglm="ChatGLMConfig",

85 colmodernvbert="ColModernVBertConfig",

86 colpali="ColPaliConfig",

vllm/transformers_utils/configs/__init__.py

@@ -18,6 +18,7 @@ _CLASS_TO_MODULE: dict[str, str] = {

18 "AfmoeConfig": "vllm.transformers_utils.configs.afmoe",

19 "AXK1Config": "vllm.transformers_utils.configs.AXK1",

20 "BagelConfig": "vllm.transformers_utils.configs.bagel",

21 + "CheersConfig": "vllm.transformers_utils.configs.cheers",

22 "ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",

23 "ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",

24 "ColPaliConfig": "vllm.transformers_utils.configs.colpali",

@@ -75,6 +76,7 @@ __all__ = [

76 "AfmoeConfig",

77 "AXK1Config",

78 "BagelConfig",

79 + "CheersConfig",

80 "ChatGLMConfig",

81 "ColModernVBertConfig",

82 "ColPaliConfig",

vllm/transformers_utils/configs/cheers.py new

+109

@@ -0,0 +1,109 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from transformers import PretrainedConfig, SiglipVisionConfig

4 +from transformers.modeling_rope_utils import rope_config_validation

5 +

6 +

7 +class CheersTextConfig(PretrainedConfig):

8 + """Qwen2-based text config with Cheers-specific defaults."""

9 +

10 + model_type = "umm"

11 + base_config_key = "text_config"

12 +

13 + def __init__(

14 + self,

15 + vocab_size=152064,

16 + hidden_size=3584,

17 + intermediate_size=18944,

18 + num_hidden_layers=28,

19 + num_attention_heads=28,

20 + num_key_value_heads=4,

21 + hidden_act="silu",

22 + max_position_embeddings=131072,

23 + initializer_range=0.02,

24 + rms_norm_eps=1e-6,

25 + use_cache=True,

26 + tie_word_embeddings=False,

27 + rope_theta=1000000.0,

28 + rope_scaling=None,

29 + use_sliding_window=False,

30 + sliding_window=131072,

31 + max_window_layers=28,

32 + layer_types=None,

33 + attention_dropout=0.0,

34 + **kwargs,

35 + ):

36 + self.vocab_size = vocab_size

37 + self.max_position_embeddings = max_position_embeddings

38 + self.hidden_size = hidden_size

39 + self.intermediate_size = intermediate_size

40 + self.num_hidden_layers = num_hidden_layers

41 + self.num_attention_heads = num_attention_heads

42 + self.use_sliding_window = use_sliding_window

43 + self.sliding_window = sliding_window if self.use_sliding_window else None

44 + self.max_window_layers = max_window_layers

45 + if num_key_value_heads is None:

46 + num_key_value_heads = num_attention_heads

47 + self.num_key_value_heads = num_key_value_heads

48 + self.hidden_act = hidden_act

49 + self.initializer_range = initializer_range

50 + self.rms_norm_eps = rms_norm_eps

51 + self.use_cache = use_cache

52 + self.rope_theta = rope_theta

53 + self.rope_scaling = rope_scaling

54 + self.attention_dropout = attention_dropout

55 + if self.rope_scaling is not None and "type" in self.rope_scaling:

56 + self.rope_scaling["rope_type"] = self.rope_scaling["type"]

57 + rope_config_validation(self)

58 +

59 + self.layer_types = layer_types

60 + if self.layer_types is None:

61 + self.layer_types = [

62 + "sliding_attention"

63 + if self.sliding_window is not None and i >= self.max_window_layers

64 + else "full_attention"

65 + for i in range(self.num_hidden_layers)

66 + ]

67 +

68 + super().__init__(

69 + tie_word_embeddings=tie_word_embeddings,

70 + **kwargs,

71 + )

72 +

73 +

74 +class CheersConfig(PretrainedConfig):

75 + """Configuration class for Cheers (UMM) model."""

76 +

77 + model_type = "umm"

78 +

79 + def __init__(

80 + self,

81 + text_config: dict | CheersTextConfig | None = None,

82 + vision_representation_config: dict | SiglipVisionConfig | None = None,

83 + vae_encoder_config: dict | None = None,

84 + vae_decoder_config: dict | None = None,

85 + **kwargs,

86 + ):

87 + super().__init__(**kwargs)

88 +

89 + if isinstance(text_config, dict):

90 + self.text_config = CheersTextConfig(**text_config)

91 + else:

92 + self.text_config = text_config or CheersTextConfig()

93 +

94 + if isinstance(vision_representation_config, dict):

95 + self.vision_representation_config = SiglipVisionConfig(

96 + **vision_representation_config

97 + )

98 + else:

99 + self.vision_representation_config = (

100 + vision_representation_config or SiglipVisionConfig()

101 + )

102 +

103 + self.vae_encoder_config = vae_encoder_config or {"resolution": 512}

104 + self.vae_decoder_config = vae_decoder_config or {"resolution": 512}

105 +

106 + @property

107 + def hidden_size(self) -> int:

108 + """Return the hidden size of the language model."""

109 + return self.text_config.hidden_size

vllm/transformers_utils/processors/__init__.py

@@ -12,6 +12,7 @@ import importlib

13 __all__ = [

14 "BagelProcessor",

15 + "CheersProcessor",

16 "CohereASRProcessor",

17 "DeepseekVLV2Processor",

18 "FireRedASR2Processor",

@@ -39,6 +40,7 @@ __all__ = [

41 _CLASS_TO_MODULE: dict[str, str] = {

42 "BagelProcessor": "vllm.transformers_utils.processors.bagel",

43 + "CheersProcessor": "vllm.transformers_utils.processors.cheers",

44 "CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",

45 "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",

46 "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",

vllm/transformers_utils/processors/cheers.py new

+89

@@ -0,0 +1,89 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Cheers (UMM) processor for image and text inputs."""

4 +

5 +from transformers import AutoProcessor

6 +from transformers.feature_extraction_utils import BatchFeature

7 +from transformers.image_utils import ImageInput

8 +from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack

9 +from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

10 +

11 +

12 +class CheersProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]

13 + _defaults = {

14 + "images_kwargs": {

15 + "return_tensors": "pt",

16 + },

17 + }

18 +

19 +

20 +class CheersProcessor(ProcessorMixin):

21 + """

22 + Constructs a Cheers processor which wraps a

23 + SigLIP image processor and a Qwen2 tokenizer.

24 + """

25 +

26 + attributes = ["image_processor", "tokenizer"]

27 + image_processor_class = "AutoImageProcessor"

28 + tokenizer_class = "AutoTokenizer"

29 +

30 + def __call__(

31 + self,

32 + text: TextInput

33 + | PreTokenizedInput

34 + | list[TextInput]

35 + | list[PreTokenizedInput] = None,

36 + images: ImageInput = None,

37 + **kwargs: Unpack[CheersProcessorKwargs],

38 + ):

39 + output_kwargs = self._merge_kwargs(

40 + CheersProcessorKwargs,

41 + tokenizer_init_kwargs=self.tokenizer.init_kwargs,

42 + **kwargs,

43 + )

44 +

45 + if images is not None:

46 + import torch

47 +

48 + if isinstance(images, (list, tuple)):

49 + all_pv = []

50 + all_ghw = []

51 + for img in images:

52 + result = self.image_processor(img, **output_kwargs["images_kwargs"])

53 + all_pv.append(result["pixel_values"])

54 + if "grid_hws" in result:

55 + all_ghw.append(result["grid_hws"])

56 + pixel_values = {

57 + "pixel_values": torch.cat(all_pv, dim=0),

58 + }

59 + if all_ghw:

60 + pixel_values["grid_hws"] = torch.cat(all_ghw, dim=0)

61 + else:

62 + pixel_values = self.image_processor(

63 + images, **output_kwargs["images_kwargs"]

64 + )

65 + else:

66 + pixel_values = {}

67 +

68 + text_inputs = (

69 + self.tokenizer(text, **output_kwargs["text_kwargs"])

70 + if text is not None

71 + else {}

72 + )

73 +

74 + return BatchFeature(data={**pixel_values, **text_inputs})

75 +

76 + def batch_decode(self, *args, **kwargs):

77 + return self.tokenizer.batch_decode(*args, **kwargs)

78 +

79 + def decode(self, *args, **kwargs):

80 + return self.tokenizer.decode(*args, **kwargs)

81 +

82 + @property

83 + def model_input_names(self):

84 + tokenizer_input_names = self.tokenizer.model_input_names

85 + image_processor_input_names = self.image_processor.model_input_names

86 + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))

87 +

88 +

89 +AutoProcessor.register("CheersProcessor", CheersProcessor)