@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4a9952ec1b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 07:30 UTC

[Bugfix] Add quant_config in ViT of Kimi-K2.5 (#34501)

Signed-off-by: LoganJane <LoganJane73@hotmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

LoganJane committed Feb 14, 2026 at 00:05 UTC 4a9952ec1b15453053f4ec443d2d81505d344075

2 files changed +26

vllm/model_executor/models/kimi_k25.py

+11

@@ -23,6 +23,10 @@ from transformers.processing_utils import ProcessorMixin

23 from vllm.config import VllmConfig

24 from vllm.config.multimodal import BaseDummyOptions

25 from vllm.logger import init_logger

26 +from vllm.model_executor.layers.quantization import QuantizationConfig

27 +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import (

28 + CompressedTensorsConfig,

29 +)

30 from vllm.model_executor.models.interfaces import (

31 SupportsMultiModal,

32 SupportsPP,

@@ -361,6 +365,7 @@ class KimiK25ForConditionalGeneration(

365 with self._mark_tower_model(vllm_config, "vision_chunk"):

366 self.vision_tower = MoonViT3dPretrainedModel(

367 config.vision_config,

368 + quant_config=self._maybe_ignore_quant_config(quant_config),

369 prefix=maybe_prefix(prefix, "vision_tower"),

370 )

371 self.vision_tower = self.vision_tower.to(

@@ -370,6 +375,7 @@ class KimiK25ForConditionalGeneration(

375 self.mm_projector = KimiK25MultiModalProjector(

376 config=config.vision_config,

377 use_data_parallel=self.use_data_parallel,

378 + quant_config=self._maybe_ignore_quant_config(quant_config),

379 prefix=maybe_prefix(prefix, "mm_projector"),

380 )

381 self.mm_projector = self.mm_projector.to(

@@ -389,6 +395,11 @@ class KimiK25ForConditionalGeneration(

395 )

396 self.media_placeholder: int = self.config.media_placeholder_token_id

397

398 + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

399 + if isinstance(quant_config, CompressedTensorsConfig):

400 + return None

401 + return quant_config

402 +

403 def _parse_and_validate_media_input(

404 self, **kwargs: object

405 ) -> KimiK25MediaPixelInputs | None:

vllm/model_executor/models/kimi_k25_vit.py

+15

@@ -28,6 +28,7 @@ from vllm.model_executor.layers.linear import (

28 ReplicatedLinear,

29 RowParallelLinear,

30 )

31 +from vllm.model_executor.layers.quantization import QuantizationConfig

32 from vllm.model_executor.models.utils import maybe_prefix

33 from vllm.model_executor.models.vision import (

34 is_vit_use_data_parallel,

@@ -304,6 +305,7 @@ class MLP2(nn.Module):

305 dims: list[int],

306 activation,

307 bias: bool = True,

308 + quant_config: QuantizationConfig | None = None,

309 prefix: str = "",

310 use_data_parallel: bool = False,

311 ):

@@ -314,6 +316,7 @@ class MLP2(nn.Module):

316 dims[0],

317 dims[1],

318 bias=bias,

319 + quant_config=quant_config,

320 prefix=maybe_prefix(prefix, "fc0"),

321 disable_tp=self.use_data_parallel,

322 )

@@ -321,6 +324,7 @@ class MLP2(nn.Module):

324 dims[1],

325 dims[2],

326 bias=bias,

327 + quant_config=quant_config,

328 prefix=maybe_prefix(prefix, "fc1"),

329 disable_tp=self.use_data_parallel,

330 )

@@ -341,6 +345,7 @@ class MoonViTEncoderLayer(nn.Module):

345 num_heads: int,

346 hidden_dim: int,

347 mlp_dim: int,

348 + quant_config: QuantizationConfig | None = None,

349 prefix: str = "",

350 *,

351 activation=F.gelu,

@@ -362,6 +367,7 @@ class MoonViTEncoderLayer(nn.Module):

367 self.mlp = MLP2(

368 [hidden_dim, mlp_dim, hidden_dim],

369 activation,

370 + quant_config=quant_config,

371 prefix=f"{prefix}.mlp",

372 use_data_parallel=self.use_data_parallel,

373 )

@@ -371,6 +377,7 @@ class MoonViTEncoderLayer(nn.Module):

377 total_num_heads=num_heads,

378 total_num_kv_heads=num_heads,

379 bias=attn_bias,

380 + quant_config=quant_config,

381 prefix=f"{prefix}.wqkv",

382 disable_tp=self.use_data_parallel,

383 )

@@ -378,6 +385,7 @@ class MoonViTEncoderLayer(nn.Module):

385 hidden_dim,

386 hidden_dim,

387 bias=attn_bias,

388 + quant_config=quant_config,

389 prefix=f"{prefix}.wo",

390 disable_tp=self.use_data_parallel,

391 )

@@ -461,6 +469,7 @@ class MoonViT3dEncoder(nn.Module):

469 num_layers: int,

470 block_cfg: dict,

471 video_attn_type: str = "spatial_temporal",

472 + quant_config: QuantizationConfig | None = None,

473 prefix: str = "",

474 ) -> None:

475 super().__init__()

@@ -476,6 +485,7 @@ class MoonViT3dEncoder(nn.Module):

485 [

486 MoonViTEncoderLayer(

487 **block_cfg,

488 + quant_config=quant_config,

489 prefix=f"{prefix}.blocks.{layer_idx}",

490 )

491 for layer_idx in range(num_layers)

@@ -544,6 +554,7 @@ class MoonViT3dPretrainedModel(nn.Module):

554 def __init__(

555 self,

556 config: KimiK25VisionConfig,

557 + quant_config: QuantizationConfig | None = None,

558 prefix: str = "",

559 ):

560 super().__init__()

@@ -573,6 +584,7 @@ class MoonViT3dPretrainedModel(nn.Module):

584 "attn_bias": True,

585 },

586 video_attn_type=config.video_attn_type,

587 + quant_config=quant_config,

588 prefix=maybe_prefix(prefix, "encoder"),

589 )

590

@@ -646,6 +658,7 @@ class KimiK25MultiModalProjector(nn.Module):

658 self,

659 config: KimiK25VisionConfig,

660 use_data_parallel: bool = False,

661 + quant_config: QuantizationConfig | None = None,

662 prefix: str = "",

663 ):

664 super().__init__()

@@ -660,12 +673,14 @@ class KimiK25MultiModalProjector(nn.Module):

673 self.hidden_size,

674 self.hidden_size,

675 bias=True,

676 + quant_config=quant_config,

677 prefix=f"{prefix}.linear_1",

678 )

679 self.linear_2 = ReplicatedLinear(

680 self.hidden_size,

681 config.mm_hidden_size,

682 bias=True,

683 + quant_config=quant_config,

684 prefix=f"{prefix}.linear_2",

685 )

686 self.act = GELUActivation()