A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
fix float16 support for kimi-vl (#17156)
Co-authored-by: zhouzaida <zhouzaida@msh.team>
Zaida Zhou committed
Apr 25, 2025 at 11:16 UTC
69bff9bc893475fbd64c47633cb8ece46cd462c7
1 file changed
+1
-2
vllm/model_executor/models/kimi_vl.py
+1
-2
@@ -340,8 +340,7 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal):
340
else:
341
pixel_values = pixel_values.reshape(-1, num_channels, patch_size,
342
patch_size)
343
- # fp32 -> bf16
344
- pixel_values = pixel_values.to(torch.bfloat16)
343
+ pixel_values = pixel_values.to(self.vision_tower.dtype)
344
# image_grid_hws.shape = (N, 2)
345
assert image_grid_hws.ndim == 2, f"unexpected shape for image_grid_hws: {image_grid_hws.shape}"
346