A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] fix bf16 multimodal model hash (#23623)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Yuekai Zhang committed
Aug 26, 2025 at 23:47 UTC
513298f1b44157f7ae2f7007ef7b17c2929d11d4
1 file changed
+13
-1
vllm/multimodal/hasher.py
+13
-1
@@ -43,7 +43,19 @@ class MultiModalHasher:
43
return cls.item_to_bytes(
44
"image", np.asarray(convert_image_mode(obj, "RGBA")))
45
if isinstance(obj, torch.Tensor):
46
- return cls.item_to_bytes("tensor", obj.cpu().numpy())
46
+ tensor_obj: torch.Tensor = obj.cpu()
47
+ tensor_dtype = tensor_obj.dtype
48
+ if tensor_dtype == torch.bfloat16:
49
+ tensor_obj = tensor_obj.contiguous()
50
+ tensor_obj = tensor_obj.view(
51
+ (tensor_obj.numel(), )).view(torch.uint8)
52
+ return cls.item_to_bytes(
53
+ "tensor", {
54
+ "original_dtype": str(tensor_dtype),
55
+ "original_shape": tuple(tensor_obj.shape),
56
+ "data": tensor_obj.numpy()
57
+ })
58
+ return cls.item_to_bytes("tensor", tensor_obj.numpy())
59
if isinstance(obj, np.ndarray):
60
# If the array is non-contiguous, we need to copy it first
61
arr_data = obj.data if obj.flags.c_contiguous else obj.tobytes()