| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import io |
| 4 | |
| 5 | import numpy as np |
| 6 | import pybase64 |
| 7 | import pytest |
| 8 | import torch |
| 9 | |
| 10 | from tests.models.utils import check_embeddings_close |
| 11 | from vllm.utils.serial_utils import ( |
| 12 | EMBED_DTYPES, |
| 13 | ENDIANNESS, |
| 14 | MM_METADATA_DTYPES, |
| 15 | EmbedDType, |
| 16 | Endianness, |
| 17 | MmMetadataDType, |
| 18 | binary2tensor, |
| 19 | numpy2base64, |
| 20 | tensor2binary, |
| 21 | ) |
| 22 | |
| 23 | FLOAT_EMBED_DTYPES = tuple(EMBED_DTYPES.keys()) |
| 24 | INTEGER_EMBED_DTYPES = tuple(MM_METADATA_DTYPES.keys()) |
| 25 | |
| 26 | |
| 27 | def test_numpy2base64_round_trip(): |
| 28 | array = np.arange(24, dtype=np.uint8).reshape(2, 3, 4) |
| 29 | |
| 30 | decoded = np.load( |
| 31 | io.BytesIO(pybase64.b64decode(numpy2base64(array))), |
| 32 | allow_pickle=False, |
| 33 | ) |
| 34 | |
| 35 | np.testing.assert_array_equal(decoded, array) |
| 36 | |
| 37 | |
| 38 | def _build_integer_tensor( |
| 39 | embed_dtype: MmMetadataDType, shape: tuple[int, ...] |
| 40 | ) -> torch.Tensor: |
| 41 | torch_dtype = MM_METADATA_DTYPES[embed_dtype].torch_dtype |
| 42 | |
| 43 | if torch_dtype is torch.bool: |
| 44 | return torch.randint(0, 2, shape, dtype=torch.int32).to(torch.bool) |
| 45 | if torch_dtype is torch.uint8: |
| 46 | return torch.randint(0, 256, shape, dtype=torch.uint8) |
| 47 | if torch_dtype is torch.int32: |
| 48 | return torch.randint(-(2**20), 2**20, shape, dtype=torch.int32) |
| 49 | if torch_dtype is torch.int64: |
| 50 | return torch.randint(-(2**62), 2**62, shape, dtype=torch.int64) |
| 51 | |
| 52 | raise AssertionError(f"Unsupported non-floating embed dtype: {embed_dtype}") |
| 53 | |
| 54 | |
| 55 | @pytest.mark.parametrize("endianness", ENDIANNESS) |
| 56 | @pytest.mark.parametrize("embed_dtype", FLOAT_EMBED_DTYPES) |
| 57 | @torch.inference_mode() |
| 58 | def test_encode_and_decode_floats(embed_dtype: EmbedDType, endianness: Endianness): |
| 59 | for i in range(10): |
| 60 | tensor = torch.rand(2, 3, 5, 7, 11, 13, device="cpu", dtype=torch.float32) |
| 61 | shape = tensor.shape |
| 62 | binary = tensor2binary(tensor, embed_dtype, endianness) |
| 63 | new_tensor = binary2tensor(binary, shape, embed_dtype, endianness).to( |
| 64 | torch.float32 |
| 65 | ) |
| 66 | |
| 67 | if embed_dtype in ["float32", "float16"]: |
| 68 | torch.testing.assert_close(tensor, new_tensor, atol=0.001, rtol=0.001) |
| 69 | elif embed_dtype == "bfloat16": |
| 70 | torch.testing.assert_close(tensor, new_tensor, atol=0.01, rtol=0.01) |
| 71 | else: # for fp8 |
| 72 | torch.testing.assert_close(tensor, new_tensor, atol=0.1, rtol=0.1) |
| 73 | |
| 74 | check_embeddings_close( |
| 75 | embeddings_0_lst=tensor.view(1, -1), |
| 76 | embeddings_1_lst=new_tensor.view(1, -1), |
| 77 | name_0="gt", |
| 78 | name_1="new", |
| 79 | tol=1e-2, |
| 80 | ) |
| 81 | |
| 82 | |
| 83 | @pytest.mark.parametrize("endianness", ENDIANNESS) |
| 84 | @pytest.mark.parametrize("embed_dtype", INTEGER_EMBED_DTYPES) |
| 85 | @torch.inference_mode() |
| 86 | def test_encode_and_decode_integers( |
| 87 | embed_dtype: MmMetadataDType, endianness: Endianness |
| 88 | ): |
| 89 | shape = (2, 3, 5, 7, 11, 13) |
| 90 | |
| 91 | for i in range(10): |
| 92 | tensor = _build_integer_tensor(embed_dtype, shape) |
| 93 | binary = tensor2binary(tensor, embed_dtype, endianness) |
| 94 | new_tensor = binary2tensor(binary, shape, embed_dtype, endianness) |
| 95 | |
| 96 | assert new_tensor.dtype == MM_METADATA_DTYPES[embed_dtype].torch_dtype |
| 97 | torch.testing.assert_close(tensor, new_tensor, atol=0, rtol=0) |