| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import io |
| 4 | import sys |
| 5 | from collections.abc import Mapping |
| 6 | from dataclasses import dataclass |
| 7 | from typing import Literal, get_args |
| 8 | |
| 9 | import numpy as np |
| 10 | import numpy.typing as npt |
| 11 | import pybase64 |
| 12 | import torch |
| 13 | |
| 14 | sys_byteorder = sys.byteorder |
| 15 | |
| 16 | |
| 17 | @dataclass(frozen=True) |
| 18 | class DTypeInfo: |
| 19 | torch_dtype: torch.dtype |
| 20 | |
| 21 | torch_view_dtype: torch.dtype |
| 22 | numpy_view_dtype: npt.DTypeLike |
| 23 | |
| 24 | @property |
| 25 | def nbytes(self) -> int: |
| 26 | return self.torch_dtype.itemsize |
| 27 | |
| 28 | |
| 29 | EmbedDType = Literal["float32", "float16", "bfloat16", "fp8_e4m3", "fp8_e5m2"] |
| 30 | MmMetadataDType = Literal["int32", "int64", "uint8", "bool"] |
| 31 | Endianness = Literal["native", "big", "little"] |
| 32 | EncodingFormat = Literal["float", "base64", "bytes", "bytes_only"] |
| 33 | |
| 34 | # I'm not sure if other platforms' CPUs support the fp8 data format. |
| 35 | # EMBED_DTYPE only uses the fp8 data representation, |
| 36 | # does not use fp8 computation, and only occurs on the CPU. |
| 37 | # Apologize for any possible break. |
| 38 | # NOTE: numpy does not support bfloat16 and fp8 |
| 39 | EMBED_DTYPES: Mapping[EmbedDType, DTypeInfo] = { |
| 40 | "float32": DTypeInfo(torch.float32, torch.float32, np.float32), |
| 41 | "float16": DTypeInfo(torch.float16, torch.float16, np.float16), |
| 42 | "bfloat16": DTypeInfo(torch.bfloat16, torch.float16, np.float16), |
| 43 | "fp8_e4m3": DTypeInfo(torch.float8_e4m3fn, torch.uint8, np.uint8), |
| 44 | "fp8_e5m2": DTypeInfo(torch.float8_e5m2, torch.uint8, np.uint8), |
| 45 | } |
| 46 | MM_METADATA_DTYPES: Mapping[MmMetadataDType, DTypeInfo] = { |
| 47 | "int32": DTypeInfo(torch.int32, torch.int32, np.int32), |
| 48 | "int64": DTypeInfo(torch.int64, torch.int64, np.int64), |
| 49 | "uint8": DTypeInfo(torch.uint8, torch.uint8, np.uint8), |
| 50 | "bool": DTypeInfo(torch.bool, torch.uint8, np.uint8), |
| 51 | } |
| 52 | _ALL_SERIAL_DTYPES: Mapping[str, DTypeInfo] = { |
| 53 | k: v for d in (EMBED_DTYPES, MM_METADATA_DTYPES) for k, v in d.items() |
| 54 | } |
| 55 | ENDIANNESS: tuple[Endianness, ...] = get_args(Endianness) |
| 56 | |
| 57 | |
| 58 | def tensor2base64(x: torch.Tensor) -> str: |
| 59 | with io.BytesIO() as buf: |
| 60 | torch.save(x, buf) |
| 61 | buf.seek(0) |
| 62 | binary_data = buf.read() |
| 63 | |
| 64 | return pybase64.b64encode(binary_data).decode("utf-8") |
| 65 | |
| 66 | |
| 67 | def numpy2base64(array: np.ndarray) -> str: |
| 68 | """Encode a NumPy array using its `.npy` representation.""" |
| 69 | with io.BytesIO() as buffer: |
| 70 | np.save(buffer, array, allow_pickle=False) |
| 71 | return pybase64.b64encode(buffer.getbuffer()).decode("ascii") |
| 72 | |
| 73 | |
| 74 | def tensor2binary( |
| 75 | tensor: torch.Tensor, |
| 76 | embed_dtype: "EmbedDType | MmMetadataDType", |
| 77 | endianness: Endianness, |
| 78 | ) -> bytes: |
| 79 | assert isinstance(tensor, torch.Tensor) |
| 80 | assert embed_dtype in _ALL_SERIAL_DTYPES |
| 81 | assert endianness in ENDIANNESS |
| 82 | |
| 83 | dtype_info = _ALL_SERIAL_DTYPES[embed_dtype] |
| 84 | |
| 85 | np_array = ( |
| 86 | tensor.to(dtype_info.torch_dtype) |
| 87 | .flatten() |
| 88 | .contiguous() |
| 89 | .view(dtype_info.torch_view_dtype) |
| 90 | .numpy() |
| 91 | ) |
| 92 | |
| 93 | if endianness != "native" and endianness != sys_byteorder: |
| 94 | np_array = np_array.byteswap() |
| 95 | |
| 96 | return np_array.tobytes() |
| 97 | |
| 98 | |
| 99 | def binary2tensor( |
| 100 | binary: bytes, |
| 101 | shape: tuple[int, ...], |
| 102 | embed_dtype: "EmbedDType | MmMetadataDType", |
| 103 | endianness: Endianness, |
| 104 | ) -> torch.Tensor: |
| 105 | assert embed_dtype in _ALL_SERIAL_DTYPES |
| 106 | assert endianness in ENDIANNESS |
| 107 | |
| 108 | dtype_info = _ALL_SERIAL_DTYPES[embed_dtype] |
| 109 | |
| 110 | np_array = np.frombuffer(binary, dtype=dtype_info.numpy_view_dtype).reshape(shape) |
| 111 | |
| 112 | if endianness != "native" and endianness != sys_byteorder: |
| 113 | np_array = np_array.byteswap() |
| 114 | |
| 115 | return torch.from_numpy(np_array).view(dtype_info.torch_dtype) |