A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[PERF] Use pybase64 to more quickly decode prompt embeddings (#22469)
Signed-off-by: Andrew Sansom <andrew@protopia.ai>
Andrew Sansom committed
Aug 7, 2025 at 21:15 UTC
e2c8f1edec24f7a89a68e3b48bc65ae683aed0cb
1 file changed
+3
-2
vllm/entrypoints/openai/serving_engine.py
+3
-2
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
-import base64
4
import io
5
import json
6
import sys
@@ -12,6 +11,7 @@ from http import HTTPStatus
11
from typing import (Annotated, Any, Callable, ClassVar, Generic, Optional,
12
TypeVar, Union, cast, overload)
13
14
+import pybase64
15
import torch
16
from fastapi import Request
17
from pydantic import BaseModel, ConfigDict, Field
@@ -1008,7 +1008,8 @@ class OpenAIServing:
1008
) -> list[EmbedsPrompt]:
1009
1010
def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:
1011
- tensor = torch.load(io.BytesIO(base64.b64decode(embed)),
1011
+ tensor = torch.load(io.BytesIO(
1012
+ pybase64.b64decode(embed, validate=True)),
1013
weights_only=True)
1014
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
1015
torch.float32,