@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b769992ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix]: Use float32 for base64 embedding (#7855)

Signed-off-by: Hollow Man <hollowman@opensuse.org>

ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟 committed Aug 26, 2024 at 06:16 UTC 0b769992ec1d780b3229c46152c6e647da113aa6
3 files changed +13 -3
examples/openai_embedding_client.py
-1
@@ -19,7 +19,6 @@ responses = client.embeddings.create(
19 "The best thing about vLLM is that it supports many different models"
20 ],
21 model=model,
22 - encoding_format="float",
22 )
23
24 for data in responses.data:
tests/entrypoints/openai/test_embedding.py
+10 -1
@@ -128,9 +128,18 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
128 for data in responses_base64.data:
129 decoded_responses_base64_data.append(
130 np.frombuffer(base64.b64decode(data.embedding),
131 - dtype="float").tolist())
131 + dtype="float32").tolist())
132
133 assert responses_float.data[0].embedding == decoded_responses_base64_data[
134 0]
135 assert responses_float.data[1].embedding == decoded_responses_base64_data[
136 1]
137 +
138 + # Default response is float32 decoded from base64 by OpenAI Client
139 + responses_default = await embedding_client.embeddings.create(
140 + input=input_texts, model=model_name)
141 +
142 + assert responses_float.data[0].embedding == responses_default.data[
143 + 0].embedding
144 + assert responses_float.data[1].embedding == responses_default.data[
145 + 1].embedding
vllm/entrypoints/openai/serving_embedding.py
+3 -1
@@ -31,7 +31,9 @@ def _get_embedding(
31 if encoding_format == "float":
32 return output.embedding
33 elif encoding_format == "base64":
34 - embedding_bytes = np.array(output.embedding).tobytes()
34 + # Force to use float32 for base64 encoding
35 + # to match the OpenAI python client behavior
36 + embedding_bytes = np.array(output.embedding, dtype="float32").tobytes()
37 return base64.b64encode(embedding_bytes).decode("utf-8")
38
39 assert_never(encoding_format)