A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix]: Use float32 for base64 embedding (#7855)
Signed-off-by: Hollow Man <hollowman@opensuse.org>
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟 committed
Aug 26, 2024 at 06:16 UTC
0b769992ec1d780b3229c46152c6e647da113aa6
3 files changed
+13
-3
examples/openai_embedding_client.py
-1
@@ -19,7 +19,6 @@ responses = client.embeddings.create(
19
"The best thing about vLLM is that it supports many different models"
20
],
21
model=model,
22
- encoding_format="float",
22
)
23
24
for data in responses.data:
tests/entrypoints/openai/test_embedding.py
+10
-1
@@ -128,9 +128,18 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
128
for data in responses_base64.data:
129
decoded_responses_base64_data.append(
130
np.frombuffer(base64.b64decode(data.embedding),
131
- dtype="float").tolist())
131
+ dtype="float32").tolist())
132
133
assert responses_float.data[0].embedding == decoded_responses_base64_data[
134
0]
135
assert responses_float.data[1].embedding == decoded_responses_base64_data[
136
1]
137
+
138
+ # Default response is float32 decoded from base64 by OpenAI Client
139
+ responses_default = await embedding_client.embeddings.create(
140
+ input=input_texts, model=model_name)
141
+
142
+ assert responses_float.data[0].embedding == responses_default.data[
143
+ 0].embedding
144
+ assert responses_float.data[1].embedding == responses_default.data[
145
+ 1].embedding
vllm/entrypoints/openai/serving_embedding.py
+3
-1
@@ -31,7 +31,9 @@ def _get_embedding(
31
if encoding_format == "float":
32
return output.embedding
33
elif encoding_format == "base64":
34
- embedding_bytes = np.array(output.embedding).tobytes()
34
+ # Force to use float32 for base64 encoding
35
+ # to match the OpenAI python client behavior
36
+ embedding_bytes = np.array(output.embedding, dtype="float32").tobytes()
37
return base64.b64encode(embedding_bytes).decode("utf-8")
38
39
assert_never(encoding_format)