A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Frontend]: Support base64 embedding (#5935)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
llmpros committed
Jun 30, 2024 at 08:53 UTC
c6c240aa0a2d4bed821282a07d50f6710cd99eed
3 files changed
+47
-14
tests/entrypoints/openai/test_embedding.py
+33
@@ -1,3 +1,6 @@
1
+import base64
2
+
3
+import numpy as np
4
import openai
5
import pytest
6
import ray
@@ -109,3 +112,33 @@ async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
112
assert embeddings.usage.completion_tokens == 0
113
assert embeddings.usage.prompt_tokens == 17
114
assert embeddings.usage.total_tokens == 17
115
+
116
+
117
+@pytest.mark.asyncio
118
+@pytest.mark.parametrize(
119
+ "model_name",
120
+ [EMBEDDING_MODEL_NAME],
121
+)
122
+async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
123
+ model_name: str):
124
+ input_texts = [
125
+ "Hello my name is",
126
+ "The best thing about vLLM is that it supports many different models"
127
+ ]
128
+
129
+ responses_float = await embedding_client.embeddings.create(
130
+ input=input_texts, model=model_name, encoding_format="float")
131
+
132
+ responses_base64 = await embedding_client.embeddings.create(
133
+ input=input_texts, model=model_name, encoding_format="base64")
134
+
135
+ decoded_responses_base64_data = []
136
+ for data in responses_base64.data:
137
+ decoded_responses_base64_data.append(
138
+ np.frombuffer(base64.b64decode(data.embedding),
139
+ dtype="float").tolist())
140
+
141
+ assert responses_float.data[0].embedding == decoded_responses_base64_data[
142
+ 0]
143
+ assert responses_float.data[1].embedding == decoded_responses_base64_data[
144
+ 1]
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -580,7 +580,7 @@ class CompletionStreamResponse(OpenAIBaseModel):
580
class EmbeddingResponseData(BaseModel):
581
index: int
582
object: str = "embedding"
583
- embedding: List[float]
583
+ embedding: Union[List[float], str]
584
585
586
class EmbeddingResponse(BaseModel):
vllm/entrypoints/openai/serving_embedding.py
+13
-13
@@ -1,6 +1,8 @@
1
+import base64
2
import time
3
from typing import AsyncIterator, List, Optional, Tuple
4
5
+import numpy as np
6
from fastapi import Request
7
8
from vllm.config import ModelConfig
@@ -20,19 +22,18 @@ TypeTokenIDs = List[int]
22
23
24
def request_output_to_embedding_response(
23
- final_res_batch: List[EmbeddingRequestOutput],
24
- request_id: str,
25
- created_time: int,
26
- model_name: str,
27
-) -> EmbeddingResponse:
25
+ final_res_batch: List[EmbeddingRequestOutput], request_id: str,
26
+ created_time: int, model_name: str,
27
+ encoding_format: str) -> EmbeddingResponse:
28
data: List[EmbeddingResponseData] = []
29
num_prompt_tokens = 0
30
for idx, final_res in enumerate(final_res_batch):
31
assert final_res is not None
32
prompt_token_ids = final_res.prompt_token_ids
33
-
34
- embedding_data = EmbeddingResponseData(
35
- index=idx, embedding=final_res.outputs.embedding)
33
+ embedding = final_res.outputs.embedding
34
+ if encoding_format == "base64":
35
+ embedding = base64.b64encode(np.array(embedding))
36
+ embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
37
data.append(embedding_data)
38
39
num_prompt_tokens += len(prompt_token_ids)
@@ -72,10 +73,8 @@ class OpenAIServingEmbedding(OpenAIServing):
73
if error_check_ret is not None:
74
return error_check_ret
75
75
- # Return error for unsupported features.
76
- if request.encoding_format == "base64":
77
- return self.create_error_response(
78
- "base64 encoding is not currently supported")
76
+ encoding_format = (request.encoding_format
77
+ if request.encoding_format else "float")
78
if request.dimensions is not None:
79
return self.create_error_response(
80
"dimensions is currently not supported")
@@ -129,7 +128,8 @@ class OpenAIServingEmbedding(OpenAIServing):
128
return self.create_error_response("Client disconnected")
129
final_res_batch[i] = res
130
response = request_output_to_embedding_response(
132
- final_res_batch, request_id, created_time, model_name)
131
+ final_res_batch, request_id, created_time, model_name,
132
+ encoding_format)
133
except ValueError as e:
134
# TODO: Use a vllm-specific Validation Error
135
return self.create_error_response(str(e))