@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6c240aa0a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Frontend]: Support base64 embedding (#5935)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

llmpros committed Jun 30, 2024 at 08:53 UTC c6c240aa0a2d4bed821282a07d50f6710cd99eed
3 files changed +47 -14
tests/entrypoints/openai/test_embedding.py
+33
@@ -1,3 +1,6 @@
1 +import base64
2 +
3 +import numpy as np
4 import openai
5 import pytest
6 import ray
@@ -109,3 +112,33 @@ async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
112 assert embeddings.usage.completion_tokens == 0
113 assert embeddings.usage.prompt_tokens == 17
114 assert embeddings.usage.total_tokens == 17
115 +
116 +
117 +@pytest.mark.asyncio
118 +@pytest.mark.parametrize(
119 + "model_name",
120 + [EMBEDDING_MODEL_NAME],
121 +)
122 +async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
123 + model_name: str):
124 + input_texts = [
125 + "Hello my name is",
126 + "The best thing about vLLM is that it supports many different models"
127 + ]
128 +
129 + responses_float = await embedding_client.embeddings.create(
130 + input=input_texts, model=model_name, encoding_format="float")
131 +
132 + responses_base64 = await embedding_client.embeddings.create(
133 + input=input_texts, model=model_name, encoding_format="base64")
134 +
135 + decoded_responses_base64_data = []
136 + for data in responses_base64.data:
137 + decoded_responses_base64_data.append(
138 + np.frombuffer(base64.b64decode(data.embedding),
139 + dtype="float").tolist())
140 +
141 + assert responses_float.data[0].embedding == decoded_responses_base64_data[
142 + 0]
143 + assert responses_float.data[1].embedding == decoded_responses_base64_data[
144 + 1]
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -580,7 +580,7 @@ class CompletionStreamResponse(OpenAIBaseModel):
580 class EmbeddingResponseData(BaseModel):
581 index: int
582 object: str = "embedding"
583 - embedding: List[float]
583 + embedding: Union[List[float], str]
584
585
586 class EmbeddingResponse(BaseModel):
vllm/entrypoints/openai/serving_embedding.py
+13 -13
@@ -1,6 +1,8 @@
1 +import base64
2 import time
3 from typing import AsyncIterator, List, Optional, Tuple
4
5 +import numpy as np
6 from fastapi import Request
7
8 from vllm.config import ModelConfig
@@ -20,19 +22,18 @@ TypeTokenIDs = List[int]
22
23
24 def request_output_to_embedding_response(
23 - final_res_batch: List[EmbeddingRequestOutput],
24 - request_id: str,
25 - created_time: int,
26 - model_name: str,
27 -) -> EmbeddingResponse:
25 + final_res_batch: List[EmbeddingRequestOutput], request_id: str,
26 + created_time: int, model_name: str,
27 + encoding_format: str) -> EmbeddingResponse:
28 data: List[EmbeddingResponseData] = []
29 num_prompt_tokens = 0
30 for idx, final_res in enumerate(final_res_batch):
31 assert final_res is not None
32 prompt_token_ids = final_res.prompt_token_ids
33 -
34 - embedding_data = EmbeddingResponseData(
35 - index=idx, embedding=final_res.outputs.embedding)
33 + embedding = final_res.outputs.embedding
34 + if encoding_format == "base64":
35 + embedding = base64.b64encode(np.array(embedding))
36 + embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
37 data.append(embedding_data)
38
39 num_prompt_tokens += len(prompt_token_ids)
@@ -72,10 +73,8 @@ class OpenAIServingEmbedding(OpenAIServing):
73 if error_check_ret is not None:
74 return error_check_ret
75
75 - # Return error for unsupported features.
76 - if request.encoding_format == "base64":
77 - return self.create_error_response(
78 - "base64 encoding is not currently supported")
76 + encoding_format = (request.encoding_format
77 + if request.encoding_format else "float")
78 if request.dimensions is not None:
79 return self.create_error_response(
80 "dimensions is currently not supported")
@@ -129,7 +128,8 @@ class OpenAIServingEmbedding(OpenAIServing):
128 return self.create_error_response("Client disconnected")
129 final_res_batch[i] = res
130 response = request_output_to_embedding_response(
132 - final_res_batch, request_id, created_time, model_name)
131 + final_res_batch, request_id, created_time, model_name,
132 + encoding_format)
133 except ValueError as e:
134 # TODO: Use a vllm-specific Validation Error
135 return self.create_error_response(str(e))