@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0dcc8cbe5a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

Adds truncate_prompt_tokens param for embeddings creation (#8999)

Signed-off-by: Flavia Beo <flavia.beo@ibm.com>

Flávia Béo committed Oct 4, 2024 at 15:31 UTC 0dcc8cbe5abd4f2fafd495bd1c65fdd75d8dd919
3 files changed +76 -5
tests/entrypoints/openai/test_embedding.py
+61
@@ -144,3 +144,64 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
144 0].embedding
145 assert responses_float.data[1].embedding == responses_default.data[
146 1].embedding
147 +
148 +
149 +@pytest.mark.asyncio
150 +@pytest.mark.parametrize(
151 + "model_name",
152 + [EMBEDDING_MODEL_NAME],
153 +)
154 +async def test_single_embedding_truncation(
155 + embedding_client: openai.AsyncOpenAI, model_name: str):
156 + input_texts = [
157 + "Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
158 + ]
159 +
160 + # test single embedding
161 + embeddings = await embedding_client.embeddings.create(
162 + model=model_name,
163 + input=input_texts,
164 + extra_body={"truncate_prompt_tokens": 10})
165 + assert embeddings.id is not None
166 + assert len(embeddings.data) == 1
167 + assert len(embeddings.data[0].embedding) == 4096
168 + assert embeddings.usage.completion_tokens == 0
169 + assert embeddings.usage.prompt_tokens == 10
170 + assert embeddings.usage.total_tokens == 10
171 +
172 + input_tokens = [
173 + 1, 24428, 289, 18341, 26165, 285, 19323, 283, 289, 26789, 3871, 28728,
174 + 9901, 340, 2229, 385, 340, 315, 28741, 28804, 2
175 + ]
176 + embeddings = await embedding_client.embeddings.create(
177 + model=model_name,
178 + input=input_tokens,
179 + extra_body={"truncate_prompt_tokens": 10})
180 +
181 + assert embeddings.id is not None
182 + assert len(embeddings.data) == 1
183 + assert len(embeddings.data[0].embedding) == 4096
184 + assert embeddings.usage.completion_tokens == 0
185 + assert embeddings.usage.prompt_tokens == 10
186 + assert embeddings.usage.total_tokens == 10
187 +
188 +
189 +@pytest.mark.asyncio
190 +@pytest.mark.parametrize(
191 + "model_name",
192 + [EMBEDDING_MODEL_NAME],
193 +)
194 +async def test_single_embedding_truncation_invalid(
195 + embedding_client: openai.AsyncOpenAI, model_name: str):
196 + input_texts = [
197 + "Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
198 + ]
199 +
200 + with pytest.raises(openai.BadRequestError):
201 + embeddings = await embedding_client.embeddings.create(
202 + model=model_name,
203 + input=input_texts,
204 + extra_body={"truncate_prompt_tokens": 8193})
205 + assert "error" in embeddings.object
206 + assert "truncate_prompt_tokens value is greater than max_model_len. "\
207 + "Please, select a smaller truncation size." in embeddings.message
vllm/entrypoints/openai/protocol.py
+1
@@ -671,6 +671,7 @@ class EmbeddingRequest(OpenAIBaseModel):
671 encoding_format: Literal["float", "base64"] = "float"
672 dimensions: Optional[int] = None
673 user: Optional[str] = None
674 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
675
676 # doc: begin-embedding-pooling-params
677 additional_data: Optional[Any] = None
vllm/entrypoints/openai/serving_embedding.py
+14 -5
@@ -110,6 +110,17 @@ class OpenAIServingEmbedding(OpenAIServing):
110 request_id = f"embd-{random_uuid()}"
111 created_time = int(time.monotonic())
112
113 + truncate_prompt_tokens = None
114 +
115 + if request.truncate_prompt_tokens is not None:
116 + if request.truncate_prompt_tokens <= self.max_model_len:
117 + truncate_prompt_tokens = request.truncate_prompt_tokens
118 + else:
119 + return self.create_error_response(
120 + "truncate_prompt_tokens value is "
121 + "greater than max_model_len."
122 + " Please, select a smaller truncation size.")
123 +
124 # Schedule the request and get the result generator.
125 generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
126 try:
@@ -123,11 +134,9 @@ class OpenAIServingEmbedding(OpenAIServing):
134 pooling_params = request.to_pooling_params()
135
136 prompts = list(
126 - self._tokenize_prompt_input_or_inputs(
127 - request,
128 - tokenizer,
129 - request.input,
130 - ))
137 + self._tokenize_prompt_input_or_inputs(request, tokenizer,
138 + request.input,
139 + truncate_prompt_tokens))
140
141 for i, prompt_inputs in enumerate(prompts):
142 request_id_item = f"{request_id}-{i}"