@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f878c8feb0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Feature]: Add OpenAI server prompt_logprobs support #6508 (#7453)

Grant Pinkert committed Aug 16, 2024 at 12:38 UTC f878c8feb06b33fa68dd1bdd687637b326eccfde
4 files changed +154 -3
tests/entrypoints/openai/test_completion.py
+124 -1
@@ -3,7 +3,7 @@ import json
3 import re
4 import shutil
5 from tempfile import TemporaryDirectory
6 -from typing import List
6 +from typing import Dict, List
7
8 import jsonschema
9 import openai # use the official client for correctness check
@@ -130,6 +130,7 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
130 temperature=0.0,
131 )
132 assert len(completion.choices[0].text) >= 1
133 + assert completion.choices[0].prompt_logprobs is None
134
135
136 @pytest.mark.asyncio
@@ -267,6 +268,128 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
268 assert len(completion.choices[0].text) >= 0
269
270
271 +@pytest.mark.asyncio
272 +@pytest.mark.parametrize(
273 + "model_name, prompt_logprobs",
274 + [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
275 +)
276 +async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
277 + model_name: str, prompt_logprobs: int):
278 + params: Dict = {
279 + "messages": [{
280 + "role": "system",
281 + "content": "You are a helpful assistant."
282 + }, {
283 + "role": "user",
284 + "content": "Who won the world series in 2020?"
285 + }, {
286 + "role":
287 + "assistant",
288 + "content":
289 + "The Los Angeles Dodgers won the World Series in 2020."
290 + }, {
291 + "role": "user",
292 + "content": "Where was it played?"
293 + }],
294 + "model":
295 + model_name
296 + }
297 +
298 + if prompt_logprobs is not None:
299 + params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
300 +
301 + if prompt_logprobs and prompt_logprobs < 0:
302 + with pytest.raises(BadRequestError) as err_info:
303 + await client.chat.completions.create(**params)
304 + expected_err_string = (
305 + "Error code: 400 - {'object': 'error', 'message': "
306 + "'Prompt_logprobs set to invalid negative value: -1',"
307 + " 'type': 'BadRequestError', 'param': None, 'code': 400}")
308 + assert str(err_info.value) == expected_err_string
309 + else:
310 + completion = await client.chat.completions.create(**params)
311 + if prompt_logprobs and prompt_logprobs > 0:
312 + assert completion.prompt_logprobs is not None
313 + assert len(completion.prompt_logprobs) > 0
314 + else:
315 + assert completion.prompt_logprobs is None
316 +
317 +
318 +@pytest.mark.asyncio
319 +@pytest.mark.parametrize(
320 + "model_name",
321 + [MODEL_NAME],
322 +)
323 +async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
324 + model_name: str):
325 + params: Dict = {
326 + "messages": [{
327 + "role": "system",
328 + "content": "You are a helpful assistant."
329 + }, {
330 + "role": "user",
331 + "content": "Who won the world series in 2020?"
332 + }, {
333 + "role":
334 + "assistant",
335 + "content":
336 + "The Los Angeles Dodgers won the World Series in 2020."
337 + }, {
338 + "role": "user",
339 + "content": "Where was it played?"
340 + }],
341 + "model":
342 + model_name,
343 + "extra_body": {
344 + "prompt_logprobs": 1
345 + }
346 + }
347 +
348 + completion_1 = await client.chat.completions.create(**params)
349 +
350 + params["extra_body"] = {"prompt_logprobs": 2}
351 + completion_2 = await client.chat.completions.create(**params)
352 +
353 + assert len(completion_1.prompt_logprobs[3]) == 1
354 + assert len(completion_2.prompt_logprobs[3]) == 2
355 +
356 +
357 +@pytest.mark.asyncio
358 +@pytest.mark.parametrize("model_name, prompt_logprobs", [(MODEL_NAME, -1),
359 + (MODEL_NAME, 0),
360 + (MODEL_NAME, 1),
361 + (MODEL_NAME, None)])
362 +async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
363 + model_name: str,
364 + prompt_logprobs: int):
365 + params: Dict = {
366 + "prompt": ["A robot may not injure another robot", "My name is"],
367 + "model": model_name,
368 + }
369 + if prompt_logprobs is not None:
370 + params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
371 +
372 + if prompt_logprobs and prompt_logprobs < 0:
373 + with pytest.raises(BadRequestError) as err_info:
374 + await client.completions.create(**params)
375 + expected_err_string = (
376 + "Error code: 400 - {'object': 'error', 'message': "
377 + "'Prompt_logprobs set to invalid negative value: -1',"
378 + " 'type': 'BadRequestError', 'param': None, 'code': 400}")
379 + assert str(err_info.value) == expected_err_string
380 + else:
381 + completion = await client.completions.create(**params)
382 + if prompt_logprobs and prompt_logprobs > 0:
383 + assert completion.choices[0].prompt_logprobs is not None
384 + assert len(completion.choices[0].prompt_logprobs) > 0
385 +
386 + assert completion.choices[1].prompt_logprobs is not None
387 + assert len(completion.choices[1].prompt_logprobs) > 0
388 +
389 + else:
390 + assert completion.choices[0].prompt_logprobs is None
391 +
392 +
393 @pytest.mark.asyncio
394 @pytest.mark.parametrize(
395 "model_name",
vllm/entrypoints/openai/protocol.py
+9 -2
@@ -13,6 +13,7 @@ from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13 from vllm.entrypoints.openai.logits_processors import get_logits_processors
14 from vllm.pooling_params import PoolingParams
15 from vllm.sampling_params import LogitsProcessor, SamplingParams
16 +from vllm.sequence import Logprob
17 from vllm.utils import random_uuid
18
19 # torch is mocked during docs generation,
@@ -152,6 +153,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
153 skip_special_tokens: bool = True
154 spaces_between_special_tokens: bool = True
155 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
156 + prompt_logprobs: Optional[int] = None
157 # doc: end-chat-completion-sampling-params
158
159 # doc: begin-chat-completion-extra-params
@@ -263,7 +265,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
265 stop=self.stop,
266 stop_token_ids=self.stop_token_ids,
267 logprobs=self.top_logprobs if self.logprobs else None,
266 - prompt_logprobs=self.top_logprobs if self.echo else None,
268 + prompt_logprobs=self.prompt_logprobs if self.prompt_logprobs else
269 + (self.top_logprobs if self.echo else None),
270 ignore_eos=self.ignore_eos,
271 max_tokens=max_tokens,
272 min_tokens=self.min_tokens,
@@ -368,6 +371,7 @@ class CompletionRequest(OpenAIBaseModel):
371 spaces_between_special_tokens: bool = True
372 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
373 allowed_token_ids: Optional[List[int]] = None
374 + prompt_logprobs: Optional[int] = None
375 # doc: end-completion-sampling-params
376
377 # doc: begin-completion-extra-params
@@ -454,7 +458,8 @@ class CompletionRequest(OpenAIBaseModel):
458 min_tokens=self.min_tokens,
459 use_beam_search=self.use_beam_search,
460 early_stopping=self.early_stopping,
457 - prompt_logprobs=self.logprobs if self.echo else None,
461 + prompt_logprobs=self.prompt_logprobs
462 + if self.prompt_logprobs else self.logprobs if self.echo else None,
463 skip_special_tokens=self.skip_special_tokens,
464 spaces_between_special_tokens=self.spaces_between_special_tokens,
465 include_stop_str_in_output=self.include_stop_str_in_output,
@@ -532,6 +537,7 @@ class CompletionResponseChoice(OpenAIBaseModel):
537 "to stop, None if the completion finished for some other reason "
538 "including encountering the EOS token"),
539 )
540 + prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
541
542
543 class CompletionResponse(OpenAIBaseModel):
@@ -627,6 +633,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
633 model: str
634 choices: List[ChatCompletionResponseChoice]
635 usage: UsageInfo
636 + prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
637
638
639 class DeltaMessage(OpenAIBaseModel):
vllm/entrypoints/openai/serving_chat.py
+11
@@ -83,6 +83,16 @@ class OpenAIServingChat(OpenAIServing):
83 if error_check_ret is not None:
84 return error_check_ret
85
86 + if request.prompt_logprobs is not None:
87 + if request.stream and request.prompt_logprobs > 0:
88 + return self.create_error_response(
89 + "Prompt_logprobs are not available when stream is enabled")
90 +
91 + if request.prompt_logprobs < 0:
92 + return self.create_error_response(
93 + f"Prompt_logprobs set to invalid "
94 + f"negative value: {request.prompt_logprobs}")
95 +
96 try:
97 (
98 lora_request,
@@ -506,6 +516,7 @@ class OpenAIServingChat(OpenAIServing):
516 model=model_name,
517 choices=choices,
518 usage=usage,
519 + prompt_logprobs=final_res.prompt_logprobs,
520 )
521
522 return response
vllm/entrypoints/openai/serving_completion.py
+10
@@ -84,6 +84,15 @@ class OpenAIServingCompletion(OpenAIServing):
84 request_id = f"cmpl-{random_uuid()}"
85 created_time = int(time.time())
86
87 + if request.prompt_logprobs is not None:
88 + if request.stream and request.prompt_logprobs > 0:
89 + return self.create_error_response(
90 + "Prompt_logprobs are not available when stream is enabled")
91 + elif request.prompt_logprobs < 0:
92 + return self.create_error_response(
93 + f"Prompt_logprobs set to invalid negative "
94 + f"value: {request.prompt_logprobs}")
95 +
96 # Schedule the request and get the result generator.
97 generators: List[AsyncGenerator[RequestOutput, None]] = []
98 try:
@@ -377,6 +386,7 @@ class OpenAIServingCompletion(OpenAIServing):
386 logprobs=logprobs,
387 finish_reason=output.finish_reason,
388 stop_reason=output.stop_reason,
389 + prompt_logprobs=final_res.prompt_logprobs,
390 )
391 choices.append(choice_data)
392