A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Feature]: Add OpenAI server prompt_logprobs support #6508 (#7453)
Grant Pinkert committed
Aug 16, 2024 at 12:38 UTC
f878c8feb06b33fa68dd1bdd687637b326eccfde
4 files changed
+154
-3
tests/entrypoints/openai/test_completion.py
+124
-1
@@ -3,7 +3,7 @@ import json
3
import re
4
import shutil
5
from tempfile import TemporaryDirectory
6
-from typing import List
6
+from typing import Dict, List
7
8
import jsonschema
9
import openai # use the official client for correctness check
@@ -130,6 +130,7 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
130
temperature=0.0,
131
)
132
assert len(completion.choices[0].text) >= 1
133
+ assert completion.choices[0].prompt_logprobs is None
134
135
136
@pytest.mark.asyncio
@@ -267,6 +268,128 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
268
assert len(completion.choices[0].text) >= 0
269
270
271
+@pytest.mark.asyncio
272
+@pytest.mark.parametrize(
273
+ "model_name, prompt_logprobs",
274
+ [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
275
+)
276
+async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
277
+ model_name: str, prompt_logprobs: int):
278
+ params: Dict = {
279
+ "messages": [{
280
+ "role": "system",
281
+ "content": "You are a helpful assistant."
282
+ }, {
283
+ "role": "user",
284
+ "content": "Who won the world series in 2020?"
285
+ }, {
286
+ "role":
287
+ "assistant",
288
+ "content":
289
+ "The Los Angeles Dodgers won the World Series in 2020."
290
+ }, {
291
+ "role": "user",
292
+ "content": "Where was it played?"
293
+ }],
294
+ "model":
295
+ model_name
296
+ }
297
+
298
+ if prompt_logprobs is not None:
299
+ params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
300
+
301
+ if prompt_logprobs and prompt_logprobs < 0:
302
+ with pytest.raises(BadRequestError) as err_info:
303
+ await client.chat.completions.create(**params)
304
+ expected_err_string = (
305
+ "Error code: 400 - {'object': 'error', 'message': "
306
+ "'Prompt_logprobs set to invalid negative value: -1',"
307
+ " 'type': 'BadRequestError', 'param': None, 'code': 400}")
308
+ assert str(err_info.value) == expected_err_string
309
+ else:
310
+ completion = await client.chat.completions.create(**params)
311
+ if prompt_logprobs and prompt_logprobs > 0:
312
+ assert completion.prompt_logprobs is not None
313
+ assert len(completion.prompt_logprobs) > 0
314
+ else:
315
+ assert completion.prompt_logprobs is None
316
+
317
+
318
+@pytest.mark.asyncio
319
+@pytest.mark.parametrize(
320
+ "model_name",
321
+ [MODEL_NAME],
322
+)
323
+async def test_more_than_one_prompt_logprobs_chat(client: openai.AsyncOpenAI,
324
+ model_name: str):
325
+ params: Dict = {
326
+ "messages": [{
327
+ "role": "system",
328
+ "content": "You are a helpful assistant."
329
+ }, {
330
+ "role": "user",
331
+ "content": "Who won the world series in 2020?"
332
+ }, {
333
+ "role":
334
+ "assistant",
335
+ "content":
336
+ "The Los Angeles Dodgers won the World Series in 2020."
337
+ }, {
338
+ "role": "user",
339
+ "content": "Where was it played?"
340
+ }],
341
+ "model":
342
+ model_name,
343
+ "extra_body": {
344
+ "prompt_logprobs": 1
345
+ }
346
+ }
347
+
348
+ completion_1 = await client.chat.completions.create(**params)
349
+
350
+ params["extra_body"] = {"prompt_logprobs": 2}
351
+ completion_2 = await client.chat.completions.create(**params)
352
+
353
+ assert len(completion_1.prompt_logprobs[3]) == 1
354
+ assert len(completion_2.prompt_logprobs[3]) == 2
355
+
356
+
357
+@pytest.mark.asyncio
358
+@pytest.mark.parametrize("model_name, prompt_logprobs", [(MODEL_NAME, -1),
359
+ (MODEL_NAME, 0),
360
+ (MODEL_NAME, 1),
361
+ (MODEL_NAME, None)])
362
+async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
363
+ model_name: str,
364
+ prompt_logprobs: int):
365
+ params: Dict = {
366
+ "prompt": ["A robot may not injure another robot", "My name is"],
367
+ "model": model_name,
368
+ }
369
+ if prompt_logprobs is not None:
370
+ params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
371
+
372
+ if prompt_logprobs and prompt_logprobs < 0:
373
+ with pytest.raises(BadRequestError) as err_info:
374
+ await client.completions.create(**params)
375
+ expected_err_string = (
376
+ "Error code: 400 - {'object': 'error', 'message': "
377
+ "'Prompt_logprobs set to invalid negative value: -1',"
378
+ " 'type': 'BadRequestError', 'param': None, 'code': 400}")
379
+ assert str(err_info.value) == expected_err_string
380
+ else:
381
+ completion = await client.completions.create(**params)
382
+ if prompt_logprobs and prompt_logprobs > 0:
383
+ assert completion.choices[0].prompt_logprobs is not None
384
+ assert len(completion.choices[0].prompt_logprobs) > 0
385
+
386
+ assert completion.choices[1].prompt_logprobs is not None
387
+ assert len(completion.choices[1].prompt_logprobs) > 0
388
+
389
+ else:
390
+ assert completion.choices[0].prompt_logprobs is None
391
+
392
+
393
@pytest.mark.asyncio
394
@pytest.mark.parametrize(
395
"model_name",
vllm/entrypoints/openai/protocol.py
+9
-2
@@ -13,6 +13,7 @@ from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.entrypoints.openai.logits_processors import get_logits_processors
14
from vllm.pooling_params import PoolingParams
15
from vllm.sampling_params import LogitsProcessor, SamplingParams
16
+from vllm.sequence import Logprob
17
from vllm.utils import random_uuid
18
19
# torch is mocked during docs generation,
@@ -152,6 +153,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
153
skip_special_tokens: bool = True
154
spaces_between_special_tokens: bool = True
155
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
156
+ prompt_logprobs: Optional[int] = None
157
# doc: end-chat-completion-sampling-params
158
159
# doc: begin-chat-completion-extra-params
@@ -263,7 +265,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
265
stop=self.stop,
266
stop_token_ids=self.stop_token_ids,
267
logprobs=self.top_logprobs if self.logprobs else None,
266
- prompt_logprobs=self.top_logprobs if self.echo else None,
268
+ prompt_logprobs=self.prompt_logprobs if self.prompt_logprobs else
269
+ (self.top_logprobs if self.echo else None),
270
ignore_eos=self.ignore_eos,
271
max_tokens=max_tokens,
272
min_tokens=self.min_tokens,
@@ -368,6 +371,7 @@ class CompletionRequest(OpenAIBaseModel):
371
spaces_between_special_tokens: bool = True
372
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
373
allowed_token_ids: Optional[List[int]] = None
374
+ prompt_logprobs: Optional[int] = None
375
# doc: end-completion-sampling-params
376
377
# doc: begin-completion-extra-params
@@ -454,7 +458,8 @@ class CompletionRequest(OpenAIBaseModel):
458
min_tokens=self.min_tokens,
459
use_beam_search=self.use_beam_search,
460
early_stopping=self.early_stopping,
457
- prompt_logprobs=self.logprobs if self.echo else None,
461
+ prompt_logprobs=self.prompt_logprobs
462
+ if self.prompt_logprobs else self.logprobs if self.echo else None,
463
skip_special_tokens=self.skip_special_tokens,
464
spaces_between_special_tokens=self.spaces_between_special_tokens,
465
include_stop_str_in_output=self.include_stop_str_in_output,
@@ -532,6 +537,7 @@ class CompletionResponseChoice(OpenAIBaseModel):
537
"to stop, None if the completion finished for some other reason "
538
"including encountering the EOS token"),
539
)
540
+ prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
541
542
543
class CompletionResponse(OpenAIBaseModel):
@@ -627,6 +633,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
633
model: str
634
choices: List[ChatCompletionResponseChoice]
635
usage: UsageInfo
636
+ prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
637
638
639
class DeltaMessage(OpenAIBaseModel):
vllm/entrypoints/openai/serving_chat.py
+11
@@ -83,6 +83,16 @@ class OpenAIServingChat(OpenAIServing):
83
if error_check_ret is not None:
84
return error_check_ret
85
86
+ if request.prompt_logprobs is not None:
87
+ if request.stream and request.prompt_logprobs > 0:
88
+ return self.create_error_response(
89
+ "Prompt_logprobs are not available when stream is enabled")
90
+
91
+ if request.prompt_logprobs < 0:
92
+ return self.create_error_response(
93
+ f"Prompt_logprobs set to invalid "
94
+ f"negative value: {request.prompt_logprobs}")
95
+
96
try:
97
(
98
lora_request,
@@ -506,6 +516,7 @@ class OpenAIServingChat(OpenAIServing):
516
model=model_name,
517
choices=choices,
518
usage=usage,
519
+ prompt_logprobs=final_res.prompt_logprobs,
520
)
521
522
return response
vllm/entrypoints/openai/serving_completion.py
+10
@@ -84,6 +84,15 @@ class OpenAIServingCompletion(OpenAIServing):
84
request_id = f"cmpl-{random_uuid()}"
85
created_time = int(time.time())
86
87
+ if request.prompt_logprobs is not None:
88
+ if request.stream and request.prompt_logprobs > 0:
89
+ return self.create_error_response(
90
+ "Prompt_logprobs are not available when stream is enabled")
91
+ elif request.prompt_logprobs < 0:
92
+ return self.create_error_response(
93
+ f"Prompt_logprobs set to invalid negative "
94
+ f"value: {request.prompt_logprobs}")
95
+
96
# Schedule the request and get the result generator.
97
generators: List[AsyncGenerator[RequestOutput, None]] = []
98
try:
@@ -377,6 +386,7 @@ class OpenAIServingCompletion(OpenAIServing):
386
logprobs=logprobs,
387
finish_reason=output.finish_reason,
388
stop_reason=output.stop_reason,
389
+ prompt_logprobs=final_res.prompt_logprobs,
390
)
391
choices.append(choice_data)
392