A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[BUGFIX] [FRONTEND] Correct chat logprobs (#5029)
Co-authored-by: Breno Faria <breno.faria@intrafind.com>
Breno Faria committed
May 30, 2024 at 11:52 UTC
87d41c849d2cde9279fb08a3a0d97123e3d8fe2f
6 files changed
+361
-98
tests/async_engine/test_openapi_server_ray.py
+4
-2
@@ -94,8 +94,10 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI):
94
chat_completion.choices) == 1
95
assert chat_completion.choices[0].message is not None
96
assert chat_completion.choices[0].logprobs is not None
97
- assert chat_completion.choices[0].logprobs.top_logprobs is not None
98
- assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
97
+ assert chat_completion.choices[0].logprobs.content[
98
+ 0].top_logprobs is not None
99
+ assert len(
100
+ chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
101
message = chat_completion.choices[0].message
102
assert message.content is not None and len(message.content) >= 10
103
assert message.role == "assistant"
tests/entrypoints/test_openai_server.py
+181
-28
@@ -184,6 +184,26 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
184
completion.choices[0].text) >= 5
185
186
187
+@pytest.mark.asyncio
188
+@pytest.mark.parametrize(
189
+ # first test base model, then test loras
190
+ "model_name",
191
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
192
+)
193
+async def test_no_logprobs(server, client: openai.AsyncOpenAI,
194
+ model_name: str):
195
+ # test using token IDs
196
+ completion = await client.completions.create(
197
+ model=MODEL_NAME,
198
+ prompt=[0, 0, 0, 0, 0],
199
+ max_tokens=5,
200
+ temperature=0.0,
201
+ logprobs=None,
202
+ )
203
+ choice = completion.choices[0]
204
+ assert choice.logprobs is None
205
+
206
+
207
@pytest.mark.asyncio
208
@pytest.mark.parametrize(
209
# first test base model, then test loras
@@ -203,7 +223,72 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
223
choice = completion.choices[0]
224
assert choice.logprobs is not None
225
assert choice.logprobs.token_logprobs is not None
206
- assert choice.logprobs.top_logprobs is None
226
+ assert choice.logprobs.top_logprobs is not None
227
+ assert len(choice.logprobs.top_logprobs[0]) <= 1
228
+
229
+
230
+@pytest.mark.asyncio
231
+@pytest.mark.parametrize(
232
+ "model_name",
233
+ [MODEL_NAME, "zephyr-lora"],
234
+)
235
+async def test_some_logprobs(server, client: openai.AsyncOpenAI,
236
+ model_name: str):
237
+ # test using token IDs
238
+ completion = await client.completions.create(
239
+ model=MODEL_NAME,
240
+ prompt=[0, 0, 0, 0, 0],
241
+ max_tokens=5,
242
+ temperature=0.0,
243
+ logprobs=5,
244
+ )
245
+ choice = completion.choices[0]
246
+ assert choice.logprobs is not None
247
+ assert choice.logprobs.token_logprobs is not None
248
+ assert choice.logprobs.top_logprobs is not None
249
+ assert len(choice.logprobs.top_logprobs[0]) <= 6
250
+
251
+
252
+@pytest.mark.asyncio
253
+@pytest.mark.parametrize(
254
+ "model_name",
255
+ [MODEL_NAME, "zephyr-lora"],
256
+)
257
+async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
258
+ model_name: str):
259
+
260
+ with pytest.raises(
261
+ (openai.BadRequestError, openai.APIError)): # test using token IDs
262
+ await client.completions.create(
263
+ model=MODEL_NAME,
264
+ prompt=[0, 0, 0, 0, 0],
265
+ max_tokens=5,
266
+ temperature=0.0,
267
+ logprobs=6,
268
+ )
269
+ ...
270
+ with pytest.raises(
271
+ (openai.BadRequestError, openai.APIError)): # test using token IDs
272
+ stream = await client.completions.create(
273
+ model=MODEL_NAME,
274
+ prompt=[0, 0, 0, 0, 0],
275
+ max_tokens=5,
276
+ temperature=0.0,
277
+ logprobs=6,
278
+ stream=True,
279
+ )
280
+ async for chunk in stream:
281
+ ...
282
+
283
+ # the server should still work afterwards
284
+ completion = await client.completions.create(
285
+ model=model_name,
286
+ prompt=[0, 0, 0, 0, 0],
287
+ max_tokens=5,
288
+ temperature=0.0,
289
+ )
290
+ completion = completion.choices[0].text
291
+ assert completion is not None and len(completion) >= 0
292
293
294
@pytest.mark.asyncio
@@ -233,8 +318,10 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
318
chat_completion.choices) == 1
319
assert chat_completion.choices[0].message is not None
320
assert chat_completion.choices[0].logprobs is not None
236
- assert chat_completion.choices[0].logprobs.top_logprobs is not None
237
- assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
321
+ assert chat_completion.choices[0].logprobs.content[
322
+ 0].top_logprobs is not None
323
+ assert len(
324
+ chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
325
message = chat_completion.choices[0].message
326
assert message.content is not None and len(message.content) >= 10
327
assert message.role == "assistant"
@@ -251,10 +338,93 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
338
assert message.content is not None and len(message.content) >= 0
339
340
341
+@pytest.mark.asyncio
342
+@pytest.mark.parametrize(
343
+ # first test base model, then test loras
344
+ "model_name",
345
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
346
+)
347
+async def test_no_logprobs_chat(server, client: openai.AsyncOpenAI,
348
+ model_name: str):
349
+ messages = [{
350
+ "role": "system",
351
+ "content": "you are a helpful assistant"
352
+ }, {
353
+ "role": "user",
354
+ "content": "what is 1+1?"
355
+ }]
356
+
357
+ chat_completion = await client.chat.completions.create(model=model_name,
358
+ messages=messages,
359
+ max_tokens=5,
360
+ temperature=0.0,
361
+ logprobs=False)
362
+
363
+ choice = chat_completion.choices[0]
364
+ assert choice.logprobs is None
365
+
366
+
367
+@pytest.mark.asyncio
368
+@pytest.mark.parametrize(
369
+ # just test 1 lora hereafter
370
+ "model_name",
371
+ [MODEL_NAME, "zephyr-lora"],
372
+)
373
+async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
374
+ model_name: str):
375
+ messages = [{
376
+ "role": "system",
377
+ "content": "you are a helpful assistant"
378
+ }, {
379
+ "role": "user",
380
+ "content": "what is 1+1?"
381
+ }]
382
+
383
+ chat_completion = await client.chat.completions.create(model=model_name,
384
+ messages=messages,
385
+ max_tokens=5,
386
+ temperature=0.0,
387
+ logprobs=True,
388
+ top_logprobs=0)
389
+
390
+ choice = chat_completion.choices[0]
391
+ assert choice.logprobs is not None
392
+ assert choice.logprobs.content is not None
393
+ assert len(choice.logprobs.content[0].top_logprobs) <= 1
394
+
395
+
396
+@pytest.mark.asyncio
397
+@pytest.mark.parametrize(
398
+ "model_name",
399
+ [MODEL_NAME, "zephyr-lora"],
400
+)
401
+async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
402
+ model_name: str):
403
+ messages = [{
404
+ "role": "system",
405
+ "content": "you are a helpful assistant"
406
+ }, {
407
+ "role": "user",
408
+ "content": "what is 1+1?"
409
+ }]
410
+
411
+ chat_completion = await client.chat.completions.create(model=model_name,
412
+ messages=messages,
413
+ max_tokens=5,
414
+ temperature=0.0,
415
+ logprobs=True,
416
+ top_logprobs=5)
417
+
418
+ choice = chat_completion.choices[0]
419
+ assert choice.logprobs is not None
420
+ assert choice.logprobs.content is not None
421
+ assert len(choice.logprobs.content[0].top_logprobs) <= 6
422
+
423
+
424
@pytest.mark.asyncio
425
@pytest.mark.parametrize("model_name", [MODEL_NAME])
256
-async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
257
- model_name: str):
426
+async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
427
+ model_name: str):
428
messages = [{
429
"role": "system",
430
"content": "you are a helpful assistant"
@@ -263,13 +433,13 @@ async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
433
"content": "what is 1+1?"
434
}]
435
266
- # Default max_logprobs is 5, so this should raise an error
436
+ # Default max_logprobs is 20, so this should raise an error
437
with pytest.raises((openai.BadRequestError, openai.APIError)):
438
stream = await client.chat.completions.create(model=model_name,
439
messages=messages,
440
max_tokens=10,
441
logprobs=True,
272
- top_logprobs=10,
442
+ top_logprobs=21,
443
stream=True)
444
async for chunk in stream:
445
...
@@ -279,25 +449,9 @@ async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
449
messages=messages,
450
max_tokens=10,
451
logprobs=True,
282
- top_logprobs=10,
452
+ top_logprobs=30,
453
stream=False)
454
285
- with pytest.raises((openai.BadRequestError, openai.APIError)):
286
- stream = await client.completions.create(model=model_name,
287
- prompt="Test",
288
- max_tokens=10,
289
- logprobs=10,
290
- stream=True)
291
- async for chunk in stream:
292
- ...
293
-
294
- with pytest.raises(openai.BadRequestError):
295
- await client.completions.create(model=model_name,
296
- prompt="Test",
297
- max_tokens=10,
298
- logprobs=10,
299
- stream=False)
300
-
455
# the server should still work afterwards
456
chat_completion = await client.chat.completions.create(model=model_name,
457
messages=messages,
@@ -744,13 +898,12 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
898
top_logprobs=5,
899
extra_body=dict(guided_choice=TEST_CHOICE,
900
guided_decoding_backend=guided_decoding_backend))
747
- top_logprobs = chat_completion.choices[0].logprobs.top_logprobs
901
+ top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
902
903
# -9999.0 is the minimum logprob returned by OpenAI
904
assert all(
751
- isinstance(logprob, float) and logprob >= -9999.0
752
- for token_dict in top_logprobs
753
- for token, logprob in token_dict.items())
905
+ isinstance(token.logprob, float) and token.logprob >= -9999.0
906
+ for token in top_logprobs)
907
908
909
@pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+43
-7
@@ -250,6 +250,19 @@ class ChatCompletionRequest(OpenAIBaseModel):
250
"('guided_json', 'guided_regex' or 'guided_choice').")
251
return data
252
253
+ @model_validator(mode="before")
254
+ @classmethod
255
+ def check_logprobs(cls, data):
256
+ if "top_logprobs" in data and data["top_logprobs"] is not None:
257
+ if "logprobs" not in data or data["logprobs"] is False:
258
+ raise ValueError(
259
+ "when using `top_logprobs`, `logprobs` must be set to true."
260
+ )
261
+ elif not 0 <= data["top_logprobs"] <= 20:
262
+ raise ValueError(
263
+ "`top_logprobs` must be a value in the interval [0, 20].")
264
+ return data
265
+
266
267
class CompletionRequest(OpenAIBaseModel):
268
# Ordered by official OpenAI API documentation
@@ -396,6 +409,15 @@ class CompletionRequest(OpenAIBaseModel):
409
"('guided_json', 'guided_regex' or 'guided_choice').")
410
return data
411
412
+ @model_validator(mode="before")
413
+ @classmethod
414
+ def check_logprobs(cls, data):
415
+ if "logprobs" in data and data[
416
+ "logprobs"] is not None and not 0 <= data["logprobs"] <= 5:
417
+ raise ValueError(("if passed, `logprobs` must be a value",
418
+ " in the interval [0, 5]."))
419
+ return data
420
+
421
422
class EmbeddingRequest(BaseModel):
423
# Ordered by official OpenAI API documentation
@@ -415,7 +437,7 @@ class EmbeddingRequest(BaseModel):
437
return PoolingParams(additional_data=self.additional_data)
438
439
418
-class LogProbs(OpenAIBaseModel):
440
+class CompletionLogProbs(OpenAIBaseModel):
441
text_offset: List[int] = Field(default_factory=list)
442
token_logprobs: List[Optional[float]] = Field(default_factory=list)
443
tokens: List[str] = Field(default_factory=list)
@@ -425,7 +447,7 @@ class LogProbs(OpenAIBaseModel):
447
class CompletionResponseChoice(OpenAIBaseModel):
448
index: int
449
text: str
428
- logprobs: Optional[LogProbs] = None
450
+ logprobs: Optional[CompletionLogProbs] = None
451
finish_reason: Optional[str] = None
452
stop_reason: Optional[Union[int, str]] = Field(
453
default=None,
@@ -448,7 +470,7 @@ class CompletionResponse(OpenAIBaseModel):
470
class CompletionResponseStreamChoice(OpenAIBaseModel):
471
index: int
472
text: str
451
- logprobs: Optional[LogProbs] = None
473
+ logprobs: Optional[CompletionLogProbs] = None
474
finish_reason: Optional[str] = None
475
stop_reason: Optional[Union[int, str]] = Field(
476
default=None,
@@ -488,11 +510,25 @@ class ChatMessage(OpenAIBaseModel):
510
content: str
511
512
513
+class ChatCompletionLogProb(OpenAIBaseModel):
514
+ token: str
515
+ logprob: float = -9999.0
516
+ bytes: Optional[List[int]] = None
517
+
518
+
519
+class ChatCompletionLogProbsContent(ChatCompletionLogProb):
520
+ top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)
521
+
522
+
523
+class ChatCompletionLogProbs(OpenAIBaseModel):
524
+ content: Optional[List[ChatCompletionLogProbsContent]] = None
525
+
526
+
527
class ChatCompletionResponseChoice(OpenAIBaseModel):
528
index: int
529
message: ChatMessage
494
- logprobs: Optional[LogProbs] = None
495
- finish_reason: Optional[str] = None
530
+ logprobs: Optional[ChatCompletionLogProbs] = None
531
+ finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
532
stop_reason: Optional[Union[int, str]] = None
533
534
@@ -513,8 +549,8 @@ class DeltaMessage(OpenAIBaseModel):
549
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
550
index: int
551
delta: DeltaMessage
516
- logprobs: Optional[LogProbs] = None
517
- finish_reason: Optional[str] = None
552
+ logprobs: Optional[ChatCompletionLogProbs] = None
553
+ finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
554
stop_reason: Optional[Union[int, str]] = None
555
556
vllm/entrypoints/openai/serving_chat.py
+59
-9
@@ -1,8 +1,10 @@
1
import codecs
2
import time
3
from dataclasses import dataclass
4
-from typing import (AsyncGenerator, AsyncIterator, Iterable, List, Optional,
5
- TypedDict, Union, cast, final)
4
+from typing import (AsyncGenerator, AsyncIterator, Dict, Iterable, List,
5
+ Optional)
6
+from typing import Sequence as GenericSequence
7
+from typing import TypedDict, Union, cast, final
8
9
from fastapi import Request
10
from openai.types.chat import ChatCompletionContentPartTextParam
@@ -10,8 +12,9 @@ from openai.types.chat import ChatCompletionContentPartTextParam
12
from vllm.config import ModelConfig
13
from vllm.engine.async_llm_engine import AsyncLLMEngine
14
from vllm.entrypoints.openai.protocol import (
13
- ChatCompletionContentPartParam, ChatCompletionMessageParam,
14
- ChatCompletionRequest, ChatCompletionResponse,
15
+ ChatCompletionContentPartParam, ChatCompletionLogProb,
16
+ ChatCompletionLogProbs, ChatCompletionLogProbsContent,
17
+ ChatCompletionMessageParam, ChatCompletionRequest, ChatCompletionResponse,
18
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
19
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
20
UsageInfo)
@@ -21,6 +24,7 @@ from vllm.logger import init_logger
24
from vllm.model_executor.guided_decoding import (
25
get_guided_decoding_logits_processor)
26
from vllm.outputs import RequestOutput
27
+from vllm.sequence import Logprob
28
from vllm.utils import random_uuid
29
30
logger = init_logger(__name__)
@@ -283,11 +287,10 @@ class OpenAIServingChat(OpenAIServing):
287
previous_num_tokens[i]:] if output.logprobs else None
288
289
if request.logprobs:
286
- logprobs = self._create_logprobs(
290
+ logprobs = self._create_chat_logprobs(
291
token_ids=delta_token_ids,
292
top_logprobs=top_logprobs,
293
num_output_top_logprobs=request.top_logprobs,
290
- initial_text_offset=len(previous_texts[i]),
294
)
295
else:
296
logprobs = None
@@ -370,7 +373,7 @@ class OpenAIServingChat(OpenAIServing):
373
top_logprobs = output.logprobs
374
375
if request.logprobs:
373
- logprobs = self._create_logprobs(
376
+ logprobs = self._create_chat_logprobs(
377
token_ids=token_ids,
378
top_logprobs=top_logprobs,
379
num_output_top_logprobs=request.top_logprobs,
@@ -383,8 +386,7 @@ class OpenAIServingChat(OpenAIServing):
386
message=ChatMessage(role=role, content=output.text),
387
logprobs=logprobs,
388
finish_reason=output.finish_reason,
386
- stop_reason=output.stop_reason,
387
- )
389
+ stop_reason=output.stop_reason)
390
choices.append(choice_data)
391
392
if request.echo:
@@ -414,3 +416,51 @@ class OpenAIServingChat(OpenAIServing):
416
)
417
418
return response
419
+
420
+ def _get_top_logprobs(
421
+ self, logprobs: Dict[int, Logprob],
422
+ top_logprobs: Optional[int]) -> List[ChatCompletionLogProb]:
423
+ return [
424
+ ChatCompletionLogProb(
425
+ token=self._get_decoded_token(p[1], p[0]),
426
+ logprob=max(p[1].logprob, -9999.0),
427
+ bytes=list(
428
+ self._get_decoded_token(p[1],
429
+ p[0]).encode("utf-8",
430
+ errors="replace")))
431
+ for i, p in enumerate(logprobs.items())
432
+ if top_logprobs and i < top_logprobs
433
+ ]
434
+
435
+ def _create_chat_logprobs(
436
+ self,
437
+ token_ids: GenericSequence[int],
438
+ top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
439
+ num_output_top_logprobs: Optional[int] = None,
440
+ ) -> ChatCompletionLogProbs:
441
+ """Create OpenAI-style logprobs."""
442
+
443
+ logprobs_content = []
444
+
445
+ for i, token_id in enumerate(token_ids):
446
+ step_top_logprobs = top_logprobs[i]
447
+ if step_top_logprobs is None:
448
+ logprobs_content.append(
449
+ ChatCompletionLogProbsContent(
450
+ token=self.tokenizer.decode(token_id),
451
+ bytes=list(
452
+ self.tokenizer.decode(token_id).encode(
453
+ "utf-8", errors="replace"))))
454
+ else:
455
+ logprobs_content.append(
456
+ ChatCompletionLogProbsContent(
457
+ token=step_top_logprobs[token_id].decoded_token,
458
+ logprob=max(step_top_logprobs[token_id].logprob,
459
+ -9999.0),
460
+ bytes=list(
461
+ step_top_logprobs[token_id].decoded_token.encode(
462
+ "utf-8", errors="replace")),
463
+ top_logprobs=self._get_top_logprobs(
464
+ step_top_logprobs, num_output_top_logprobs)))
465
+
466
+ return ChatCompletionLogProbs(content=logprobs_content)
vllm/entrypoints/openai/serving_completion.py
+68
-6
@@ -1,23 +1,29 @@
1
import time
2
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
3
- Optional, Tuple)
3
+ Optional)
4
+from typing import Sequence as GenericSequence
5
+from typing import Tuple
6
7
from fastapi import Request
8
9
from vllm.config import ModelConfig
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
9
-from vllm.entrypoints.openai.protocol import (CompletionRequest,
11
+# yapf: disable
12
+from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
13
+ CompletionRequest,
14
CompletionResponse,
15
CompletionResponseChoice,
16
CompletionResponseStreamChoice,
17
CompletionStreamResponse,
14
- LogProbs, UsageInfo)
18
+ UsageInfo)
19
+# yapf: enable
20
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
21
OpenAIServing)
22
from vllm.logger import init_logger
23
from vllm.model_executor.guided_decoding import (
24
get_guided_decoding_logits_processor)
25
from vllm.outputs import RequestOutput
26
+from vllm.sequence import Logprob
27
from vllm.utils import merge_async_iterators, random_uuid
28
29
logger = init_logger(__name__)
@@ -25,7 +31,7 @@ logger = init_logger(__name__)
31
TypeTokenIDs = List[int]
32
TypeTopLogProbs = List[Optional[Dict[int, float]]]
33
TypeCreateLogProbsFn = Callable[
28
- [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
34
+ [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs]
35
36
37
def parse_prompt_format(prompt) -> Tuple[bool, list]:
@@ -235,7 +241,7 @@ class OpenAIServingCompletion(OpenAIServing):
241
i]:] if output.logprobs else None
242
243
if request.logprobs is not None:
238
- logprobs = self._create_logprobs(
244
+ logprobs = self._create_completion_logprobs(
245
token_ids=delta_token_ids,
246
top_logprobs=top_logprobs,
247
num_output_top_logprobs=request.logprobs,
@@ -317,7 +323,7 @@ class OpenAIServingCompletion(OpenAIServing):
323
assert top_logprobs is not None, (
324
"top_logprobs must be provided when logprobs "
325
"is requested")
320
- logprobs = self._create_logprobs(
326
+ logprobs = self._create_completion_logprobs(
327
token_ids=token_ids,
328
top_logprobs=top_logprobs,
329
num_output_top_logprobs=request.logprobs,
@@ -351,3 +357,59 @@ class OpenAIServingCompletion(OpenAIServing):
357
choices=choices,
358
usage=usage,
359
)
360
+
361
+ def _create_completion_logprobs(
362
+ self,
363
+ token_ids: GenericSequence[int],
364
+ top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
365
+ num_output_top_logprobs: int,
366
+ initial_text_offset: int = 0,
367
+ ) -> CompletionLogProbs:
368
+ """Create logprobs for OpenAI Completion API."""
369
+ out_text_offset: List[int] = []
370
+ out_token_logprobs: List[Optional[float]] = []
371
+ out_tokens: List[str] = []
372
+ out_top_logprobs: List[Optional[Dict[str, float]]] = []
373
+
374
+ last_token_len = 0
375
+
376
+ for i, token_id in enumerate(token_ids):
377
+ step_top_logprobs = top_logprobs[i]
378
+ if step_top_logprobs is None:
379
+ token = self.tokenizer.decode(token_id)
380
+ out_tokens.append(token)
381
+ out_token_logprobs.append(None)
382
+ out_top_logprobs.append(None)
383
+ else:
384
+ token = self._get_decoded_token(step_top_logprobs[token_id],
385
+ token_id)
386
+ token_logprob = max(step_top_logprobs[token_id].logprob,
387
+ -9999.0)
388
+ out_tokens.append(token)
389
+ out_token_logprobs.append(token_logprob)
390
+
391
+ # makes sure to add the top num_output_top_logprobs + 1
392
+ # logprobs, as defined in the openai API
393
+ # (cf. https://github.com/openai/openai-openapi/blob/
394
+ # 893ba52242dbd5387a97b96444ee1c742cfce9bd/openapi.yaml#L7153)
395
+ out_top_logprobs.append({
396
+ # Convert float("-inf") to the
397
+ # JSON-serializable float that OpenAI uses
398
+ self._get_decoded_token(top_lp[1], top_lp[0]):
399
+ max(top_lp[1].logprob, -9999.0)
400
+ for i, top_lp in enumerate(step_top_logprobs.items())
401
+ if num_output_top_logprobs >= i
402
+ })
403
+
404
+ if len(out_text_offset) == 0:
405
+ out_text_offset.append(initial_text_offset)
406
+ else:
407
+ out_text_offset.append(out_text_offset[-1] + last_token_len)
408
+ last_token_len = len(token)
409
+
410
+ return CompletionLogProbs(
411
+ text_offset=out_text_offset,
412
+ token_logprobs=out_token_logprobs,
413
+ tokens=out_tokens,
414
+ top_logprobs=out_top_logprobs,
415
+ )
vllm/entrypoints/openai/serving_engine.py
+6
-46
@@ -11,7 +11,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
CompletionRequest,
13
EmbeddingRequest, ErrorResponse,
14
- LogProbs, ModelCard, ModelList,
14
+ ModelCard, ModelList,
15
ModelPermission)
16
from vllm.logger import init_logger
17
from vllm.lora.request import LoRARequest
@@ -75,51 +75,6 @@ class OpenAIServing:
75
model_cards.extend(lora_cards)
76
return ModelList(data=model_cards)
77
78
- def _create_logprobs(
79
- self,
80
- token_ids: List[int],
81
- top_logprobs: List[Optional[Dict[int, Logprob]]],
82
- num_output_top_logprobs: Optional[int] = None,
83
- initial_text_offset: int = 0,
84
- ) -> LogProbs:
85
- """Create OpenAI-style logprobs."""
86
- logprobs = LogProbs()
87
- last_token_len = 0
88
- if num_output_top_logprobs:
89
- logprobs.top_logprobs = []
90
-
91
- for i, token_id in enumerate(token_ids):
92
- step_top_logprobs = top_logprobs[i]
93
- if step_top_logprobs is None:
94
- token = self.tokenizer.decode(token_id)
95
- logprobs.tokens.append(token)
96
- logprobs.token_logprobs.append(None)
97
- assert logprobs.top_logprobs is not None
98
- logprobs.top_logprobs.append(None)
99
- else:
100
- token_logprob = step_top_logprobs[token_id].logprob
101
- token = step_top_logprobs[token_id].decoded_token
102
- logprobs.tokens.append(token)
103
- token_logprob = max(token_logprob, -9999.0)
104
- logprobs.token_logprobs.append(token_logprob)
105
-
106
- if num_output_top_logprobs:
107
- assert logprobs.top_logprobs is not None
108
- logprobs.top_logprobs.append({
109
- # Convert float("-inf") to the
110
- # JSON-serializable float that OpenAI uses
111
- p.decoded_token: max(p.logprob, -9999.0)
112
- for i, p in step_top_logprobs.items()
113
- } if step_top_logprobs else None)
114
-
115
- if len(logprobs.text_offset) == 0:
116
- logprobs.text_offset.append(initial_text_offset)
117
- else:
118
- logprobs.text_offset.append(logprobs.text_offset[-1] +
119
- last_token_len)
120
- last_token_len = len(token)
121
- return logprobs
122
-
78
def create_error_response(
79
self,
80
message: str,
@@ -235,3 +190,8 @@ class OpenAIServing:
190
f"Please reduce the length of the messages or completion.", )
191
else:
192
return input_ids, input_text
193
+
194
+ def _get_decoded_token(self, logprob: Logprob, token_id: int) -> str:
195
+ if logprob.decoded_token is not None:
196
+ return logprob.decoded_token
197
+ return self.tokenizer.decode(token_id)