@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 87d41c849d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[BUGFIX] [FRONTEND] Correct chat logprobs (#5029)

Co-authored-by: Breno Faria <breno.faria@intrafind.com>

Breno Faria committed May 30, 2024 at 11:52 UTC 87d41c849d2cde9279fb08a3a0d97123e3d8fe2f
6 files changed +361 -98
tests/async_engine/test_openapi_server_ray.py
+4 -2
@@ -94,8 +94,10 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI):
94 chat_completion.choices) == 1
95 assert chat_completion.choices[0].message is not None
96 assert chat_completion.choices[0].logprobs is not None
97 - assert chat_completion.choices[0].logprobs.top_logprobs is not None
98 - assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
97 + assert chat_completion.choices[0].logprobs.content[
98 + 0].top_logprobs is not None
99 + assert len(
100 + chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
101 message = chat_completion.choices[0].message
102 assert message.content is not None and len(message.content) >= 10
103 assert message.role == "assistant"
tests/entrypoints/test_openai_server.py
+181 -28
@@ -184,6 +184,26 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
184 completion.choices[0].text) >= 5
185
186
187 +@pytest.mark.asyncio
188 +@pytest.mark.parametrize(
189 + # first test base model, then test loras
190 + "model_name",
191 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
192 +)
193 +async def test_no_logprobs(server, client: openai.AsyncOpenAI,
194 + model_name: str):
195 + # test using token IDs
196 + completion = await client.completions.create(
197 + model=MODEL_NAME,
198 + prompt=[0, 0, 0, 0, 0],
199 + max_tokens=5,
200 + temperature=0.0,
201 + logprobs=None,
202 + )
203 + choice = completion.choices[0]
204 + assert choice.logprobs is None
205 +
206 +
207 @pytest.mark.asyncio
208 @pytest.mark.parametrize(
209 # first test base model, then test loras
@@ -203,7 +223,72 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
223 choice = completion.choices[0]
224 assert choice.logprobs is not None
225 assert choice.logprobs.token_logprobs is not None
206 - assert choice.logprobs.top_logprobs is None
226 + assert choice.logprobs.top_logprobs is not None
227 + assert len(choice.logprobs.top_logprobs[0]) <= 1
228 +
229 +
230 +@pytest.mark.asyncio
231 +@pytest.mark.parametrize(
232 + "model_name",
233 + [MODEL_NAME, "zephyr-lora"],
234 +)
235 +async def test_some_logprobs(server, client: openai.AsyncOpenAI,
236 + model_name: str):
237 + # test using token IDs
238 + completion = await client.completions.create(
239 + model=MODEL_NAME,
240 + prompt=[0, 0, 0, 0, 0],
241 + max_tokens=5,
242 + temperature=0.0,
243 + logprobs=5,
244 + )
245 + choice = completion.choices[0]
246 + assert choice.logprobs is not None
247 + assert choice.logprobs.token_logprobs is not None
248 + assert choice.logprobs.top_logprobs is not None
249 + assert len(choice.logprobs.top_logprobs[0]) <= 6
250 +
251 +
252 +@pytest.mark.asyncio
253 +@pytest.mark.parametrize(
254 + "model_name",
255 + [MODEL_NAME, "zephyr-lora"],
256 +)
257 +async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
258 + model_name: str):
259 +
260 + with pytest.raises(
261 + (openai.BadRequestError, openai.APIError)): # test using token IDs
262 + await client.completions.create(
263 + model=MODEL_NAME,
264 + prompt=[0, 0, 0, 0, 0],
265 + max_tokens=5,
266 + temperature=0.0,
267 + logprobs=6,
268 + )
269 + ...
270 + with pytest.raises(
271 + (openai.BadRequestError, openai.APIError)): # test using token IDs
272 + stream = await client.completions.create(
273 + model=MODEL_NAME,
274 + prompt=[0, 0, 0, 0, 0],
275 + max_tokens=5,
276 + temperature=0.0,
277 + logprobs=6,
278 + stream=True,
279 + )
280 + async for chunk in stream:
281 + ...
282 +
283 + # the server should still work afterwards
284 + completion = await client.completions.create(
285 + model=model_name,
286 + prompt=[0, 0, 0, 0, 0],
287 + max_tokens=5,
288 + temperature=0.0,
289 + )
290 + completion = completion.choices[0].text
291 + assert completion is not None and len(completion) >= 0
292
293
294 @pytest.mark.asyncio
@@ -233,8 +318,10 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
318 chat_completion.choices) == 1
319 assert chat_completion.choices[0].message is not None
320 assert chat_completion.choices[0].logprobs is not None
236 - assert chat_completion.choices[0].logprobs.top_logprobs is not None
237 - assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
321 + assert chat_completion.choices[0].logprobs.content[
322 + 0].top_logprobs is not None
323 + assert len(
324 + chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
325 message = chat_completion.choices[0].message
326 assert message.content is not None and len(message.content) >= 10
327 assert message.role == "assistant"
@@ -251,10 +338,93 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
338 assert message.content is not None and len(message.content) >= 0
339
340
341 +@pytest.mark.asyncio
342 +@pytest.mark.parametrize(
343 + # first test base model, then test loras
344 + "model_name",
345 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
346 +)
347 +async def test_no_logprobs_chat(server, client: openai.AsyncOpenAI,
348 + model_name: str):
349 + messages = [{
350 + "role": "system",
351 + "content": "you are a helpful assistant"
352 + }, {
353 + "role": "user",
354 + "content": "what is 1+1?"
355 + }]
356 +
357 + chat_completion = await client.chat.completions.create(model=model_name,
358 + messages=messages,
359 + max_tokens=5,
360 + temperature=0.0,
361 + logprobs=False)
362 +
363 + choice = chat_completion.choices[0]
364 + assert choice.logprobs is None
365 +
366 +
367 +@pytest.mark.asyncio
368 +@pytest.mark.parametrize(
369 + # just test 1 lora hereafter
370 + "model_name",
371 + [MODEL_NAME, "zephyr-lora"],
372 +)
373 +async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
374 + model_name: str):
375 + messages = [{
376 + "role": "system",
377 + "content": "you are a helpful assistant"
378 + }, {
379 + "role": "user",
380 + "content": "what is 1+1?"
381 + }]
382 +
383 + chat_completion = await client.chat.completions.create(model=model_name,
384 + messages=messages,
385 + max_tokens=5,
386 + temperature=0.0,
387 + logprobs=True,
388 + top_logprobs=0)
389 +
390 + choice = chat_completion.choices[0]
391 + assert choice.logprobs is not None
392 + assert choice.logprobs.content is not None
393 + assert len(choice.logprobs.content[0].top_logprobs) <= 1
394 +
395 +
396 +@pytest.mark.asyncio
397 +@pytest.mark.parametrize(
398 + "model_name",
399 + [MODEL_NAME, "zephyr-lora"],
400 +)
401 +async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
402 + model_name: str):
403 + messages = [{
404 + "role": "system",
405 + "content": "you are a helpful assistant"
406 + }, {
407 + "role": "user",
408 + "content": "what is 1+1?"
409 + }]
410 +
411 + chat_completion = await client.chat.completions.create(model=model_name,
412 + messages=messages,
413 + max_tokens=5,
414 + temperature=0.0,
415 + logprobs=True,
416 + top_logprobs=5)
417 +
418 + choice = chat_completion.choices[0]
419 + assert choice.logprobs is not None
420 + assert choice.logprobs.content is not None
421 + assert len(choice.logprobs.content[0].top_logprobs) <= 6
422 +
423 +
424 @pytest.mark.asyncio
425 @pytest.mark.parametrize("model_name", [MODEL_NAME])
256 -async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
257 - model_name: str):
426 +async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
427 + model_name: str):
428 messages = [{
429 "role": "system",
430 "content": "you are a helpful assistant"
@@ -263,13 +433,13 @@ async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
433 "content": "what is 1+1?"
434 }]
435
266 - # Default max_logprobs is 5, so this should raise an error
436 + # Default max_logprobs is 20, so this should raise an error
437 with pytest.raises((openai.BadRequestError, openai.APIError)):
438 stream = await client.chat.completions.create(model=model_name,
439 messages=messages,
440 max_tokens=10,
441 logprobs=True,
272 - top_logprobs=10,
442 + top_logprobs=21,
443 stream=True)
444 async for chunk in stream:
445 ...
@@ -279,25 +449,9 @@ async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
449 messages=messages,
450 max_tokens=10,
451 logprobs=True,
282 - top_logprobs=10,
452 + top_logprobs=30,
453 stream=False)
454
285 - with pytest.raises((openai.BadRequestError, openai.APIError)):
286 - stream = await client.completions.create(model=model_name,
287 - prompt="Test",
288 - max_tokens=10,
289 - logprobs=10,
290 - stream=True)
291 - async for chunk in stream:
292 - ...
293 -
294 - with pytest.raises(openai.BadRequestError):
295 - await client.completions.create(model=model_name,
296 - prompt="Test",
297 - max_tokens=10,
298 - logprobs=10,
299 - stream=False)
300 -
455 # the server should still work afterwards
456 chat_completion = await client.chat.completions.create(model=model_name,
457 messages=messages,
@@ -744,13 +898,12 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
898 top_logprobs=5,
899 extra_body=dict(guided_choice=TEST_CHOICE,
900 guided_decoding_backend=guided_decoding_backend))
747 - top_logprobs = chat_completion.choices[0].logprobs.top_logprobs
901 + top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
902
903 # -9999.0 is the minimum logprob returned by OpenAI
904 assert all(
751 - isinstance(logprob, float) and logprob >= -9999.0
752 - for token_dict in top_logprobs
753 - for token, logprob in token_dict.items())
905 + isinstance(token.logprob, float) and token.logprob >= -9999.0
906 + for token in top_logprobs)
907
908
909 @pytest.mark.asyncio
vllm/entrypoints/openai/protocol.py
+43 -7
@@ -250,6 +250,19 @@ class ChatCompletionRequest(OpenAIBaseModel):
250 "('guided_json', 'guided_regex' or 'guided_choice').")
251 return data
252
253 + @model_validator(mode="before")
254 + @classmethod
255 + def check_logprobs(cls, data):
256 + if "top_logprobs" in data and data["top_logprobs"] is not None:
257 + if "logprobs" not in data or data["logprobs"] is False:
258 + raise ValueError(
259 + "when using `top_logprobs`, `logprobs` must be set to true."
260 + )
261 + elif not 0 <= data["top_logprobs"] <= 20:
262 + raise ValueError(
263 + "`top_logprobs` must be a value in the interval [0, 20].")
264 + return data
265 +
266
267 class CompletionRequest(OpenAIBaseModel):
268 # Ordered by official OpenAI API documentation
@@ -396,6 +409,15 @@ class CompletionRequest(OpenAIBaseModel):
409 "('guided_json', 'guided_regex' or 'guided_choice').")
410 return data
411
412 + @model_validator(mode="before")
413 + @classmethod
414 + def check_logprobs(cls, data):
415 + if "logprobs" in data and data[
416 + "logprobs"] is not None and not 0 <= data["logprobs"] <= 5:
417 + raise ValueError(("if passed, `logprobs` must be a value",
418 + " in the interval [0, 5]."))
419 + return data
420 +
421
422 class EmbeddingRequest(BaseModel):
423 # Ordered by official OpenAI API documentation
@@ -415,7 +437,7 @@ class EmbeddingRequest(BaseModel):
437 return PoolingParams(additional_data=self.additional_data)
438
439
418 -class LogProbs(OpenAIBaseModel):
440 +class CompletionLogProbs(OpenAIBaseModel):
441 text_offset: List[int] = Field(default_factory=list)
442 token_logprobs: List[Optional[float]] = Field(default_factory=list)
443 tokens: List[str] = Field(default_factory=list)
@@ -425,7 +447,7 @@ class LogProbs(OpenAIBaseModel):
447 class CompletionResponseChoice(OpenAIBaseModel):
448 index: int
449 text: str
428 - logprobs: Optional[LogProbs] = None
450 + logprobs: Optional[CompletionLogProbs] = None
451 finish_reason: Optional[str] = None
452 stop_reason: Optional[Union[int, str]] = Field(
453 default=None,
@@ -448,7 +470,7 @@ class CompletionResponse(OpenAIBaseModel):
470 class CompletionResponseStreamChoice(OpenAIBaseModel):
471 index: int
472 text: str
451 - logprobs: Optional[LogProbs] = None
473 + logprobs: Optional[CompletionLogProbs] = None
474 finish_reason: Optional[str] = None
475 stop_reason: Optional[Union[int, str]] = Field(
476 default=None,
@@ -488,11 +510,25 @@ class ChatMessage(OpenAIBaseModel):
510 content: str
511
512
513 +class ChatCompletionLogProb(OpenAIBaseModel):
514 + token: str
515 + logprob: float = -9999.0
516 + bytes: Optional[List[int]] = None
517 +
518 +
519 +class ChatCompletionLogProbsContent(ChatCompletionLogProb):
520 + top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)
521 +
522 +
523 +class ChatCompletionLogProbs(OpenAIBaseModel):
524 + content: Optional[List[ChatCompletionLogProbsContent]] = None
525 +
526 +
527 class ChatCompletionResponseChoice(OpenAIBaseModel):
528 index: int
529 message: ChatMessage
494 - logprobs: Optional[LogProbs] = None
495 - finish_reason: Optional[str] = None
530 + logprobs: Optional[ChatCompletionLogProbs] = None
531 + finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
532 stop_reason: Optional[Union[int, str]] = None
533
534
@@ -513,8 +549,8 @@ class DeltaMessage(OpenAIBaseModel):
549 class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
550 index: int
551 delta: DeltaMessage
516 - logprobs: Optional[LogProbs] = None
517 - finish_reason: Optional[str] = None
552 + logprobs: Optional[ChatCompletionLogProbs] = None
553 + finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
554 stop_reason: Optional[Union[int, str]] = None
555
556
vllm/entrypoints/openai/serving_chat.py
+59 -9
@@ -1,8 +1,10 @@
1 import codecs
2 import time
3 from dataclasses import dataclass
4 -from typing import (AsyncGenerator, AsyncIterator, Iterable, List, Optional,
5 - TypedDict, Union, cast, final)
4 +from typing import (AsyncGenerator, AsyncIterator, Dict, Iterable, List,
5 + Optional)
6 +from typing import Sequence as GenericSequence
7 +from typing import TypedDict, Union, cast, final
8
9 from fastapi import Request
10 from openai.types.chat import ChatCompletionContentPartTextParam
@@ -10,8 +12,9 @@ from openai.types.chat import ChatCompletionContentPartTextParam
12 from vllm.config import ModelConfig
13 from vllm.engine.async_llm_engine import AsyncLLMEngine
14 from vllm.entrypoints.openai.protocol import (
13 - ChatCompletionContentPartParam, ChatCompletionMessageParam,
14 - ChatCompletionRequest, ChatCompletionResponse,
15 + ChatCompletionContentPartParam, ChatCompletionLogProb,
16 + ChatCompletionLogProbs, ChatCompletionLogProbsContent,
17 + ChatCompletionMessageParam, ChatCompletionRequest, ChatCompletionResponse,
18 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
19 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
20 UsageInfo)
@@ -21,6 +24,7 @@ from vllm.logger import init_logger
24 from vllm.model_executor.guided_decoding import (
25 get_guided_decoding_logits_processor)
26 from vllm.outputs import RequestOutput
27 +from vllm.sequence import Logprob
28 from vllm.utils import random_uuid
29
30 logger = init_logger(__name__)
@@ -283,11 +287,10 @@ class OpenAIServingChat(OpenAIServing):
287 previous_num_tokens[i]:] if output.logprobs else None
288
289 if request.logprobs:
286 - logprobs = self._create_logprobs(
290 + logprobs = self._create_chat_logprobs(
291 token_ids=delta_token_ids,
292 top_logprobs=top_logprobs,
293 num_output_top_logprobs=request.top_logprobs,
290 - initial_text_offset=len(previous_texts[i]),
294 )
295 else:
296 logprobs = None
@@ -370,7 +373,7 @@ class OpenAIServingChat(OpenAIServing):
373 top_logprobs = output.logprobs
374
375 if request.logprobs:
373 - logprobs = self._create_logprobs(
376 + logprobs = self._create_chat_logprobs(
377 token_ids=token_ids,
378 top_logprobs=top_logprobs,
379 num_output_top_logprobs=request.top_logprobs,
@@ -383,8 +386,7 @@ class OpenAIServingChat(OpenAIServing):
386 message=ChatMessage(role=role, content=output.text),
387 logprobs=logprobs,
388 finish_reason=output.finish_reason,
386 - stop_reason=output.stop_reason,
387 - )
389 + stop_reason=output.stop_reason)
390 choices.append(choice_data)
391
392 if request.echo:
@@ -414,3 +416,51 @@ class OpenAIServingChat(OpenAIServing):
416 )
417
418 return response
419 +
420 + def _get_top_logprobs(
421 + self, logprobs: Dict[int, Logprob],
422 + top_logprobs: Optional[int]) -> List[ChatCompletionLogProb]:
423 + return [
424 + ChatCompletionLogProb(
425 + token=self._get_decoded_token(p[1], p[0]),
426 + logprob=max(p[1].logprob, -9999.0),
427 + bytes=list(
428 + self._get_decoded_token(p[1],
429 + p[0]).encode("utf-8",
430 + errors="replace")))
431 + for i, p in enumerate(logprobs.items())
432 + if top_logprobs and i < top_logprobs
433 + ]
434 +
435 + def _create_chat_logprobs(
436 + self,
437 + token_ids: GenericSequence[int],
438 + top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
439 + num_output_top_logprobs: Optional[int] = None,
440 + ) -> ChatCompletionLogProbs:
441 + """Create OpenAI-style logprobs."""
442 +
443 + logprobs_content = []
444 +
445 + for i, token_id in enumerate(token_ids):
446 + step_top_logprobs = top_logprobs[i]
447 + if step_top_logprobs is None:
448 + logprobs_content.append(
449 + ChatCompletionLogProbsContent(
450 + token=self.tokenizer.decode(token_id),
451 + bytes=list(
452 + self.tokenizer.decode(token_id).encode(
453 + "utf-8", errors="replace"))))
454 + else:
455 + logprobs_content.append(
456 + ChatCompletionLogProbsContent(
457 + token=step_top_logprobs[token_id].decoded_token,
458 + logprob=max(step_top_logprobs[token_id].logprob,
459 + -9999.0),
460 + bytes=list(
461 + step_top_logprobs[token_id].decoded_token.encode(
462 + "utf-8", errors="replace")),
463 + top_logprobs=self._get_top_logprobs(
464 + step_top_logprobs, num_output_top_logprobs)))
465 +
466 + return ChatCompletionLogProbs(content=logprobs_content)
vllm/entrypoints/openai/serving_completion.py
+68 -6
@@ -1,23 +1,29 @@
1 import time
2 from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
3 - Optional, Tuple)
3 + Optional)
4 +from typing import Sequence as GenericSequence
5 +from typing import Tuple
6
7 from fastapi import Request
8
9 from vllm.config import ModelConfig
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
9 -from vllm.entrypoints.openai.protocol import (CompletionRequest,
11 +# yapf: disable
12 +from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
13 + CompletionRequest,
14 CompletionResponse,
15 CompletionResponseChoice,
16 CompletionResponseStreamChoice,
17 CompletionStreamResponse,
14 - LogProbs, UsageInfo)
18 + UsageInfo)
19 +# yapf: enable
20 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
21 OpenAIServing)
22 from vllm.logger import init_logger
23 from vllm.model_executor.guided_decoding import (
24 get_guided_decoding_logits_processor)
25 from vllm.outputs import RequestOutput
26 +from vllm.sequence import Logprob
27 from vllm.utils import merge_async_iterators, random_uuid
28
29 logger = init_logger(__name__)
@@ -25,7 +31,7 @@ logger = init_logger(__name__)
31 TypeTokenIDs = List[int]
32 TypeTopLogProbs = List[Optional[Dict[int, float]]]
33 TypeCreateLogProbsFn = Callable[
28 - [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
34 + [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs]
35
36
37 def parse_prompt_format(prompt) -> Tuple[bool, list]:
@@ -235,7 +241,7 @@ class OpenAIServingCompletion(OpenAIServing):
241 i]:] if output.logprobs else None
242
243 if request.logprobs is not None:
238 - logprobs = self._create_logprobs(
244 + logprobs = self._create_completion_logprobs(
245 token_ids=delta_token_ids,
246 top_logprobs=top_logprobs,
247 num_output_top_logprobs=request.logprobs,
@@ -317,7 +323,7 @@ class OpenAIServingCompletion(OpenAIServing):
323 assert top_logprobs is not None, (
324 "top_logprobs must be provided when logprobs "
325 "is requested")
320 - logprobs = self._create_logprobs(
326 + logprobs = self._create_completion_logprobs(
327 token_ids=token_ids,
328 top_logprobs=top_logprobs,
329 num_output_top_logprobs=request.logprobs,
@@ -351,3 +357,59 @@ class OpenAIServingCompletion(OpenAIServing):
357 choices=choices,
358 usage=usage,
359 )
360 +
361 + def _create_completion_logprobs(
362 + self,
363 + token_ids: GenericSequence[int],
364 + top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
365 + num_output_top_logprobs: int,
366 + initial_text_offset: int = 0,
367 + ) -> CompletionLogProbs:
368 + """Create logprobs for OpenAI Completion API."""
369 + out_text_offset: List[int] = []
370 + out_token_logprobs: List[Optional[float]] = []
371 + out_tokens: List[str] = []
372 + out_top_logprobs: List[Optional[Dict[str, float]]] = []
373 +
374 + last_token_len = 0
375 +
376 + for i, token_id in enumerate(token_ids):
377 + step_top_logprobs = top_logprobs[i]
378 + if step_top_logprobs is None:
379 + token = self.tokenizer.decode(token_id)
380 + out_tokens.append(token)
381 + out_token_logprobs.append(None)
382 + out_top_logprobs.append(None)
383 + else:
384 + token = self._get_decoded_token(step_top_logprobs[token_id],
385 + token_id)
386 + token_logprob = max(step_top_logprobs[token_id].logprob,
387 + -9999.0)
388 + out_tokens.append(token)
389 + out_token_logprobs.append(token_logprob)
390 +
391 + # makes sure to add the top num_output_top_logprobs + 1
392 + # logprobs, as defined in the openai API
393 + # (cf. https://github.com/openai/openai-openapi/blob/
394 + # 893ba52242dbd5387a97b96444ee1c742cfce9bd/openapi.yaml#L7153)
395 + out_top_logprobs.append({
396 + # Convert float("-inf") to the
397 + # JSON-serializable float that OpenAI uses
398 + self._get_decoded_token(top_lp[1], top_lp[0]):
399 + max(top_lp[1].logprob, -9999.0)
400 + for i, top_lp in enumerate(step_top_logprobs.items())
401 + if num_output_top_logprobs >= i
402 + })
403 +
404 + if len(out_text_offset) == 0:
405 + out_text_offset.append(initial_text_offset)
406 + else:
407 + out_text_offset.append(out_text_offset[-1] + last_token_len)
408 + last_token_len = len(token)
409 +
410 + return CompletionLogProbs(
411 + text_offset=out_text_offset,
412 + token_logprobs=out_token_logprobs,
413 + tokens=out_tokens,
414 + top_logprobs=out_top_logprobs,
415 + )
vllm/entrypoints/openai/serving_engine.py
+6 -46
@@ -11,7 +11,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 CompletionRequest,
13 EmbeddingRequest, ErrorResponse,
14 - LogProbs, ModelCard, ModelList,
14 + ModelCard, ModelList,
15 ModelPermission)
16 from vllm.logger import init_logger
17 from vllm.lora.request import LoRARequest
@@ -75,51 +75,6 @@ class OpenAIServing:
75 model_cards.extend(lora_cards)
76 return ModelList(data=model_cards)
77
78 - def _create_logprobs(
79 - self,
80 - token_ids: List[int],
81 - top_logprobs: List[Optional[Dict[int, Logprob]]],
82 - num_output_top_logprobs: Optional[int] = None,
83 - initial_text_offset: int = 0,
84 - ) -> LogProbs:
85 - """Create OpenAI-style logprobs."""
86 - logprobs = LogProbs()
87 - last_token_len = 0
88 - if num_output_top_logprobs:
89 - logprobs.top_logprobs = []
90 -
91 - for i, token_id in enumerate(token_ids):
92 - step_top_logprobs = top_logprobs[i]
93 - if step_top_logprobs is None:
94 - token = self.tokenizer.decode(token_id)
95 - logprobs.tokens.append(token)
96 - logprobs.token_logprobs.append(None)
97 - assert logprobs.top_logprobs is not None
98 - logprobs.top_logprobs.append(None)
99 - else:
100 - token_logprob = step_top_logprobs[token_id].logprob
101 - token = step_top_logprobs[token_id].decoded_token
102 - logprobs.tokens.append(token)
103 - token_logprob = max(token_logprob, -9999.0)
104 - logprobs.token_logprobs.append(token_logprob)
105 -
106 - if num_output_top_logprobs:
107 - assert logprobs.top_logprobs is not None
108 - logprobs.top_logprobs.append({
109 - # Convert float("-inf") to the
110 - # JSON-serializable float that OpenAI uses
111 - p.decoded_token: max(p.logprob, -9999.0)
112 - for i, p in step_top_logprobs.items()
113 - } if step_top_logprobs else None)
114 -
115 - if len(logprobs.text_offset) == 0:
116 - logprobs.text_offset.append(initial_text_offset)
117 - else:
118 - logprobs.text_offset.append(logprobs.text_offset[-1] +
119 - last_token_len)
120 - last_token_len = len(token)
121 - return logprobs
122 -
78 def create_error_response(
79 self,
80 message: str,
@@ -235,3 +190,8 @@ class OpenAIServing:
190 f"Please reduce the length of the messages or completion.", )
191 else:
192 return input_ids, input_text
193 +
194 + def _get_decoded_token(self, logprob: Logprob, token_id: int) -> str:
195 + if logprob.decoded_token is not None:
196 + return logprob.decoded_token
197 + return self.tokenizer.decode(token_id)