@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f510395bbf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[BugFix][Frontend] Fix completion logprobs=0 error (#3731)

Roy committed Mar 30, 2024 at 00:38 UTC f510395bbf4177dad55f53b85a671ba4e4aba60d
5 files changed +26 -7
tests/entrypoints/test_openai_server.py
+21
@@ -199,6 +199,27 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
199 completion.choices[0].text) >= 5
200
201
202 +@pytest.mark.parametrize(
203 + # first test base model, then test loras
204 + "model_name",
205 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
206 +)
207 +async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
208 + model_name: str):
209 + # test using token IDs
210 + completion = await client.completions.create(
211 + model=MODEL_NAME,
212 + prompt=[0, 0, 0, 0, 0],
213 + max_tokens=5,
214 + temperature=0.0,
215 + logprobs=0,
216 + )
217 + choice = completion.choices[0]
218 + assert choice.logprobs is not None
219 + assert choice.logprobs.token_logprobs is not None
220 + assert choice.logprobs.top_logprobs is None
221 +
222 +
223 @pytest.mark.parametrize(
224 # just test 1 lora hereafter
225 "model_name",
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -330,7 +330,7 @@ class LogProbs(BaseModel):
330 text_offset: List[int] = Field(default_factory=list)
331 token_logprobs: List[Optional[float]] = Field(default_factory=list)
332 tokens: List[str] = Field(default_factory=list)
333 - top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None
333 + top_logprobs: Optional[List[Optional[Dict[str, float]]]] = None
334
335
336 class CompletionResponseChoice(BaseModel):
vllm/entrypoints/openai/serving_completion.py
-3
@@ -251,9 +251,6 @@ class OpenAIServingCompletion(OpenAIServing):
251 i]:] if output.logprobs else None
252
253 if request.logprobs is not None:
254 - assert top_logprobs is not None, (
255 - "top_logprobs must be provided when logprobs "
256 - "is requested")
254 logprobs = self._create_logprobs(
255 token_ids=delta_token_ids,
256 top_logprobs=top_logprobs,
vllm/model_executor/layers/sampler.py
+3 -2
@@ -534,7 +534,8 @@ def _get_logprobs(
534 # Prepare query indices
535 batched_logprobs_query_seq_indices: List[int] = []
536 batched_logprobs_query_token_indices: List[int] = []
537 - largest_num_logprobs = 0
537 + # at least get one logprob for each token
538 + largest_num_logprobs = 1
539 sample_idx = 0
540 for i, (seq_group, sample_result) in enumerate(
541 zip(sampling_metadata.seq_groups, sample_results)):
@@ -643,7 +644,7 @@ def _get_logprobs(
644 batched_ranks_query_result[query_result_idx].item())
645 }
646 query_result_idx += 1
646 - if num_logprobs > 0:
647 + if num_logprobs >= 0:
648 sample_logprobs_dict.update(
649 zip(
650 top_token_ids[sample_idx +
vllm/outputs.py
+1 -1
@@ -111,7 +111,7 @@ class RequestOutput:
111 # NOTE: We need omit logprobs here explicitly because the sequence
112 # always has the logprobs of the sampled tokens even if the
113 # logprobs are not requested.
114 - include_logprobs = seq_group.sampling_params.logprobs
114 + include_logprobs = seq_group.sampling_params.logprobs is not None
115 outputs = [
116 CompletionOutput(seqs.index(seq), seq.output_text,
117 seq.get_output_token_ids(),