A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[BugFix][Frontend] Fix completion logprobs=0 error (#3731)
Roy committed
Mar 30, 2024 at 00:38 UTC
f510395bbf4177dad55f53b85a671ba4e4aba60d
5 files changed
+26
-7
tests/entrypoints/test_openai_server.py
+21
@@ -199,6 +199,27 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
199
completion.choices[0].text) >= 5
200
201
202
+@pytest.mark.parametrize(
203
+ # first test base model, then test loras
204
+ "model_name",
205
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
206
+)
207
+async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
208
+ model_name: str):
209
+ # test using token IDs
210
+ completion = await client.completions.create(
211
+ model=MODEL_NAME,
212
+ prompt=[0, 0, 0, 0, 0],
213
+ max_tokens=5,
214
+ temperature=0.0,
215
+ logprobs=0,
216
+ )
217
+ choice = completion.choices[0]
218
+ assert choice.logprobs is not None
219
+ assert choice.logprobs.token_logprobs is not None
220
+ assert choice.logprobs.top_logprobs is None
221
+
222
+
223
@pytest.mark.parametrize(
224
# just test 1 lora hereafter
225
"model_name",
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -330,7 +330,7 @@ class LogProbs(BaseModel):
330
text_offset: List[int] = Field(default_factory=list)
331
token_logprobs: List[Optional[float]] = Field(default_factory=list)
332
tokens: List[str] = Field(default_factory=list)
333
- top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None
333
+ top_logprobs: Optional[List[Optional[Dict[str, float]]]] = None
334
335
336
class CompletionResponseChoice(BaseModel):
vllm/entrypoints/openai/serving_completion.py
-3
@@ -251,9 +251,6 @@ class OpenAIServingCompletion(OpenAIServing):
251
i]:] if output.logprobs else None
252
253
if request.logprobs is not None:
254
- assert top_logprobs is not None, (
255
- "top_logprobs must be provided when logprobs "
256
- "is requested")
254
logprobs = self._create_logprobs(
255
token_ids=delta_token_ids,
256
top_logprobs=top_logprobs,
vllm/model_executor/layers/sampler.py
+3
-2
@@ -534,7 +534,8 @@ def _get_logprobs(
534
# Prepare query indices
535
batched_logprobs_query_seq_indices: List[int] = []
536
batched_logprobs_query_token_indices: List[int] = []
537
- largest_num_logprobs = 0
537
+ # at least get one logprob for each token
538
+ largest_num_logprobs = 1
539
sample_idx = 0
540
for i, (seq_group, sample_result) in enumerate(
541
zip(sampling_metadata.seq_groups, sample_results)):
@@ -643,7 +644,7 @@ def _get_logprobs(
644
batched_ranks_query_result[query_result_idx].item())
645
}
646
query_result_idx += 1
646
- if num_logprobs > 0:
647
+ if num_logprobs >= 0:
648
sample_logprobs_dict.update(
649
zip(
650
top_token_ids[sample_idx +
vllm/outputs.py
+1
-1
@@ -111,7 +111,7 @@ class RequestOutput:
111
# NOTE: We need omit logprobs here explicitly because the sequence
112
# always has the logprobs of the sampled tokens even if the
113
# logprobs are not requested.
114
- include_logprobs = seq_group.sampling_params.logprobs
114
+ include_logprobs = seq_group.sampling_params.logprobs is not None
115
outputs = [
116
CompletionOutput(seqs.index(seq), seq.output_text,
117
seq.get_output_token_ids(),