A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Support `prompt_logprobs==0` (#5217)
Toshiki Kataoka committed
Jun 4, 2024 at 09:59 UTC
06b2550cbb56f4a538f43c56addf20f4d2d19cad
4 files changed
+11
-7
tests/entrypoints/test_openai_server.py
+8
-4
@@ -224,7 +224,7 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
224
assert choice.logprobs is not None
225
assert choice.logprobs.token_logprobs is not None
226
assert choice.logprobs.top_logprobs is not None
227
- assert len(choice.logprobs.top_logprobs[0]) <= 1
227
+ assert len(choice.logprobs.top_logprobs[0]) == 1
228
229
230
@pytest.mark.asyncio
@@ -246,7 +246,7 @@ async def test_some_logprobs(server, client: openai.AsyncOpenAI,
246
assert choice.logprobs is not None
247
assert choice.logprobs.token_logprobs is not None
248
assert choice.logprobs.top_logprobs is not None
249
- assert len(choice.logprobs.top_logprobs[0]) <= 6
249
+ assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
250
251
252
@pytest.mark.asyncio
@@ -1217,8 +1217,9 @@ number: "1" | "2"
1217
"model_name",
1218
[MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
1219
)
1220
+@pytest.mark.parametrize("logprobs_arg", [1, 0])
1221
async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1221
- model_name: str):
1222
+ model_name: str, logprobs_arg: int):
1223
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
1224
# test using text and token IDs
1225
for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
@@ -1227,7 +1228,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1228
max_tokens=5,
1229
temperature=0.0,
1230
echo=True,
1230
- logprobs=1)
1231
+ logprobs=logprobs_arg)
1232
1233
prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
1234
list) else prompt
@@ -1240,6 +1241,9 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1241
and logprobs.token_logprobs[0] is None)
1242
assert (len(logprobs.top_logprobs) > 5
1243
and logprobs.top_logprobs[0] is None)
1244
+ for top_logprobs in logprobs.top_logprobs[1:]:
1245
+ assert max(logprobs_arg,
1246
+ 1) <= len(top_logprobs) <= logprobs_arg + 1
1247
assert len(logprobs.tokens) > 5
1248
1249
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -312,7 +312,7 @@ class OpenAIServingCompletion(OpenAIServing):
312
elif request.echo and request.max_tokens > 0:
313
token_ids = prompt_token_ids + output.token_ids
314
top_logprobs = (prompt_logprobs + output.logprobs
315
- if request.logprobs else None)
315
+ if request.logprobs is not None else None)
316
output_text = prompt_text + output.text
317
else:
318
token_ids = output.token_ids
vllm/model_executor/sampling_metadata.py
+1
-1
@@ -233,7 +233,7 @@ def _prepare_seq_groups(
233
logits = hidden_states[selected_token_indices]
234
"""
235
236
- if sampling_params.prompt_logprobs:
236
+ if sampling_params.prompt_logprobs is not None:
237
selected_token_indices.extend(
238
range(model_output_idx, model_output_idx + prompt_logprob_len))
239
model_output_idx += prompt_logprob_len
vllm/worker/model_runner.py
+1
-1
@@ -427,7 +427,7 @@ class ModelRunner:
427
[lora_id] *
428
(query_len if seq_group_metadata.sampling_params
429
and seq_group_metadata.sampling_params.prompt_logprobs
430
- else 1))
430
+ is not None else 1))
431
432
mm_data = seq_group_metadata.multi_modal_data
433
if mm_data is not None: