@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06b2550cbb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Support `prompt_logprobs==0` (#5217)

Toshiki Kataoka committed Jun 4, 2024 at 09:59 UTC 06b2550cbb56f4a538f43c56addf20f4d2d19cad
4 files changed +11 -7
tests/entrypoints/test_openai_server.py
+8 -4
@@ -224,7 +224,7 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
224 assert choice.logprobs is not None
225 assert choice.logprobs.token_logprobs is not None
226 assert choice.logprobs.top_logprobs is not None
227 - assert len(choice.logprobs.top_logprobs[0]) <= 1
227 + assert len(choice.logprobs.top_logprobs[0]) == 1
228
229
230 @pytest.mark.asyncio
@@ -246,7 +246,7 @@ async def test_some_logprobs(server, client: openai.AsyncOpenAI,
246 assert choice.logprobs is not None
247 assert choice.logprobs.token_logprobs is not None
248 assert choice.logprobs.top_logprobs is not None
249 - assert len(choice.logprobs.top_logprobs[0]) <= 6
249 + assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
250
251
252 @pytest.mark.asyncio
@@ -1217,8 +1217,9 @@ number: "1" | "2"
1217 "model_name",
1218 [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
1219 )
1220 +@pytest.mark.parametrize("logprobs_arg", [1, 0])
1221 async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1221 - model_name: str):
1222 + model_name: str, logprobs_arg: int):
1223 tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
1224 # test using text and token IDs
1225 for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
@@ -1227,7 +1228,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1228 max_tokens=5,
1229 temperature=0.0,
1230 echo=True,
1230 - logprobs=1)
1231 + logprobs=logprobs_arg)
1232
1233 prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
1234 list) else prompt
@@ -1240,6 +1241,9 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1241 and logprobs.token_logprobs[0] is None)
1242 assert (len(logprobs.top_logprobs) > 5
1243 and logprobs.top_logprobs[0] is None)
1244 + for top_logprobs in logprobs.top_logprobs[1:]:
1245 + assert max(logprobs_arg,
1246 + 1) <= len(top_logprobs) <= logprobs_arg + 1
1247 assert len(logprobs.tokens) > 5
1248
1249
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -312,7 +312,7 @@ class OpenAIServingCompletion(OpenAIServing):
312 elif request.echo and request.max_tokens > 0:
313 token_ids = prompt_token_ids + output.token_ids
314 top_logprobs = (prompt_logprobs + output.logprobs
315 - if request.logprobs else None)
315 + if request.logprobs is not None else None)
316 output_text = prompt_text + output.text
317 else:
318 token_ids = output.token_ids
vllm/model_executor/sampling_metadata.py
+1 -1
@@ -233,7 +233,7 @@ def _prepare_seq_groups(
233 logits = hidden_states[selected_token_indices]
234 """
235
236 - if sampling_params.prompt_logprobs:
236 + if sampling_params.prompt_logprobs is not None:
237 selected_token_indices.extend(
238 range(model_output_idx, model_output_idx + prompt_logprob_len))
239 model_output_idx += prompt_logprob_len
vllm/worker/model_runner.py
+1 -1
@@ -427,7 +427,7 @@ class ModelRunner:
427 [lora_id] *
428 (query_len if seq_group_metadata.sampling_params
429 and seq_group_metadata.sampling_params.prompt_logprobs
430 - else 1))
430 + is not None else 1))
431
432 mm_data = seq_group_metadata.multi_modal_data
433 if mm_data is not None: