@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e25fee57c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[BugFix] Fix server crash on empty prompt (#7746)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

Maximilien de Bayser committed Aug 23, 2024 at 10:12 UTC e25fee57c2e69161bd261f5986dc5aeb198bbd42
3 files changed +39
tests/entrypoints/llm/test_prompt_validation.py new
+9
@@ -0,0 +1,9 @@
1 +import pytest
2 +
3 +from vllm import LLM
4 +
5 +
6 +def test_empty_prompt():
7 + llm = LLM(model="gpt2")
8 + with pytest.raises(ValueError, match='Prompt cannot be empty'):
9 + llm.generate([""])
tests/entrypoints/openai/test_prompt_validation.py new
+22
@@ -0,0 +1,22 @@
1 +# imports for guided decoding tests
2 +import re
3 +
4 +import openai
5 +import pytest
6 +
7 +from ...utils import RemoteOpenAIServer
8 +
9 +
10 +@pytest.mark.asyncio
11 +async def test_empty_prompt():
12 + model_name = "gpt2"
13 + server_args = ["--enforce-eager"]
14 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
15 + client = remote_server.get_async_client()
16 +
17 + with pytest.raises(openai.BadRequestError,
18 + match=re.compile('.+Prompt cannot be empty.+')):
19 + await client.completions.create(model=model_name,
20 + prompt="",
21 + max_tokens=5,
22 + temperature=0.0)
vllm/engine/llm_engine.py
+8
@@ -591,6 +591,7 @@ class LLMEngine:
591 prompt_adapter_request: Optional[PromptAdapterRequest],
592 trace_headers: Optional[Mapping[str, str]] = None,
593 ) -> None:
594 + self._validate_model_inputs(processed_inputs)
595 # Create the sequences.
596 block_size = self.cache_config.block_size
597 seq_id = next(self.seq_counter)
@@ -1647,3 +1648,10 @@ class LLMEngine:
1648
1649 def is_embedding_model(self):
1650 return self.model_config.is_embedding_model
1651 +
1652 + def _validate_model_inputs(self, inputs: Union[LLMInputs,
1653 + EncoderDecoderLLMInputs]):
1654 + prompt_key = "encoder_prompt_token_ids" \
1655 + if self.is_encoder_decoder_model() else "prompt_token_ids"
1656 + if not inputs.get(prompt_key):
1657 + raise ValueError("Prompt cannot be empty")