A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[BugFix] Fix server crash on empty prompt (#7746)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Aug 23, 2024 at 10:12 UTC
e25fee57c2e69161bd261f5986dc5aeb198bbd42
3 files changed
+39
tests/entrypoints/llm/test_prompt_validation.py
new
+9
@@ -0,0 +1,9 @@
1
+import pytest
2
+
3
+from vllm import LLM
4
+
5
+
6
+def test_empty_prompt():
7
+ llm = LLM(model="gpt2")
8
+ with pytest.raises(ValueError, match='Prompt cannot be empty'):
9
+ llm.generate([""])
tests/entrypoints/openai/test_prompt_validation.py
new
+22
@@ -0,0 +1,22 @@
1
+# imports for guided decoding tests
2
+import re
3
+
4
+import openai
5
+import pytest
6
+
7
+from ...utils import RemoteOpenAIServer
8
+
9
+
10
+@pytest.mark.asyncio
11
+async def test_empty_prompt():
12
+ model_name = "gpt2"
13
+ server_args = ["--enforce-eager"]
14
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
15
+ client = remote_server.get_async_client()
16
+
17
+ with pytest.raises(openai.BadRequestError,
18
+ match=re.compile('.+Prompt cannot be empty.+')):
19
+ await client.completions.create(model=model_name,
20
+ prompt="",
21
+ max_tokens=5,
22
+ temperature=0.0)
vllm/engine/llm_engine.py
+8
@@ -591,6 +591,7 @@ class LLMEngine:
591
prompt_adapter_request: Optional[PromptAdapterRequest],
592
trace_headers: Optional[Mapping[str, str]] = None,
593
) -> None:
594
+ self._validate_model_inputs(processed_inputs)
595
# Create the sequences.
596
block_size = self.cache_config.block_size
597
seq_id = next(self.seq_counter)
@@ -1647,3 +1648,10 @@ class LLMEngine:
1648
1649
def is_embedding_model(self):
1650
return self.model_config.is_embedding_model
1651
+
1652
+ def _validate_model_inputs(self, inputs: Union[LLMInputs,
1653
+ EncoderDecoderLLMInputs]):
1654
+ prompt_key = "encoder_prompt_token_ids" \
1655
+ if self.is_encoder_decoder_model() else "prompt_token_ids"
1656
+ if not inputs.get(prompt_key):
1657
+ raise ValueError("Prompt cannot be empty")