A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Add validation for seed (#4529)
sasha0552 committed
May 1, 2024 at 19:31 UTC
c47ba4aaa94d067bbb0437526cae9a33c698c717
2 files changed
+26
-2
tests/entrypoints/test_openai_server.py
+20
@@ -13,6 +13,7 @@ import pytest
13
# and debugging.
14
import ray
15
import requests
16
+import torch
17
# downloading lora to test lora requests
18
from huggingface_hub import snapshot_download
19
from openai import BadRequestError
@@ -870,5 +871,24 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
871
assert len(logprobs.tokens) > 5
872
873
874
+async def test_long_seed(server, client: openai.AsyncOpenAI):
875
+ for seed in [
876
+ torch.iinfo(torch.long).min - 1,
877
+ torch.iinfo(torch.long).max + 1
878
+ ]:
879
+ with pytest.raises(BadRequestError) as exc_info:
880
+ await client.chat.completions.create(
881
+ model=MODEL_NAME,
882
+ messages=[{
883
+ "role": "system",
884
+ "content": "You are a helpful assistant.",
885
+ }],
886
+ temperature=0,
887
+ seed=seed)
888
+
889
+ assert ("greater_than_equal" in exc_info.value.message
890
+ or "less_than_equal" in exc_info.value.message)
891
+
892
+
893
if __name__ == "__main__":
894
pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+6
-2
@@ -79,7 +79,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
79
n: Optional[int] = 1
80
presence_penalty: Optional[float] = 0.0
81
response_format: Optional[ResponseFormat] = None
82
- seed: Optional[int] = None
82
+ seed: Optional[int] = Field(None,
83
+ ge=torch.iinfo(torch.long).min,
84
+ le=torch.iinfo(torch.long).max)
85
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
86
stream: Optional[bool] = False
87
temperature: Optional[float] = 0.7
@@ -228,7 +230,9 @@ class CompletionRequest(OpenAIBaseModel):
230
max_tokens: Optional[int] = 16
231
n: int = 1
232
presence_penalty: Optional[float] = 0.0
231
- seed: Optional[int] = None
233
+ seed: Optional[int] = Field(None,
234
+ ge=torch.iinfo(torch.long).min,
235
+ le=torch.iinfo(torch.long).max)
236
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
237
stream: Optional[bool] = False
238
suffix: Optional[str] = None