@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c47ba4aaa9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Add validation for seed (#4529)

sasha0552 committed May 1, 2024 at 19:31 UTC c47ba4aaa94d067bbb0437526cae9a33c698c717
2 files changed +26 -2
tests/entrypoints/test_openai_server.py
+20
@@ -13,6 +13,7 @@ import pytest
13 # and debugging.
14 import ray
15 import requests
16 +import torch
17 # downloading lora to test lora requests
18 from huggingface_hub import snapshot_download
19 from openai import BadRequestError
@@ -870,5 +871,24 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
871 assert len(logprobs.tokens) > 5
872
873
874 +async def test_long_seed(server, client: openai.AsyncOpenAI):
875 + for seed in [
876 + torch.iinfo(torch.long).min - 1,
877 + torch.iinfo(torch.long).max + 1
878 + ]:
879 + with pytest.raises(BadRequestError) as exc_info:
880 + await client.chat.completions.create(
881 + model=MODEL_NAME,
882 + messages=[{
883 + "role": "system",
884 + "content": "You are a helpful assistant.",
885 + }],
886 + temperature=0,
887 + seed=seed)
888 +
889 + assert ("greater_than_equal" in exc_info.value.message
890 + or "less_than_equal" in exc_info.value.message)
891 +
892 +
893 if __name__ == "__main__":
894 pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+6 -2
@@ -79,7 +79,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
79 n: Optional[int] = 1
80 presence_penalty: Optional[float] = 0.0
81 response_format: Optional[ResponseFormat] = None
82 - seed: Optional[int] = None
82 + seed: Optional[int] = Field(None,
83 + ge=torch.iinfo(torch.long).min,
84 + le=torch.iinfo(torch.long).max)
85 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
86 stream: Optional[bool] = False
87 temperature: Optional[float] = 0.7
@@ -228,7 +230,9 @@ class CompletionRequest(OpenAIBaseModel):
230 max_tokens: Optional[int] = 16
231 n: int = 1
232 presence_penalty: Optional[float] = 0.0
231 - seed: Optional[int] = None
233 + seed: Optional[int] = Field(None,
234 + ge=torch.iinfo(torch.long).min,
235 + le=torch.iinfo(torch.long).max)
236 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
237 stream: Optional[bool] = False
238 suffix: Optional[str] = None