A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] Fix the issue where the model name is empty string, causing no response with the model name. (#15938)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Apr 2, 2025 at 21:33 UTC
594a8b9030a64b9807f987762c950ee1595f3849
2 files changed
+29
-4
tests/entrypoints/openai/test_chat.py
+28
-3
@@ -11,7 +11,7 @@ import pytest
11
import pytest_asyncio
12
import requests
13
import torch
14
-from openai import BadRequestError
14
+from openai import BadRequestError, OpenAI
15
16
from ...utils import RemoteOpenAIServer
17
from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
@@ -1054,7 +1054,7 @@ async def test_long_seed(client: openai.AsyncOpenAI):
1054
1055
1056
@pytest.mark.asyncio
1057
-async def test_http_chat_wo_model_name(server: RemoteOpenAIServer):
1057
+async def test_http_chat_no_model_name_with_curl(server: RemoteOpenAIServer):
1058
url = f"http://localhost:{server.port}/v1/chat/completions"
1059
headers = {
1060
"Content-Type": "application/json",
@@ -1075,10 +1075,35 @@ async def test_http_chat_wo_model_name(server: RemoteOpenAIServer):
1075
response = requests.post(url, headers=headers, json=data)
1076
response_data = response.json()
1077
print(response_data)
1078
-
1078
+ assert response_data.get("model") == MODEL_NAME
1079
choice = response_data.get("choices")[0]
1080
message = choice.get("message")
1081
assert message is not None
1082
content = message.get("content")
1083
assert content is not None
1084
assert len(content) > 0
1085
+
1086
+
1087
+@pytest.mark.asyncio
1088
+@pytest.mark.parametrize("model_name", [MODEL_NAME, ""])
1089
+async def test_http_chat_no_model_name_with_openai(server: RemoteOpenAIServer,
1090
+ model_name: str):
1091
+
1092
+ openai_api_key = "EMPTY"
1093
+ openai_api_base = f"http://localhost:{server.port}/v1"
1094
+
1095
+ client = OpenAI(
1096
+ api_key=openai_api_key,
1097
+ base_url=openai_api_base,
1098
+ )
1099
+ messages = [
1100
+ {
1101
+ "role": "user",
1102
+ "content": "Hello, vLLM!"
1103
+ },
1104
+ ]
1105
+ response = client.chat.completions.create(
1106
+ model="", # empty string
1107
+ messages=messages,
1108
+ )
1109
+ assert response.model == MODEL_NAME
vllm/entrypoints/openai/serving_engine.py
+1
-1
@@ -537,7 +537,7 @@ class OpenAIServing:
537
lora_request: Optional[LoRARequest] = None) -> str:
538
if lora_request:
539
return lora_request.lora_name
540
- if model_name is None:
540
+ if not model_name:
541
return self.models.base_model_paths[0].name
542
return model_name
543