A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix model name included in responses (#24663)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Sep 11, 2025 at 18:47 UTC
c1eda615ba3aee7dd815adee74e058c2cc8c3ebb
10 files changed
+50
-74
tests/entrypoints/openai/test_chat.py
+1
-54
@@ -12,7 +12,7 @@ import pytest_asyncio
12
import regex as re
13
import requests
14
import torch
15
-from openai import BadRequestError, OpenAI
15
+from openai import BadRequestError
16
17
from ...utils import RemoteOpenAIServer
18
@@ -968,59 +968,6 @@ async def test_long_seed(client: openai.AsyncOpenAI):
968
or "less_than_equal" in exc_info.value.message)
969
970
971
-@pytest.mark.asyncio
972
-async def test_http_chat_no_model_name_with_curl(server: RemoteOpenAIServer):
973
- url = f"http://localhost:{server.port}/v1/chat/completions"
974
- headers = {
975
- "Content-Type": "application/json",
976
- }
977
- data = {
978
- # model_name is avoided here.
979
- "messages": [{
980
- "role": "system",
981
- "content": "You are a helpful assistant."
982
- }, {
983
- "role": "user",
984
- "content": "what is 1+1?"
985
- }],
986
- "max_tokens":
987
- 5
988
- }
989
-
990
- response = requests.post(url, headers=headers, json=data)
991
- response_data = response.json()
992
- print(response_data)
993
- assert response_data.get("model") == MODEL_NAME
994
- choice = response_data.get("choices")[0]
995
- message = choice.get("message")
996
- assert message is not None
997
- content = message.get("content")
998
- assert content is not None
999
- assert len(content) > 0
1000
-
1001
-
1002
-@pytest.mark.asyncio
1003
-async def test_http_chat_no_model_name_with_openai(server: RemoteOpenAIServer):
1004
- openai_api_key = "EMPTY"
1005
- openai_api_base = f"http://localhost:{server.port}/v1"
1006
-
1007
- client = OpenAI(
1008
- api_key=openai_api_key,
1009
- base_url=openai_api_base,
1010
- )
1011
- messages = [
1012
- {
1013
- "role": "user",
1014
- "content": "Hello, vLLM!"
1015
- },
1016
- ]
1017
- response = client.chat.completions.create(
1018
- model="", # empty string
1019
- messages=messages,
1020
- )
1021
- assert response.model == MODEL_NAME
1022
-
1023
-
971
@pytest.mark.asyncio
972
async def test_invocations(server: RemoteOpenAIServer,
973
client: openai.AsyncOpenAI):
tests/entrypoints/openai/test_serving_chat.py
+41
-1
@@ -213,8 +213,12 @@ async def test_gpt_oss_multi_turn_chat(gptoss_client: OpenAI,
213
214
215
MODEL_NAME = "openai-community/gpt2"
216
+MODEL_NAME_SHORT = "gpt2"
217
CHAT_TEMPLATE = "Dummy chat template for testing {}"
217
-BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
218
+BASE_MODEL_PATHS = [
219
+ BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
220
+ BaseModelPath(name=MODEL_NAME_SHORT, model_path=MODEL_NAME_SHORT)
221
+]
222
223
224
@dataclass
@@ -270,6 +274,42 @@ def test_async_serving_chat_init():
274
assert serving_completion.chat_template == CHAT_TEMPLATE
275
276
277
+@pytest.mark.asyncio
278
+async def test_serving_chat_returns_correct_model_name():
279
+ mock_engine = MagicMock(spec=MQLLMEngineClient)
280
+ mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
281
+ mock_engine.errored = False
282
+
283
+ models = OpenAIServingModels(engine_client=mock_engine,
284
+ base_model_paths=BASE_MODEL_PATHS,
285
+ model_config=MockModelConfig())
286
+ serving_chat = OpenAIServingChat(mock_engine,
287
+ MockModelConfig(),
288
+ models,
289
+ response_role="assistant",
290
+ chat_template=CHAT_TEMPLATE,
291
+ chat_template_content_format="auto",
292
+ request_logger=None)
293
+ messages = [{"role": "user", "content": "what is 1+1?"}]
294
+
295
+ async def return_model_name(*args):
296
+ return args[3]
297
+
298
+ serving_chat.chat_completion_full_generator = return_model_name
299
+
300
+ # Test that full name is returned when short name is requested
301
+ req = ChatCompletionRequest(model=MODEL_NAME_SHORT, messages=messages)
302
+ assert await serving_chat.create_chat_completion(req) == MODEL_NAME
303
+
304
+ # Test that full name is returned when empty string is specified
305
+ req = ChatCompletionRequest(model="", messages=messages)
306
+ assert await serving_chat.create_chat_completion(req) == MODEL_NAME
307
+
308
+ # Test that full name is returned when no model is specified
309
+ req = ChatCompletionRequest(messages=messages)
310
+ assert await serving_chat.create_chat_completion(req) == MODEL_NAME
311
+
312
+
313
@pytest.mark.asyncio
314
async def test_serving_chat_should_set_correct_max_tokens():
315
mock_engine = MagicMock(spec=MQLLMEngineClient)
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -186,7 +186,7 @@ class OpenAIServingChat(OpenAIServing):
186
lora_request = self._maybe_get_adapters(
187
request, supports_default_mm_loras=True)
188
189
- model_name = self._get_model_name(request.model, lora_request)
189
+ model_name = self.models.model_name(lora_request)
190
191
tokenizer = await self.engine_client.get_tokenizer(lora_request)
192
vllm/entrypoints/openai/serving_classification.py
+1
-1
@@ -146,7 +146,7 @@ class ServingClassification(ClassificationMixin):
146
request: ClassificationRequest,
147
raw_request: Request,
148
) -> Union[ClassificationResponse, ErrorResponse]:
149
- model_name = self._get_model_name(request.model)
149
+ model_name = self.models.model_name()
150
request_id = (f"{self.request_id_prefix}-"
151
f"{self._base_request_id(raw_request)}")
152
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -232,7 +232,7 @@ class OpenAIServingCompletion(OpenAIServing):
232
233
result_generator = merge_async_iterators(*generators)
234
235
- model_name = self._get_model_name(request.model, lora_request)
235
+ model_name = self.models.model_name(lora_request)
236
num_prompts = len(engine_prompts)
237
238
# Similar to the OpenAI API, when n != best_of, we do not stream the
vllm/entrypoints/openai/serving_embedding.py
+1
-1
@@ -599,7 +599,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
599
See https://platform.openai.com/docs/api-reference/embeddings/create
600
for the API specification. This API mimics the OpenAI Embedding API.
601
"""
602
- model_name = self._get_model_name(request.model)
602
+ model_name = self.models.model_name()
603
request_id = (
604
f"{self.request_id_prefix}-"
605
f"{self._base_request_id(raw_request, request.request_id)}")
vllm/entrypoints/openai/serving_engine.py
-11
@@ -980,17 +980,6 @@ class OpenAIServing:
980
return True
981
return self.models.is_base_model(model_name)
982
983
- def _get_model_name(
984
- self,
985
- model_name: Optional[str] = None,
986
- lora_request: Optional[LoRARequest] = None,
987
- ) -> str:
988
- if lora_request:
989
- return lora_request.lora_name
990
- if not model_name:
991
- return self.models.base_model_paths[0].name
992
- return model_name
993
-
983
984
def clamp_prompt_logprobs(
985
prompt_logprobs: Union[PromptLogprobs,
vllm/entrypoints/openai/serving_pooling.py
+1
-1
@@ -91,7 +91,7 @@ class OpenAIServingPooling(OpenAIServing):
91
if error_check_ret is not None:
92
return error_check_ret
93
94
- model_name = self._get_model_name(request.model)
94
+ model_name = self.models.model_name()
95
96
request_id = f"pool-{self._base_request_id(raw_request)}"
97
created_time = int(time.time())
vllm/entrypoints/openai/serving_responses.py
+1
-1
@@ -237,7 +237,7 @@ class OpenAIServingResponses(OpenAIServing):
237
238
try:
239
lora_request = self._maybe_get_adapters(request)
240
- model_name = self._get_model_name(request.model, lora_request)
240
+ model_name = self.models.model_name(lora_request)
241
tokenizer = await self.engine_client.get_tokenizer(lora_request)
242
243
if self.use_harmony:
vllm/entrypoints/openai/serving_score.py
+2
-2
@@ -353,7 +353,7 @@ class ServingScores(OpenAIServing):
353
final_res_batch,
354
request_id,
355
created_time,
356
- self._get_model_name(request.model),
356
+ self.models.model_name(),
357
)
358
except asyncio.CancelledError:
359
return self.create_error_response("Client disconnected")
@@ -399,7 +399,7 @@ class ServingScores(OpenAIServing):
399
return self.request_output_to_rerank_response(
400
final_res_batch,
401
request_id,
402
- self._get_model_name(request.model),
402
+ self.models.model_name(),
403
documents,
404
top_n,
405
)