@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ffdf8ce0c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 22:15 UTC

[HTTP Server] Make model param optional in request (#13568)

Keyun Tong committed Feb 21, 2025 at 21:55 UTC 0ffdf8ce0ce2a59426b8ef6b7f4d96f8458b8c14

9 files changed +61 -18

tests/entrypoints/openai/test_chat.py

+32

@@ -9,6 +9,7 @@ import jsonschema

9 import openai # use the official client for correctness check

10 import pytest

11 import pytest_asyncio

12 +import requests

13 import torch

14 from openai import BadRequestError

@@ -996,3 +997,34 @@ async def test_long_seed(client: openai.AsyncOpenAI):

997

998 assert ("greater_than_equal" in exc_info.value.message

999 or "less_than_equal" in exc_info.value.message)

1000 +

1001 +

1002 +@pytest.mark.asyncio

1003 +async def test_http_chat_wo_model_name(server: RemoteOpenAIServer):

1004 + url = f"http://localhost:{server.port}/v1/chat/completions"

1005 + headers = {

1006 + "Content-Type": "application/json",

1007 + }

1008 + data = {

1009 + # model_name is avoided here.

1010 + "messages": [{

1011 + "role": "system",

1012 + "content": "You are a helpful assistant."

1013 + }, {

1014 + "role": "user",

1015 + "content": "what is 1+1?"

1016 + }],

1017 + "max_tokens":

1018 + 5

1019 + }

1020 +

1021 + response = requests.post(url, headers=headers, json=data)

1022 + response_data = response.json()

1023 + print(response_data)

1024 +

1025 + choice = response_data.get("choices")[0]

1026 + message = choice.get("message")

1027 + assert message is not None

1028 + content = message.get("content")

1029 + assert content is not None

1030 + assert len(content) > 0

vllm/entrypoints/openai/protocol.py

+10 -10

@@ -213,7 +213,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

213 # Ordered by official OpenAI API documentation

214 # https://platform.openai.com/docs/api-reference/chat/create

215 messages: List[ChatCompletionMessageParam]

216 - model: str

216 + model: Optional[str] = None

217 frequency_penalty: Optional[float] = 0.0

218 logit_bias: Optional[Dict[str, float]] = None

219 logprobs: Optional[bool] = False

@@ -642,7 +642,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

642 class CompletionRequest(OpenAIBaseModel):

643 # Ordered by official OpenAI API documentation

644 # https://platform.openai.com/docs/api-reference/completions/create

645 - model: str

645 + model: Optional[str] = None

646 prompt: Union[List[int], List[List[int]], str, List[str]]

647 best_of: Optional[int] = None

648 echo: Optional[bool] = False

@@ -907,7 +907,7 @@ class CompletionRequest(OpenAIBaseModel):

907 class EmbeddingCompletionRequest(OpenAIBaseModel):

908 # Ordered by official OpenAI API documentation

909 # https://platform.openai.com/docs/api-reference/embeddings

910 - model: str

910 + model: Optional[str] = None

911 input: Union[List[int], List[List[int]], str, List[str]]

912 encoding_format: Literal["float", "base64"] = "float"

913 dimensions: Optional[int] = None

@@ -939,7 +939,7 @@ class EmbeddingCompletionRequest(OpenAIBaseModel):

939

940

941 class EmbeddingChatRequest(OpenAIBaseModel):

942 - model: str

942 + model: Optional[str] = None

943 messages: List[ChatCompletionMessageParam]

944

945 encoding_format: Literal["float", "base64"] = "float"

@@ -1007,7 +1007,7 @@ PoolingRequest = Union[PoolingCompletionRequest, PoolingChatRequest]

1007

1008

1009 class ScoreRequest(OpenAIBaseModel):

1010 - model: str

1010 + model: Optional[str] = None

1011 text_1: Union[List[str], str]

1012 text_2: Union[List[str], str]

1013 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

@@ -1031,7 +1031,7 @@ class ScoreRequest(OpenAIBaseModel):

1031

1032

1033 class RerankRequest(OpenAIBaseModel):

1034 - model: str

1034 + model: Optional[str] = None

1035 query: str

1036 documents: List[str]

1037 top_n: int = Field(default_factory=lambda: 0)

@@ -1345,7 +1345,7 @@ class BatchRequestOutput(OpenAIBaseModel):

1345

1346

1347 class TokenizeCompletionRequest(OpenAIBaseModel):

1348 - model: str

1348 + model: Optional[str] = None

1349 prompt: str

1350

1351 add_special_tokens: bool = Field(

@@ -1357,7 +1357,7 @@ class TokenizeCompletionRequest(OpenAIBaseModel):

1357

1358

1359 class TokenizeChatRequest(OpenAIBaseModel):

1360 - model: str

1360 + model: Optional[str] = None

1361 messages: List[ChatCompletionMessageParam]

1362

1363 add_generation_prompt: bool = Field(

@@ -1423,7 +1423,7 @@ class TokenizeResponse(OpenAIBaseModel):

1423

1424

1425 class DetokenizeRequest(OpenAIBaseModel):

1426 - model: str

1426 + model: Optional[str] = None

1427 tokens: List[int]

1428

1429

@@ -1456,7 +1456,7 @@ class TranscriptionRequest(OpenAIBaseModel):

1456 formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.

1457 """

1458

1459 - model: str

1459 + model: Optional[str] = None

1460 """ID of the model to use.

1461 """

1462

vllm/entrypoints/openai/serving_chat.py

+1 -1

@@ -141,7 +141,7 @@ class OpenAIServingChat(OpenAIServing):

141 prompt_adapter_request,

142 ) = self._maybe_get_adapters(request)

143

144 - model_name = self.models.model_name(lora_request)

144 + model_name = self._get_model_name(request.model, lora_request)

145

146 tokenizer = await self.engine_client.get_tokenizer(lora_request)

147

vllm/entrypoints/openai/serving_completion.py

+1 -1

@@ -166,7 +166,7 @@ class OpenAIServingCompletion(OpenAIServing):

166

167 result_generator = merge_async_iterators(*generators)

168

169 - model_name = self.models.model_name(lora_request)

169 + model_name = self._get_model_name(request.model, lora_request)

170 num_prompts = len(engine_prompts)

171

172 # Similar to the OpenAI API, when n != best_of, we do not stream the

vllm/entrypoints/openai/serving_embedding.py

+1 -1

@@ -83,7 +83,7 @@ class OpenAIServingEmbedding(OpenAIServing):

83 return self.create_error_response(

84 "dimensions is currently not supported")

86 - model_name = request.model

86 + model_name = self._get_model_name(request.model)

87 request_id = f"embd-{self._base_request_id(raw_request)}"

88 created_time = int(time.time())

vllm/entrypoints/openai/serving_engine.py

+12 -1

@@ -523,5 +523,16 @@ class OpenAIServing:

523 return logprob.decoded_token

524 return tokenizer.decode(token_id)

525

526 - def _is_model_supported(self, model_name):

526 + def _is_model_supported(self, model_name) -> bool:

527 + if not model_name:

528 + return True

529 return self.models.is_base_model(model_name)

530 +

531 + def _get_model_name(self,

532 + model_name: Optional[str] = None,

533 + lora_request: Optional[LoRARequest] = None) -> str:

534 + if lora_request:

535 + return lora_request.lora_name

536 + if model_name is None:

537 + return self.models.base_model_paths[0].name

538 + return model_name

vllm/entrypoints/openai/serving_models.py

+1 -1

@@ -95,7 +95,7 @@ class OpenAIServingModels:

95 if isinstance(load_result, ErrorResponse):

96 raise ValueError(load_result.message)

98 - def is_base_model(self, model_name):

98 + def is_base_model(self, model_name) -> bool:

99 return any(model.name == model_name for model in self.base_model_paths)

100

101 def model_name(self, lora_request: Optional[LoRARequest] = None) -> str:

vllm/entrypoints/openai/serving_pooling.py

+1 -1

@@ -79,7 +79,7 @@ class OpenAIServingPooling(OpenAIServing):

79 return self.create_error_response(

80 "dimensions is currently not supported")

82 - model_name = request.model

82 + model_name = self._get_model_name(request.model)

83 request_id = f"pool-{self._base_request_id(raw_request)}"

84 created_time = int(time.time())

vllm/entrypoints/openai/serving_score.py

+2 -2

@@ -318,7 +318,7 @@ class ServingScores(OpenAIServing):

318 final_res_batch,

319 request_id,

320 created_time,

321 - request.model,

321 + self._get_model_name(request.model),

322 )

323 except asyncio.CancelledError:

324 return self.create_error_response("Client disconnected")

@@ -358,7 +358,7 @@ class ServingScores(OpenAIServing):

358 request.truncate_prompt_tokens,

359 )

360 return self.request_output_to_rerank_response(

361 - final_res_batch, request_id, request.model, documents, top_n)

361 + final_res_batch, request_id, self._get_model_name(request.model), documents, top_n)

362 except asyncio.CancelledError:

363 return self.create_error_response("Client disconnected")

364 except ValueError as e: