A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend][OpenAI] Support for returning max_model_len on /v1/models response (#4643)
Avinash Raj committed
Jun 2, 2024 at 13:36 UTC
f790ad3c50f050778af1fd31170746b7c68ca2fc
2 files changed
+2
vllm/entrypoints/openai/protocol.py
+1
@@ -82,6 +82,7 @@ class ModelCard(OpenAIBaseModel):
82
owned_by: str = "vllm"
83
root: Optional[str] = None
84
parent: Optional[str] = None
85
+ max_model_len: Optional[int] = None
86
permission: List[ModelPermission] = Field(default_factory=list)
87
88
vllm/entrypoints/openai/serving_engine.py
+1
@@ -62,6 +62,7 @@ class OpenAIServing:
62
"""Show available models. Right now we only have one model."""
63
model_cards = [
64
ModelCard(id=served_model_name,
65
+ max_model_len=self.max_model_len,
66
root=self.served_model_names[0],
67
permission=[ModelPermission()])
68
for served_model_name in self.served_model_names