@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f790ad3c50
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Frontend][OpenAI] Support for returning max_model_len on /v1/models response (#4643)

Avinash Raj committed Jun 2, 2024 at 13:36 UTC f790ad3c50f050778af1fd31170746b7c68ca2fc
2 files changed +2
vllm/entrypoints/openai/protocol.py
+1
@@ -82,6 +82,7 @@ class ModelCard(OpenAIBaseModel):
82 owned_by: str = "vllm"
83 root: Optional[str] = None
84 parent: Optional[str] = None
85 + max_model_len: Optional[int] = None
86 permission: List[ModelPermission] = Field(default_factory=list)
87
88
vllm/entrypoints/openai/serving_engine.py
+1
@@ -62,6 +62,7 @@ class OpenAIServing:
62 """Show available models. Right now we only have one model."""
63 model_cards = [
64 ModelCard(id=served_model_name,
65 + max_model_len=self.max_model_len,
66 root=self.served_model_names[0],
67 permission=[ModelPermission()])
68 for served_model_name in self.served_model_names