A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[benchmark] use model card root instead of id (#31329)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Dec 26, 2025 at 10:55 UTC
3b8f31b362e5f94e5ffd620e5a0fa29c041171eb
1 file changed
+6
-6
vllm/benchmarks/serve.py
+6
-6
@@ -61,7 +61,7 @@ TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) a
61
62
async def get_first_model_from_server(
63
base_url: str, headers: dict | None = None
64
-) -> str:
64
+) -> tuple[str, str]:
65
"""Fetch the first model from the server's /v1/models endpoint."""
66
models_url = f"{base_url}/v1/models"
67
async with aiohttp.ClientSession() as session:
@@ -70,7 +70,7 @@ async def get_first_model_from_server(
70
response.raise_for_status()
71
data = await response.json()
72
if "data" in data and len(data["data"]) > 0:
73
- return data["data"][0]["id"]
73
+ return data["data"][0]["id"], data["data"][0]["root"]
74
else:
75
raise ValueError(
76
f"No models found on the server at {base_url}. "
@@ -1157,7 +1157,7 @@ def add_cli_args(parser: argparse.ArgumentParser):
1157
"--save-detailed",
1158
action="store_true",
1159
help="When saving the results, whether to include per request "
1160
- "information such as response, error, ttfs, tpots, etc.",
1160
+ "information such as response, error, ttfts, tpots, etc.",
1161
)
1162
parser.add_argument(
1163
"--append-result",
@@ -1396,12 +1396,12 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1396
# Fetch model from server if not specified
1397
if args.model is None:
1398
print("Model not specified, fetching first model from server...")
1399
- model_id = await get_first_model_from_server(base_url, headers)
1400
- print(f"Using model: {model_id}")
1399
+ model_name, model_id = await get_first_model_from_server(base_url, headers)
1400
+ print(f"First model name: {model_name}, first model id: {model_id}")
1401
else:
1402
+ model_name = args.served_model_name
1403
model_id = args.model
1404
1404
- model_name = args.served_model_name
1405
tokenizer_id = args.tokenizer if args.tokenizer is not None else model_id
1406
tokenizer_mode = args.tokenizer_mode
1407