A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Fix] Benchmark `"EngineClient" has no attribute "model_config"` (#17976)
Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca>
Brayden Zhong committed
May 12, 2025 at 01:55 UTC
891b9d33de7ee7b3ee95b9bd7bb8a9cffae0e08c
2 files changed
+7
-5
benchmarks/benchmark_throughput.py
+4
-3
@@ -146,9 +146,10 @@ async def run_vllm_async(
146
147
async with build_async_engine_client_from_engine_args(
148
engine_args, disable_frontend_multiprocessing) as llm:
149
+ model_config = await llm.get_model_config()
150
assert all(
150
- llm.model_config.max_model_len >= (request.prompt_len +
151
- request.expected_output_len)
151
+ model_config.max_model_len >= (request.prompt_len +
152
+ request.expected_output_len)
153
for request in requests), (
154
"Please ensure that max_model_len is greater than the sum of"
155
" prompt_len and expected_output_len for all requests.")
@@ -599,7 +600,7 @@ if __name__ == "__main__":
600
"--lora-path",
601
type=str,
602
default=None,
602
- help="Path to the lora adapters to use. This can be an absolute path, "
603
+ help="Path to the LoRA adapters to use. This can be an absolute path, "
604
"a relative path, or a Hugging Face model identifier.")
605
parser.add_argument(
606
"--prefix-len",
vllm/benchmarks/throughput.py
+3
-2
@@ -148,9 +148,10 @@ async def run_vllm_async(
148
149
async with build_async_engine_client_from_engine_args(
150
engine_args, disable_frontend_multiprocessing) as llm:
151
+ model_config = await llm.get_model_config()
152
assert all(
152
- llm.model_config.max_model_len >= (request.prompt_len +
153
- request.expected_output_len)
153
+ model_config.max_model_len >= (request.prompt_len +
154
+ request.expected_output_len)
155
for request in requests), (
156
"Please ensure that max_model_len is greater than the sum of"
157
" prompt_len and expected_output_len for all requests.")