A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bugfix][Benchmarks] Allow benchmark of deepspeed-mii backend to select a model (#17285)
Signed-off-by: Teruaki Ishizaki <teruaki.ishizaki@ntt.com>
Teruaki Ishizaki committed
May 1, 2025 at 20:54 UTC
86a1f67a3b63977b55bf5ef2e550893d7750bfa0
1 file changed
+1
benchmarks/backend_request_func.py
+1
@@ -201,6 +201,7 @@ async def async_request_deepspeed_mii(
201
timeout=AIOHTTP_TIMEOUT) as session:
202
203
payload = {
204
+ "model": request_func_input.model,
205
"prompt": request_func_input.prompt,
206
"max_tokens": request_func_input.output_len,
207
"temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.