A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix][Benchmarks] Fix a benchmark of deepspeed-mii backend to use api_key (#17291)
Signed-off-by: Teruaki Ishizaki <teruaki.ishizaki@ntt.com>
Teruaki Ishizaki committed
May 23, 2025 at 13:30 UTC
4be2255c81528c75dd033b25ffffe1803b20311a
1 file changed
+8
-1
benchmarks/backend_request_func.py
+8
-1
@@ -194,6 +194,11 @@ async def async_request_deepspeed_mii(
194
request_func_input: RequestFuncInput,
195
pbar: Optional[tqdm] = None,
196
) -> RequestFuncOutput:
197
+ api_url = request_func_input.api_url
198
+ assert api_url.endswith(("completions", "profile")), (
199
+ "OpenAI Completions API URL must end with 'completions' or 'profile'."
200
+ )
201
+
202
async with aiohttp.ClientSession(
203
trust_env=True, timeout=AIOHTTP_TIMEOUT
204
) as session:
@@ -204,6 +209,8 @@ async def async_request_deepspeed_mii(
209
"temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.
210
"top_p": 1.0,
211
}
212
+ headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
213
+
214
output = RequestFuncOutput()
215
output.prompt_len = request_func_input.prompt_len
216
@@ -215,7 +222,7 @@ async def async_request_deepspeed_mii(
222
st = time.perf_counter()
223
try:
224
async with session.post(
218
- url=request_func_input.api_url, json=payload
225
+ url=api_url, json=payload, headers=headers
226
) as response:
227
if response.status == 200:
228
parsed_resp = await response.json()