A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix the openai benchmarking requests to work with latest OpenAI apis (#2992)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Chen Wang committed
Mar 4, 2024 at 18:51 UTC
9a4548bae73a8831f668116d8a6e88491d933a4e
1 file changed
+70
benchmarks/backend_request_func.py
+70
@@ -275,10 +275,80 @@ async def async_request_openai_completions(
275
return output
276
277
278
+async def async_request_openai_chat_completions(
279
+ request_func_input: RequestFuncInput,
280
+ pbar: Optional[tqdm] = None,
281
+) -> RequestFuncOutput:
282
+ api_url = request_func_input.api_url
283
+ assert api_url.endswith(
284
+ "v1/chat/completions"
285
+ ), "OpenAI Chat API URL must end with 'v1/chat/completions'."
286
+
287
+ async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
288
+ assert not request_func_input.use_beam_search
289
+ payload = {
290
+ "model": request_func_input.model,
291
+ "messages": [
292
+ {
293
+ "role": "user",
294
+ "content": request_func_input.prompt,
295
+ },
296
+ ],
297
+ "temperature": 0.0,
298
+ "max_tokens": request_func_input.output_len,
299
+ "stream": True,
300
+ }
301
+ headers = {
302
+ "Content-Type": "application/json",
303
+ "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
304
+ }
305
+
306
+ output = RequestFuncOutput()
307
+ output.prompt_len = request_func_input.prompt_len
308
+
309
+ generated_text = ""
310
+ ttft = 0
311
+ st = time.perf_counter()
312
+ try:
313
+ async with session.post(url=api_url, json=payload,
314
+ headers=headers) as response:
315
+ if response.status == 200:
316
+ async for chunk in response.content:
317
+ if ttft == 0:
318
+ ttft = time.perf_counter() - st
319
+ output.ttft = ttft
320
+
321
+ chunk = chunk.strip()
322
+ if not chunk:
323
+ continue
324
+
325
+ chunk = chunk.decode("utf-8").lstrip("data: ")
326
+ if chunk == "[DONE]":
327
+ latency = time.perf_counter() - st
328
+ else:
329
+ body = json.loads(chunk)
330
+ if "content" in body["choices"][0]["delta"]:
331
+ generated_text += body["choices"][0]["delta"][
332
+ "content"]
333
+
334
+ output.generated_text = generated_text
335
+ output.success = True
336
+ output.latency = latency
337
+ else:
338
+ output.success = False
339
+ except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
340
+ output.success = False
341
+
342
+ if pbar:
343
+ pbar.update(1)
344
+ return output
345
+
346
+
347
ASYNC_REQUEST_FUNCS = {
348
"tgi": async_request_tgi,
349
"vllm": async_request_vllm,
350
"deepspeed-mii": async_request_deepspeed_mii,
351
"openai": async_request_openai_completions,
352
+ "openai-chat": async_request_openai_chat_completions,
353
"tensorrt-llm": async_request_trt_llm,
354
}