@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a4548bae7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix the openai benchmarking requests to work with latest OpenAI apis (#2992)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Chen Wang committed Mar 4, 2024 at 18:51 UTC 9a4548bae73a8831f668116d8a6e88491d933a4e
1 file changed +70
benchmarks/backend_request_func.py
+70
@@ -275,10 +275,80 @@ async def async_request_openai_completions(
275 return output
276
277
278 +async def async_request_openai_chat_completions(
279 + request_func_input: RequestFuncInput,
280 + pbar: Optional[tqdm] = None,
281 +) -> RequestFuncOutput:
282 + api_url = request_func_input.api_url
283 + assert api_url.endswith(
284 + "v1/chat/completions"
285 + ), "OpenAI Chat API URL must end with 'v1/chat/completions'."
286 +
287 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
288 + assert not request_func_input.use_beam_search
289 + payload = {
290 + "model": request_func_input.model,
291 + "messages": [
292 + {
293 + "role": "user",
294 + "content": request_func_input.prompt,
295 + },
296 + ],
297 + "temperature": 0.0,
298 + "max_tokens": request_func_input.output_len,
299 + "stream": True,
300 + }
301 + headers = {
302 + "Content-Type": "application/json",
303 + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
304 + }
305 +
306 + output = RequestFuncOutput()
307 + output.prompt_len = request_func_input.prompt_len
308 +
309 + generated_text = ""
310 + ttft = 0
311 + st = time.perf_counter()
312 + try:
313 + async with session.post(url=api_url, json=payload,
314 + headers=headers) as response:
315 + if response.status == 200:
316 + async for chunk in response.content:
317 + if ttft == 0:
318 + ttft = time.perf_counter() - st
319 + output.ttft = ttft
320 +
321 + chunk = chunk.strip()
322 + if not chunk:
323 + continue
324 +
325 + chunk = chunk.decode("utf-8").lstrip("data: ")
326 + if chunk == "[DONE]":
327 + latency = time.perf_counter() - st
328 + else:
329 + body = json.loads(chunk)
330 + if "content" in body["choices"][0]["delta"]:
331 + generated_text += body["choices"][0]["delta"][
332 + "content"]
333 +
334 + output.generated_text = generated_text
335 + output.success = True
336 + output.latency = latency
337 + else:
338 + output.success = False
339 + except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
340 + output.success = False
341 +
342 + if pbar:
343 + pbar.update(1)
344 + return output
345 +
346 +
347 ASYNC_REQUEST_FUNCS = {
348 "tgi": async_request_tgi,
349 "vllm": async_request_vllm,
350 "deepspeed-mii": async_request_deepspeed_mii,
351 "openai": async_request_openai_completions,
352 + "openai-chat": async_request_openai_chat_completions,
353 "tensorrt-llm": async_request_trt_llm,
354 }