@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3bb8e2c9a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Enable proxy support in benchmark script (#12356)

Signed-off-by: Junichi Sato <junichi.sato@sbintuitions.co.jp>

Junichi Sato committed Jan 24, 2025 at 23:58 UTC 3bb8e2c9a2a2dadfe61f35bcde15f72c43f4635f
1 file changed +10 -5
benchmarks/backend_request_func.py
+10 -5
@@ -51,7 +51,8 @@ async def async_request_tgi(
51 api_url = request_func_input.api_url
52 assert api_url.endswith("generate_stream")
53
54 - async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
54 + async with aiohttp.ClientSession(trust_env=True,
55 + timeout=AIOHTTP_TIMEOUT) as session:
56 params = {
57 "best_of": request_func_input.best_of,
58 "max_new_tokens": request_func_input.output_len,
@@ -123,7 +124,8 @@ async def async_request_trt_llm(
124 api_url = request_func_input.api_url
125 assert api_url.endswith("generate_stream")
126
126 - async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
127 + async with aiohttp.ClientSession(trust_env=True,
128 + timeout=AIOHTTP_TIMEOUT) as session:
129 assert request_func_input.best_of == 1
130 payload = {
131 "accumulate_tokens": True,
@@ -187,7 +189,8 @@ async def async_request_deepspeed_mii(
189 request_func_input: RequestFuncInput,
190 pbar: Optional[tqdm] = None,
191 ) -> RequestFuncOutput:
190 - async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
192 + async with aiohttp.ClientSession(trust_env=True,
193 + timeout=AIOHTTP_TIMEOUT) as session:
194 assert request_func_input.best_of == 1
195
196 payload = {
@@ -235,7 +238,8 @@ async def async_request_openai_completions(
238 ("completions", "profile")
239 ), "OpenAI Completions API URL must end with 'completions' or 'profile'."
240
238 - async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
241 + async with aiohttp.ClientSession(trust_env=True,
242 + timeout=AIOHTTP_TIMEOUT) as session:
243 payload = {
244 "model": request_func_input.model_name \
245 if request_func_input.model_name else request_func_input.model,
@@ -333,7 +337,8 @@ async def async_request_openai_chat_completions(
337 "chat/completions"
338 ), "OpenAI Chat Completions API URL must end with 'chat/completions'."
339
336 - async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
340 + async with aiohttp.ClientSession(trust_env=True,
341 + timeout=AIOHTTP_TIMEOUT) as session:
342 content = [{"type": "text", "text": request_func_input.prompt}]
343 if request_func_input.multi_modal_content:
344 content.append(request_func_input.multi_modal_content)