A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Enable proxy support in benchmark script (#12356)
Signed-off-by: Junichi Sato <junichi.sato@sbintuitions.co.jp>
Junichi Sato committed
Jan 24, 2025 at 23:58 UTC
3bb8e2c9a2a2dadfe61f35bcde15f72c43f4635f
1 file changed
+10
-5
benchmarks/backend_request_func.py
+10
-5
@@ -51,7 +51,8 @@ async def async_request_tgi(
51
api_url = request_func_input.api_url
52
assert api_url.endswith("generate_stream")
53
54
- async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
54
+ async with aiohttp.ClientSession(trust_env=True,
55
+ timeout=AIOHTTP_TIMEOUT) as session:
56
params = {
57
"best_of": request_func_input.best_of,
58
"max_new_tokens": request_func_input.output_len,
@@ -123,7 +124,8 @@ async def async_request_trt_llm(
124
api_url = request_func_input.api_url
125
assert api_url.endswith("generate_stream")
126
126
- async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
127
+ async with aiohttp.ClientSession(trust_env=True,
128
+ timeout=AIOHTTP_TIMEOUT) as session:
129
assert request_func_input.best_of == 1
130
payload = {
131
"accumulate_tokens": True,
@@ -187,7 +189,8 @@ async def async_request_deepspeed_mii(
189
request_func_input: RequestFuncInput,
190
pbar: Optional[tqdm] = None,
191
) -> RequestFuncOutput:
190
- async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
192
+ async with aiohttp.ClientSession(trust_env=True,
193
+ timeout=AIOHTTP_TIMEOUT) as session:
194
assert request_func_input.best_of == 1
195
196
payload = {
@@ -235,7 +238,8 @@ async def async_request_openai_completions(
238
("completions", "profile")
239
), "OpenAI Completions API URL must end with 'completions' or 'profile'."
240
238
- async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
241
+ async with aiohttp.ClientSession(trust_env=True,
242
+ timeout=AIOHTTP_TIMEOUT) as session:
243
payload = {
244
"model": request_func_input.model_name \
245
if request_func_input.model_name else request_func_input.model,
@@ -333,7 +337,8 @@ async def async_request_openai_chat_completions(
337
"chat/completions"
338
), "OpenAI Chat Completions API URL must end with 'chat/completions'."
339
336
- async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
340
+ async with aiohttp.ClientSession(trust_env=True,
341
+ timeout=AIOHTTP_TIMEOUT) as session:
342
content = [{"type": "text", "text": request_func_input.prompt}]
343
if request_func_input.multi_modal_content:
344
content.append(request_func_input.multi_modal_content)