A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmark] Enable benchmark to run with `encoding_format="bytes"` (#27467)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 24, 2025 at 19:16 UTC
b7030d962b082077e11c863746708866c4c9d83f
1 file changed
+9
-2
vllm/benchmarks/lib/endpoint_request_func.py
+9
-2
@@ -498,10 +498,17 @@ async def _run_pooling_request(
498
async with session.post(url=api_url, headers=headers, json=payload) as response:
499
if response.status == 200:
500
output.ttft = output.latency = time.perf_counter() - st
501
- data = await response.json()
501
+
502
+ if payload.get("encoding_format", "float") == "bytes":
503
+ metadata = json.loads(response.headers["metadata"])
504
+ usage = metadata.get("usage", {})
505
+ else:
506
+ data = await response.json()
507
+ usage = data.get("usage", {})
508
+
509
output.success = True
510
output.generated_text = ""
504
- output.prompt_len = data.get("usage", {}).get("prompt_tokens", 0)
511
+ output.prompt_len = usage.get("prompt_tokens", 0)
512
else:
513
output.success = False
514
output.error = response.reason or ""