A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
fix "Total generated tokens:" is 0 if using --backend tgi and --endpo… (#14673)
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Wang, Yi committed
Mar 20, 2025 at 11:56 UTC
40828ce5fea04a66e219675f8018e60f9479646b
1 file changed
+5
-1
benchmarks/backend_request_func.py
+5
-1
@@ -63,7 +63,7 @@ async def async_request_tgi(
63
"temperature": 0.01, # TGI does not accept 0.0 temperature.
64
"top_p": 0.99, # TGI does not accept 1.0 top_p.
65
"truncate": request_func_input.prompt_len,
66
- # TGI does not accept ignore_eos flag.
66
+ "ignore_eos_token": request_func_input.ignore_eos,
67
}
68
payload = {
69
"inputs": request_func_input.prompt,
@@ -71,6 +71,10 @@ async def async_request_tgi(
71
}
72
output = RequestFuncOutput()
73
output.prompt_len = request_func_input.prompt_len
74
+ if request_func_input.ignore_eos:
75
+ output.output_tokens = request_func_input.output_len
76
+ else:
77
+ output.output_tokens = None
78
79
ttft = 0.0
80
st = time.perf_counter()