A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
fix the issue that len(tokenizer(prompt)["input_ids"]) > prompt_len (#10524)
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Wang, Yi committed
Nov 21, 2024 at 19:15 UTC
8a93a598d9ac265882e55432e7aef55c8bff23f4
1 file changed
+1
benchmarks/backend_request_func.py
+1
@@ -54,6 +54,7 @@ async def async_request_tgi(
54
"do_sample": True,
55
"temperature": 0.01, # TGI does not accept 0.0 temperature.
56
"top_p": 0.99, # TGI does not accept 1.0 top_p.
57
+ "truncate": request_func_input.prompt_len,
58
# TGI does not accept ignore_eos flag.
59
}
60
payload = {