@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a93a598d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

fix the issue that len(tokenizer(prompt)["input_ids"]) > prompt_len (#10524)

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

Wang, Yi committed Nov 21, 2024 at 19:15 UTC 8a93a598d9ac265882e55432e7aef55c8bff23f4
1 file changed +1
benchmarks/backend_request_func.py
+1
@@ -54,6 +54,7 @@ async def async_request_tgi(
54 "do_sample": True,
55 "temperature": 0.01, # TGI does not accept 0.0 temperature.
56 "top_p": 0.99, # TGI does not accept 1.0 top_p.
57 + "truncate": request_func_input.prompt_len,
58 # TGI does not accept ignore_eos flag.
59 }
60 payload = {