A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Misc][Benchmark] Add support for different `tokenizer_mode` (#15040)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: Russell Bryant <rbryant@redhat.com>
Aaron Pham committed
Mar 19, 2025 at 10:56 UTC
6c5a3195db126cedf7c891d1af3cac8080f8b759
2 files changed
+13
-2
benchmarks/benchmark_serving_structured_output.py
+12
-2
@@ -732,8 +732,11 @@ def main(args: argparse.Namespace):
732
api_url = f"http://{args.host}:{args.port}{args.endpoint}"
733
base_url = f"http://{args.host}:{args.port}"
734
735
- tokenizer = get_tokenizer(tokenizer_id,
736
- trust_remote_code=args.trust_remote_code)
735
+ tokenizer = get_tokenizer(
736
+ tokenizer_id,
737
+ trust_remote_code=args.trust_remote_code,
738
+ tokenizer_mode=args.tokenizer_mode,
739
+ )
740
741
if args.dataset == 'grammar':
742
args.structure_type = 'guided_grammar'
@@ -876,6 +879,13 @@ if __name__ == "__main__":
879
help=
880
"Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
881
)
882
+ parser.add_argument(
883
+ "--tokenizer-mode",
884
+ type=str,
885
+ default="auto",
886
+ help=
887
+ "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
888
+ )
889
parser.add_argument(
890
"--num-prompts",
891
type=int,
benchmarks/run_structured_output_benchmark.sh
+1
@@ -54,6 +54,7 @@ for qps in "${QPS_VALUES[@]}"; do
54
python "$SCRIPT_DIR/benchmark_serving_structured_output.py" $COMMON_PARAMS \
55
--request-rate $qps \
56
--result-filename "$FILENAME" \
57
+ --tokenizer-mode ${TOKENIZER_MODE:-"auto"} \
58
--port ${PORT:-8000}
59
60
echo "Completed benchmark with QPS: $qps"