@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 238c0d93b4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Add tokenizer_mode param to benchmark_serving.py (#11174)

Signed-off-by: Alexander Matveev <alexm@neuralmagic.com>

Alexander Matveev committed Dec 13, 2024 at 11:19 UTC 238c0d93b40008244fae64530d82f1860b1f9121
1 file changed +12
benchmarks/benchmark_serving.py
+12
@@ -781,6 +781,7 @@ def main(args: argparse.Namespace):
781 backend = args.backend
782 model_id = args.model
783 tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
784 + tokenizer_mode = args.tokenizer_mode
785
786 if args.base_url is not None:
787 api_url = f"{args.base_url}{args.endpoint}"
@@ -790,6 +791,7 @@ def main(args: argparse.Namespace):
791 base_url = f"http://{args.host}:{args.port}"
792
793 tokenizer = get_tokenizer(tokenizer_id,
794 + tokenizer_mode=tokenizer_mode,
795 trust_remote_code=args.trust_remote_code)
796
797 if args.dataset is not None:
@@ -1210,5 +1212,15 @@ if __name__ == "__main__":
1212 "from the sampled HF dataset.",
1213 )
1214
1215 + parser.add_argument(
1216 + '--tokenizer-mode',
1217 + type=str,
1218 + default="auto",
1219 + choices=['auto', 'slow', 'mistral'],
1220 + help='The tokenizer mode.\n\n* "auto" will use the '
1221 + 'fast tokenizer if available.\n* "slow" will '
1222 + 'always use the slow tokenizer. \n* '
1223 + '"mistral" will always use the `mistral_common` tokenizer.')
1224 +
1225 args = parser.parse_args()
1226 main(args)