A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Add tokenizer_mode param to benchmark_serving.py (#11174)
Signed-off-by: Alexander Matveev <alexm@neuralmagic.com>
Alexander Matveev committed
Dec 13, 2024 at 11:19 UTC
238c0d93b40008244fae64530d82f1860b1f9121
1 file changed
+12
benchmarks/benchmark_serving.py
+12
@@ -781,6 +781,7 @@ def main(args: argparse.Namespace):
781
backend = args.backend
782
model_id = args.model
783
tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
784
+ tokenizer_mode = args.tokenizer_mode
785
786
if args.base_url is not None:
787
api_url = f"{args.base_url}{args.endpoint}"
@@ -790,6 +791,7 @@ def main(args: argparse.Namespace):
791
base_url = f"http://{args.host}:{args.port}"
792
793
tokenizer = get_tokenizer(tokenizer_id,
794
+ tokenizer_mode=tokenizer_mode,
795
trust_remote_code=args.trust_remote_code)
796
797
if args.dataset is not None:
@@ -1210,5 +1212,15 @@ if __name__ == "__main__":
1212
"from the sampled HF dataset.",
1213
)
1214
1215
+ parser.add_argument(
1216
+ '--tokenizer-mode',
1217
+ type=str,
1218
+ default="auto",
1219
+ choices=['auto', 'slow', 'mistral'],
1220
+ help='The tokenizer mode.\n\n* "auto" will use the '
1221
+ 'fast tokenizer if available.\n* "slow" will '
1222
+ 'always use the slow tokenizer. \n* '
1223
+ '"mistral" will always use the `mistral_common` tokenizer.')
1224
+
1225
args = parser.parse_args()
1226
main(args)