A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
bugfix: Fix signature mismatch in benchmark's `get_tokenizer` function (#11982)
Signed-off-by: elijah <f1renze.142857@gmail.com>
elijah committed
Jan 13, 2025 at 23:22 UTC
c6db21313cef3f1f9aa5714efba0d60b5f1a8dad
1 file changed
+24
-3
benchmarks/backend_request_func.py
+24
-3
@@ -417,14 +417,35 @@ def get_model(pretrained_model_name_or_path: str) -> str:
417
418
419
def get_tokenizer(
420
- pretrained_model_name_or_path: str, trust_remote_code: bool
420
+ pretrained_model_name_or_path: str,
421
+ tokenizer_mode: str = "auto",
422
+ trust_remote_code: bool = False,
423
+ **kwargs,
424
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
425
if pretrained_model_name_or_path is not None and not os.path.exists(
426
pretrained_model_name_or_path):
427
pretrained_model_name_or_path = get_model(
428
pretrained_model_name_or_path)
426
- return AutoTokenizer.from_pretrained(pretrained_model_name_or_path,
427
- trust_remote_code=trust_remote_code)
429
+ if tokenizer_mode == "slow":
430
+ if kwargs.get("use_fast", False):
431
+ raise ValueError(
432
+ "Cannot use the fast tokenizer in slow tokenizer mode.")
433
+ kwargs["use_fast"] = False
434
+ if tokenizer_mode == "mistral":
435
+ try:
436
+ from vllm.transformers_utils.tokenizer import MistralTokenizer
437
+ except ImportError as e:
438
+ raise ImportError("MistralTokenizer requires vllm package.\n"
439
+ "Please install it with `pip install vllm` "
440
+ "to use mistral tokenizer mode.") from e
441
+ return MistralTokenizer.from_pretrained(
442
+ str(pretrained_model_name_or_path))
443
+ else:
444
+ return AutoTokenizer.from_pretrained(
445
+ pretrained_model_name_or_path,
446
+ trust_remote_code=trust_remote_code,
447
+ **kwargs,
448
+ )
449
450
451
ASYNC_REQUEST_FUNCS = {