@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d1cfbfc74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Delete Llama tokenizer warnings (#2146)

Woosuk Kwon committed Dec 16, 2023 at 22:05 UTC 3d1cfbfc745b8644661263071c341dfa3684dde2
1 file changed -17
vllm/transformers_utils/tokenizer.py
-17
@@ -8,9 +8,6 @@ from vllm.transformers_utils.tokenizers import *
8
9 logger = init_logger(__name__)
10
11 -# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file.
12 -_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
13 -
11
12 def get_tokenizer(
13 tokenizer_name: str,
@@ -27,13 +24,6 @@ def get_tokenizer(
24 "Cannot use the fast tokenizer in slow tokenizer mode.")
25 kwargs["use_fast"] = False
26
30 - if ("llama" in tokenizer_name.lower() and kwargs.get("use_fast", True)
31 - and tokenizer_name != _FAST_LLAMA_TOKENIZER):
32 - logger.info(
33 - "For some LLaMA V1 models, initializing the fast tokenizer may "
34 - "take a long time. To reduce the initialization time, consider "
35 - f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
36 - "tokenizer.")
27 try:
28 tokenizer = AutoTokenizer.from_pretrained(
29 tokenizer_name,
@@ -41,13 +31,6 @@ def get_tokenizer(
31 trust_remote_code=trust_remote_code,
32 tokenizer_revision=tokenizer_revision,
33 **kwargs)
44 - except TypeError as e:
45 - # The LLaMA tokenizer causes a protobuf error in some environments.
46 - err_msg = (
47 - "Failed to load the tokenizer. If you are using a LLaMA V1 model "
48 - f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the "
49 - "original tokenizer.")
50 - raise RuntimeError(err_msg) from e
34 except ValueError as e:
35 # If the error pertains to the tokenizer class not existing or not
36 # currently being imported, suggest using the --trust-remote-code flag.