A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] Delete Llama tokenizer warnings (#2146)
Woosuk Kwon committed
Dec 16, 2023 at 22:05 UTC
3d1cfbfc745b8644661263071c341dfa3684dde2
1 file changed
-17
vllm/transformers_utils/tokenizer.py
-17
@@ -8,9 +8,6 @@ from vllm.transformers_utils.tokenizers import *
8
9
logger = init_logger(__name__)
10
11
-# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file.
12
-_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
13
-
11
12
def get_tokenizer(
13
tokenizer_name: str,
@@ -27,13 +24,6 @@ def get_tokenizer(
24
"Cannot use the fast tokenizer in slow tokenizer mode.")
25
kwargs["use_fast"] = False
26
30
- if ("llama" in tokenizer_name.lower() and kwargs.get("use_fast", True)
31
- and tokenizer_name != _FAST_LLAMA_TOKENIZER):
32
- logger.info(
33
- "For some LLaMA V1 models, initializing the fast tokenizer may "
34
- "take a long time. To reduce the initialization time, consider "
35
- f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
36
- "tokenizer.")
27
try:
28
tokenizer = AutoTokenizer.from_pretrained(
29
tokenizer_name,
@@ -41,13 +31,6 @@ def get_tokenizer(
31
trust_remote_code=trust_remote_code,
32
tokenizer_revision=tokenizer_revision,
33
**kwargs)
44
- except TypeError as e:
45
- # The LLaMA tokenizer causes a protobuf error in some environments.
46
- err_msg = (
47
- "Failed to load the tokenizer. If you are using a LLaMA V1 model "
48
- f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the "
49
- "original tokenizer.")
50
- raise RuntimeError(err_msg) from e
34
except ValueError as e:
35
# If the error pertains to the tokenizer class not existing or not
36
# currently being imported, suggest using the --trust-remote-code flag.