A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fix parameter name in `get_tokenizer` (#4107)
Cyrus Leung committed
Apr 26, 2024 at 10:10 UTC
a74dee9b62d10767eb0580f196f5e508e9e80a2d
2 files changed
+26
-5
tests/tokenization/test_tokenizer.py
new
+20
@@ -0,0 +1,20 @@
1
+import pytest
2
+from transformers import PreTrainedTokenizerBase
3
+
4
+from vllm.transformers_utils.tokenizer import get_tokenizer
5
+
6
+TOKENIZER_NAMES = [
7
+ "facebook/opt-125m",
8
+ "gpt2",
9
+]
10
+
11
+
12
+@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES)
13
+def test_tokenizer_revision(tokenizer_name: str):
14
+ # Assume that "main" branch always exists
15
+ tokenizer = get_tokenizer(tokenizer_name, revision="main")
16
+ assert isinstance(tokenizer, PreTrainedTokenizerBase)
17
+
18
+ # Assume that "never" branch always does not exist
19
+ with pytest.raises(OSError, match='not a valid git identifier'):
20
+ get_tokenizer(tokenizer_name, revision="never")
vllm/transformers_utils/tokenizer.py
+6
-5
@@ -58,11 +58,12 @@ def get_tokenizer(
58
*args,
59
tokenizer_mode: str = "auto",
60
trust_remote_code: bool = False,
61
- tokenizer_revision: Optional[str] = None,
61
+ revision: Optional[str] = None,
62
download_dir: Optional[str] = None,
63
**kwargs,
64
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
65
- """Gets a tokenizer for the given model name via Huggingface/modelscope."""
65
+ """Gets a tokenizer for the given model name via HuggingFace or ModelScope.
66
+ """
67
if VLLM_USE_MODELSCOPE:
68
# download model from ModelScope hub,
69
# lazy import so that modelscope is not required for normal use.
@@ -74,7 +75,7 @@ def get_tokenizer(
75
tokenizer_path = snapshot_download(
76
model_id=tokenizer_name,
77
cache_dir=download_dir,
77
- revision=tokenizer_revision,
78
+ revision=revision,
79
# Ignore weights - we only need the tokenizer.
80
ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
81
tokenizer_name = tokenizer_path
@@ -90,7 +91,7 @@ def get_tokenizer(
91
tokenizer_name,
92
*args,
93
trust_remote_code=trust_remote_code,
93
- tokenizer_revision=tokenizer_revision,
94
+ revision=revision,
95
**kwargs)
96
except ValueError as e:
97
# If the error pertains to the tokenizer class not existing or not
@@ -114,7 +115,7 @@ def get_tokenizer(
115
tokenizer_name,
116
*args,
117
trust_remote_code=trust_remote_code,
117
- tokenizer_revision=tokenizer_revision,
118
+ revision=revision,
119
**kwargs)
120
else:
121
raise e