@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a74dee9b62
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fix parameter name in `get_tokenizer` (#4107)

Cyrus Leung committed Apr 26, 2024 at 10:10 UTC a74dee9b62d10767eb0580f196f5e508e9e80a2d
2 files changed +26 -5
tests/tokenization/test_tokenizer.py new
+20
@@ -0,0 +1,20 @@
1 +import pytest
2 +from transformers import PreTrainedTokenizerBase
3 +
4 +from vllm.transformers_utils.tokenizer import get_tokenizer
5 +
6 +TOKENIZER_NAMES = [
7 + "facebook/opt-125m",
8 + "gpt2",
9 +]
10 +
11 +
12 +@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES)
13 +def test_tokenizer_revision(tokenizer_name: str):
14 + # Assume that "main" branch always exists
15 + tokenizer = get_tokenizer(tokenizer_name, revision="main")
16 + assert isinstance(tokenizer, PreTrainedTokenizerBase)
17 +
18 + # Assume that "never" branch always does not exist
19 + with pytest.raises(OSError, match='not a valid git identifier'):
20 + get_tokenizer(tokenizer_name, revision="never")
vllm/transformers_utils/tokenizer.py
+6 -5
@@ -58,11 +58,12 @@ def get_tokenizer(
58 *args,
59 tokenizer_mode: str = "auto",
60 trust_remote_code: bool = False,
61 - tokenizer_revision: Optional[str] = None,
61 + revision: Optional[str] = None,
62 download_dir: Optional[str] = None,
63 **kwargs,
64 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
65 - """Gets a tokenizer for the given model name via Huggingface/modelscope."""
65 + """Gets a tokenizer for the given model name via HuggingFace or ModelScope.
66 + """
67 if VLLM_USE_MODELSCOPE:
68 # download model from ModelScope hub,
69 # lazy import so that modelscope is not required for normal use.
@@ -74,7 +75,7 @@ def get_tokenizer(
75 tokenizer_path = snapshot_download(
76 model_id=tokenizer_name,
77 cache_dir=download_dir,
77 - revision=tokenizer_revision,
78 + revision=revision,
79 # Ignore weights - we only need the tokenizer.
80 ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
81 tokenizer_name = tokenizer_path
@@ -90,7 +91,7 @@ def get_tokenizer(
91 tokenizer_name,
92 *args,
93 trust_remote_code=trust_remote_code,
93 - tokenizer_revision=tokenizer_revision,
94 + revision=revision,
95 **kwargs)
96 except ValueError as e:
97 # If the error pertains to the tokenizer class not existing or not
@@ -114,7 +115,7 @@ def get_tokenizer(
115 tokenizer_name,
116 *args,
117 trust_remote_code=trust_remote_code,
117 - tokenizer_revision=tokenizer_revision,
118 + revision=revision,
119 **kwargs)
120 else:
121 raise e