@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66d18a7fb0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

add support for tokenizer revision (#1163)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Federico Cassano committed Oct 2, 2023 at 22:19 UTC 66d18a7fb07d3a9d6e6e46352aea05355a51cfed
5 files changed +23 -1
vllm/config.py
+5
@@ -41,6 +41,9 @@ class ModelConfig:
41 revision: The specific model version to use. It can be a branch name,
42 a tag name, or a commit id. If unspecified, will use the default
43 version.
44 + tokenizer_revision: The specific tokenizer version to use. It can be a
45 + branch name, a tag name, or a commit id. If unspecified, will use
46 + the default version.
47 max_model_len: Maximum length of a sequence (including prompt and
48 output). If None, will be derived from the model.
49 quantization: Quantization method that was used to quantize the model
@@ -58,6 +61,7 @@ class ModelConfig:
61 dtype: str,
62 seed: int,
63 revision: Optional[str] = None,
64 + tokenizer_revision: Optional[str] = None,
65 max_model_len: Optional[int] = None,
66 quantization: Optional[str] = None,
67 ) -> None:
@@ -69,6 +73,7 @@ class ModelConfig:
73 self.load_format = load_format
74 self.seed = seed
75 self.revision = revision
76 + self.tokenizer_revision = tokenizer_revision
77 self.quantization = quantization
78
79 self.hf_config = get_config(model, trust_remote_code, revision)
vllm/engine/arg_utils.py
+10 -1
@@ -29,6 +29,7 @@ class EngineArgs:
29 max_num_seqs: int = 256
30 disable_log_stats: bool = False
31 revision: Optional[str] = None
32 + tokenizer_revision: Optional[str] = None
33 quantization: Optional[str] = None
34
35 def __post_init__(self):
@@ -57,6 +58,13 @@ class EngineArgs:
58 help='the specific model version to use. It can be a branch '
59 'name, a tag name, or a commit id. If unspecified, will use '
60 'the default version.')
61 + parser.add_argument(
62 + '--tokenizer-revision',
63 + type=str,
64 + default=None,
65 + help='the specific tokenizer version to use. It can be a branch '
66 + 'name, a tag name, or a commit id. If unspecified, will use '
67 + 'the default version.')
68 parser.add_argument('--tokenizer-mode',
69 type=str,
70 default=EngineArgs.tokenizer_mode,
@@ -175,7 +183,8 @@ class EngineArgs:
183 self.tokenizer_mode, self.trust_remote_code,
184 self.download_dir, self.load_format,
185 self.dtype, self.seed, self.revision,
178 - self.max_model_len, self.quantization)
186 + self.tokenizer_revision, self.max_model_len,
187 + self.quantization)
188 cache_config = CacheConfig(
189 self.block_size, self.gpu_memory_utilization, self.swap_space,
190 getattr(model_config.hf_config, 'sliding_window', None))
vllm/engine/llm_engine.py
+2
@@ -75,6 +75,7 @@ class LLMEngine:
75 f"tokenizer={model_config.tokenizer!r}, "
76 f"tokenizer_mode={model_config.tokenizer_mode}, "
77 f"revision={model_config.revision}, "
78 + f"tokenizer_revision={model_config.tokenizer_revision}, "
79 f"trust_remote_code={model_config.trust_remote_code}, "
80 f"dtype={model_config.dtype}, "
81 f"max_seq_len={model_config.max_model_len}, "
@@ -98,6 +99,7 @@ class LLMEngine:
99 model_config.tokenizer,
100 tokenizer_mode=model_config.tokenizer_mode,
101 trust_remote_code=model_config.trust_remote_code,
102 + tokenizer_revision=model_config.tokenizer_revision,
103 revision=model_config.revision)
104 self.seq_counter = Counter()
105
vllm/entrypoints/llm.py
+4
@@ -42,6 +42,8 @@ class LLM:
42 quantized and use `dtype` to determine the data type of the weights.
43 revision: The specific model version to use. It can be a branch name,
44 a tag name, or a commit id.
45 + tokenizer_revision: The specific tokenizer version to use. It can be a
46 + branch name, a tag name, or a commit id.
47 seed: The seed to initialize the random number generator for sampling.
48 gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
49 reserve for the model weights, activations, and KV cache. Higher
@@ -65,6 +67,7 @@ class LLM:
67 dtype: str = "auto",
68 quantization: Optional[str] = None,
69 revision: Optional[str] = None,
70 + tokenizer_revision: Optional[str] = None,
71 seed: int = 0,
72 gpu_memory_utilization: float = 0.9,
73 swap_space: int = 4,
@@ -81,6 +84,7 @@ class LLM:
84 dtype=dtype,
85 quantization=quantization,
86 revision=revision,
87 + tokenizer_revision=tokenizer_revision,
88 seed=seed,
89 gpu_memory_utilization=gpu_memory_utilization,
90 swap_space=swap_space,
vllm/transformers_utils/tokenizer.py
+2
@@ -16,6 +16,7 @@ def get_tokenizer(
16 *args,
17 tokenizer_mode: str = "auto",
18 trust_remote_code: bool = False,
19 + tokenizer_revision: Optional[str] = None,
20 **kwargs,
21 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
22 """Gets a tokenizer for the given model name via Huggingface."""
@@ -37,6 +38,7 @@ def get_tokenizer(
38 tokenizer_name,
39 *args,
40 trust_remote_code=trust_remote_code,
41 + tokenizer_revision=tokenizer_revision,
42 **kwargs)
43 except TypeError as e:
44 # The LLaMA tokenizer causes a protobuf error in some environments.