A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add trust-remote-code flag to handle remote tokenizers (#364)
codethazine committed
Jul 7, 2023 at 20:04 UTC
a945fcc2aec97d5fd85f5a7264814de04bbf5154
5 files changed
+39
-6
vllm/config.py
+4
@@ -20,6 +20,8 @@ class ModelConfig:
20
tokenizer: Name or path of the huggingface tokenizer to use.
21
tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
22
available, and "slow" will always use the slow tokenizer.
23
+ trust_remote_code: Trust remote code (e.g., from HuggingFace) when
24
+ downloading the model and tokenizer.
25
download_dir: Directory to download and load the weights, default to the
26
default cache directory of huggingface.
27
use_np_weights: Save a numpy copy of model weights for faster loading.
@@ -36,6 +38,7 @@ class ModelConfig:
38
model: str,
39
tokenizer: str,
40
tokenizer_mode: str,
41
+ trust_remote_code: bool,
42
download_dir: Optional[str],
43
use_np_weights: bool,
44
use_dummy_weights: bool,
@@ -45,6 +48,7 @@ class ModelConfig:
48
self.model = model
49
self.tokenizer = tokenizer
50
self.tokenizer_mode = tokenizer_mode
51
+ self.trust_remote_code = trust_remote_code
52
self.download_dir = download_dir
53
self.use_np_weights = use_np_weights
54
self.use_dummy_weights = use_dummy_weights
vllm/engine/arg_utils.py
+8
-3
@@ -13,6 +13,7 @@ class EngineArgs:
13
model: str
14
tokenizer: Optional[str] = None
15
tokenizer_mode: str = 'auto'
16
+ trust_remote_code: bool = False
17
download_dir: Optional[str] = None
18
use_np_weights: bool = False
19
use_dummy_weights: bool = False
@@ -55,6 +56,9 @@ class EngineArgs:
56
help='tokenizer mode. "auto" will use the fast '
57
'tokenizer if available, and "slow" will '
58
'always use the slow tokenizer.')
59
+ parser.add_argument('--trust-remote-code',
60
+ action='store_true',
61
+ help='trust remote code from huggingface')
62
parser.add_argument('--download-dir',
63
type=str,
64
default=EngineArgs.download_dir,
@@ -141,9 +145,10 @@ class EngineArgs:
145
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
146
# Initialize the configs.
147
model_config = ModelConfig(self.model, self.tokenizer,
144
- self.tokenizer_mode, self.download_dir,
145
- self.use_np_weights, self.use_dummy_weights,
146
- self.dtype, self.seed)
148
+ self.tokenizer_mode, self.trust_remote_code,
149
+ self.download_dir, self.use_np_weights,
150
+ self.use_dummy_weights, self.dtype,
151
+ self.seed)
152
cache_config = CacheConfig(self.block_size,
153
self.gpu_memory_utilization,
154
self.swap_space)
vllm/engine/llm_engine.py
+4
-1
@@ -62,6 +62,7 @@ class LLMEngine:
62
f"model={model_config.model!r}, "
63
f"tokenizer={model_config.tokenizer!r}, "
64
f"tokenizer_mode={model_config.tokenizer_mode}, "
65
+ f"trust_remote_code={model_config.trust_remote_code}, "
66
f"dtype={model_config.dtype}, "
67
f"use_dummy_weights={model_config.use_dummy_weights}, "
68
f"download_dir={model_config.download_dir!r}, "
@@ -78,7 +79,9 @@ class LLMEngine:
79
self._verify_args()
80
81
self.tokenizer = get_tokenizer(
81
- model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
82
+ model_config.tokenizer,
83
+ tokenizer_mode=model_config.tokenizer_mode,
84
+ trust_remote_code=model_config.trust_remote_code)
85
self.seq_counter = Counter()
86
87
# Create the parallel GPU workers.
vllm/entrypoints/llm.py
+4
@@ -28,6 +28,8 @@ class LLM:
28
tokenizer: The name or path of a HuggingFace Transformers tokenizer.
29
tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
30
if available, and "slow" will always use the slow tokenizer.
31
+ trust_remote_code: Trust remote code (e.g., from HuggingFace) when
32
+ downloading the model and tokenizer.
33
tensor_parallel_size: The number of GPUs to use for distributed
34
execution with tensor parallelism.
35
dtype: The data type for the model weights and activations. Currently,
@@ -43,6 +45,7 @@ class LLM:
45
model: str,
46
tokenizer: Optional[str] = None,
47
tokenizer_mode: str = "auto",
48
+ trust_remote_code: bool = False,
49
tensor_parallel_size: int = 1,
50
dtype: str = "auto",
51
seed: int = 0,
@@ -54,6 +57,7 @@ class LLM:
57
model=model,
58
tokenizer=tokenizer,
59
tokenizer_mode=tokenizer_mode,
60
+ trust_remote_code=trust_remote_code,
61
tensor_parallel_size=tensor_parallel_size,
62
dtype=dtype,
63
seed=seed,
vllm/transformers_utils/tokenizer.py
+19
-2
@@ -15,6 +15,7 @@ def get_tokenizer(
15
tokenizer_name: str,
16
*args,
17
tokenizer_mode: str = "auto",
18
+ trust_remote_code: bool = False,
19
**kwargs,
20
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
21
"""Gets a tokenizer for the given model name via Huggingface."""
@@ -31,8 +32,11 @@ def get_tokenizer(
32
f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
33
"tokenizer.")
34
try:
34
- tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, *args,
35
- **kwargs)
35
+ tokenizer = AutoTokenizer.from_pretrained(
36
+ tokenizer_name,
37
+ trust_remote_code=trust_remote_code,
38
+ *args,
39
+ **kwargs)
40
except TypeError as e:
41
# The LLaMA tokenizer causes a protobuf error in some environments.
42
err_msg = (
@@ -40,6 +44,19 @@ def get_tokenizer(
44
f"model, use '{_FAST_LLAMA_TOKENIZER}' instead of the original "
45
"tokenizer.")
46
raise RuntimeError(err_msg) from e
47
+ except ValueError as e:
48
+ # If the error pertains to the tokenizer class not existing or not
49
+ # currently being imported, suggest using the --trust-remote-code flag.
50
+ if (e is not None and
51
+ ("does not exist or is not currently imported." in str(e)
52
+ or "requires you to execute the tokenizer file" in str(e))):
53
+ err_msg = (
54
+ "Failed to load the tokenizer. If the tokenizer is a custom "
55
+ "tokenizer not yet available in the HuggingFace transformers "
56
+ "library, consider using the --trust-remote-code flag.")
57
+ raise RuntimeError(err_msg) from e
58
+ else:
59
+ raise e
60
61
if not isinstance(tokenizer, PreTrainedTokenizerFast):
62
logger.warning(