A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add trust_remote_code arg to get_config (#405)
Woosuk Kwon committed
Jul 8, 2023 at 15:24 UTC
ddfdf470ae721d5be668af97d5a2b5d40ce4c15c
3 files changed
+20
-6
vllm/config.py
+1
-1
@@ -54,7 +54,7 @@ class ModelConfig:
54
self.use_dummy_weights = use_dummy_weights
55
self.seed = seed
56
57
- self.hf_config = get_config(model)
57
+ self.hf_config = get_config(model, trust_remote_code)
58
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
59
self._verify_tokenizer_mode()
60
vllm/transformers_utils/config.py
+15
-2
@@ -7,8 +7,21 @@ _CONFIG_REGISTRY = {
7
}
8
9
10
-def get_config(model: str) -> PretrainedConfig:
11
- config = AutoConfig.from_pretrained(model, trust_remote_code=True)
10
+def get_config(model: str, trust_remote_code: bool) -> PretrainedConfig:
11
+ try:
12
+ config = AutoConfig.from_pretrained(
13
+ model, trust_remote_code=trust_remote_code)
14
+ except ValueError as e:
15
+ if (not trust_remote_code and
16
+ "requires you to execute the configuration file" in str(e)):
17
+ err_msg = (
18
+ "Failed to load the model config. If the model is a custom "
19
+ "model not yet available in the HuggingFace transformers "
20
+ "library, consider setting `trust_remote_code=True` in LLM "
21
+ "or using the `--trust-remote-code` flag in the CLI.")
22
+ raise RuntimeError(err_msg) from e
23
+ else:
24
+ raise e
25
if config.model_type in _CONFIG_REGISTRY:
26
config_class = _CONFIG_REGISTRY[config.model_type]
27
config = config_class.from_pretrained(model)
vllm/transformers_utils/tokenizer.py
+4
-3
@@ -34,8 +34,8 @@ def get_tokenizer(
34
try:
35
tokenizer = AutoTokenizer.from_pretrained(
36
tokenizer_name,
37
- trust_remote_code=trust_remote_code,
37
*args,
38
+ trust_remote_code=trust_remote_code,
39
**kwargs)
40
except TypeError as e:
41
# The LLaMA tokenizer causes a protobuf error in some environments.
@@ -47,13 +47,14 @@ def get_tokenizer(
47
except ValueError as e:
48
# If the error pertains to the tokenizer class not existing or not
49
# currently being imported, suggest using the --trust-remote-code flag.
50
- if (e is not None and
50
+ if (not trust_remote_code and
51
("does not exist or is not currently imported." in str(e)
52
or "requires you to execute the tokenizer file" in str(e))):
53
err_msg = (
54
"Failed to load the tokenizer. If the tokenizer is a custom "
55
"tokenizer not yet available in the HuggingFace transformers "
56
- "library, consider using the --trust-remote-code flag.")
56
+ "library, consider setting `trust_remote_code=True` in LLM "
57
+ "or using the `--trust-remote-code` flag in the CLI.")
58
raise RuntimeError(err_msg) from e
59
else:
60
raise e