@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ddfdf470ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add trust_remote_code arg to get_config (#405)

Woosuk Kwon committed Jul 8, 2023 at 15:24 UTC ddfdf470ae721d5be668af97d5a2b5d40ce4c15c
3 files changed +20 -6
vllm/config.py
+1 -1
@@ -54,7 +54,7 @@ class ModelConfig:
54 self.use_dummy_weights = use_dummy_weights
55 self.seed = seed
56
57 - self.hf_config = get_config(model)
57 + self.hf_config = get_config(model, trust_remote_code)
58 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
59 self._verify_tokenizer_mode()
60
vllm/transformers_utils/config.py
+15 -2
@@ -7,8 +7,21 @@ _CONFIG_REGISTRY = {
7 }
8
9
10 -def get_config(model: str) -> PretrainedConfig:
11 - config = AutoConfig.from_pretrained(model, trust_remote_code=True)
10 +def get_config(model: str, trust_remote_code: bool) -> PretrainedConfig:
11 + try:
12 + config = AutoConfig.from_pretrained(
13 + model, trust_remote_code=trust_remote_code)
14 + except ValueError as e:
15 + if (not trust_remote_code and
16 + "requires you to execute the configuration file" in str(e)):
17 + err_msg = (
18 + "Failed to load the model config. If the model is a custom "
19 + "model not yet available in the HuggingFace transformers "
20 + "library, consider setting `trust_remote_code=True` in LLM "
21 + "or using the `--trust-remote-code` flag in the CLI.")
22 + raise RuntimeError(err_msg) from e
23 + else:
24 + raise e
25 if config.model_type in _CONFIG_REGISTRY:
26 config_class = _CONFIG_REGISTRY[config.model_type]
27 config = config_class.from_pretrained(model)
vllm/transformers_utils/tokenizer.py
+4 -3
@@ -34,8 +34,8 @@ def get_tokenizer(
34 try:
35 tokenizer = AutoTokenizer.from_pretrained(
36 tokenizer_name,
37 - trust_remote_code=trust_remote_code,
37 *args,
38 + trust_remote_code=trust_remote_code,
39 **kwargs)
40 except TypeError as e:
41 # The LLaMA tokenizer causes a protobuf error in some environments.
@@ -47,13 +47,14 @@ def get_tokenizer(
47 except ValueError as e:
48 # If the error pertains to the tokenizer class not existing or not
49 # currently being imported, suggest using the --trust-remote-code flag.
50 - if (e is not None and
50 + if (not trust_remote_code and
51 ("does not exist or is not currently imported." in str(e)
52 or "requires you to execute the tokenizer file" in str(e))):
53 err_msg = (
54 "Failed to load the tokenizer. If the tokenizer is a custom "
55 "tokenizer not yet available in the HuggingFace transformers "
56 - "library, consider using the --trust-remote-code flag.")
56 + "library, consider setting `trust_remote_code=True` in LLM "
57 + "or using the `--trust-remote-code` flag in the CLI.")
58 raise RuntimeError(err_msg) from e
59 else:
60 raise e