@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a945fcc2ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add trust-remote-code flag to handle remote tokenizers (#364)

codethazine committed Jul 7, 2023 at 20:04 UTC a945fcc2aec97d5fd85f5a7264814de04bbf5154
5 files changed +39 -6
vllm/config.py
+4
@@ -20,6 +20,8 @@ class ModelConfig:
20 tokenizer: Name or path of the huggingface tokenizer to use.
21 tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
22 available, and "slow" will always use the slow tokenizer.
23 + trust_remote_code: Trust remote code (e.g., from HuggingFace) when
24 + downloading the model and tokenizer.
25 download_dir: Directory to download and load the weights, default to the
26 default cache directory of huggingface.
27 use_np_weights: Save a numpy copy of model weights for faster loading.
@@ -36,6 +38,7 @@ class ModelConfig:
38 model: str,
39 tokenizer: str,
40 tokenizer_mode: str,
41 + trust_remote_code: bool,
42 download_dir: Optional[str],
43 use_np_weights: bool,
44 use_dummy_weights: bool,
@@ -45,6 +48,7 @@ class ModelConfig:
48 self.model = model
49 self.tokenizer = tokenizer
50 self.tokenizer_mode = tokenizer_mode
51 + self.trust_remote_code = trust_remote_code
52 self.download_dir = download_dir
53 self.use_np_weights = use_np_weights
54 self.use_dummy_weights = use_dummy_weights
vllm/engine/arg_utils.py
+8 -3
@@ -13,6 +13,7 @@ class EngineArgs:
13 model: str
14 tokenizer: Optional[str] = None
15 tokenizer_mode: str = 'auto'
16 + trust_remote_code: bool = False
17 download_dir: Optional[str] = None
18 use_np_weights: bool = False
19 use_dummy_weights: bool = False
@@ -55,6 +56,9 @@ class EngineArgs:
56 help='tokenizer mode. "auto" will use the fast '
57 'tokenizer if available, and "slow" will '
58 'always use the slow tokenizer.')
59 + parser.add_argument('--trust-remote-code',
60 + action='store_true',
61 + help='trust remote code from huggingface')
62 parser.add_argument('--download-dir',
63 type=str,
64 default=EngineArgs.download_dir,
@@ -141,9 +145,10 @@ class EngineArgs:
145 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
146 # Initialize the configs.
147 model_config = ModelConfig(self.model, self.tokenizer,
144 - self.tokenizer_mode, self.download_dir,
145 - self.use_np_weights, self.use_dummy_weights,
146 - self.dtype, self.seed)
148 + self.tokenizer_mode, self.trust_remote_code,
149 + self.download_dir, self.use_np_weights,
150 + self.use_dummy_weights, self.dtype,
151 + self.seed)
152 cache_config = CacheConfig(self.block_size,
153 self.gpu_memory_utilization,
154 self.swap_space)
vllm/engine/llm_engine.py
+4 -1
@@ -62,6 +62,7 @@ class LLMEngine:
62 f"model={model_config.model!r}, "
63 f"tokenizer={model_config.tokenizer!r}, "
64 f"tokenizer_mode={model_config.tokenizer_mode}, "
65 + f"trust_remote_code={model_config.trust_remote_code}, "
66 f"dtype={model_config.dtype}, "
67 f"use_dummy_weights={model_config.use_dummy_weights}, "
68 f"download_dir={model_config.download_dir!r}, "
@@ -78,7 +79,9 @@ class LLMEngine:
79 self._verify_args()
80
81 self.tokenizer = get_tokenizer(
81 - model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
82 + model_config.tokenizer,
83 + tokenizer_mode=model_config.tokenizer_mode,
84 + trust_remote_code=model_config.trust_remote_code)
85 self.seq_counter = Counter()
86
87 # Create the parallel GPU workers.
vllm/entrypoints/llm.py
+4
@@ -28,6 +28,8 @@ class LLM:
28 tokenizer: The name or path of a HuggingFace Transformers tokenizer.
29 tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
30 if available, and "slow" will always use the slow tokenizer.
31 + trust_remote_code: Trust remote code (e.g., from HuggingFace) when
32 + downloading the model and tokenizer.
33 tensor_parallel_size: The number of GPUs to use for distributed
34 execution with tensor parallelism.
35 dtype: The data type for the model weights and activations. Currently,
@@ -43,6 +45,7 @@ class LLM:
45 model: str,
46 tokenizer: Optional[str] = None,
47 tokenizer_mode: str = "auto",
48 + trust_remote_code: bool = False,
49 tensor_parallel_size: int = 1,
50 dtype: str = "auto",
51 seed: int = 0,
@@ -54,6 +57,7 @@ class LLM:
57 model=model,
58 tokenizer=tokenizer,
59 tokenizer_mode=tokenizer_mode,
60 + trust_remote_code=trust_remote_code,
61 tensor_parallel_size=tensor_parallel_size,
62 dtype=dtype,
63 seed=seed,
vllm/transformers_utils/tokenizer.py
+19 -2
@@ -15,6 +15,7 @@ def get_tokenizer(
15 tokenizer_name: str,
16 *args,
17 tokenizer_mode: str = "auto",
18 + trust_remote_code: bool = False,
19 **kwargs,
20 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
21 """Gets a tokenizer for the given model name via Huggingface."""
@@ -31,8 +32,11 @@ def get_tokenizer(
32 f"using '{_FAST_LLAMA_TOKENIZER}' instead of the original "
33 "tokenizer.")
34 try:
34 - tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, *args,
35 - **kwargs)
35 + tokenizer = AutoTokenizer.from_pretrained(
36 + tokenizer_name,
37 + trust_remote_code=trust_remote_code,
38 + *args,
39 + **kwargs)
40 except TypeError as e:
41 # The LLaMA tokenizer causes a protobuf error in some environments.
42 err_msg = (
@@ -40,6 +44,19 @@ def get_tokenizer(
44 f"model, use '{_FAST_LLAMA_TOKENIZER}' instead of the original "
45 "tokenizer.")
46 raise RuntimeError(err_msg) from e
47 + except ValueError as e:
48 + # If the error pertains to the tokenizer class not existing or not
49 + # currently being imported, suggest using the --trust-remote-code flag.
50 + if (e is not None and
51 + ("does not exist or is not currently imported." in str(e)
52 + or "requires you to execute the tokenizer file" in str(e))):
53 + err_msg = (
54 + "Failed to load the tokenizer. If the tokenizer is a custom "
55 + "tokenizer not yet available in the HuggingFace transformers "
56 + "library, consider using the --trust-remote-code flag.")
57 + raise RuntimeError(err_msg) from e
58 + else:
59 + raise e
60
61 if not isinstance(tokenizer, PreTrainedTokenizerFast):
62 logger.warning(