A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[BUG] Remove token param #10921 (#11022)
Signed-off-by: Flavia Beo <flavia.beo@ibm.com>
Flávia Béo committed
Dec 10, 2024 at 14:38 UTC
250ee65d72a0c7b86ec5cea9cbe9377da21d6439
1 file changed
+29
-34
vllm/transformers_utils/config.py
+29
-34
@@ -1,5 +1,6 @@
1
import enum
2
import json
3
+import os
4
from pathlib import Path
5
from typing import Any, Dict, Optional, Type, Union
6
@@ -41,6 +42,7 @@ else:
42
from transformers import AutoConfig
43
44
MISTRAL_CONFIG_NAME = "params.json"
45
+HF_TOKEN = os.getenv('HF_TOKEN', None)
46
47
logger = init_logger(__name__)
48
@@ -77,8 +79,8 @@ class ConfigFormat(str, enum.Enum):
79
MISTRAL = "mistral"
80
81
80
-def file_or_path_exists(model: Union[str, Path], config_name, revision,
81
- token) -> bool:
82
+def file_or_path_exists(model: Union[str, Path], config_name: str,
83
+ revision: Optional[str]) -> bool:
84
if Path(model).exists():
85
return (Path(model) / config_name).is_file()
86
@@ -93,7 +95,10 @@ def file_or_path_exists(model: Union[str, Path], config_name, revision,
95
# NB: file_exists will only check for the existence of the config file on
96
# hf_hub. This will fail in offline mode.
97
try:
96
- return file_exists(model, config_name, revision=revision, token=token)
98
+ return file_exists(model,
99
+ config_name,
100
+ revision=revision,
101
+ token=HF_TOKEN)
102
except huggingface_hub.errors.OfflineModeIsEnabled:
103
# Don't raise in offline mode, all we know is that we don't have this
104
# file cached.
@@ -161,7 +166,6 @@ def get_config(
166
revision: Optional[str] = None,
167
code_revision: Optional[str] = None,
168
config_format: ConfigFormat = ConfigFormat.AUTO,
164
- token: Optional[str] = None,
169
**kwargs,
170
) -> PretrainedConfig:
171
# Separate model folder from file path for GGUF models
@@ -173,19 +177,20 @@ def get_config(
177
178
if config_format == ConfigFormat.AUTO:
179
if is_gguf or file_or_path_exists(
176
- model, HF_CONFIG_NAME, revision=revision, token=token):
180
+ model, HF_CONFIG_NAME, revision=revision):
181
config_format = ConfigFormat.HF
178
- elif file_or_path_exists(model,
179
- MISTRAL_CONFIG_NAME,
180
- revision=revision,
181
- token=token):
182
+ elif file_or_path_exists(model, MISTRAL_CONFIG_NAME,
183
+ revision=revision):
184
config_format = ConfigFormat.MISTRAL
185
else:
186
# If we're in offline mode and found no valid config format, then
187
# raise an offline mode error to indicate to the user that they
188
# don't have files cached and may need to go online.
189
# This is conveniently triggered by calling file_exists().
188
- file_exists(model, HF_CONFIG_NAME, revision=revision, token=token)
190
+ file_exists(model,
191
+ HF_CONFIG_NAME,
192
+ revision=revision,
193
+ token=HF_TOKEN)
194
195
raise ValueError(f"No supported config format found in {model}")
196
@@ -194,7 +199,7 @@ def get_config(
199
model,
200
revision=revision,
201
code_revision=code_revision,
197
- token=token,
202
+ token=HF_TOKEN,
203
**kwargs,
204
)
205
@@ -206,7 +211,7 @@ def get_config(
211
model,
212
revision=revision,
213
code_revision=code_revision,
209
- token=token,
214
+ token=HF_TOKEN,
215
**kwargs,
216
)
217
else:
@@ -216,7 +221,7 @@ def get_config(
221
trust_remote_code=trust_remote_code,
222
revision=revision,
223
code_revision=code_revision,
219
- token=token,
224
+ token=HF_TOKEN,
225
**kwargs,
226
)
227
except ValueError as e:
@@ -234,7 +239,7 @@ def get_config(
239
raise e
240
241
elif config_format == ConfigFormat.MISTRAL:
237
- config = load_params_config(model, revision, token=token, **kwargs)
242
+ config = load_params_config(model, revision, token=HF_TOKEN, **kwargs)
243
else:
244
raise ValueError(f"Unsupported config format: {config_format}")
245
@@ -256,8 +261,7 @@ def get_config(
261
262
def get_hf_file_to_dict(file_name: str,
263
model: Union[str, Path],
259
- revision: Optional[str] = 'main',
260
- token: Optional[str] = None):
264
+ revision: Optional[str] = 'main'):
265
"""
266
Downloads a file from the Hugging Face Hub and returns
267
its contents as a dictionary.
@@ -266,7 +270,6 @@ def get_hf_file_to_dict(file_name: str,
270
- file_name (str): The name of the file to download.
271
- model (str): The name of the model on the Hugging Face Hub.
272
- revision (str): The specific version of the model.
269
- - token (str): The Hugging Face authentication token.
273
274
Returns:
275
- config_dict (dict): A dictionary containing
@@ -276,8 +279,7 @@ def get_hf_file_to_dict(file_name: str,
279
280
if file_or_path_exists(model=model,
281
config_name=file_name,
279
- revision=revision,
280
- token=token):
282
+ revision=revision):
283
284
if not file_path.is_file():
285
try:
@@ -296,9 +298,7 @@ def get_hf_file_to_dict(file_name: str,
298
return None
299
300
299
-def get_pooling_config(model: str,
300
- revision: Optional[str] = 'main',
301
- token: Optional[str] = None):
301
+def get_pooling_config(model: str, revision: Optional[str] = 'main'):
302
"""
303
This function gets the pooling and normalize
304
config from the model - only applies to
@@ -315,8 +315,7 @@ def get_pooling_config(model: str,
315
"""
316
317
modules_file_name = "modules.json"
318
- modules_dict = get_hf_file_to_dict(modules_file_name, model, revision,
319
- token)
318
+ modules_dict = get_hf_file_to_dict(modules_file_name, model, revision)
319
320
if modules_dict is None:
321
return None
@@ -332,8 +331,7 @@ def get_pooling_config(model: str,
331
if pooling:
332
333
pooling_file_name = "{}/config.json".format(pooling["path"])
335
- pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision,
336
- token)
334
+ pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision)
335
pooling_type_name = next(
336
(item for item, val in pooling_dict.items() if val is True), None)
337
@@ -368,8 +366,8 @@ def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
366
367
368
def get_sentence_transformer_tokenizer_config(model: str,
371
- revision: Optional[str] = 'main',
372
- token: Optional[str] = None):
369
+ revision: Optional[str] = 'main'
370
+ ):
371
"""
372
Returns the tokenization configuration dictionary for a
373
given Sentence Transformer BERT model.
@@ -379,7 +377,6 @@ def get_sentence_transformer_tokenizer_config(model: str,
377
BERT model.
378
- revision (str, optional): The revision of the m
379
odel to use. Defaults to 'main'.
382
- - token (str): A Hugging Face access token.
380
381
Returns:
382
- dict: A dictionary containing the configuration parameters
@@ -394,7 +391,7 @@ def get_sentence_transformer_tokenizer_config(model: str,
391
"sentence_xlm-roberta_config.json",
392
"sentence_xlnet_config.json",
393
]:
397
- encoder_dict = get_hf_file_to_dict(config_name, model, revision, token)
394
+ encoder_dict = get_hf_file_to_dict(config_name, model, revision)
395
if encoder_dict:
396
break
397
@@ -474,16 +471,14 @@ def maybe_register_config_serialize_by_value() -> None:
471
exc_info=e)
472
473
477
-def load_params_config(model: Union[str, Path],
478
- revision: Optional[str],
479
- token: Optional[str] = None,
474
+def load_params_config(model: Union[str, Path], revision: Optional[str],
475
**kwargs) -> PretrainedConfig:
476
# This function loads a params.json config which
477
# should be used when loading models in mistral format
478
479
config_file_name = "params.json"
480
486
- config_dict = get_hf_file_to_dict(config_file_name, model, revision, token)
481
+ config_dict = get_hf_file_to_dict(config_file_name, model, revision)
482
assert isinstance(config_dict, dict)
483
484
config_mapping = {