A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Further reduce the HTTP calls to huggingface.co (#13107)
Maximilien de Bayser committed
Feb 12, 2025 at 07:34 UTC
7c4033acd46749f9da48aa8e347648d47e2f4876
1 file changed
+79
-56
vllm/transformers_utils/config.py
+79
-56
@@ -4,12 +4,14 @@ import enum
4
import json
5
import os
6
import time
7
+from functools import cache
8
from pathlib import Path
8
-from typing import Any, Dict, Literal, Optional, Type, Union
9
+from typing import Any, Callable, Dict, Literal, Optional, Type, Union
10
11
import huggingface_hub
11
-from huggingface_hub import (file_exists, hf_hub_download, list_repo_files,
12
- try_to_load_from_cache)
12
+from huggingface_hub import hf_hub_download
13
+from huggingface_hub import list_repo_files as hf_list_repo_files
14
+from huggingface_hub import try_to_load_from_cache
15
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
16
HFValidationError, LocalEntryNotFoundError,
17
RepositoryNotFoundError,
@@ -86,6 +88,65 @@ class ConfigFormat(str, enum.Enum):
88
MISTRAL = "mistral"
89
90
91
+def with_retry(func: Callable[[], Any],
92
+ log_msg: str,
93
+ max_retries: int = 2,
94
+ retry_delay: int = 2):
95
+ for attempt in range(max_retries):
96
+ try:
97
+ return func()
98
+ except Exception as e:
99
+ if attempt == max_retries - 1:
100
+ logger.error("%s: %s", log_msg, e)
101
+ raise
102
+ logger.error("%s: %s, retrying %d of %d", log_msg, e, attempt + 1,
103
+ max_retries)
104
+ time.sleep(retry_delay)
105
+ retry_delay *= 2
106
+
107
+
108
+# @cache doesn't cache exceptions
109
+@cache
110
+def list_repo_files(
111
+ repo_id: str,
112
+ *,
113
+ revision: Optional[str] = None,
114
+ repo_type: Optional[str] = None,
115
+ token: Union[str, bool, None] = None,
116
+) -> list[str]:
117
+
118
+ def lookup_files():
119
+ try:
120
+ return hf_list_repo_files(repo_id,
121
+ revision=revision,
122
+ repo_type=repo_type,
123
+ token=token)
124
+ except huggingface_hub.errors.OfflineModeIsEnabled:
125
+ # Don't raise in offline mode,
126
+ # all we know is that we don't have this
127
+ # file cached.
128
+ return []
129
+
130
+ return with_retry(lookup_files, "Error retrieving file list")
131
+
132
+
133
+def file_exists(
134
+ repo_id: str,
135
+ file_name: str,
136
+ *,
137
+ repo_type: Optional[str] = None,
138
+ revision: Optional[str] = None,
139
+ token: Union[str, bool, None] = None,
140
+) -> bool:
141
+
142
+ file_list = list_repo_files(repo_id,
143
+ repo_type=repo_type,
144
+ revision=revision,
145
+ token=token)
146
+ return file_name in file_list
147
+
148
+
149
+# In offline mode the result can be a false negative
150
def file_or_path_exists(model: Union[str, Path], config_name: str,
151
revision: Optional[str]) -> bool:
152
if Path(model).exists():
@@ -103,31 +164,10 @@ def file_or_path_exists(model: Union[str, Path], config_name: str,
164
# hf_hub. This will fail in offline mode.
165
166
# Call HF to check if the file exists
106
- # 2 retries and exponential backoff
107
- max_retries = 2
108
- retry_delay = 2
109
- for attempt in range(max_retries):
110
- try:
111
- return file_exists(model,
112
- config_name,
113
- revision=revision,
114
- token=HF_TOKEN)
115
- except huggingface_hub.errors.OfflineModeIsEnabled:
116
- # Don't raise in offline mode,
117
- # all we know is that we don't have this
118
- # file cached.
119
- return False
120
- except Exception as e:
121
- logger.error(
122
- "Error checking file existence: %s, retrying %d of %d", e,
123
- attempt + 1, max_retries)
124
- if attempt == max_retries - 1:
125
- logger.error("Error checking file existence: %s", e)
126
- raise
127
- time.sleep(retry_delay)
128
- retry_delay *= 2
129
- continue
130
- return False
167
+ return file_exists(str(model),
168
+ config_name,
169
+ revision=revision,
170
+ token=HF_TOKEN)
171
172
173
def patch_rope_scaling(config: PretrainedConfig) -> None:
@@ -208,32 +248,7 @@ def get_config(
248
revision=revision):
249
config_format = ConfigFormat.MISTRAL
250
else:
211
- # If we're in offline mode and found no valid config format, then
212
- # raise an offline mode error to indicate to the user that they
213
- # don't have files cached and may need to go online.
214
- # This is conveniently triggered by calling file_exists().
215
-
216
- # Call HF to check if the file exists
217
- # 2 retries and exponential backoff
218
- max_retries = 2
219
- retry_delay = 2
220
- for attempt in range(max_retries):
221
- try:
222
- file_exists(model,
223
- HF_CONFIG_NAME,
224
- revision=revision,
225
- token=HF_TOKEN)
226
- except Exception as e:
227
- logger.error(
228
- "Error checking file existence: %s, retrying %d of %d",
229
- e, attempt + 1, max_retries)
230
- if attempt == max_retries:
231
- logger.error("Error checking file existence: %s", e)
232
- raise e
233
- time.sleep(retry_delay)
234
- retry_delay *= 2
235
-
236
- raise ValueError(f"No supported config format found in {model}")
251
+ raise ValueError(f"No supported config format found in {model}.")
252
253
if config_format == ConfigFormat.HF:
254
config_dict, _ = PretrainedConfig.get_config_dict(
@@ -339,10 +354,11 @@ def get_hf_file_to_dict(file_name: str,
354
file_name=file_name,
355
revision=revision)
356
342
- if file_path is None and file_or_path_exists(
343
- model=model, config_name=file_name, revision=revision):
357
+ if file_path is None:
358
try:
359
hf_hub_file = hf_hub_download(model, file_name, revision=revision)
360
+ except huggingface_hub.errors.OfflineModeIsEnabled:
361
+ return None
362
except (RepositoryNotFoundError, RevisionNotFoundError,
363
EntryNotFoundError, LocalEntryNotFoundError) as e:
364
logger.debug("File or repository not found in hf_hub_download", e)
@@ -363,6 +379,7 @@ def get_hf_file_to_dict(file_name: str,
379
return None
380
381
382
+@cache
383
def get_pooling_config(model: str, revision: Optional[str] = 'main'):
384
"""
385
This function gets the pooling and normalize
@@ -390,6 +407,8 @@ def get_pooling_config(model: str, revision: Optional[str] = 'main'):
407
if modules_dict is None:
408
return None
409
410
+ logger.info("Found sentence-transformers modules configuration.")
411
+
412
pooling = next((item for item in modules_dict
413
if item["type"] == "sentence_transformers.models.Pooling"),
414
None)
@@ -408,6 +427,7 @@ def get_pooling_config(model: str, revision: Optional[str] = 'main'):
427
if pooling_type_name is not None:
428
pooling_type_name = get_pooling_config_name(pooling_type_name)
429
430
+ logger.info("Found pooling configuration.")
431
return {"pooling_type": pooling_type_name, "normalize": normalize}
432
433
return None
@@ -435,6 +455,7 @@ def get_pooling_config_name(pooling_name: str) -> Union[str, None]:
455
return None
456
457
458
+@cache
459
def get_sentence_transformer_tokenizer_config(model: str,
460
revision: Optional[str] = 'main'
461
):
@@ -491,6 +512,8 @@ def get_sentence_transformer_tokenizer_config(model: str,
512
if not encoder_dict:
513
return None
514
515
+ logger.info("Found sentence-transformers tokenize configuration.")
516
+
517
if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")):
518
return encoder_dict
519
return None