A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[misc] Add retries with exponential backoff for HF file existence check (#13008)
Kevin H. Luu committed
Feb 10, 2025 at 01:15 UTC
fde71262e0c235fa5ad80677b3ba65df7f5110de
1 file changed
+48
-13
vllm/transformers_utils/config.py
+48
-13
@@ -3,6 +3,7 @@
3
import enum
4
import json
5
import os
6
+import time
7
from pathlib import Path
8
from typing import Any, Dict, Literal, Optional, Type, Union
9
@@ -100,15 +101,33 @@ def file_or_path_exists(model: Union[str, Path], config_name: str,
101
102
# NB: file_exists will only check for the existence of the config file on
103
# hf_hub. This will fail in offline mode.
103
- try:
104
- return file_exists(model,
105
- config_name,
106
- revision=revision,
107
- token=HF_TOKEN)
108
- except huggingface_hub.errors.OfflineModeIsEnabled:
109
- # Don't raise in offline mode, all we know is that we don't have this
110
- # file cached.
111
- return False
104
+
105
+ # Call HF to check if the file exists
106
+ # 2 retries and exponential backoff
107
+ max_retries = 2
108
+ retry_delay = 2
109
+ for attempt in range(max_retries):
110
+ try:
111
+ return file_exists(model,
112
+ config_name,
113
+ revision=revision,
114
+ token=HF_TOKEN)
115
+ except huggingface_hub.errors.OfflineModeIsEnabled:
116
+ # Don't raise in offline mode,
117
+ # all we know is that we don't have this
118
+ # file cached.
119
+ return False
120
+ except Exception as e:
121
+ logger.error(
122
+ "Error checking file existence: %s, retrying %d of %d", e,
123
+ attempt + 1, max_retries)
124
+ if attempt == max_retries - 1:
125
+ logger.error("Error checking file existence: %s", e)
126
+ raise
127
+ time.sleep(retry_delay)
128
+ retry_delay *= 2
129
+ continue
130
+ return False
131
132
133
def patch_rope_scaling(config: PretrainedConfig) -> None:
@@ -193,10 +212,26 @@ def get_config(
212
# raise an offline mode error to indicate to the user that they
213
# don't have files cached and may need to go online.
214
# This is conveniently triggered by calling file_exists().
196
- file_exists(model,
197
- HF_CONFIG_NAME,
198
- revision=revision,
199
- token=HF_TOKEN)
215
+
216
+ # Call HF to check if the file exists
217
+ # 2 retries and exponential backoff
218
+ max_retries = 2
219
+ retry_delay = 2
220
+ for attempt in range(max_retries):
221
+ try:
222
+ file_exists(model,
223
+ HF_CONFIG_NAME,
224
+ revision=revision,
225
+ token=HF_TOKEN)
226
+ except Exception as e:
227
+ logger.error(
228
+ "Error checking file existence: %s, retrying %d of %d",
229
+ e, attempt + 1, max_retries)
230
+ if attempt == max_retries:
231
+ logger.error("Error checking file existence: %s", e)
232
+ raise e
233
+ time.sleep(retry_delay)
234
+ retry_delay *= 2
235
236
raise ValueError(f"No supported config format found in {model}")
237