A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Core] Loading model from S3 using RunAI Model Streamer as optional loader (#10192)
Signed-off-by: OmerD <omer@run.ai>
omer-dayan committed
Dec 20, 2024 at 18:46 UTC
995f56236bc08300ea11fc8cd3d66029ffec8678
13 files changed
+457
-3
Dockerfile
+2
-2
@@ -240,9 +240,9 @@ FROM vllm-base AS vllm-openai
240
# install additional dependencies for openai api server
241
RUN --mount=type=cache,target=/root/.cache/pip \
242
if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
243
- pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10'; \
243
+ pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
244
else \
245
- pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10'; \
245
+ pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
246
fi
247
248
ENV VLLM_USAGE_SOURCE production-docker-image
docs/source/index.rst
+1
@@ -88,6 +88,7 @@ Documentation
88
serving/metrics
89
serving/integrations
90
serving/tensorizer
91
+ serving/runai_model_streamer
92
93
.. toctree::
94
:maxdepth: 1
docs/source/serving/runai_model_streamer.rst
new
+53
@@ -0,0 +1,53 @@
1
+.. _runai_model_streamer:
2
+
3
+Loading Models with Run:ai Model Streamer
4
+=========================================
5
+Run:ai Model Streamer is a library to read tensors in concurrency, while streaming it to GPU memory.
6
+Further reading can be found in `Run:ai Model Streamer Documentation <https://github.com/run-ai/runai-model-streamer/blob/master/docs/README.md>`_.
7
+
8
+vLLM supports loading weights in Safetensors format using the Run:ai Model Streamer.
9
+You first need to install vLLM RunAI optional dependency:
10
+
11
+.. code-block:: console
12
+
13
+ $ pip3 install vllm[runai]
14
+
15
+To run it as an OpenAI-compatible server, add the `--load-format runai_streamer` flag:
16
+
17
+.. code-block:: console
18
+
19
+ $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer
20
+
21
+To run model from AWS S3 object store run:
22
+
23
+.. code-block:: console
24
+
25
+ $ vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
26
+
27
+
28
+To run model from a S3 compatible object store run:
29
+
30
+.. code-block:: console
31
+
32
+ $ RUNAI_STREAMER_S3_USE_VIRTUAL_ADDRESSING=0 AWS_EC2_METADATA_DISABLED=true AWS_ENDPOINT_URL=https://storage.googleapis.com vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
33
+
34
+Tunable parameters
35
+------------------
36
+You can tune parameters using `--model-loader-extra-config`:
37
+
38
+You can tune `concurrency` that controls the level of concurrency and number of OS threads reading tensors from the file to the CPU buffer.
39
+For reading from S3, it will be the number of client instances the host is opening to the S3 server.
40
+
41
+ .. code-block:: console
42
+
43
+ $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"concurrency":16}'
44
+
45
+You can controls the size of the CPU Memory buffer to which tensors are read from the file, and limit this size.
46
+You can read further about CPU buffer memory limiting `here <https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md#runai_streamer_memory_limit>`_.
47
+
48
+ .. code-block:: console
49
+
50
+ $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"memory_limit":5368709120}'
51
+
52
+.. note::
53
+ For further instructions about tunable parameters and additional parameters configurable through environment variables, read the `Environment Variables Documentation <https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md>`_.
setup.py
+1
@@ -630,6 +630,7 @@ setup(
630
ext_modules=ext_modules,
631
extras_require={
632
"tensorizer": ["tensorizer>=2.9.0"],
633
+ "runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
634
"audio": ["librosa", "soundfile"], # Required for audio processing
635
"video": ["decord"] # Required for video processing
636
},
tests/runai_model_streamer/__init__.py
tests/runai_model_streamer/test_runai_model_streamer_loader.py
new
+31
@@ -0,0 +1,31 @@
1
+from vllm import SamplingParams
2
+from vllm.config import LoadConfig, LoadFormat
3
+from vllm.model_executor.model_loader.loader import (RunaiModelStreamerLoader,
4
+ get_model_loader)
5
+
6
+test_model = "openai-community/gpt2"
7
+
8
+prompts = [
9
+ "Hello, my name is",
10
+ "The president of the United States is",
11
+ "The capital of France is",
12
+ "The future of AI is",
13
+]
14
+# Create a sampling params object.
15
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
16
+
17
+
18
+def get_runai_model_loader():
19
+ load_config = LoadConfig(load_format=LoadFormat.RUNAI_STREAMER)
20
+ return get_model_loader(load_config)
21
+
22
+
23
+def test_get_model_loader_with_runai_flag():
24
+ model_loader = get_runai_model_loader()
25
+ assert isinstance(model_loader, RunaiModelStreamerLoader)
26
+
27
+
28
+def test_runai_model_loader_download_files(vllm_runner):
29
+ with vllm_runner(test_model, load_format=LoadFormat.RUNAI_STREAMER) as llm:
30
+ deserialized_outputs = llm.generate(prompts, sampling_params)
31
+ assert deserialized_outputs
tests/runai_model_streamer/test_weight_utils.py
new
+39
@@ -0,0 +1,39 @@
1
+import glob
2
+import tempfile
3
+
4
+import huggingface_hub.constants
5
+import torch
6
+
7
+from vllm.model_executor.model_loader.weight_utils import (
8
+ download_weights_from_hf, runai_safetensors_weights_iterator,
9
+ safetensors_weights_iterator)
10
+
11
+
12
+def test_runai_model_loader():
13
+ with tempfile.TemporaryDirectory() as tmpdir:
14
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
15
+ download_weights_from_hf("openai-community/gpt2",
16
+ allow_patterns=["*.safetensors"],
17
+ cache_dir=tmpdir)
18
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
19
+ assert len(safetensors) > 0
20
+
21
+ runai_model_streamer_tensors = {}
22
+ hf_safetensors_tensors = {}
23
+
24
+ for name, tensor in runai_safetensors_weights_iterator(safetensors):
25
+ runai_model_streamer_tensors[name] = tensor
26
+
27
+ for name, tensor in safetensors_weights_iterator(safetensors):
28
+ hf_safetensors_tensors[name] = tensor
29
+
30
+ assert len(runai_model_streamer_tensors) == len(hf_safetensors_tensors)
31
+
32
+ for name, runai_tensor in runai_model_streamer_tensors.items():
33
+ assert runai_tensor.dtype == hf_safetensors_tensors[name].dtype
34
+ assert runai_tensor.shape == hf_safetensors_tensors[name].shape
35
+ assert torch.all(runai_tensor.eq(hf_safetensors_tensors[name]))
36
+
37
+
38
+if __name__ == "__main__":
39
+ test_runai_model_loader()
vllm/config.py
+37
@@ -29,6 +29,7 @@ from vllm.transformers_utils.config import (
29
get_hf_text_config, get_pooling_config,
30
get_sentence_transformer_tokenizer_config, is_encoder_decoder,
31
try_get_generation_config, uses_mrope)
32
+from vllm.transformers_utils.utils import is_s3
33
from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
34
get_cpu_memory, print_warning_once, random_uuid,
35
resolve_obj_by_qualname)
@@ -256,6 +257,8 @@ class ModelConfig:
257
f"'Please instead use `--hf-overrides '{hf_override!r}'`")
258
warnings.warn(DeprecationWarning(msg), stacklevel=2)
259
260
+ self.maybe_pull_model_tokenizer_for_s3(model, tokenizer)
261
+
262
# The tokenizer version is consistent with the model version by default.
263
if tokenizer_revision is None:
264
self.tokenizer_revision = revision
@@ -357,6 +360,39 @@ class ModelConfig:
360
self._verify_cuda_graph()
361
self._verify_bnb_config()
362
363
+ def maybe_pull_model_tokenizer_for_s3(self, model: str,
364
+ tokenizer: str) -> None:
365
+ """
366
+ Pull the model config or tokenizer to a temporary
367
+ directory in case of S3.
368
+
369
+ Args:
370
+ model: The model name or path.
371
+ tokenizer: The tokenizer name or path.
372
+
373
+ """
374
+ if is_s3(model) or is_s3(tokenizer):
375
+ try:
376
+ from vllm.transformers_utils.s3_utils import S3Model
377
+ except ImportError as err:
378
+ raise ImportError(
379
+ "Please install Run:ai optional dependency "
380
+ "to use the S3 capabilities. "
381
+ "You can install it with: pip install vllm[runai]"
382
+ ) from err
383
+
384
+ if is_s3(model):
385
+ self.s3_model = S3Model()
386
+ self.s3_model.pull_files(model, allow_pattern=["*config.json"])
387
+ self.model_weights = self.model
388
+ self.model = self.s3_model.dir
389
+
390
+ if is_s3(tokenizer):
391
+ self.s3_tokenizer = S3Model()
392
+ self.s3_tokenizer.pull_files(
393
+ model, ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
394
+ self.tokenizer = self.s3_tokenizer.dir
395
+
396
def _init_multimodal_config(
397
self, limit_mm_per_prompt: Optional[Mapping[str, int]]
398
) -> Optional["MultiModalConfig"]:
@@ -1099,6 +1135,7 @@ class LoadFormat(str, enum.Enum):
1135
GGUF = "gguf"
1136
BITSANDBYTES = "bitsandbytes"
1137
MISTRAL = "mistral"
1138
+ RUNAI_STREAMER = "runai_streamer"
1139
1140
1141
@dataclass
vllm/engine/arg_utils.py
+2
@@ -316,6 +316,8 @@ class EngineArgs:
316
'* "tensorizer" will load the weights using tensorizer from '
317
'CoreWeave. See the Tensorize vLLM Model script in the Examples '
318
'section for more information.\n'
319
+ '* "runai_streamer" will load the Safetensors weights using Run:ai'
320
+ 'Model Streamer \n'
321
'* "bitsandbytes" will load the weights using bitsandbytes '
322
'quantization.\n')
323
parser.add_argument(
vllm/model_executor/model_loader/loader.py
+117
-1
@@ -45,9 +45,10 @@ from vllm.model_executor.model_loader.weight_utils import (
45
filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
46
get_gguf_extra_tensor_names, gguf_quant_weights_iterator,
47
initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
48
- safetensors_weights_iterator)
48
+ runai_safetensors_weights_iterator, safetensors_weights_iterator)
49
from vllm.model_executor.utils import set_weight_attrs
50
from vllm.platforms import current_platform
51
+from vllm.transformers_utils.utils import is_s3
52
from vllm.utils import is_pin_memory_available
53
54
@@ -1234,6 +1235,118 @@ class GGUFModelLoader(BaseModelLoader):
1235
return model
1236
1237
1238
+class RunaiModelStreamerLoader(BaseModelLoader):
1239
+ """
1240
+ Model loader that can load safetensors
1241
+ files from local FS or S3 bucket.
1242
+ """
1243
+
1244
+ def __init__(self, load_config: LoadConfig):
1245
+ super().__init__(load_config)
1246
+ if load_config.model_loader_extra_config:
1247
+ extra_config = load_config.model_loader_extra_config
1248
+
1249
+ if ("concurrency" in extra_config
1250
+ and isinstance(extra_config.get("concurrency"), int)):
1251
+ os.environ["RUNAI_STREAMER_CONCURRENCY"] = str(
1252
+ extra_config.get("concurrency"))
1253
+
1254
+ if ("memory_limit" in extra_config
1255
+ and isinstance(extra_config.get("memory_limit"), int)):
1256
+ os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] = str(
1257
+ extra_config.get("memory_limit"))
1258
+
1259
+ runai_streamer_s3_endpoint = os.getenv(
1260
+ 'RUNAI_STREAMER_S3_ENDPOINT')
1261
+ aws_endpoint_url = os.getenv('AWS_ENDPOINT_URL')
1262
+ if (runai_streamer_s3_endpoint is None
1263
+ and aws_endpoint_url is not None):
1264
+ os.environ["RUNAI_STREAMER_S3_ENDPOINT"] = aws_endpoint_url
1265
+
1266
+ def _prepare_weights(self, model_name_or_path: str,
1267
+ revision: Optional[str]) -> List[str]:
1268
+ """Prepare weights for the model.
1269
+
1270
+ If the model is not local, it will be downloaded."""
1271
+ is_s3_path = is_s3(model_name_or_path)
1272
+ if is_s3_path:
1273
+ try:
1274
+ from vllm.transformers_utils.s3_utils import glob as s3_glob
1275
+ except ImportError as err:
1276
+ raise ImportError(
1277
+ "Please install Run:ai optional dependency "
1278
+ "to use the S3 capabilities. "
1279
+ "You can install it with: pip install vllm[runai]"
1280
+ ) from err
1281
+
1282
+ is_local = os.path.isdir(model_name_or_path)
1283
+ safetensors_pattern = "*.safetensors"
1284
+ index_file = SAFE_WEIGHTS_INDEX_NAME
1285
+
1286
+ hf_folder = (model_name_or_path if
1287
+ (is_local or is_s3_path) else download_weights_from_hf(
1288
+ model_name_or_path,
1289
+ self.load_config.download_dir,
1290
+ [safetensors_pattern],
1291
+ revision,
1292
+ ignore_patterns=self.load_config.ignore_patterns,
1293
+ ))
1294
+
1295
+ if is_s3_path:
1296
+ hf_weights_files = s3_glob(path=hf_folder,
1297
+ allow_pattern=[safetensors_pattern])
1298
+ else:
1299
+ hf_weights_files = glob.glob(
1300
+ os.path.join(hf_folder, safetensors_pattern))
1301
+
1302
+ if not is_local and not is_s3_path:
1303
+ download_safetensors_index_file_from_hf(
1304
+ model_name_or_path, index_file, self.load_config.download_dir,
1305
+ revision)
1306
+
1307
+ if not hf_weights_files:
1308
+ raise RuntimeError(
1309
+ f"Cannot find any safetensors model weights with "
1310
+ f"`{model_name_or_path}`")
1311
+
1312
+ return hf_weights_files
1313
+
1314
+ def _get_weights_iterator(
1315
+ self, model_or_path: str,
1316
+ revision: str) -> Generator[Tuple[str, torch.Tensor], None, None]:
1317
+ """Get an iterator for the model weights based on the load format."""
1318
+ hf_weights_files = self._prepare_weights(model_or_path, revision)
1319
+ return runai_safetensors_weights_iterator(hf_weights_files)
1320
+
1321
+ def download_model(self, model_config: ModelConfig) -> None:
1322
+ """Download model if necessary"""
1323
+ self._prepare_weights(model_config.model, model_config.revision)
1324
+
1325
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
1326
+ """Perform streaming of the model to destination"""
1327
+ device_config = vllm_config.device_config
1328
+ model_config = vllm_config.model_config
1329
+
1330
+ target_device = torch.device(device_config.device)
1331
+ with set_default_torch_dtype(model_config.dtype):
1332
+ with target_device:
1333
+ model = _initialize_model(vllm_config=vllm_config)
1334
+
1335
+ model_weights = model_config.model
1336
+ if hasattr(model_config, "model_weights"):
1337
+ model_weights = model_config.model_weights
1338
+ model.load_weights(
1339
+ self._get_weights_iterator(model_weights,
1340
+ model_config.revision))
1341
+
1342
+ for _, module in model.named_modules():
1343
+ quant_method = getattr(module, "quant_method", None)
1344
+ if quant_method is not None:
1345
+ with device_loading_context(module, target_device):
1346
+ quant_method.process_weights_after_loading(module)
1347
+ return model.eval()
1348
+
1349
+
1350
def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1351
"""Get a model loader based on the load format."""
1352
@@ -1255,4 +1368,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1368
if load_config.load_format == LoadFormat.GGUF:
1369
return GGUFModelLoader(load_config)
1370
1371
+ if load_config.load_format == LoadFormat.RUNAI_STREAMER:
1372
+ return RunaiModelStreamerLoader(load_config)
1373
+
1374
return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/weight_utils.py
+24
@@ -410,6 +410,30 @@ def safetensors_weights_iterator(
410
yield name, param
411
412
413
+def runai_safetensors_weights_iterator(
414
+ hf_weights_files: List[str]
415
+) -> Generator[Tuple[str, torch.Tensor], None, None]:
416
+ """Iterate over the weights in the model safetensor files."""
417
+ try:
418
+ from runai_model_streamer import SafetensorsStreamer
419
+ except ImportError as err:
420
+ raise ImportError(
421
+ "Please install Run:ai optional dependency."
422
+ "You can install it with: pip install vllm[runai]") from err
423
+
424
+ enable_tqdm = not torch.distributed.is_initialized(
425
+ ) or torch.distributed.get_rank() == 0
426
+ with SafetensorsStreamer() as streamer:
427
+ for st_file in tqdm(
428
+ hf_weights_files,
429
+ desc="Loading safetensors using Runai Model Streamer",
430
+ disable=not enable_tqdm,
431
+ bar_format=_BAR_FORMAT,
432
+ ):
433
+ streamer.stream_file(st_file)
434
+ yield from streamer.get_tensors()
435
+
436
+
437
def pt_weights_iterator(
438
hf_weights_files: List[str]
439
) -> Generator[Tuple[str, torch.Tensor], None, None]:
vllm/transformers_utils/s3_utils.py
new
+146
@@ -0,0 +1,146 @@
1
+import fnmatch
2
+import os
3
+import shutil
4
+import signal
5
+import tempfile
6
+from pathlib import Path
7
+from typing import Optional
8
+
9
+import boto3
10
+
11
+
12
+def _filter_allow(paths: list[str], patterns: list[str]) -> list[str]:
13
+ return [
14
+ path for path in paths if any(
15
+ fnmatch.fnmatch(path, pattern) for pattern in patterns)
16
+ ]
17
+
18
+
19
+def _filter_ignore(paths: list[str], patterns: list[str]) -> list[str]:
20
+ return [
21
+ path for path in paths
22
+ if not any(fnmatch.fnmatch(path, pattern) for pattern in patterns)
23
+ ]
24
+
25
+
26
+def glob(s3=None,
27
+ path: str = "",
28
+ allow_pattern: Optional[list[str]] = None) -> list[str]:
29
+ """
30
+ List full file names from S3 path and filter by allow pattern.
31
+
32
+ Args:
33
+ s3: S3 client to use.
34
+ path: The S3 path to list from.
35
+ allow_pattern: A list of patterns of which files to pull.
36
+
37
+ Returns:
38
+ list[str]: List of full S3 paths allowed by the pattern
39
+ """
40
+ if s3 is None:
41
+ s3 = boto3.client("s3")
42
+ bucket_name, _, paths = list_files(s3,
43
+ path=path,
44
+ allow_pattern=allow_pattern)
45
+ return [f"s3://{bucket_name}/{path}" for path in paths]
46
+
47
+
48
+def list_files(
49
+ s3,
50
+ path: str,
51
+ allow_pattern: Optional[list[str]] = None,
52
+ ignore_pattern: Optional[list[str]] = None
53
+) -> tuple[str, str, list[str]]:
54
+ """
55
+ List files from S3 path and filter by pattern.
56
+
57
+ Args:
58
+ s3: S3 client to use.
59
+ path: The S3 path to list from.
60
+ allow_pattern: A list of patterns of which files to pull.
61
+ ignore_pattern: A list of patterns of which files not to pull.
62
+
63
+ Returns:
64
+ tuple[str, str, list[str]]: A tuple where:
65
+ - The first element is the bucket name
66
+ - The second element is string represent the bucket
67
+ and the prefix as a dir like string
68
+ - The third element is a list of files allowed or
69
+ disallowed by pattern
70
+ """
71
+ parts = path.removeprefix('s3://').split('/')
72
+ prefix = '/'.join(parts[1:])
73
+ bucket_name = parts[0]
74
+
75
+ objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
76
+ paths = [obj['Key'] for obj in objects.get('Contents', [])]
77
+
78
+ paths = _filter_ignore(paths, ["*/"])
79
+ if allow_pattern is not None:
80
+ paths = _filter_allow(paths, allow_pattern)
81
+
82
+ if ignore_pattern is not None:
83
+ paths = _filter_ignore(paths, ignore_pattern)
84
+
85
+ return bucket_name, prefix, paths
86
+
87
+
88
+class S3Model:
89
+ """
90
+ A class representing a S3 model mirrored into a temporary directory.
91
+
92
+ Attributes:
93
+ s3: S3 client.
94
+ dir: The temporary created directory.
95
+
96
+ Methods:
97
+ pull_files(): Pull model from S3 to the temporary directory.
98
+ """
99
+
100
+ def __init__(self) -> None:
101
+ self.s3 = boto3.client('s3')
102
+ for sig in (signal.SIGINT, signal.SIGTERM):
103
+ existing_handler = signal.getsignal(sig)
104
+ signal.signal(sig, self._close_by_signal(existing_handler))
105
+ self.dir = tempfile.mkdtemp()
106
+
107
+ def __del__(self):
108
+ self._close()
109
+
110
+ def _close(self) -> None:
111
+ if os.path.exists(self.dir):
112
+ shutil.rmtree(self.dir)
113
+
114
+ def _close_by_signal(self, existing_handler=None):
115
+
116
+ def new_handler(signum, frame):
117
+ self._close()
118
+ if existing_handler:
119
+ existing_handler(signum, frame)
120
+
121
+ return new_handler
122
+
123
+ def pull_files(self,
124
+ s3_model_path: str = "",
125
+ allow_pattern: Optional[list[str]] = None,
126
+ ignore_pattern: Optional[list[str]] = None) -> None:
127
+ """
128
+ Pull files from S3 storage into the temporary directory.
129
+
130
+ Args:
131
+ s3_model_path: The S3 path of the model.
132
+ allow_pattern: A list of patterns of which files to pull.
133
+ ignore_pattern: A list of patterns of which files not to pull.
134
+
135
+ """
136
+ bucket_name, base_dir, files = list_files(self.s3, s3_model_path,
137
+ allow_pattern,
138
+ ignore_pattern)
139
+ if len(files) == 0:
140
+ return
141
+
142
+ for file in files:
143
+ destination_file = self.dir + file.removeprefix(base_dir)
144
+ local_dir = Path(destination_file).parent
145
+ os.makedirs(local_dir, exist_ok=True)
146
+ self.s3.download_file(bucket_name, file, destination_file)
vllm/transformers_utils/utils.py
+4
@@ -3,6 +3,10 @@ from pathlib import Path
3
from typing import Union
4
5
6
+def is_s3(model_or_path: str) -> bool:
7
+ return model_or_path.lower().startswith('s3://')
8
+
9
+
10
def check_gguf_file(model: Union[str, PathLike]) -> bool:
11
"""Check if the file is a GGUF model."""
12
model = Path(model)