@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e152f29502
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[misc] Reduce number of config file requests to HuggingFace (#12797)

Signed-off-by: EC2 Default User <ec2-user@ip-172-31-20-117.us-west-2.compute.internal> Signed-off-by: <> Co-authored-by: EC2 Default User <ec2-user@ip-172-31-20-117.us-west-2.compute.internal>

Kevin H. Luu committed Feb 6, 2025 at 06:59 UTC e152f295020ea2a7ca37be9cabadfaef78464274
1 file changed +23 -13
vllm/transformers_utils/config.py
+23 -13
@@ -7,7 +7,7 @@ from pathlib import Path
7 from typing import Any, Dict, Optional, Type, Union
8
9 import huggingface_hub
10 -from huggingface_hub import (file_exists, hf_hub_download,
10 +from huggingface_hub import (file_exists, hf_hub_download, list_repo_files,
11 try_to_load_from_cache)
12 from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
13 LocalEntryNotFoundError,
@@ -395,18 +395,28 @@ def get_sentence_transformer_tokenizer_config(model: str,
395 - dict: A dictionary containing the configuration parameters
396 for the Sentence Transformer BERT model.
397 """
398 - for config_name in [
399 - "sentence_bert_config.json",
400 - "sentence_roberta_config.json",
401 - "sentence_distilbert_config.json",
402 - "sentence_camembert_config.json",
403 - "sentence_albert_config.json",
404 - "sentence_xlm-roberta_config.json",
405 - "sentence_xlnet_config.json",
406 - ]:
407 - encoder_dict = get_hf_file_to_dict(config_name, model, revision)
408 - if encoder_dict:
409 - break
398 + sentence_transformer_config_files = [
399 + "sentence_bert_config.json",
400 + "sentence_roberta_config.json",
401 + "sentence_distilbert_config.json",
402 + "sentence_camembert_config.json",
403 + "sentence_albert_config.json",
404 + "sentence_xlm-roberta_config.json",
405 + "sentence_xlnet_config.json",
406 + ]
407 + try:
408 + # If model is on HuggingfaceHub, get the repo files
409 + repo_files = list_repo_files(model, revision=revision, token=HF_TOKEN)
410 + except Exception as e:
411 + logger.debug("Error getting repo files", e)
412 + repo_files = []
413 +
414 + encoder_dict = None
415 + for config_name in sentence_transformer_config_files:
416 + if config_name in repo_files or Path(model).exists():
417 + encoder_dict = get_hf_file_to_dict(config_name, model, revision)
418 + if encoder_dict:
419 + break
420
421 if not encoder_dict:
422 return None