A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Fix VLLM_USE_MODELSCOPE issue (#13384)
r.4ntix committed
Feb 17, 2025 at 22:22 UTC
ce77eb9410c694000c5da5abfa638500c6c72aeb
2 files changed
+32
-7
vllm/transformers_utils/config.py
+12
-6
@@ -117,6 +117,12 @@ def list_repo_files(
117
118
def lookup_files():
119
try:
120
+ if VLLM_USE_MODELSCOPE:
121
+ from vllm.transformers_utils.utils import (
122
+ modelscope_list_repo_files)
123
+ return modelscope_list_repo_files(repo_id,
124
+ revision=revision,
125
+ token=token)
126
return hf_list_repo_files(repo_id,
127
revision=revision,
128
repo_type=repo_type,
@@ -382,17 +388,17 @@ def get_hf_file_to_dict(file_name: str,
388
@cache
389
def get_pooling_config(model: str, revision: Optional[str] = 'main'):
390
"""
385
- This function gets the pooling and normalize
386
- config from the model - only applies to
387
- sentence-transformers models.
391
+ This function gets the pooling and normalize
392
+ config from the model - only applies to
393
+ sentence-transformers models.
394
395
Args:
396
model (str): The name of the Hugging Face model.
391
- revision (str, optional): The specific version
397
+ revision (str, optional): The specific version
398
of the model to use. Defaults to 'main'.
399
400
Returns:
395
- dict: A dictionary containing the pooling
401
+ dict: A dictionary containing the pooling
402
type and whether normalization is used.
403
"""
404
@@ -499,7 +505,7 @@ def get_sentence_transformer_tokenizer_config(model: str,
505
revision=revision,
506
token=HF_TOKEN)
507
except Exception as e:
502
- logger.debug("Error getting repo files", e)
508
+ logger.error("Error getting repo files", e)
509
repo_files = []
510
511
for config_name in sentence_transformer_config_files:
vllm/transformers_utils/utils.py
+20
-1
@@ -2,7 +2,7 @@
2
3
from os import PathLike
4
from pathlib import Path
5
-from typing import Union
5
+from typing import List, Optional, Union
6
7
8
def is_s3(model_or_path: str) -> bool:
@@ -20,3 +20,22 @@ def check_gguf_file(model: Union[str, PathLike]) -> bool:
20
with open(model, "rb") as f:
21
header = f.read(4)
22
return header == b"GGUF"
23
+
24
+
25
+def modelscope_list_repo_files(
26
+ repo_id: str,
27
+ revision: Optional[str] = None,
28
+ token: Union[str, bool, None] = None,
29
+) -> List[str]:
30
+ """List files in a modelscope repo."""
31
+ from modelscope.hub.api import HubApi
32
+ from modelscope.utils.hf_util import _try_login
33
+ _try_login(token)
34
+ api = HubApi()
35
+ # same as huggingface_hub.list_repo_files
36
+ files = [
37
+ file['Path'] for file in api.get_model_files(
38
+ model_id=repo_id, revision=revision, recursive=True)
39
+ if file['Type'] == 'blob'
40
+ ]
41
+ return files