A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] remove peft as dependency for prompt models (#8162)
Prashant Gupta committed
Sep 10, 2024 at 14:21 UTC
04e7c4e77118159e0b892681acd04a1b50a7ea6e
3 files changed
+94
-9
vllm/config.py
-8
@@ -1558,14 +1558,6 @@ class PromptAdapterConfig:
1558
prompt_adapter_dtype: Optional[torch.dtype] = None
1559
1560
def __post_init__(self):
1561
- library_name = 'peft'
1562
- try:
1563
- __import__(library_name)
1564
- except ImportError as e:
1565
- raise ImportError(
1566
- f"'{library_name}' is not installed for prompt adapter support."
1567
- f"Please install it using 'pip install {library_name}'."
1568
- ) from e
1561
1562
if self.max_prompt_adapters < 1:
1563
raise ValueError(f"max_prompt_adapters "
vllm/prompt_adapter/models.py
+1
-1
@@ -14,6 +14,7 @@ from vllm.config import PromptAdapterConfig
14
from vllm.prompt_adapter.layers import (
15
VocabParallelEmbeddingWithPromptAdapter) # yapf: disable
16
from vllm.prompt_adapter.layers import PromptAdapterMapping
17
+from vllm.prompt_adapter.utils import load_peft_weights
18
19
logger = logging.getLogger(__name__)
20
@@ -90,7 +91,6 @@ class PromptAdapterModel(AdapterModel):
91
config: PromptAdapterConfig,
92
device: str = "cuda",
93
) -> "PromptAdapterModel":
93
- from peft.utils import load_peft_weights
94
95
if num_virtual_tokens > config.max_prompt_adapter_token:
96
raise ValueError(
vllm/prompt_adapter/utils.py
new
+93
@@ -0,0 +1,93 @@
1
+# code borrowed from: https://github.com/huggingface/peft/blob/v0.12.0/src/peft/utils/save_and_load.py#L420
2
+
3
+import os
4
+from typing import Optional
5
+
6
+import torch
7
+from huggingface_hub import file_exists, hf_hub_download
8
+from huggingface_hub.utils import EntryNotFoundError
9
+from safetensors.torch import load_file as safe_load_file
10
+
11
+WEIGHTS_NAME = "adapter_model.bin"
12
+SAFETENSORS_WEIGHTS_NAME = "adapter_model.safetensors"
13
+
14
+
15
+# Get current device name based on available devices
16
+def infer_device() -> str:
17
+ if torch.cuda.is_available():
18
+ return "cuda"
19
+ return "cpu"
20
+
21
+
22
+def load_peft_weights(model_id: str,
23
+ device: Optional[str] = None,
24
+ **hf_hub_download_kwargs) -> dict:
25
+ r"""
26
+ A helper method to load the PEFT weights from the HuggingFace Hub or locally
27
+
28
+ Args:
29
+ model_id (`str`):
30
+ The local path to the adapter weights or the name of the adapter to
31
+ load from the HuggingFace Hub.
32
+ device (`str`):
33
+ The device to load the weights onto.
34
+ hf_hub_download_kwargs (`dict`):
35
+ Additional arguments to pass to the `hf_hub_download` method when
36
+ loading from the HuggingFace Hub.
37
+ """
38
+ path = (os.path.join(model_id, hf_hub_download_kwargs["subfolder"])
39
+ if hf_hub_download_kwargs.get("subfolder", None) is not None else
40
+ model_id)
41
+
42
+ if device is None:
43
+ device = infer_device()
44
+
45
+ if os.path.exists(os.path.join(path, SAFETENSORS_WEIGHTS_NAME)):
46
+ filename = os.path.join(path, SAFETENSORS_WEIGHTS_NAME)
47
+ use_safetensors = True
48
+ elif os.path.exists(os.path.join(path, WEIGHTS_NAME)):
49
+ filename = os.path.join(path, WEIGHTS_NAME)
50
+ use_safetensors = False
51
+ else:
52
+ token = hf_hub_download_kwargs.get("token", None)
53
+ if token is None:
54
+ token = hf_hub_download_kwargs.get("use_auth_token", None)
55
+
56
+ hub_filename = (os.path.join(hf_hub_download_kwargs["subfolder"],
57
+ SAFETENSORS_WEIGHTS_NAME)
58
+ if hf_hub_download_kwargs.get("subfolder", None)
59
+ is not None else SAFETENSORS_WEIGHTS_NAME)
60
+ has_remote_safetensors_file = file_exists(
61
+ repo_id=model_id,
62
+ filename=hub_filename,
63
+ revision=hf_hub_download_kwargs.get("revision", None),
64
+ repo_type=hf_hub_download_kwargs.get("repo_type", None),
65
+ token=token,
66
+ )
67
+ use_safetensors = has_remote_safetensors_file
68
+
69
+ if has_remote_safetensors_file:
70
+ # Priority 1: load safetensors weights
71
+ filename = hf_hub_download(
72
+ model_id,
73
+ SAFETENSORS_WEIGHTS_NAME,
74
+ **hf_hub_download_kwargs,
75
+ )
76
+ else:
77
+ try:
78
+ filename = hf_hub_download(model_id, WEIGHTS_NAME,
79
+ **hf_hub_download_kwargs)
80
+ except EntryNotFoundError:
81
+ raise ValueError( # noqa: B904
82
+ f"Can't find weights for {model_id} in {model_id} or \
83
+ in the Hugging Face Hub. "
84
+ f"Please check that the file {WEIGHTS_NAME} or \
85
+ {SAFETENSORS_WEIGHTS_NAME} is present at {model_id}.")
86
+
87
+ if use_safetensors:
88
+ adapters_weights = safe_load_file(filename, device=device)
89
+ else:
90
+ adapters_weights = torch.load(filename,
91
+ map_location=torch.device(device))
92
+
93
+ return adapters_weights