A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Model] Support GGUF models newly added in `transformers` 4.46.0 (#9685)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Jan 13, 2025 at 08:13 UTC
d14e98d924724b284dc5eaf8070d935e214e50c0
7 files changed
+162
-87
examples/offline_inference/gguf_inference.py
+8
-14
@@ -3,27 +3,20 @@ from huggingface_hub import hf_hub_download
3
from vllm import LLM, SamplingParams
4
5
6
-def run_gguf_inference(model_path):
7
- PROMPT_TEMPLATE = "<|system|>\n{system_message}</s>\n<|user|>\n{prompt}</s>\n<|assistant|>\n" # noqa: E501
8
- system_message = "You are a friendly chatbot who always responds in the style of a pirate." # noqa: E501
6
+def run_gguf_inference(model_path, tokenizer):
7
# Sample prompts.
8
prompts = [
9
"How many helicopters can a human eat in one sitting?",
10
"What's the future of AI?",
11
]
14
- prompts = [
15
- PROMPT_TEMPLATE.format(system_message=system_message, prompt=prompt)
16
- for prompt in prompts
17
- ]
12
+ prompts = [[{"role": "user", "content": prompt}] for prompt in prompts]
13
# Create a sampling params object.
14
sampling_params = SamplingParams(temperature=0, max_tokens=128)
15
16
# Create an LLM.
22
- llm = LLM(model=model_path,
23
- tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
24
- gpu_memory_utilization=0.95)
17
+ llm = LLM(model=model_path, tokenizer=tokenizer)
18
26
- outputs = llm.generate(prompts, sampling_params)
19
+ outputs = llm.chat(prompts, sampling_params)
20
# Print the outputs.
21
for output in outputs:
22
prompt = output.prompt
@@ -32,7 +25,8 @@ def run_gguf_inference(model_path):
25
26
27
if __name__ == "__main__":
35
- repo_id = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
36
- filename = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
28
+ repo_id = "bartowski/Phi-3-medium-4k-instruct-GGUF"
29
+ filename = "Phi-3-medium-4k-instruct-IQ2_M.gguf"
30
+ tokenizer = "microsoft/Phi-3-medium-4k-instruct"
31
model = hf_hub_download(repo_id, filename=filename)
38
- run_gguf_inference(model)
32
+ run_gguf_inference(model, tokenizer)
tests/models/decoder_only/language/test_gguf.py
+74
-31
@@ -4,6 +4,7 @@ Note: To pass the test, quantization higher than Q4 should be used
4
"""
5
6
import os
7
+from typing import List, NamedTuple, Type
8
9
import pytest
10
from huggingface_hub import hf_hub_download
@@ -11,6 +12,7 @@ from transformers import AutoTokenizer
12
13
from tests.quantization.utils import is_quant_method_supported
14
15
+from ....conftest import VllmRunner
16
from ...utils import check_logprobs_close
17
18
os.environ["TOKENIZERS_PARALLELISM"] = "true"
@@ -18,31 +20,74 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
20
MAX_MODEL_LEN = 1024
21
22
23
+class GGUFTestConfig(NamedTuple):
24
+ original_model: str
25
+ gguf_repo: str
26
+ gguf_filename: str
27
+
28
+ @property
29
+ def gguf_model(self):
30
+ return hf_hub_download(self.gguf_repo, filename=self.gguf_filename)
31
+
32
+
33
+LLAMA_CONFIG = GGUFTestConfig(
34
+ original_model="meta-llama/Llama-3.2-1B-Instruct",
35
+ gguf_repo="bartowski/Llama-3.2-1B-Instruct-GGUF",
36
+ gguf_filename="Llama-3.2-1B-Instruct-IQ4_XS.gguf",
37
+)
38
+
39
+QWEN2_CONFIG = GGUFTestConfig(
40
+ original_model="Qwen/Qwen2.5-1.5B-Instruct",
41
+ gguf_repo="Qwen/Qwen2.5-1.5B-Instruct-GGUF",
42
+ gguf_filename="qwen2.5-1.5b-instruct-q6_k.gguf",
43
+)
44
+
45
+PHI3_CONFIG = GGUFTestConfig(
46
+ original_model="microsoft/Phi-3.5-mini-instruct",
47
+ gguf_repo="bartowski/Phi-3.5-mini-instruct-GGUF",
48
+ gguf_filename="Phi-3.5-mini-instruct-IQ4_XS.gguf",
49
+)
50
+
51
+GPT2_CONFIG = GGUFTestConfig(
52
+ original_model="openai-community/gpt2-large",
53
+ gguf_repo="QuantFactory/gpt2-large-GGUF",
54
+ gguf_filename="gpt2-large.Q4_K_M.gguf",
55
+)
56
+
57
+STABLELM_CONFIG = GGUFTestConfig(
58
+ original_model="stabilityai/stablelm-3b-4e1t",
59
+ gguf_repo="afrideva/stablelm-3b-4e1t-GGUF",
60
+ gguf_filename="stablelm-3b-4e1t.q4_k_m.gguf",
61
+)
62
+
63
+STARCODER_CONFIG = GGUFTestConfig(
64
+ original_model="bigcode/starcoder2-3b",
65
+ gguf_repo="QuantFactory/starcoder2-3b-GGUF",
66
+ gguf_filename="starcoder2-3b.Q6_K.gguf",
67
+)
68
+
69
+MODELS = [
70
+ LLAMA_CONFIG,
71
+ QWEN2_CONFIG,
72
+ PHI3_CONFIG,
73
+ GPT2_CONFIG,
74
+ STABLELM_CONFIG,
75
+ # STARCODER_CONFIG, # broken
76
+]
77
+
78
+
79
@pytest.mark.skipif(not is_quant_method_supported("gguf"),
80
reason="gguf is not supported on this GPU type.")
23
-@pytest.mark.parametrize(("original_model", "gguf_id", "gguf_path"), [
24
- ("meta-llama/Llama-3.2-1B-Instruct",
25
- "bartowski/Llama-3.2-1B-Instruct-GGUF",
26
- "Llama-3.2-1B-Instruct-Q4_K_M.gguf"),
27
- ("meta-llama/Llama-3.2-1B-Instruct",
28
- "bartowski/Llama-3.2-1B-Instruct-GGUF",
29
- "Llama-3.2-1B-Instruct-IQ4_XS.gguf"),
30
- ("Qwen/Qwen2-1.5B-Instruct", "Qwen/Qwen2-1.5B-Instruct-GGUF",
31
- "qwen2-1_5b-instruct-q4_k_m.gguf"),
32
- ("Qwen/Qwen2-1.5B-Instruct", "legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
33
- "Qwen2-1.5B-Instruct.IQ4_XS.gguf"),
34
-])
81
+@pytest.mark.parametrize("model", MODELS)
82
@pytest.mark.parametrize("dtype", ["half"])
83
@pytest.mark.parametrize("max_tokens", [32])
84
@pytest.mark.parametrize("num_logprobs", [5])
85
@pytest.mark.parametrize("tp_size", [1, 2])
86
def test_models(
40
- num_gpus_available,
41
- vllm_runner,
42
- example_prompts,
43
- original_model,
44
- gguf_id,
45
- gguf_path,
87
+ num_gpus_available: int,
88
+ vllm_runner: Type[VllmRunner],
89
+ example_prompts: List[str],
90
+ model: GGUFTestConfig,
91
dtype: str,
92
max_tokens: int,
93
num_logprobs: int,
@@ -51,28 +96,26 @@ def test_models(
96
if num_gpus_available < tp_size:
97
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
98
54
- gguf_model = hf_hub_download(gguf_id, filename=gguf_path)
55
-
56
- tokenizer = AutoTokenizer.from_pretrained(original_model)
57
- messages = [[{
58
- 'role': 'user',
59
- 'content': prompt
60
- }] for prompt in example_prompts]
61
- example_prompts = tokenizer.apply_chat_template(messages,
62
- tokenize=False,
63
- add_generation_prompt=True)
99
+ tokenizer = AutoTokenizer.from_pretrained(model.original_model)
100
+ if tokenizer.chat_template is not None:
101
+ messages = [[{
102
+ 'role': 'user',
103
+ 'content': prompt
104
+ }] for prompt in example_prompts]
105
+ example_prompts = tokenizer.apply_chat_template(
106
+ messages, tokenize=False, add_generation_prompt=True)
107
108
# Run unquantized model.
66
- with vllm_runner(model_name=original_model,
109
+ with vllm_runner(model_name=model.original_model,
110
dtype=dtype,
111
max_model_len=MAX_MODEL_LEN,
112
tensor_parallel_size=tp_size) as original_model:
70
-
113
original_outputs = original_model.generate_greedy_logprobs(
114
example_prompts[:-1], max_tokens, num_logprobs)
115
116
# Run gguf model.
75
- with vllm_runner(model_name=gguf_model,
117
+ with vllm_runner(model_name=model.gguf_model,
118
+ tokenizer_name=model.original_model,
119
dtype=dtype,
120
max_model_len=MAX_MODEL_LEN,
121
tensor_parallel_size=tp_size) as gguf_model:
vllm/model_executor/layers/linear.py
+35
-23
@@ -447,8 +447,14 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
447
is_gguf_weight = getattr(param, "is_gguf_weight", False)
448
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
449
if is_gguf_weight_type:
450
- param.data[loaded_shard_id].copy_(loaded_weight)
451
- param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
450
+ if loaded_shard_id is not None:
451
+ param.data[loaded_shard_id].copy_(loaded_weight)
452
+ param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
453
+ else:
454
+ param.shard_weight_type = {
455
+ i: loaded_weight.item()
456
+ for i, _ in enumerate(self.output_sizes)
457
+ }
458
return
459
460
if is_gguf_weight:
@@ -459,15 +465,15 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
465
shard_size = loaded_weight.size(output_dim) // tp_size
466
start_idx = tp_rank * shard_size
467
462
- loaded_weight = loaded_weight.narrow(output_dim, start_idx,
463
- shard_size)
464
-
465
- param.shard_id.append(loaded_shard_id)
466
- param.shard_id_map[loaded_shard_id] = len(param.data_container)
467
- param.data_container.append(loaded_weight)
468
- if len(param.data_container) == 2:
469
- self.qweight = param.materialize_nested()
470
- return
468
+ if loaded_shard_id is not None:
469
+ loaded_weight = loaded_weight.narrow(output_dim, start_idx,
470
+ shard_size)
471
+ param.shard_id.append(loaded_shard_id)
472
+ param.shard_id_map[loaded_shard_id] = len(param.data_container)
473
+ param.data_container.append(loaded_weight)
474
+ if len(param.data_container) == 2:
475
+ self.qweight = param.materialize_nested()
476
+ return
477
478
param_data = param.data
479
output_dim = getattr(param, "output_dim", None)
@@ -811,10 +817,16 @@ class QKVParallelLinear(ColumnParallelLinear):
817
# initialize GGUF param after we know the quantize type
818
is_gguf_weight = getattr(param, "is_gguf_weight", False)
819
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
814
- if is_gguf_weight_type and loaded_shard_id is not None:
820
+ if is_gguf_weight_type:
821
idx_map = {"q": 0, "k": 1, "v": 2}
816
- param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
817
- param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
822
+ if loaded_shard_id is not None:
823
+ param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
824
+ param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
825
+ else:
826
+ param.shard_weight_type = {
827
+ k: loaded_weight.item()
828
+ for k in idx_map
829
+ }
830
return
831
832
if is_gguf_weight:
@@ -825,15 +837,15 @@ class QKVParallelLinear(ColumnParallelLinear):
837
shard_size = loaded_weight.size(output_dim) // tp_size
838
start_idx = tp_rank * shard_size
839
828
- loaded_weight = loaded_weight.narrow(output_dim, start_idx,
829
- shard_size)
830
-
831
- param.shard_id.append(loaded_shard_id)
832
- param.shard_id_map[loaded_shard_id] = len(param.data_container)
833
- param.data_container.append(loaded_weight)
834
- if len(param.data_container) == 3:
835
- self.qweight = param.materialize_nested()
836
- return
840
+ if loaded_shard_id is not None:
841
+ loaded_weight = loaded_weight.narrow(output_dim, start_idx,
842
+ shard_size)
843
+ param.shard_id.append(loaded_shard_id)
844
+ param.shard_id_map[loaded_shard_id] = len(param.data_container)
845
+ param.data_container.append(loaded_weight)
846
+ if len(param.data_container) == 3:
847
+ self.qweight = param.materialize_nested()
848
+ return
849
850
param_data = param.data
851
output_dim = getattr(param, "output_dim", None)
vllm/model_executor/models/gpt2.py
+8
-3
@@ -198,7 +198,10 @@ class GPT2Model(nn.Module):
198
assert not config.scale_attn_by_inverse_layer_idx
199
assert not config.reorder_and_upcast_attn
200
self.embed_dim = config.hidden_size
201
- self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
201
+ self.wte = VocabParallelEmbedding(config.vocab_size,
202
+ self.embed_dim,
203
+ quant_config=quant_config,
204
+ prefix=f"{prefix}.wte")
205
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
206
self.start_layer, self.end_layer, self.h = make_layers(
207
config.num_hidden_layers,
@@ -259,7 +262,9 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
262
self.lm_head = self.transformer.wte
263
else:
264
self.lm_head = ParallelLMHead(self.config.vocab_size,
262
- self.config.hidden_size)
265
+ self.config.hidden_size,
266
+ quant_config=quant_config,
267
+ prefix=f"{prefix}.lm_head")
268
self.logits_processor = LogitsProcessor(config.vocab_size)
269
self.sampler = get_sampler()
270
self.make_empty_intermediate_tensors = (
@@ -304,7 +309,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
309
params_dict = dict(self.named_parameters(remove_duplicate=False))
310
loaded_params: Set[str] = set()
311
for name, loaded_weight in weights:
307
- if "lm_head.weight" in name:
312
+ if name.startswith("lm_head"):
313
# GPT-2 ties the weights of the embedding layer and the final
314
# linear layer.
315
continue
vllm/model_executor/models/llama.py
+2
-1
@@ -156,7 +156,8 @@ class LlamaAttention(nn.Module):
156
)
157
158
is_neox_style = True
159
- if quant_config is not None and quant_config.get_name() == "gguf":
159
+ is_gguf = quant_config and quant_config.get_name() == "gguf"
160
+ if is_gguf and config.model_type == "llama":
161
is_neox_style = False
162
163
self.rotary_emb = get_rope(
vllm/model_executor/models/stablelm.py
+20
-11
@@ -22,7 +22,7 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
22
23
import torch
24
from torch import nn
25
-from transformers import PretrainedConfig
25
+from transformers import StableLmConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
from vllm.config import CacheConfig, VllmConfig
@@ -50,8 +50,9 @@ from .utils import (is_pp_missing_parameter,
50
class StablelmMLP(nn.Module):
51
52
def __init__(self,
53
- config: PretrainedConfig,
54
- quant_config: Optional[QuantizationConfig] = None) -> None:
53
+ config: StableLmConfig,
54
+ quant_config: Optional[QuantizationConfig] = None,
55
+ prefix: str = "") -> None:
56
super().__init__()
57
self.config = config
58
self.hidden_size = config.hidden_size
@@ -59,10 +60,13 @@ class StablelmMLP(nn.Module):
60
self.gate_up_proj = MergedColumnParallelLinear(
61
config.hidden_size, [config.intermediate_size] * 2,
62
bias=False,
62
- quant_config=quant_config)
63
+ quant_config=quant_config,
64
+ prefix=f"{prefix}.gate_up_proj")
65
self.down_proj = RowParallelLinear(config.intermediate_size,
66
config.hidden_size,
65
- bias=False)
67
+ bias=False,
68
+ quant_config=quant_config,
69
+ prefix=f"{prefix}.down_proj")
70
self.act_fn = SiluAndMul()
71
72
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -75,7 +79,7 @@ class StablelmMLP(nn.Module):
79
class StablelmAttention(nn.Module):
80
81
def __init__(self,
78
- config: PretrainedConfig,
82
+ config: StableLmConfig,
83
cache_config: Optional[CacheConfig] = None,
84
quant_config: Optional[QuantizationConfig] = None,
85
prefix: str = "") -> None:
@@ -116,11 +120,13 @@ class StablelmAttention(nn.Module):
120
self.total_num_heads,
121
self.total_num_key_value_heads,
122
self.qkv_bias,
119
- quant_config=quant_config)
123
+ quant_config=quant_config,
124
+ prefix=f"{prefix}.qkv_proj")
125
self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
126
self.hidden_size,
127
bias=False,
123
- quant_config=quant_config)
128
+ quant_config=quant_config,
129
+ prefix=f"{prefix}.o_proj")
130
self.rotary_emb = get_rope(
131
self.head_dim,
132
rotary_dim=self.rotary_ndims,
@@ -154,7 +160,7 @@ class StablelmDecoderLayer(nn.Module):
160
161
def __init__(
162
self,
157
- config: PretrainedConfig,
163
+ config: StableLmConfig,
164
cache_config: Optional[CacheConfig] = None,
165
quant_config: Optional[QuantizationConfig] = None,
166
prefix: str = "",
@@ -164,7 +170,7 @@ class StablelmDecoderLayer(nn.Module):
170
cache_config,
171
quant_config,
172
prefix=f"{prefix}.self_attn")
167
- self.mlp = StablelmMLP(config, quant_config)
173
+ self.mlp = StablelmMLP(config, quant_config, prefix=f"{prefix}.mlp")
174
norm_eps = getattr(config, "norm_eps",
175
getattr(config, "layer_norm_eps", 1e-05))
176
self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
@@ -210,6 +216,8 @@ class StableLMEpochModel(nn.Module):
216
self.embed_tokens = VocabParallelEmbedding(
217
config.vocab_size,
218
config.hidden_size,
219
+ quant_config=quant_config,
220
+ prefix=f"{prefix}.embed_tokens",
221
)
222
self.start_layer, self.end_layer, self.layers = make_layers(
223
config.num_hidden_layers,
@@ -270,7 +278,8 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
278
prefix=maybe_prefix(prefix, "model"))
279
self.lm_head = ParallelLMHead(config.vocab_size,
280
config.hidden_size,
273
- quant_config=quant_config)
281
+ quant_config=quant_config,
282
+ prefix=f"{prefix}.lm_head")
283
if self.config.tie_word_embeddings:
284
self.lm_head.weight = self.model.embed_tokens.weight
285
self.logits_processor = LogitsProcessor(config.vocab_size)
vllm/model_executor/models/starcoder2.py
+15
-4
@@ -88,12 +88,14 @@ class Starcoder2Attention(nn.Module):
88
self.total_num_kv_heads,
89
bias=self.use_bias,
90
quant_config=quant_config,
91
+ prefix=f"{prefix}.qkv_proj",
92
)
93
self.o_proj = RowParallelLinear(
94
self.total_num_heads * self.head_dim,
95
self.hidden_size,
96
bias=self.use_bias,
97
quant_config=quant_config,
98
+ prefix=f"{prefix}.o_proj",
99
)
100
self.rotary_emb = get_rope(
101
self.head_dim,
@@ -129,19 +131,22 @@ class Starcoder2MLP(nn.Module):
131
132
def __init__(self,
133
config: Starcoder2Config,
132
- quant_config: Optional[QuantizationConfig] = None):
134
+ quant_config: Optional[QuantizationConfig] = None,
135
+ prefix: str = ""):
136
super().__init__()
137
self.c_fc = ColumnParallelLinear(
138
config.hidden_size,
139
config.intermediate_size,
140
bias=config.use_bias,
141
quant_config=quant_config,
142
+ prefix=f"{prefix}.c_fc",
143
)
144
self.c_proj = RowParallelLinear(
145
config.intermediate_size,
146
config.hidden_size,
147
bias=config.use_bias,
148
quant_config=quant_config,
149
+ prefix=f"{prefix}.c_proj",
150
)
151
self.act = get_act_fn(config.hidden_act)
152
@@ -165,7 +170,9 @@ class Starcoder2DecoderLayer(nn.Module):
170
cache_config,
171
quant_config=quant_config,
172
prefix=f"{prefix}.self_attn")
168
- self.mlp = Starcoder2MLP(config, quant_config=quant_config)
173
+ self.mlp = Starcoder2MLP(config,
174
+ quant_config=quant_config,
175
+ prefix=f"{prefix}.mlp")
176
self.input_layernorm = nn.LayerNorm(config.hidden_size,
177
eps=config.norm_epsilon)
178
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
@@ -213,8 +220,11 @@ class Starcoder2Model(nn.Module):
220
self.vocab_size = config.vocab_size
221
222
# TODO: consider padding_idx (currently removed)
216
- self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
217
- config.hidden_size)
223
+ self.embed_tokens = VocabParallelEmbedding(
224
+ config.vocab_size,
225
+ config.hidden_size,
226
+ quant_config=quant_config,
227
+ prefix=f"{prefix}.embed_tokens")
228
self.start_layer, self.end_layer, self.layers = make_layers(
229
config.num_hidden_layers,
230
lambda prefix: Starcoder2DecoderLayer(
@@ -279,6 +289,7 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
289
org_num_embeddings=config.vocab_size,
290
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
291
quant_config=quant_config,
292
+ prefix=f"{prefix}.lm_head",
293
)
294
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
295
config.vocab_size)