@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d14e98d924
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model] Support GGUF models newly added in `transformers` 4.46.0 (#9685)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Jan 13, 2025 at 08:13 UTC d14e98d924724b284dc5eaf8070d935e214e50c0
7 files changed +162 -87
examples/offline_inference/gguf_inference.py
+8 -14
@@ -3,27 +3,20 @@ from huggingface_hub import hf_hub_download
3 from vllm import LLM, SamplingParams
4
5
6 -def run_gguf_inference(model_path):
7 - PROMPT_TEMPLATE = "<|system|>\n{system_message}</s>\n<|user|>\n{prompt}</s>\n<|assistant|>\n" # noqa: E501
8 - system_message = "You are a friendly chatbot who always responds in the style of a pirate." # noqa: E501
6 +def run_gguf_inference(model_path, tokenizer):
7 # Sample prompts.
8 prompts = [
9 "How many helicopters can a human eat in one sitting?",
10 "What's the future of AI?",
11 ]
14 - prompts = [
15 - PROMPT_TEMPLATE.format(system_message=system_message, prompt=prompt)
16 - for prompt in prompts
17 - ]
12 + prompts = [[{"role": "user", "content": prompt}] for prompt in prompts]
13 # Create a sampling params object.
14 sampling_params = SamplingParams(temperature=0, max_tokens=128)
15
16 # Create an LLM.
22 - llm = LLM(model=model_path,
23 - tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
24 - gpu_memory_utilization=0.95)
17 + llm = LLM(model=model_path, tokenizer=tokenizer)
18
26 - outputs = llm.generate(prompts, sampling_params)
19 + outputs = llm.chat(prompts, sampling_params)
20 # Print the outputs.
21 for output in outputs:
22 prompt = output.prompt
@@ -32,7 +25,8 @@ def run_gguf_inference(model_path):
25
26
27 if __name__ == "__main__":
35 - repo_id = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
36 - filename = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
28 + repo_id = "bartowski/Phi-3-medium-4k-instruct-GGUF"
29 + filename = "Phi-3-medium-4k-instruct-IQ2_M.gguf"
30 + tokenizer = "microsoft/Phi-3-medium-4k-instruct"
31 model = hf_hub_download(repo_id, filename=filename)
38 - run_gguf_inference(model)
32 + run_gguf_inference(model, tokenizer)
tests/models/decoder_only/language/test_gguf.py
+74 -31
@@ -4,6 +4,7 @@ Note: To pass the test, quantization higher than Q4 should be used
4 """
5
6 import os
7 +from typing import List, NamedTuple, Type
8
9 import pytest
10 from huggingface_hub import hf_hub_download
@@ -11,6 +12,7 @@ from transformers import AutoTokenizer
12
13 from tests.quantization.utils import is_quant_method_supported
14
15 +from ....conftest import VllmRunner
16 from ...utils import check_logprobs_close
17
18 os.environ["TOKENIZERS_PARALLELISM"] = "true"
@@ -18,31 +20,74 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
20 MAX_MODEL_LEN = 1024
21
22
23 +class GGUFTestConfig(NamedTuple):
24 + original_model: str
25 + gguf_repo: str
26 + gguf_filename: str
27 +
28 + @property
29 + def gguf_model(self):
30 + return hf_hub_download(self.gguf_repo, filename=self.gguf_filename)
31 +
32 +
33 +LLAMA_CONFIG = GGUFTestConfig(
34 + original_model="meta-llama/Llama-3.2-1B-Instruct",
35 + gguf_repo="bartowski/Llama-3.2-1B-Instruct-GGUF",
36 + gguf_filename="Llama-3.2-1B-Instruct-IQ4_XS.gguf",
37 +)
38 +
39 +QWEN2_CONFIG = GGUFTestConfig(
40 + original_model="Qwen/Qwen2.5-1.5B-Instruct",
41 + gguf_repo="Qwen/Qwen2.5-1.5B-Instruct-GGUF",
42 + gguf_filename="qwen2.5-1.5b-instruct-q6_k.gguf",
43 +)
44 +
45 +PHI3_CONFIG = GGUFTestConfig(
46 + original_model="microsoft/Phi-3.5-mini-instruct",
47 + gguf_repo="bartowski/Phi-3.5-mini-instruct-GGUF",
48 + gguf_filename="Phi-3.5-mini-instruct-IQ4_XS.gguf",
49 +)
50 +
51 +GPT2_CONFIG = GGUFTestConfig(
52 + original_model="openai-community/gpt2-large",
53 + gguf_repo="QuantFactory/gpt2-large-GGUF",
54 + gguf_filename="gpt2-large.Q4_K_M.gguf",
55 +)
56 +
57 +STABLELM_CONFIG = GGUFTestConfig(
58 + original_model="stabilityai/stablelm-3b-4e1t",
59 + gguf_repo="afrideva/stablelm-3b-4e1t-GGUF",
60 + gguf_filename="stablelm-3b-4e1t.q4_k_m.gguf",
61 +)
62 +
63 +STARCODER_CONFIG = GGUFTestConfig(
64 + original_model="bigcode/starcoder2-3b",
65 + gguf_repo="QuantFactory/starcoder2-3b-GGUF",
66 + gguf_filename="starcoder2-3b.Q6_K.gguf",
67 +)
68 +
69 +MODELS = [
70 + LLAMA_CONFIG,
71 + QWEN2_CONFIG,
72 + PHI3_CONFIG,
73 + GPT2_CONFIG,
74 + STABLELM_CONFIG,
75 + # STARCODER_CONFIG, # broken
76 +]
77 +
78 +
79 @pytest.mark.skipif(not is_quant_method_supported("gguf"),
80 reason="gguf is not supported on this GPU type.")
23 -@pytest.mark.parametrize(("original_model", "gguf_id", "gguf_path"), [
24 - ("meta-llama/Llama-3.2-1B-Instruct",
25 - "bartowski/Llama-3.2-1B-Instruct-GGUF",
26 - "Llama-3.2-1B-Instruct-Q4_K_M.gguf"),
27 - ("meta-llama/Llama-3.2-1B-Instruct",
28 - "bartowski/Llama-3.2-1B-Instruct-GGUF",
29 - "Llama-3.2-1B-Instruct-IQ4_XS.gguf"),
30 - ("Qwen/Qwen2-1.5B-Instruct", "Qwen/Qwen2-1.5B-Instruct-GGUF",
31 - "qwen2-1_5b-instruct-q4_k_m.gguf"),
32 - ("Qwen/Qwen2-1.5B-Instruct", "legraphista/Qwen2-1.5B-Instruct-IMat-GGUF",
33 - "Qwen2-1.5B-Instruct.IQ4_XS.gguf"),
34 -])
81 +@pytest.mark.parametrize("model", MODELS)
82 @pytest.mark.parametrize("dtype", ["half"])
83 @pytest.mark.parametrize("max_tokens", [32])
84 @pytest.mark.parametrize("num_logprobs", [5])
85 @pytest.mark.parametrize("tp_size", [1, 2])
86 def test_models(
40 - num_gpus_available,
41 - vllm_runner,
42 - example_prompts,
43 - original_model,
44 - gguf_id,
45 - gguf_path,
87 + num_gpus_available: int,
88 + vllm_runner: Type[VllmRunner],
89 + example_prompts: List[str],
90 + model: GGUFTestConfig,
91 dtype: str,
92 max_tokens: int,
93 num_logprobs: int,
@@ -51,28 +96,26 @@ def test_models(
96 if num_gpus_available < tp_size:
97 pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
98
54 - gguf_model = hf_hub_download(gguf_id, filename=gguf_path)
55 -
56 - tokenizer = AutoTokenizer.from_pretrained(original_model)
57 - messages = [[{
58 - 'role': 'user',
59 - 'content': prompt
60 - }] for prompt in example_prompts]
61 - example_prompts = tokenizer.apply_chat_template(messages,
62 - tokenize=False,
63 - add_generation_prompt=True)
99 + tokenizer = AutoTokenizer.from_pretrained(model.original_model)
100 + if tokenizer.chat_template is not None:
101 + messages = [[{
102 + 'role': 'user',
103 + 'content': prompt
104 + }] for prompt in example_prompts]
105 + example_prompts = tokenizer.apply_chat_template(
106 + messages, tokenize=False, add_generation_prompt=True)
107
108 # Run unquantized model.
66 - with vllm_runner(model_name=original_model,
109 + with vllm_runner(model_name=model.original_model,
110 dtype=dtype,
111 max_model_len=MAX_MODEL_LEN,
112 tensor_parallel_size=tp_size) as original_model:
70 -
113 original_outputs = original_model.generate_greedy_logprobs(
114 example_prompts[:-1], max_tokens, num_logprobs)
115
116 # Run gguf model.
75 - with vllm_runner(model_name=gguf_model,
117 + with vllm_runner(model_name=model.gguf_model,
118 + tokenizer_name=model.original_model,
119 dtype=dtype,
120 max_model_len=MAX_MODEL_LEN,
121 tensor_parallel_size=tp_size) as gguf_model:
vllm/model_executor/layers/linear.py
+35 -23
@@ -447,8 +447,14 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
447 is_gguf_weight = getattr(param, "is_gguf_weight", False)
448 is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
449 if is_gguf_weight_type:
450 - param.data[loaded_shard_id].copy_(loaded_weight)
451 - param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
450 + if loaded_shard_id is not None:
451 + param.data[loaded_shard_id].copy_(loaded_weight)
452 + param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
453 + else:
454 + param.shard_weight_type = {
455 + i: loaded_weight.item()
456 + for i, _ in enumerate(self.output_sizes)
457 + }
458 return
459
460 if is_gguf_weight:
@@ -459,15 +465,15 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
465 shard_size = loaded_weight.size(output_dim) // tp_size
466 start_idx = tp_rank * shard_size
467
462 - loaded_weight = loaded_weight.narrow(output_dim, start_idx,
463 - shard_size)
464 -
465 - param.shard_id.append(loaded_shard_id)
466 - param.shard_id_map[loaded_shard_id] = len(param.data_container)
467 - param.data_container.append(loaded_weight)
468 - if len(param.data_container) == 2:
469 - self.qweight = param.materialize_nested()
470 - return
468 + if loaded_shard_id is not None:
469 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
470 + shard_size)
471 + param.shard_id.append(loaded_shard_id)
472 + param.shard_id_map[loaded_shard_id] = len(param.data_container)
473 + param.data_container.append(loaded_weight)
474 + if len(param.data_container) == 2:
475 + self.qweight = param.materialize_nested()
476 + return
477
478 param_data = param.data
479 output_dim = getattr(param, "output_dim", None)
@@ -811,10 +817,16 @@ class QKVParallelLinear(ColumnParallelLinear):
817 # initialize GGUF param after we know the quantize type
818 is_gguf_weight = getattr(param, "is_gguf_weight", False)
819 is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
814 - if is_gguf_weight_type and loaded_shard_id is not None:
820 + if is_gguf_weight_type:
821 idx_map = {"q": 0, "k": 1, "v": 2}
816 - param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
817 - param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
822 + if loaded_shard_id is not None:
823 + param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
824 + param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
825 + else:
826 + param.shard_weight_type = {
827 + k: loaded_weight.item()
828 + for k in idx_map
829 + }
830 return
831
832 if is_gguf_weight:
@@ -825,15 +837,15 @@ class QKVParallelLinear(ColumnParallelLinear):
837 shard_size = loaded_weight.size(output_dim) // tp_size
838 start_idx = tp_rank * shard_size
839
828 - loaded_weight = loaded_weight.narrow(output_dim, start_idx,
829 - shard_size)
830 -
831 - param.shard_id.append(loaded_shard_id)
832 - param.shard_id_map[loaded_shard_id] = len(param.data_container)
833 - param.data_container.append(loaded_weight)
834 - if len(param.data_container) == 3:
835 - self.qweight = param.materialize_nested()
836 - return
840 + if loaded_shard_id is not None:
841 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
842 + shard_size)
843 + param.shard_id.append(loaded_shard_id)
844 + param.shard_id_map[loaded_shard_id] = len(param.data_container)
845 + param.data_container.append(loaded_weight)
846 + if len(param.data_container) == 3:
847 + self.qweight = param.materialize_nested()
848 + return
849
850 param_data = param.data
851 output_dim = getattr(param, "output_dim", None)
vllm/model_executor/models/gpt2.py
+8 -3
@@ -198,7 +198,10 @@ class GPT2Model(nn.Module):
198 assert not config.scale_attn_by_inverse_layer_idx
199 assert not config.reorder_and_upcast_attn
200 self.embed_dim = config.hidden_size
201 - self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
201 + self.wte = VocabParallelEmbedding(config.vocab_size,
202 + self.embed_dim,
203 + quant_config=quant_config,
204 + prefix=f"{prefix}.wte")
205 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
206 self.start_layer, self.end_layer, self.h = make_layers(
207 config.num_hidden_layers,
@@ -259,7 +262,9 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
262 self.lm_head = self.transformer.wte
263 else:
264 self.lm_head = ParallelLMHead(self.config.vocab_size,
262 - self.config.hidden_size)
265 + self.config.hidden_size,
266 + quant_config=quant_config,
267 + prefix=f"{prefix}.lm_head")
268 self.logits_processor = LogitsProcessor(config.vocab_size)
269 self.sampler = get_sampler()
270 self.make_empty_intermediate_tensors = (
@@ -304,7 +309,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
309 params_dict = dict(self.named_parameters(remove_duplicate=False))
310 loaded_params: Set[str] = set()
311 for name, loaded_weight in weights:
307 - if "lm_head.weight" in name:
312 + if name.startswith("lm_head"):
313 # GPT-2 ties the weights of the embedding layer and the final
314 # linear layer.
315 continue
vllm/model_executor/models/llama.py
+2 -1
@@ -156,7 +156,8 @@ class LlamaAttention(nn.Module):
156 )
157
158 is_neox_style = True
159 - if quant_config is not None and quant_config.get_name() == "gguf":
159 + is_gguf = quant_config and quant_config.get_name() == "gguf"
160 + if is_gguf and config.model_type == "llama":
161 is_neox_style = False
162
163 self.rotary_emb = get_rope(
vllm/model_executor/models/stablelm.py
+20 -11
@@ -22,7 +22,7 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
22
23 import torch
24 from torch import nn
25 -from transformers import PretrainedConfig
25 +from transformers import StableLmConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 from vllm.config import CacheConfig, VllmConfig
@@ -50,8 +50,9 @@ from .utils import (is_pp_missing_parameter,
50 class StablelmMLP(nn.Module):
51
52 def __init__(self,
53 - config: PretrainedConfig,
54 - quant_config: Optional[QuantizationConfig] = None) -> None:
53 + config: StableLmConfig,
54 + quant_config: Optional[QuantizationConfig] = None,
55 + prefix: str = "") -> None:
56 super().__init__()
57 self.config = config
58 self.hidden_size = config.hidden_size
@@ -59,10 +60,13 @@ class StablelmMLP(nn.Module):
60 self.gate_up_proj = MergedColumnParallelLinear(
61 config.hidden_size, [config.intermediate_size] * 2,
62 bias=False,
62 - quant_config=quant_config)
63 + quant_config=quant_config,
64 + prefix=f"{prefix}.gate_up_proj")
65 self.down_proj = RowParallelLinear(config.intermediate_size,
66 config.hidden_size,
65 - bias=False)
67 + bias=False,
68 + quant_config=quant_config,
69 + prefix=f"{prefix}.down_proj")
70 self.act_fn = SiluAndMul()
71
72 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -75,7 +79,7 @@ class StablelmMLP(nn.Module):
79 class StablelmAttention(nn.Module):
80
81 def __init__(self,
78 - config: PretrainedConfig,
82 + config: StableLmConfig,
83 cache_config: Optional[CacheConfig] = None,
84 quant_config: Optional[QuantizationConfig] = None,
85 prefix: str = "") -> None:
@@ -116,11 +120,13 @@ class StablelmAttention(nn.Module):
120 self.total_num_heads,
121 self.total_num_key_value_heads,
122 self.qkv_bias,
119 - quant_config=quant_config)
123 + quant_config=quant_config,
124 + prefix=f"{prefix}.qkv_proj")
125 self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
126 self.hidden_size,
127 bias=False,
123 - quant_config=quant_config)
128 + quant_config=quant_config,
129 + prefix=f"{prefix}.o_proj")
130 self.rotary_emb = get_rope(
131 self.head_dim,
132 rotary_dim=self.rotary_ndims,
@@ -154,7 +160,7 @@ class StablelmDecoderLayer(nn.Module):
160
161 def __init__(
162 self,
157 - config: PretrainedConfig,
163 + config: StableLmConfig,
164 cache_config: Optional[CacheConfig] = None,
165 quant_config: Optional[QuantizationConfig] = None,
166 prefix: str = "",
@@ -164,7 +170,7 @@ class StablelmDecoderLayer(nn.Module):
170 cache_config,
171 quant_config,
172 prefix=f"{prefix}.self_attn")
167 - self.mlp = StablelmMLP(config, quant_config)
173 + self.mlp = StablelmMLP(config, quant_config, prefix=f"{prefix}.mlp")
174 norm_eps = getattr(config, "norm_eps",
175 getattr(config, "layer_norm_eps", 1e-05))
176 self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
@@ -210,6 +216,8 @@ class StableLMEpochModel(nn.Module):
216 self.embed_tokens = VocabParallelEmbedding(
217 config.vocab_size,
218 config.hidden_size,
219 + quant_config=quant_config,
220 + prefix=f"{prefix}.embed_tokens",
221 )
222 self.start_layer, self.end_layer, self.layers = make_layers(
223 config.num_hidden_layers,
@@ -270,7 +278,8 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
278 prefix=maybe_prefix(prefix, "model"))
279 self.lm_head = ParallelLMHead(config.vocab_size,
280 config.hidden_size,
273 - quant_config=quant_config)
281 + quant_config=quant_config,
282 + prefix=f"{prefix}.lm_head")
283 if self.config.tie_word_embeddings:
284 self.lm_head.weight = self.model.embed_tokens.weight
285 self.logits_processor = LogitsProcessor(config.vocab_size)
vllm/model_executor/models/starcoder2.py
+15 -4
@@ -88,12 +88,14 @@ class Starcoder2Attention(nn.Module):
88 self.total_num_kv_heads,
89 bias=self.use_bias,
90 quant_config=quant_config,
91 + prefix=f"{prefix}.qkv_proj",
92 )
93 self.o_proj = RowParallelLinear(
94 self.total_num_heads * self.head_dim,
95 self.hidden_size,
96 bias=self.use_bias,
97 quant_config=quant_config,
98 + prefix=f"{prefix}.o_proj",
99 )
100 self.rotary_emb = get_rope(
101 self.head_dim,
@@ -129,19 +131,22 @@ class Starcoder2MLP(nn.Module):
131
132 def __init__(self,
133 config: Starcoder2Config,
132 - quant_config: Optional[QuantizationConfig] = None):
134 + quant_config: Optional[QuantizationConfig] = None,
135 + prefix: str = ""):
136 super().__init__()
137 self.c_fc = ColumnParallelLinear(
138 config.hidden_size,
139 config.intermediate_size,
140 bias=config.use_bias,
141 quant_config=quant_config,
142 + prefix=f"{prefix}.c_fc",
143 )
144 self.c_proj = RowParallelLinear(
145 config.intermediate_size,
146 config.hidden_size,
147 bias=config.use_bias,
148 quant_config=quant_config,
149 + prefix=f"{prefix}.c_proj",
150 )
151 self.act = get_act_fn(config.hidden_act)
152
@@ -165,7 +170,9 @@ class Starcoder2DecoderLayer(nn.Module):
170 cache_config,
171 quant_config=quant_config,
172 prefix=f"{prefix}.self_attn")
168 - self.mlp = Starcoder2MLP(config, quant_config=quant_config)
173 + self.mlp = Starcoder2MLP(config,
174 + quant_config=quant_config,
175 + prefix=f"{prefix}.mlp")
176 self.input_layernorm = nn.LayerNorm(config.hidden_size,
177 eps=config.norm_epsilon)
178 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
@@ -213,8 +220,11 @@ class Starcoder2Model(nn.Module):
220 self.vocab_size = config.vocab_size
221
222 # TODO: consider padding_idx (currently removed)
216 - self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
217 - config.hidden_size)
223 + self.embed_tokens = VocabParallelEmbedding(
224 + config.vocab_size,
225 + config.hidden_size,
226 + quant_config=quant_config,
227 + prefix=f"{prefix}.embed_tokens")
228 self.start_layer, self.end_layer, self.layers = make_layers(
229 config.num_hidden_layers,
230 lambda prefix: Starcoder2DecoderLayer(
@@ -279,6 +289,7 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
289 org_num_embeddings=config.vocab_size,
290 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
291 quant_config=quant_config,
292 + prefix=f"{prefix}.lm_head",
293 )
294 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
295 config.vocab_size)