A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model] Add AWQ quantization support for InternVL2 model (#7187)
Isotr0py committed
Aug 21, 2024 at 14:18 UTC
12e1c65bc937573f19ea53e51c18294611ba33d7
4 files changed
+123
-25
tests/models/test_internvl.py
+102
-1
@@ -1,5 +1,5 @@
1
import types
2
-from typing import List, Optional, Type
2
+from typing import List, Optional, Tuple, Type
3
4
import pytest
5
import torch
@@ -178,6 +178,74 @@ def run_test(
178
)
179
180
181
+def run_awq_test(
182
+ vllm_runner: Type[VllmRunner],
183
+ image_assets: _ImageAssets,
184
+ models: Tuple[str, str],
185
+ *,
186
+ size_factors: List[float],
187
+ dtype: str,
188
+ max_tokens: int,
189
+ num_logprobs: int,
190
+ tensor_parallel_size: int,
191
+ distributed_executor_backend: Optional[str] = None,
192
+):
193
+ source_model, quant_model = models
194
+
195
+ images = [asset.pil_image for asset in image_assets]
196
+
197
+ inputs_per_image = [(
198
+ [prompt for _ in size_factors],
199
+ [rescale_image_size(image, factor) for factor in size_factors],
200
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
201
+
202
+ # NOTE: take care of the order. run vLLM first, and then run HF.
203
+ # vLLM needs a fresh new process without cuda initialization.
204
+ # if we run HF first, the cuda initialization will be done and it
205
+ # will hurt multiprocessing backend with fork method (the default method).
206
+
207
+ # max_model_len should be greater than image_feature_size
208
+ with vllm_runner(source_model,
209
+ max_model_len=4096,
210
+ dtype=dtype,
211
+ tensor_parallel_size=tensor_parallel_size,
212
+ distributed_executor_backend=distributed_executor_backend,
213
+ enforce_eager=True) as vllm_model:
214
+ source_outputs_per_image = [
215
+ vllm_model.generate_greedy_logprobs(prompts,
216
+ max_tokens,
217
+ num_logprobs=num_logprobs,
218
+ images=images)
219
+ for prompts, images in inputs_per_image
220
+ ]
221
+
222
+ with vllm_runner(quant_model,
223
+ quantization="awq",
224
+ max_model_len=4096,
225
+ dtype=dtype,
226
+ tensor_parallel_size=tensor_parallel_size,
227
+ distributed_executor_backend=distributed_executor_backend,
228
+ enforce_eager=True) as vllm_model:
229
+ quant_outputs_per_image = [
230
+ vllm_model.generate_greedy_logprobs(prompts,
231
+ max_tokens,
232
+ num_logprobs=num_logprobs,
233
+ images=images)
234
+ for prompts, images in inputs_per_image
235
+ ]
236
+
237
+ for source_outputs, quant_outputs in zip(source_outputs_per_image,
238
+ quant_outputs_per_image):
239
+ # TODO: Check whether using original CLIPVisionModel can improve
240
+ # consistency against HF
241
+ check_logprobs_close(
242
+ outputs_0_lst=source_outputs,
243
+ outputs_1_lst=quant_outputs,
244
+ name_0="source",
245
+ name_1="awq",
246
+ )
247
+
248
+
249
target_dtype = "half"
250
if is_cpu():
251
target_dtype = "bfloat16"
@@ -214,3 +282,36 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
282
num_logprobs=num_logprobs,
283
tensor_parallel_size=1,
284
)
285
+
286
+
287
+@pytest.mark.parametrize(
288
+ "models", [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")])
289
+@pytest.mark.parametrize(
290
+ "size_factors",
291
+ [
292
+ # No image
293
+ [],
294
+ # Single-scale
295
+ [1.0],
296
+ # Single-scale, batched
297
+ [1.0, 1.0, 1.0],
298
+ # Multi-scale
299
+ [0.25, 0.5, 1.0],
300
+ ],
301
+)
302
+@pytest.mark.parametrize("dtype", ["half"])
303
+@pytest.mark.parametrize("max_tokens", [128])
304
+@pytest.mark.parametrize("num_logprobs", [5])
305
+@torch.inference_mode()
306
+def test_awq_models(vllm_runner, image_assets, models, size_factors,
307
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
308
+ run_awq_test(
309
+ vllm_runner,
310
+ image_assets,
311
+ models,
312
+ size_factors=size_factors,
313
+ dtype=dtype,
314
+ max_tokens=max_tokens,
315
+ num_logprobs=num_logprobs,
316
+ tensor_parallel_size=1,
317
+ )
vllm/model_executor/layers/linear.py
+4
-2
@@ -570,7 +570,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
570
# for the packing.
571
if isinstance(param, PackedvLLMParameter
572
) and param.packed_dim == param.output_dim:
573
- param.adjust_shard_indexes_for_packing(
573
+ shard_size, shard_offset = \
574
+ param.adjust_shard_indexes_for_packing(
575
shard_size=shard_size, shard_offset=shard_offset)
576
577
loaded_weight_shard = loaded_weight.narrow(param.output_dim,
@@ -719,7 +720,8 @@ class QKVParallelLinear(ColumnParallelLinear):
720
# for the packing.
721
if isinstance(param, PackedvLLMParameter
722
) and param.packed_dim == param.output_dim:
722
- param.adjust_shard_indexes_for_packing(
723
+ shard_size, shard_offset = \
724
+ param.adjust_shard_indexes_for_packing(
725
shard_size=shard_size, shard_offset=shard_offset)
726
727
loaded_weight_shard = loaded_weight.narrow(param.output_dim,
vllm/model_executor/model_loader/weight_utils.py
+4
@@ -131,6 +131,10 @@ def get_quant_config(model_config: ModelConfig,
131
# Read the quantization config from the HF model config, if available.
132
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
133
None)
134
+ # some vision model may keep quantization_config in their text_config
135
+ hf_text_config = getattr(model_config.hf_config, "text_config", None)
136
+ if hf_quant_config is None and hf_text_config is not None:
137
+ hf_quant_config = getattr(hf_text_config, "quantization_config", None)
138
if hf_quant_config is None:
139
# compressed-tensors uses a compressions_config
140
hf_quant_config = getattr(model_config.hf_config, "compression_config",
vllm/model_executor/models/internlm2.py
+13
-22
@@ -87,6 +87,7 @@ class InternLM2Attention(nn.Module):
87
self.head_dim = hidden_size // self.total_num_heads
88
self.q_size = self.num_heads * self.head_dim
89
self.kv_size = self.num_kv_heads * self.head_dim
90
+ self.key_value_groups = int(self.num_heads / self.num_kv_heads)
91
self.scaling = self.head_dim**-0.5
92
self.rope_theta = rope_theta
93
self.max_position_embeddings = max_position_embeddings
@@ -120,6 +121,14 @@ class InternLM2Attention(nn.Module):
121
cache_config=cache_config,
122
quant_config=quant_config)
123
124
+ def split_qkv(self, qkv: torch.Tensor):
125
+ qkv = qkv.view(-1, self.num_kv_heads, self.key_value_groups + 2, 128)
126
+ q, k, v = torch.split(qkv, [self.key_value_groups, 1, 1], dim=2)
127
+ q = q.reshape(-1, self.q_size)
128
+ k = k.reshape(-1, self.kv_size)
129
+ v = v.reshape(-1, self.kv_size)
130
+ return q, k, v
131
+
132
def forward(
133
self,
134
positions: torch.Tensor,
@@ -128,7 +137,7 @@ class InternLM2Attention(nn.Module):
137
attn_metadata: AttentionMetadata,
138
) -> torch.Tensor:
139
qkv, _ = self.wqkv(hidden_states)
131
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
140
+ q, k, v = self.split_qkv(qkv)
141
q, k = self.rotary_emb(positions, q, k)
142
attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
143
output, _ = self.wo(attn_output)
@@ -324,24 +333,6 @@ class InternLM2ForCausalLM(nn.Module):
333
if name.endswith(".bias") and name not in params_dict:
334
continue
335
param = params_dict[name]
327
- if "wqkv" in name:
328
- config = self.config
329
- kv_groups = (config.num_attention_heads //
330
- config.num_key_value_heads)
331
- head_dim = config.hidden_size // config.num_attention_heads
332
- loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
333
- head_dim,
334
- loaded_weight.shape[-1])
335
- wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
336
- dim=1)
337
- wq = wq.reshape(-1, wq.shape[-1])
338
- wk = wk.reshape(-1, wk.shape[-1])
339
- wv = wv.reshape(-1, wv.shape[-1])
340
- weight_loader = param.weight_loader
341
- weight_loader(param, wq, 'q')
342
- weight_loader(param, wk, 'k')
343
- weight_loader(param, wv, 'v')
344
- else:
345
- weight_loader = getattr(param, "weight_loader",
346
- default_weight_loader)
347
- weight_loader(param, loaded_weight)
336
+ weight_loader = getattr(param, "weight_loader",
337
+ default_weight_loader)
338
+ weight_loader(param, loaded_weight)