@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 12e1c65bc9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model] Add AWQ quantization support for InternVL2 model (#7187)

Isotr0py committed Aug 21, 2024 at 14:18 UTC 12e1c65bc937573f19ea53e51c18294611ba33d7
4 files changed +123 -25
tests/models/test_internvl.py
+102 -1
@@ -1,5 +1,5 @@
1 import types
2 -from typing import List, Optional, Type
2 +from typing import List, Optional, Tuple, Type
3
4 import pytest
5 import torch
@@ -178,6 +178,74 @@ def run_test(
178 )
179
180
181 +def run_awq_test(
182 + vllm_runner: Type[VllmRunner],
183 + image_assets: _ImageAssets,
184 + models: Tuple[str, str],
185 + *,
186 + size_factors: List[float],
187 + dtype: str,
188 + max_tokens: int,
189 + num_logprobs: int,
190 + tensor_parallel_size: int,
191 + distributed_executor_backend: Optional[str] = None,
192 +):
193 + source_model, quant_model = models
194 +
195 + images = [asset.pil_image for asset in image_assets]
196 +
197 + inputs_per_image = [(
198 + [prompt for _ in size_factors],
199 + [rescale_image_size(image, factor) for factor in size_factors],
200 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
201 +
202 + # NOTE: take care of the order. run vLLM first, and then run HF.
203 + # vLLM needs a fresh new process without cuda initialization.
204 + # if we run HF first, the cuda initialization will be done and it
205 + # will hurt multiprocessing backend with fork method (the default method).
206 +
207 + # max_model_len should be greater than image_feature_size
208 + with vllm_runner(source_model,
209 + max_model_len=4096,
210 + dtype=dtype,
211 + tensor_parallel_size=tensor_parallel_size,
212 + distributed_executor_backend=distributed_executor_backend,
213 + enforce_eager=True) as vllm_model:
214 + source_outputs_per_image = [
215 + vllm_model.generate_greedy_logprobs(prompts,
216 + max_tokens,
217 + num_logprobs=num_logprobs,
218 + images=images)
219 + for prompts, images in inputs_per_image
220 + ]
221 +
222 + with vllm_runner(quant_model,
223 + quantization="awq",
224 + max_model_len=4096,
225 + dtype=dtype,
226 + tensor_parallel_size=tensor_parallel_size,
227 + distributed_executor_backend=distributed_executor_backend,
228 + enforce_eager=True) as vllm_model:
229 + quant_outputs_per_image = [
230 + vllm_model.generate_greedy_logprobs(prompts,
231 + max_tokens,
232 + num_logprobs=num_logprobs,
233 + images=images)
234 + for prompts, images in inputs_per_image
235 + ]
236 +
237 + for source_outputs, quant_outputs in zip(source_outputs_per_image,
238 + quant_outputs_per_image):
239 + # TODO: Check whether using original CLIPVisionModel can improve
240 + # consistency against HF
241 + check_logprobs_close(
242 + outputs_0_lst=source_outputs,
243 + outputs_1_lst=quant_outputs,
244 + name_0="source",
245 + name_1="awq",
246 + )
247 +
248 +
249 target_dtype = "half"
250 if is_cpu():
251 target_dtype = "bfloat16"
@@ -214,3 +282,36 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
282 num_logprobs=num_logprobs,
283 tensor_parallel_size=1,
284 )
285 +
286 +
287 +@pytest.mark.parametrize(
288 + "models", [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")])
289 +@pytest.mark.parametrize(
290 + "size_factors",
291 + [
292 + # No image
293 + [],
294 + # Single-scale
295 + [1.0],
296 + # Single-scale, batched
297 + [1.0, 1.0, 1.0],
298 + # Multi-scale
299 + [0.25, 0.5, 1.0],
300 + ],
301 +)
302 +@pytest.mark.parametrize("dtype", ["half"])
303 +@pytest.mark.parametrize("max_tokens", [128])
304 +@pytest.mark.parametrize("num_logprobs", [5])
305 +@torch.inference_mode()
306 +def test_awq_models(vllm_runner, image_assets, models, size_factors,
307 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
308 + run_awq_test(
309 + vllm_runner,
310 + image_assets,
311 + models,
312 + size_factors=size_factors,
313 + dtype=dtype,
314 + max_tokens=max_tokens,
315 + num_logprobs=num_logprobs,
316 + tensor_parallel_size=1,
317 + )
vllm/model_executor/layers/linear.py
+4 -2
@@ -570,7 +570,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
570 # for the packing.
571 if isinstance(param, PackedvLLMParameter
572 ) and param.packed_dim == param.output_dim:
573 - param.adjust_shard_indexes_for_packing(
573 + shard_size, shard_offset = \
574 + param.adjust_shard_indexes_for_packing(
575 shard_size=shard_size, shard_offset=shard_offset)
576
577 loaded_weight_shard = loaded_weight.narrow(param.output_dim,
@@ -719,7 +720,8 @@ class QKVParallelLinear(ColumnParallelLinear):
720 # for the packing.
721 if isinstance(param, PackedvLLMParameter
722 ) and param.packed_dim == param.output_dim:
722 - param.adjust_shard_indexes_for_packing(
723 + shard_size, shard_offset = \
724 + param.adjust_shard_indexes_for_packing(
725 shard_size=shard_size, shard_offset=shard_offset)
726
727 loaded_weight_shard = loaded_weight.narrow(param.output_dim,
vllm/model_executor/model_loader/weight_utils.py
+4
@@ -131,6 +131,10 @@ def get_quant_config(model_config: ModelConfig,
131 # Read the quantization config from the HF model config, if available.
132 hf_quant_config = getattr(model_config.hf_config, "quantization_config",
133 None)
134 + # some vision model may keep quantization_config in their text_config
135 + hf_text_config = getattr(model_config.hf_config, "text_config", None)
136 + if hf_quant_config is None and hf_text_config is not None:
137 + hf_quant_config = getattr(hf_text_config, "quantization_config", None)
138 if hf_quant_config is None:
139 # compressed-tensors uses a compressions_config
140 hf_quant_config = getattr(model_config.hf_config, "compression_config",
vllm/model_executor/models/internlm2.py
+13 -22
@@ -87,6 +87,7 @@ class InternLM2Attention(nn.Module):
87 self.head_dim = hidden_size // self.total_num_heads
88 self.q_size = self.num_heads * self.head_dim
89 self.kv_size = self.num_kv_heads * self.head_dim
90 + self.key_value_groups = int(self.num_heads / self.num_kv_heads)
91 self.scaling = self.head_dim**-0.5
92 self.rope_theta = rope_theta
93 self.max_position_embeddings = max_position_embeddings
@@ -120,6 +121,14 @@ class InternLM2Attention(nn.Module):
121 cache_config=cache_config,
122 quant_config=quant_config)
123
124 + def split_qkv(self, qkv: torch.Tensor):
125 + qkv = qkv.view(-1, self.num_kv_heads, self.key_value_groups + 2, 128)
126 + q, k, v = torch.split(qkv, [self.key_value_groups, 1, 1], dim=2)
127 + q = q.reshape(-1, self.q_size)
128 + k = k.reshape(-1, self.kv_size)
129 + v = v.reshape(-1, self.kv_size)
130 + return q, k, v
131 +
132 def forward(
133 self,
134 positions: torch.Tensor,
@@ -128,7 +137,7 @@ class InternLM2Attention(nn.Module):
137 attn_metadata: AttentionMetadata,
138 ) -> torch.Tensor:
139 qkv, _ = self.wqkv(hidden_states)
131 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
140 + q, k, v = self.split_qkv(qkv)
141 q, k = self.rotary_emb(positions, q, k)
142 attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
143 output, _ = self.wo(attn_output)
@@ -324,24 +333,6 @@ class InternLM2ForCausalLM(nn.Module):
333 if name.endswith(".bias") and name not in params_dict:
334 continue
335 param = params_dict[name]
327 - if "wqkv" in name:
328 - config = self.config
329 - kv_groups = (config.num_attention_heads //
330 - config.num_key_value_heads)
331 - head_dim = config.hidden_size // config.num_attention_heads
332 - loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
333 - head_dim,
334 - loaded_weight.shape[-1])
335 - wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
336 - dim=1)
337 - wq = wq.reshape(-1, wq.shape[-1])
338 - wk = wk.reshape(-1, wk.shape[-1])
339 - wv = wv.reshape(-1, wv.shape[-1])
340 - weight_loader = param.weight_loader
341 - weight_loader(param, wq, 'q')
342 - weight_loader(param, wk, 'k')
343 - weight_loader(param, wv, 'v')
344 - else:
345 - weight_loader = getattr(param, "weight_loader",
346 - default_weight_loader)
347 - weight_loader(param, loaded_weight)
336 + weight_loader = getattr(param, "weight_loader",
337 + default_weight_loader)
338 + weight_loader(param, loaded_weight)