@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c59e120c55
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Feature add lora support for Qwen2 (#3177)

whyiug committed Mar 8, 2024 at 13:58 UTC c59e120c557743b0fc8178ee1796c8a3def78bf4
2 files changed +26
csrc/punica/bgmv/bgmv_config.h
+2
@@ -21,6 +21,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
21 f(in_T, out_T, W_T, narrow, 2048) \
22 f(in_T, out_T, W_T, narrow, 2560) \
23 f(in_T, out_T, W_T, narrow, 2752) \
24 + f(in_T, out_T, W_T, narrow, 2816) \
25 f(in_T, out_T, W_T, narrow, 3072) \
26 f(in_T, out_T, W_T, narrow, 3456) \
27 f(in_T, out_T, W_T, narrow, 3584) \
@@ -36,6 +37,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37 f(in_T, out_T, W_T, narrow, 10240) \
38 f(in_T, out_T, W_T, narrow, 11008) \
39 f(in_T, out_T, W_T, narrow, 12288) \
40 + f(in_T, out_T, W_T, narrow, 13696) \
41 f(in_T, out_T, W_T, narrow, 13824) \
42 f(in_T, out_T, W_T, narrow, 14336) \
43 f(in_T, out_T, W_T, narrow, 16384) \
vllm/model_executor/models/qwen2.py
+24
@@ -46,6 +46,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.model_executor.weight_utils import (default_weight_loader,
47 hf_model_weights_iterator)
48 from vllm.sequence import SamplerOutput
49 +from vllm.config import LoRAConfig
50
51 KVCache = Tuple[torch.Tensor, torch.Tensor]
52
@@ -264,12 +265,35 @@ class Qwen2Model(nn.Module):
265
266
267 class Qwen2ForCausalLM(nn.Module):
268 + packed_modules_mapping = {
269 + "qkv_proj": [
270 + "q_proj",
271 + "k_proj",
272 + "v_proj",
273 + ],
274 + "gate_up_proj": [
275 + "gate_proj",
276 + "up_proj",
277 + ],
278 + }
279 +
280 + # LoRA specific attributes
281 + supported_lora_modules = [
282 + "qkv_proj",
283 + "o_proj",
284 + "gate_up_proj",
285 + "down_proj",
286 + ]
287 + embedding_modules = {}
288 + embedding_padding_modules = []
289
290 def __init__(
291 self,
292 config: Qwen2Config,
293 linear_method: Optional[LinearMethodBase] = None,
294 + lora_config: Optional[LoRAConfig] = None,
295 ) -> None:
296 + del lora_config
297 super().__init__()
298 self.config = config
299 self.linear_method = linear_method