A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Feature add lora support for Qwen2 (#3177)
whyiug committed
Mar 8, 2024 at 13:58 UTC
c59e120c557743b0fc8178ee1796c8a3def78bf4
2 files changed
+26
csrc/punica/bgmv/bgmv_config.h
+2
@@ -21,6 +21,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
21
f(in_T, out_T, W_T, narrow, 2048) \
22
f(in_T, out_T, W_T, narrow, 2560) \
23
f(in_T, out_T, W_T, narrow, 2752) \
24
+ f(in_T, out_T, W_T, narrow, 2816) \
25
f(in_T, out_T, W_T, narrow, 3072) \
26
f(in_T, out_T, W_T, narrow, 3456) \
27
f(in_T, out_T, W_T, narrow, 3584) \
@@ -36,6 +37,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37
f(in_T, out_T, W_T, narrow, 10240) \
38
f(in_T, out_T, W_T, narrow, 11008) \
39
f(in_T, out_T, W_T, narrow, 12288) \
40
+ f(in_T, out_T, W_T, narrow, 13696) \
41
f(in_T, out_T, W_T, narrow, 13824) \
42
f(in_T, out_T, W_T, narrow, 14336) \
43
f(in_T, out_T, W_T, narrow, 16384) \
vllm/model_executor/models/qwen2.py
+24
@@ -46,6 +46,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.model_executor.weight_utils import (default_weight_loader,
47
hf_model_weights_iterator)
48
from vllm.sequence import SamplerOutput
49
+from vllm.config import LoRAConfig
50
51
KVCache = Tuple[torch.Tensor, torch.Tensor]
52
@@ -264,12 +265,35 @@ class Qwen2Model(nn.Module):
265
266
267
class Qwen2ForCausalLM(nn.Module):
268
+ packed_modules_mapping = {
269
+ "qkv_proj": [
270
+ "q_proj",
271
+ "k_proj",
272
+ "v_proj",
273
+ ],
274
+ "gate_up_proj": [
275
+ "gate_proj",
276
+ "up_proj",
277
+ ],
278
+ }
279
+
280
+ # LoRA specific attributes
281
+ supported_lora_modules = [
282
+ "qkv_proj",
283
+ "o_proj",
284
+ "gate_up_proj",
285
+ "down_proj",
286
+ ]
287
+ embedding_modules = {}
288
+ embedding_padding_modules = []
289
290
def __init__(
291
self,
292
config: Qwen2Config,
293
linear_method: Optional[LinearMethodBase] = None,
294
+ lora_config: Optional[LoRAConfig] = None,
295
) -> None:
296
+ del lora_config
297
super().__init__()
298
self.config = config
299
self.linear_method = linear_method