A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[MODEL] LoRA support for Jamba model (#11209)
Signed-off-by: Erez Schwartz <erezs@ai21.com>
ErezSC42 committed
Dec 27, 2024 at 19:58 UTC
55509c2114718c1292c11348f002461ba44cb23b
5 files changed
+132
-32
tests/lora/conftest.py
+24
@@ -4,6 +4,7 @@ from typing import Dict, List, TypedDict
4
from unittest.mock import MagicMock, patch
5
6
import pytest
7
+import safetensors
8
import torch
9
import torch.nn as nn
10
from huggingface_hub import snapshot_download
@@ -169,6 +170,29 @@ def mixtral_lora_files_all_target_modules():
170
return snapshot_download(repo_id="dyang415/mixtral-lora-v0")
171
172
173
+@pytest.fixture(scope="session")
174
+def jamba_lora_files():
175
+ # some of the adapters have unnecessary weights for serving,
176
+ # hence we remove them
177
+ def remove_unnecessary_weights(path):
178
+ lora_path = f"{adapter_path}/adapter_model.safetensors"
179
+ tensors = safetensors.torch.load_file(lora_path)
180
+ nonlora_keys = []
181
+ for k in list(tensors.keys()):
182
+ if "lora" not in k:
183
+ nonlora_keys.append(k)
184
+ for k in nonlora_keys:
185
+ del tensors[k]
186
+ safetensors.torch.save_file(tensors, lora_path)
187
+
188
+ adapter_path = snapshot_download(
189
+ repo_id=
190
+ "hf-100/Jamba-1.5-mini-Spellbound-StoryWriter-0.1-6583896-ckpt53-lora")
191
+
192
+ remove_unnecessary_weights(adapter_path)
193
+ return adapter_path
194
+
195
+
196
@pytest.fixture(scope="session")
197
def gemma_lora_files():
198
return snapshot_download(repo_id="wskwon/gemma-7b-test-lora")
tests/lora/test_jamba.py
new
+54
@@ -0,0 +1,54 @@
1
+from typing import List
2
+
3
+import pytest
4
+import torch
5
+
6
+import vllm
7
+from vllm.lora.request import LoRARequest
8
+
9
+MODEL_PATH = "ai21labs/AI21-Jamba-1.5-Mini"
10
+
11
+MAX_TOKENS = 40
12
+
13
+
14
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,
15
+ prompts: List[str]) -> List[str]:
16
+
17
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=MAX_TOKENS)
18
+ outputs = llm.generate(
19
+ prompts,
20
+ sampling_params,
21
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
22
+ if lora_id else None)
23
+ # Print the outputs.
24
+ generated_texts: List[str] = []
25
+ for output in outputs:
26
+ prompt = output.prompt
27
+ generated_text = output.outputs[0].text.strip()
28
+ generated_texts.append(generated_text)
29
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
30
+ return generated_texts
31
+
32
+
33
+@pytest.mark.parametrize("tp_size", [4])
34
+def test_jamba_lora(jamba_lora_files, tp_size):
35
+ """Original test, the LoRA model has the common target modules, not all"""
36
+ if torch.cuda.device_count() < tp_size:
37
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
38
+
39
+ prompts = ["Write a story about a sheep and a goat."]
40
+
41
+ llm = vllm.LLM(
42
+ MODEL_PATH,
43
+ enable_lora=True,
44
+ max_num_seqs=16,
45
+ max_loras=4,
46
+ distributed_executor_backend="ray",
47
+ tensor_parallel_size=tp_size,
48
+ )
49
+
50
+ expected_jamba_output = [
51
+ """Once upon a time, in a lush green meadow, there lived a sheep named Clara and a goat named Billy. Clara was a gentle creature, always nibbling on the soft grass and humming""" # noqa: E501
52
+ ]
53
+ assert do_sample(llm, jamba_lora_files, lora_id=1,
54
+ prompts=prompts) == expected_jamba_output
vllm/model_executor/layers/mamba/mamba_mixer.py
+18
-4
@@ -42,12 +42,14 @@ class MambaMixer(CustomOp):
42
use_rms_norm: bool,
43
rms_norm_has_weight: bool = True,
44
rms_norm_eps: float = 1e-5,
45
- activation="silu"):
45
+ activation="silu",
46
+ is_lora_enabled: bool = False):
47
super().__init__()
48
self.time_step_rank = time_step_rank
49
self.ssm_state_size = ssm_state_size
50
self.use_rms_norm = use_rms_norm
51
self.activation = activation
52
+ self.is_lora_enabled = is_lora_enabled
53
54
self.conv1d = ColumnParallelLinear(
55
input_size=conv_kernel_size,
@@ -63,6 +65,7 @@ class MambaMixer(CustomOp):
65
self.in_proj = MergedColumnParallelLinear(hidden_size,
66
[intermediate_size] * 2,
67
bias=use_bias)
68
+
69
# selective projection used to make dt, B and C input dependent
70
self.x_proj = RowParallelLinear(
71
intermediate_size,
@@ -170,7 +173,13 @@ class MambaMixer(CustomOp):
173
174
# 3. State Space Model sequence transformation
175
# 3.a. input varying initialization of time_step, B and C
173
- ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
176
+
177
+ if self.is_lora_enabled:
178
+ # lora kernel requires contiguous tensor
179
+ ssm_parameters = self.x_proj(
180
+ hidden_states.transpose(-2, -1).contiguous())[0]
181
+ else:
182
+ ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
183
184
time_step, B, C = torch.split(
185
ssm_parameters,
@@ -222,6 +231,11 @@ class MambaMixer(CustomOp):
231
scan_outputs = scan_outputs.transpose(0, 1)
232
233
# 4. Final linear projection
225
- contextualized_states = self.out_proj(scan_outputs.transpose(-2,
226
- -1))[0]
234
+ if self.is_lora_enabled:
235
+ # lora kernel requires contiguous tensor
236
+ contextualized_states = self.out_proj(
237
+ scan_outputs.transpose(-2, -1).contiguous())[0]
238
+ else:
239
+ contextualized_states = self.out_proj(
240
+ scan_outputs.transpose(-2, -1))[0]
241
return contextualized_states
vllm/model_executor/models/jamba.py
+26
-24
@@ -107,9 +107,11 @@ class JambaMambaDecoderLayer(nn.Module):
107
layer_idx: int,
108
cache_config: Optional[CacheConfig] = None,
109
quant_config: Optional[QuantizationConfig] = None,
110
- prefix: str = "") -> None:
110
+ is_lora_enabled: Optional[bool] = False,
111
+ **kwargs) -> None:
112
super().__init__()
113
self.config = config
114
+ self.is_lora_enabled = is_lora_enabled
115
self.mamba = MambaMixer(hidden_size= config.hidden_size,
116
ssm_state_size = config.mamba_d_state,
117
conv_kernel_size = config.mamba_d_conv,
@@ -120,7 +122,9 @@ class JambaMambaDecoderLayer(nn.Module):
122
use_bias = config.mamba_proj_bias,
123
use_rms_norm=True,
124
rms_norm_eps=config.rms_norm_eps,
123
- activation=config.hidden_act)
125
+ activation=config.hidden_act,
126
+ is_lora_enabled = self.is_lora_enabled
127
+ )
128
129
num_experts = config.layers_num_experts[layer_idx]
130
ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
@@ -156,14 +160,13 @@ class JambaMambaDecoderLayer(nn.Module):
160
161
class JambaAttentionDecoderLayer(nn.Module):
162
159
- def __init__(
160
- self,
161
- config: JambaConfig,
162
- layer_idx: int,
163
- cache_config: Optional[CacheConfig] = None,
164
- quant_config: Optional[QuantizationConfig] = None,
165
- prefix: str = "",
166
- ) -> None:
163
+ def __init__(self,
164
+ config: JambaConfig,
165
+ layer_idx: int,
166
+ cache_config: Optional[CacheConfig] = None,
167
+ quant_config: Optional[QuantizationConfig] = None,
168
+ prefix: str = "",
169
+ **kwargs) -> None:
170
super().__init__()
171
self.hidden_size = config.hidden_size
172
tp_size = get_tensor_model_parallel_world_size()
@@ -287,17 +290,18 @@ class JambaModel(nn.Module):
290
org_num_embeddings=config.vocab_size,
291
)
292
293
+ extra_kwargs = {"is_lora_enabled": bool(vllm_config.lora_config)}
294
+
295
def get_layer(prefix: str):
296
layer_idx = int(prefix.rsplit(".", 1)[1])
297
layer_class = ALL_DECODER_LAYER_TYPES[
298
config.layers_block_type[layer_idx]]
294
- return layer_class(
295
- config,
296
- layer_idx,
297
- cache_config,
298
- quant_config=quant_config,
299
- prefix=prefix,
300
- )
299
+ return layer_class(config,
300
+ layer_idx,
301
+ cache_config,
302
+ quant_config=quant_config,
303
+ prefix=prefix,
304
+ **extra_kwargs)
305
306
self.start_layer, self.end_layer, self.layers = make_layers(
307
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers")
@@ -371,14 +375,13 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
375
"k_proj",
376
"v_proj",
377
],
378
+ "in_proj": ["in_proj"],
379
}
380
381
# LoRA specific attributes
382
supported_lora_modules = [
378
- "qkv_proj",
379
- "o_proj",
380
- "embed_tokens",
381
- "lm_head",
383
+ "qkv_proj", "o_proj", "embed_tokens", "lm_head", "up_proj",
384
+ "down_proj", "gate_proj", "out_proj", "in_proj", "x_proj"
385
]
386
embedding_modules = {
387
"embed_tokens": "input_embeddings",
@@ -423,9 +426,9 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
426
self.make_empty_intermediate_tensors = (
427
self.model.make_empty_intermediate_tensors)
428
if self.scheduler_config is not None and \
426
- not self.model_config.enforce_eager:
429
+ not self.model_config.enforce_eager:
430
if self.scheduler_config.max_num_seqs > \
428
- vllm_config.compilation_config.max_capture_size:
431
+ vllm_config.compilation_config.max_capture_size:
432
self.max_batch_size = \
433
vllm_config.compilation_config.max_capture_size
434
else:
@@ -446,7 +449,6 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
449
inputs_embeds: Optional[torch.Tensor] = None,
450
**kwargs):
451
if self.mamba_cache is None:
449
-
452
num_mamba_layers = self.model_config.get_num_layers_by_block_type(
453
self.vllm_config.parallel_config, LayerBlockType.mamba)
454
self.mamba_cache = MambaCacheManager(
vllm/model_executor/models/mamba.py
+10
-4
@@ -38,10 +38,12 @@ class MambaDecoderLayer(nn.Module):
38
def __init__(self,
39
config: MambaConfig,
40
cache_config: Optional[CacheConfig] = None,
41
- quant_config: Optional[QuantizationConfig] = None) -> None:
41
+ quant_config: Optional[QuantizationConfig] = None,
42
+ is_lora_enabled: Optional[bool] = False) -> None:
43
super().__init__()
44
self.config = config
45
self.is_falcon_mamba = config.model_type == "falcon_mamba"
46
+ self.is_lora_enabled = is_lora_enabled
47
mixer_rms_eps = config.mixer_rms_eps if self.is_falcon_mamba else None
48
self.mixer = MambaMixer(hidden_size=config.hidden_size,
49
ssm_state_size=config.state_size,
@@ -53,7 +55,8 @@ class MambaDecoderLayer(nn.Module):
55
use_rms_norm=self.is_falcon_mamba,
56
rms_norm_has_weight=not self.is_falcon_mamba,
57
rms_norm_eps=mixer_rms_eps,
56
- activation=config.hidden_act)
58
+ activation=config.hidden_act,
59
+ is_lora_enabled=self.is_lora_enabled)
60
61
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
62
@@ -85,6 +88,7 @@ class MambaModel(nn.Module):
88
cache_config = vllm_config.cache_config
89
quant_config = vllm_config.quant_config
90
lora_config = vllm_config.lora_config
91
+ is_lora_enabled = bool(lora_config)
92
93
self.config = config
94
self.padding_idx = config.pad_token_id
@@ -101,8 +105,10 @@ class MambaModel(nn.Module):
105
106
self.start_layer, self.end_layer, self.layers = make_layers(
107
config.num_hidden_layers,
104
- lambda prefix: MambaDecoderLayer(
105
- config, cache_config=cache_config, quant_config=quant_config),
108
+ lambda prefix: MambaDecoderLayer(config,
109
+ cache_config=cache_config,
110
+ quant_config=quant_config,
111
+ is_lora_enabled=is_lora_enabled),
112
prefix=f"{prefix}.layers")
113
114
self.norm_f = RMSNorm(config.hidden_size,