A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix fully sharded LoRAs with Mixtral (#11390)
Signed-off-by: Jason Greene <jason.greene@redhat.com>
Jason T. Greene committed
Dec 22, 2024 at 09:25 UTC
f1d1bf6288abfe051ec4ad891c5a96575e347bfc
2 files changed
+5
-2
tests/lora/test_mixtral.py
+3
-1
@@ -62,8 +62,9 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
62
63
64
@pytest.mark.parametrize("tp_size", [4])
65
+@pytest.mark.parametrize("fully_shard", [True, False])
66
def test_mixtral_lora_all_target_modules(mixtral_lora_files_all_target_modules,
66
- tp_size):
67
+ tp_size, fully_shard):
68
"""This LoRA model has all supported Mixtral target modules"""
69
70
if torch.cuda.device_count() < tp_size:
@@ -82,6 +83,7 @@ def test_mixtral_lora_all_target_modules(mixtral_lora_files_all_target_modules,
83
max_loras=4,
84
distributed_executor_backend="ray",
85
tensor_parallel_size=tp_size,
86
+ fully_sharded_loras=fully_shard,
87
max_lora_rank=32,
88
)
89
vllm/lora/layers.py
+2
-1
@@ -425,8 +425,9 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
425
if self.base_layer.skip_bias_add else None)
426
return output, output_bias
427
428
+ # ReplicatedLinear should always be replaced, regardless of the fully
429
+ # sharded LoRAs setting, because it is, by definition, copied per GPU.
430
@classmethod
429
- @_not_fully_sharded_can_replace
431
def can_replace_layer(
432
cls,
433
source_layer: nn.Module,