@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f1d1bf6288
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix fully sharded LoRAs with Mixtral (#11390)

Signed-off-by: Jason Greene <jason.greene@redhat.com>

Jason T. Greene committed Dec 22, 2024 at 09:25 UTC f1d1bf6288abfe051ec4ad891c5a96575e347bfc
2 files changed +5 -2
tests/lora/test_mixtral.py
+3 -1
@@ -62,8 +62,9 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
62
63
64 @pytest.mark.parametrize("tp_size", [4])
65 +@pytest.mark.parametrize("fully_shard", [True, False])
66 def test_mixtral_lora_all_target_modules(mixtral_lora_files_all_target_modules,
66 - tp_size):
67 + tp_size, fully_shard):
68 """This LoRA model has all supported Mixtral target modules"""
69
70 if torch.cuda.device_count() < tp_size:
@@ -82,6 +83,7 @@ def test_mixtral_lora_all_target_modules(mixtral_lora_files_all_target_modules,
83 max_loras=4,
84 distributed_executor_backend="ray",
85 tensor_parallel_size=tp_size,
86 + fully_sharded_loras=fully_shard,
87 max_lora_rank=32,
88 )
89
vllm/lora/layers.py
+2 -1
@@ -425,8 +425,9 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
425 if self.base_layer.skip_bias_add else None)
426 return output, output_bias
427
428 + # ReplicatedLinear should always be replaced, regardless of the fully
429 + # sharded LoRAs setting, because it is, by definition, copied per GPU.
430 @classmethod
429 - @_not_fully_sharded_can_replace
431 def can_replace_layer(
432 cls,
433 source_layer: nn.Module,