A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] Remove LoRA log (#15388)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 25, 2025 at 11:43 UTC
6db94571d74e3dbc6e38bec7b4bd2913eea36cc9
2 files changed
+5
-15
vllm/config.py
-6
@@ -2373,12 +2373,6 @@ class LoRAConfig:
2373
self.lora_dtype = model_config.dtype
2374
elif isinstance(self.lora_dtype, str):
2375
self.lora_dtype = getattr(torch, self.lora_dtype)
2376
- if model_config.quantization and model_config.quantization not in [
2377
- "awq", "gptq"
2378
- ]:
2379
- # TODO support marlin
2380
- logger.warning("%s quantization is not tested with LoRA yet.",
2381
- model_config.quantization)
2376
2377
def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
2378
# Reminder: Please update docs/source/features/compatibility_matrix.md
vllm/lora/punica_wrapper/punica_gpu.py
+5
-9
@@ -78,10 +78,6 @@ class PunicaWrapperGPU(PunicaWrapperBase):
78
...], scale: float, **kwargs):
79
"""
80
Performs GEMM for multiple slices of lora_a.
81
- When `is_prefill is` true, it indicates that it is currently the
82
- prefill stage, and the `_shrink_prefill` function should be called.
83
- Otherwise, it is the decode stage, and the _shrink_decode function
84
- should be called.
81
82
Semantics:
83
for i in range(len(lora_a_stacked)):
@@ -129,7 +125,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
125
lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
126
bias's weight
127
output_slices (Tuple[int, ...]): Every slice's size
132
- add_inputs (bool): Defaults to True.
128
+ add_inputs (bool): Defaults to True.
129
"""
130
y_org = y
131
y = y.view(-1, y.shape[-1])
@@ -226,7 +222,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
222
223
if buffer is None:
224
r = lora_b_stacked[0].size(-1)
229
- # We set the buffer to be float32 by default ,refer to:
225
+ # We set the buffer to be float32 by default, refer to:
226
# https://github.com/triton-lang/triton/issues/1387
227
buffer = torch.zeros( # type: ignore
228
(len(output_slices), x.size(0), r),
@@ -268,16 +264,16 @@ class PunicaWrapperGPU(PunicaWrapperBase):
264
y (torch.Tensor): Output tensor.
265
x (torch.Tensor): Input tensor.
266
lora_a_stacked (torch.Tensor): lora_a's weights.
271
- lora_b_stacked (torch.Tensor):lora_b's weights.
267
+ lora_b_stacked (torch.Tensor): lora_b's weights.
268
scale (float): Scaling factor.
273
- buffer (Optional[torch.Tensor]):Default to None.
269
+ buffer (Optional[torch.Tensor]): Default to None.
270
"""
271
y_org = y
272
y = y.view(-1, y.shape[-1])
273
x = x.view(-1, x.shape[-1])
274
r = lora_b_stacked.size(-1)
275
if buffer is None:
280
- # We set the buffer to be float32 by default ,refer to:
276
+ # We set the buffer to be float32 by default, refer to:
277
# https://github.com/triton-lang/triton/issues/1387
278
buffer = torch.zeros((x.size(0), r),
279
dtype=torch.float32,