A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] LoRA : Fix the order in which the kernels process LoRAs (#16040)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Apr 6, 2025 at 10:04 UTC
3a100b9278d1d2b27537fc86f14fb88d8c27466f
1 file changed
+1
-1
vllm/lora/ops/triton_ops/lora_kernel_metadata.py
+1
-1
@@ -111,7 +111,7 @@ class LoRAKernelMeta:
111
112
# active_lora_ids, num_tokens_per_lora
113
lora_ids, num_tokens_per_lora = torch.unique(token_lora_mapping,
114
- sorted=False,
114
+ sorted=True,
115
return_counts=True)
116
self.active_lora_ids[:lora_ids.size(0)].copy_(lora_ids,
117
non_blocking=True)