A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] remove max_num_batched_tokens limit for lora (#7288)
Cherilyn Buren committed
Aug 8, 2024 at 14:17 UTC
48abee9e5492924a69551d859d66d98874d72d60
1 file changed
-5
vllm/config.py
-5
@@ -1377,11 +1377,6 @@ class LoRAConfig:
1377
model_config.quantization)
1378
1379
def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
1380
- if scheduler_config.max_num_batched_tokens > 65528:
1381
- raise ValueError(
1382
- "Due to limitations of the custom LoRA CUDA kernel, "
1383
- "max_num_batched_tokens must be <= 65528 when "
1384
- "LoRA is enabled.")
1380
if scheduler_config.chunked_prefill_enabled:
1381
raise ValueError("LoRA is not supported with chunked prefill yet.")
1382