@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48abee9e54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] remove max_num_batched_tokens limit for lora (#7288)

Cherilyn Buren committed Aug 8, 2024 at 14:17 UTC 48abee9e5492924a69551d859d66d98874d72d60
1 file changed -5
vllm/config.py
-5
@@ -1377,11 +1377,6 @@ class LoRAConfig:
1377 model_config.quantization)
1378
1379 def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
1380 - if scheduler_config.max_num_batched_tokens > 65528:
1381 - raise ValueError(
1382 - "Due to limitations of the custom LoRA CUDA kernel, "
1383 - "max_num_batched_tokens must be <= 65528 when "
1384 - "LoRA is enabled.")
1380 if scheduler_config.chunked_prefill_enabled:
1381 raise ValueError("LoRA is not supported with chunked prefill yet.")
1382