A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix][Model] Jamba assertions and no chunked prefill by default for Jamba (#6784)
tomeras91 committed
Jul 27, 2024 at 06:45 UTC
ed94e4f427bce8611e198d051dbd3b0097b448e8
2 files changed
+10
-1
vllm/engine/arg_utils.py
+5
-1
@@ -754,10 +754,14 @@ class EngineArgs:
754
use_sliding_window = (model_config.get_sliding_window()
755
is not None)
756
use_spec_decode = self.speculative_model is not None
757
+ has_seqlen_agnostic_layers = (
758
+ model_config.contains_seqlen_agnostic_layers(
759
+ parallel_config))
760
if (is_gpu and not use_sliding_window and not use_spec_decode
761
and not self.enable_lora
762
and not self.enable_prompt_adapter
760
- and not self.enable_prefix_caching):
763
+ and not self.enable_prefix_caching
764
+ and not has_seqlen_agnostic_layers):
765
self.enable_chunked_prefill = True
766
logger.warning(
767
"Chunked prefill is enabled by default for models with "
vllm/model_executor/models/jamba.py
+5
@@ -644,6 +644,11 @@ class JambaForCausalLM(nn.Module, HasInnerState):
644
lora_config: Optional[LoRAConfig] = None,
645
scheduler_config: Optional[SchedulerConfig] = None,
646
) -> None:
647
+ assert not scheduler_config.chunked_prefill_enabled, \
648
+ "Jamba currently does not support chunked prefill"
649
+ assert not cache_config.enable_prefix_caching, \
650
+ "Jamba currently does not support prefix caching"
651
+
652
super().__init__()
653
self.config = config
654
self.scheduler_config = scheduler_config