@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ed94e4f427
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix][Model] Jamba assertions and no chunked prefill by default for Jamba (#6784)

tomeras91 committed Jul 27, 2024 at 06:45 UTC ed94e4f427bce8611e198d051dbd3b0097b448e8
2 files changed +10 -1
vllm/engine/arg_utils.py
+5 -1
@@ -754,10 +754,14 @@ class EngineArgs:
754 use_sliding_window = (model_config.get_sliding_window()
755 is not None)
756 use_spec_decode = self.speculative_model is not None
757 + has_seqlen_agnostic_layers = (
758 + model_config.contains_seqlen_agnostic_layers(
759 + parallel_config))
760 if (is_gpu and not use_sliding_window and not use_spec_decode
761 and not self.enable_lora
762 and not self.enable_prompt_adapter
760 - and not self.enable_prefix_caching):
763 + and not self.enable_prefix_caching
764 + and not has_seqlen_agnostic_layers):
765 self.enable_chunked_prefill = True
766 logger.warning(
767 "Chunked prefill is enabled by default for models with "
vllm/model_executor/models/jamba.py
+5
@@ -644,6 +644,11 @@ class JambaForCausalLM(nn.Module, HasInnerState):
644 lora_config: Optional[LoRAConfig] = None,
645 scheduler_config: Optional[SchedulerConfig] = None,
646 ) -> None:
647 + assert not scheduler_config.chunked_prefill_enabled, \
648 + "Jamba currently does not support chunked prefill"
649 + assert not cache_config.enable_prefix_caching, \
650 + "Jamba currently does not support prefix caching"
651 +
652 super().__init__()
653 self.config = config
654 self.scheduler_config = scheduler_config