@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07ab160741
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model][Jamba] Mamba cache single buffer (#6739)

Co-authored-by: Mor Zusman <morz@ai21.com>

Mor Zusman committed Aug 9, 2024 at 17:07 UTC 07ab160741a486bbef23efbf26aaf2ea8a785ae1
2 files changed +148 -124
vllm/model_executor/models/jamba.py
+148 -121
@@ -609,12 +609,8 @@ class JambaForCausalLM(nn.Module, HasInnerState):
609 # compatibility
610 if not lora_config else lora_config.lora_vocab_padding_size,
611 )
612 - # Current step used indices
613 - self.current_indices: List[int] = []
612 # Used to track and store by the Mamba cache between steps.
613 self.mamba_cache: Tuple[torch.Tensor, torch.Tensor] = tuple()
616 - # Used as an input_buffer for the CUDA graph runs.
617 - self.mamba_gc_cache_buffer: Tuple[torch.Tensor, torch.Tensor] = tuple()
614 # Maps between the request id and a dict that maps between the seq_id
615 # and its index inside the self.mamba_cache
616 self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
@@ -644,95 +640,148 @@ class JambaForCausalLM(nn.Module, HasInnerState):
640 batch_size = input_ids.shape[0]
641 if attn_metadata.prefill_metadata:
642 batch_size = len(request_ids_to_seq_ids)
647 - (
648 - current_seqlen_agnostic_cache,
649 - indices,
650 - ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
651 - batch_size,
652 - finished_requests_ids)
643 + mamba_cache = self._prepare_current_run_mamba_cache(
644 + request_ids_to_seq_ids, batch_size, finished_requests_ids)
645 else:
646 # CUDA graph capturing runs
655 - current_seqlen_agnostic_cache, indices = (
656 - kwargs["seqlen_agnostic_capture_inputs"],
657 - [],
658 - )
659 - self.current_indices = indices
647 + mamba_cache = kwargs["seqlen_agnostic_capture_inputs"]
648
649 hidden_states = self.model(input_ids, positions, kv_caches,
662 - attn_metadata,
663 - current_seqlen_agnostic_cache[0],
664 - current_seqlen_agnostic_cache[1])
665 -
666 - if "seqlen_agnostic_capture_inputs" not in kwargs:
667 - self._copy_mamba_cache_by_indices(self.current_indices,
668 - current_seqlen_agnostic_cache)
669 -
650 + attn_metadata, mamba_cache[0],
651 + mamba_cache[1])
652 return hidden_states
653
672 - def _copy_mamba_cache_by_indices(
673 - self, indices: List[int],
674 - current_seqlen_agnostic_cache: Tuple[torch.Tensor, torch.Tensor]):
675 - for i, offset in enumerate(indices):
676 - self._copy_mamba_cache(offset, i, current_seqlen_agnostic_cache)
654 + def _swap_mamba_cache(self, from_index: int, to_index: int):
655 + assert len(self.mamba_cache) > 0
656 + for cache_t in self.mamba_cache:
657 + cache_t[:, [to_index,from_index]] = \
658 + cache_t[:, [from_index,to_index]]
659
678 - def _copy_mamba_cache(self, index_to: int, index_from: int,
679 - from_buffer: Tuple[torch.Tensor, torch.Tensor]):
660 + def _copy_mamba_cache(self, from_index: int, to_index: int):
661 assert len(self.mamba_cache) > 0
681 - for (cache_t, from_buffer_t) in zip(self.mamba_cache, from_buffer):
682 - cache_t[:, index_to].copy_(from_buffer_t[:, index_from],
662 + for cache_t in self.mamba_cache:
663 + cache_t[:, to_index].copy_(cache_t[:, from_index],
664 non_blocking=True)
665
685 - def _assign_seq_id_to_mamba_cache(self, cur_rid: str,
686 - seqs_id: List[int]) -> List[int]:
687 - indices_for_current_run = []
688 - for seq_id in seqs_id:
689 - if cur_rid not in self.mamba_cache_indices_mapping:
690 - self.mamba_cache_indices_mapping[cur_rid] = {}
691 - first_free_index = self._first_free_index_in_mamba_cache()
692 - self.mamba_cache_indices_mapping[cur_rid][
693 - seq_id] = first_free_index
694 - index_for_current_run = first_free_index
695 - ## case of decoding n>1, copy prefill cache to decoding indices
696 - elif seq_id not in (seq_ids2indices :=
697 - self.mamba_cache_indices_mapping[cur_rid]):
698 - first_free_index = self._first_free_index_in_mamba_cache()
699 - index_exist = list(seq_ids2indices.values())[0]
700 - self._copy_mamba_cache(index_from=index_exist,
701 - index_to=first_free_index,
702 - from_buffer=self.mamba_cache)
703 - self.mamba_cache_indices_mapping[cur_rid][
704 - seq_id] = first_free_index
705 - index_for_current_run = first_free_index
706 - else:
707 - index_for_current_run = self.mamba_cache_indices_mapping[
708 - cur_rid][seq_id]
709 -
710 - indices_for_current_run.append(index_for_current_run)
711 - return indices_for_current_run
666 + def _move_out_if_already_occupied(self, index: int,
667 + all_occupied_indices: List[int]):
668 + if index in all_occupied_indices:
669 + first_free_index = self._first_free_index_in_mamba_cache()
670 + # In case occupied, move the occupied to a new empty block
671 + self._move_cache_index_and_mappings(from_index=index,
672 + to_index=first_free_index)
673 +
674 + def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
675 + seq_id: int,
676 + destination_index: int):
677 + """
678 + Assign (req_id,seq_id) pair to a `destination_index` index, if
679 + already occupied, move the occupying index to a free index.
680 + """
681 + all_occupied_indices = self._get_all_occupied_indices()
682 + if cur_rid not in self.mamba_cache_indices_mapping:
683 + self._move_out_if_already_occupied(
684 + index=destination_index,
685 + all_occupied_indices=all_occupied_indices)
686 + self.mamba_cache_indices_mapping[cur_rid] = {
687 + seq_id: destination_index
688 + }
689 + elif seq_id not in (seq_ids2indices :=
690 + self.mamba_cache_indices_mapping[cur_rid]):
691 + # parallel sampling , where n > 1, assume prefill have
692 + # already happened now we only need to copy the already
693 + # existing cache into the siblings seq_ids caches
694 + self._move_out_if_already_occupied(
695 + index=destination_index,
696 + all_occupied_indices=all_occupied_indices)
697 + index_exists = list(seq_ids2indices.values())[0]
698 + # case of decoding n>1, copy prefill cache to decoding indices
699 + self._copy_mamba_cache(from_index=index_exists,
700 + to_index=destination_index)
701 + self.mamba_cache_indices_mapping[cur_rid][
702 + seq_id] = destination_index
703 + else:
704 + # already exists
705 + cache_index_already_exists = self.mamba_cache_indices_mapping[
706 + cur_rid][seq_id]
707 + if cache_index_already_exists != destination_index:
708 + # In case the seq id already exists but not in
709 + # the right destination, swap it with what's occupying it
710 + self._swap_pair_indices_and_mappings(
711 + from_index=cache_index_already_exists,
712 + to_index=destination_index)
713
714 def _prepare_current_run_mamba_cache(
714 - self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int,
715 - finished_requests_ids: List[str]
716 - ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], List[int]]:
717 - indices_for_current_run = []
718 - for request_id, seqs_id in request_ids_to_seq_ids.items():
715 + self, request_ids_to_seq_ids: Dict[str, list[int]],
716 + batch_size: int, finished_requests_ids: List[str]):
717 + running_indices = []
718 + request_ids_to_seq_ids_flatten = [
719 + (req_id, seq_id)
720 + for req_id, seq_ids in request_ids_to_seq_ids.items()
721 + for seq_id in seq_ids
722 + ]
723 + for dest_index, (request_id,
724 + seq_id) in enumerate(request_ids_to_seq_ids_flatten):
725 if request_id in finished_requests_ids:
720 - # Do not allocate cache for requests that run
726 + # Do not allocate cache index for requests that run
727 # and finish right after
728 continue
723 - indices_for_current_run += self._assign_seq_id_to_mamba_cache(
724 - request_id, seqs_id)
725 - ## Pad the batch in case of running batch that was not captured via CG
726 - padded_indices = indices_for_current_run.copy()
727 - pad_index = self._first_free_index_in_mamba_cache()
729 + self._assign_seq_id_to_mamba_cache_in_specific_dest(
730 + request_id, seq_id, dest_index)
731 + running_indices.append(dest_index)
732
729 - for _ in range(batch_size - len(indices_for_current_run)):
730 - padded_indices.append(pad_index)
733 + self._clean_up_first_bs_blocks(batch_size, running_indices)
734 + conv_state = self.mamba_cache[0][:, :batch_size]
735 + temporal_state = self.mamba_cache[1][:, :batch_size]
736
732 - conv_state = self.mamba_cache[0][:, padded_indices]
733 - temporal_state = self.mamba_cache[1][:, padded_indices]
737 + return (conv_state, temporal_state)
738
735 - return (conv_state, temporal_state), indices_for_current_run
739 + def _get_all_occupied_indices(self):
740 + return [
741 + cache_idx
742 + for seq_ids2indices in self.mamba_cache_indices_mapping.values()
743 + for cache_idx in seq_ids2indices.values()
744 + ]
745 +
746 + def _clean_up_first_bs_blocks(self, batch_size: int,
747 + indices_for_current_run: List[int]):
748 + # move out all of the occupied but currently not running blocks
749 + # outside of the first n blocks
750 + destination_indices = set([range(batch_size)])
751 + max_possible_batch_size = self.mamba_cache[0].shape[1]
752 + for destination_index in destination_indices:
753 + if destination_index in self._get_all_occupied_indices() and \
754 + destination_index not in indices_for_current_run:
755 + # move not running indices outside of the batch
756 + all_other_indices = list(
757 + range(batch_size, max_possible_batch_size))
758 + first_avail_index = self._first_free_index_in_mamba_cache(
759 + all_other_indices)
760 + self._swap_indices(from_index=destination_index,
761 + to_index=first_avail_index)
762 +
763 + def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
764 + self._copy_mamba_cache(from_index=from_index, to_index=to_index)
765 + self._update_mapping_index(from_index=from_index, to_index=to_index)
766 +
767 + def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
768 + self._swap_mamba_cache(from_index=from_index, to_index=to_index)
769 + self._swap_mapping_index(from_index=from_index, to_index=to_index)
770 +
771 + def _swap_mapping_index(self, from_index: int, to_index: int):
772 + for seq_ids2index in self.mamba_cache_indices_mapping.values():
773 + for seq_id, index in seq_ids2index.items():
774 + if from_index == index:
775 + seq_ids2index.update({seq_id: to_index})
776 + elif to_index == index:
777 + seq_ids2index.update({seq_id: from_index})
778 +
779 + def _update_mapping_index(self, from_index: int, to_index: int):
780 + for seq_ids2index in self.mamba_cache_indices_mapping.values():
781 + for seq_id, index in seq_ids2index.items():
782 + if from_index == index:
783 + seq_ids2index.update({seq_id: to_index})
784 + return
785
786 def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
787 """
@@ -747,28 +796,9 @@ class JambaForCausalLM(nn.Module, HasInnerState):
796 self._release_mamba_cache(finished_requests_ids)
797 request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
798 cg_batch_size = input_buffers['input_ids'].shape[0]
750 - (
751 - current_mamba_cache,
752 - indices,
753 - ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
754 - cg_batch_size,
755 - finished_requests_ids)
756 - self.current_indices = indices
757 -
758 - for input_buffer, current_cache_buffer in zip(
759 - input_buffers["seqlen_agnostic_capture_inputs"],
760 - current_mamba_cache):
761 - input_buffer.copy_(current_cache_buffer, non_blocking=True)
762 -
763 - def copy_outputs_after_cuda_graphs(self, input_buffers, **kwargs):
764 - """
765 - Copy the relevant Mamba cache from the CUDA graph input_buffers
766 - back to the JambaForCausalLM.mamba_cache after CUDA
767 - graph replay run is done.
768 - """
769 - self._copy_mamba_cache_by_indices(
770 - self.current_indices,
771 - input_buffers["seqlen_agnostic_capture_inputs"])
799 + self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
800 + cg_batch_size,
801 + finished_requests_ids)
802
803 def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
804 """
@@ -776,26 +806,25 @@ class JambaForCausalLM(nn.Module, HasInnerState):
806 The buffer is used to maintain the Mamba Cache during the CUDA graph
807 replay runs.
808 """
779 - return tuple(buffer[:, :batch_size]
780 - for buffer in self.mamba_gc_cache_buffer)
809 + return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
810
811 def _release_mamba_cache(self, finished_seq_groups_req_ids: List[str]):
812 for req_id in finished_seq_groups_req_ids:
813 if req_id in self.mamba_cache_indices_mapping:
814 self.mamba_cache_indices_mapping.pop(req_id)
815
787 - def _first_free_index_in_mamba_cache(self) -> int:
788 - if self.mamba_cache:
816 + def _first_free_index_in_mamba_cache(
817 + self, indices_range: Optional[List[int]] = None) -> int:
818 + assert self.mamba_cache is not None
819 + if indices_range is None:
820 max_possible_batch_size = self.mamba_cache[0].shape[1]
790 - occupied = [
791 - id for seq_ids in self.mamba_cache_indices_mapping.values()
792 - for id in seq_ids.values()
793 - ]
794 - first_free_index = [
795 - i not in occupied for i in range(max_possible_batch_size)
796 - ].index(True)
797 - return first_free_index
798 - return 0
821 + indices_range = list(range(max_possible_batch_size))
822 + all_occupied_indices = self._get_all_occupied_indices()
823 + for i in indices_range:
824 + if i not in all_occupied_indices:
825 + return i
826 + raise Exception("Couldn't find a free spot in the mamba cache! This"
827 + "should never happen")
828
829 def _get_mamba_cache_shape(
830 self
@@ -819,20 +848,18 @@ class JambaForCausalLM(nn.Module, HasInnerState):
848 [layer_type == "mamba" for layer_type in layers_type])
849 max_batch_size = (_get_graph_batch_size(
850 self.scheduler_config.max_num_seqs) if self.scheduler_config else
822 - max(_BATCH_SIZES_TO_CAPTURE)) + 10
851 + max(_BATCH_SIZES_TO_CAPTURE) + 2)
852 conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
853 assert conv_state_shape is not None and temporal_state_shape is not None
854
826 - for buffername in ["mamba_cache", "mamba_gc_cache_buffer"]:
827 - buffer = (torch.empty(size=(mamba_layers, max_batch_size) +
828 - conv_state_shape,
829 - dtype=dtype,
830 - device="cuda"),
831 - torch.empty(size=(mamba_layers, max_batch_size) +
832 - temporal_state_shape,
833 - dtype=dtype,
834 - device="cuda"))
835 - setattr(self, buffername, buffer)
855 + self.mamba_cache = (torch.empty(size=(mamba_layers, max_batch_size) +
856 + conv_state_shape,
857 + dtype=dtype,
858 + device="cuda"),
859 + torch.empty(size=(mamba_layers, max_batch_size) +
860 + temporal_state_shape,
861 + dtype=dtype,
862 + device="cuda"))
863
864 def compute_logits(self, hidden_states: torch.Tensor,
865 sampling_metadata: SamplingMetadata) -> torch.Tensor:
vllm/worker/model_runner.py
-3
@@ -1711,9 +1711,6 @@ class CUDAGraphRunner:
1711 non_blocking=True)
1712 # Run the graph.
1713 self.graph.replay()
1714 - if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1715 - self.model.copy_outputs_after_cuda_graphs(self.input_buffers,
1716 - **kwargs)
1714 # Return the output tensor.
1715 if get_pp_group().is_last_rank:
1716 return self.output_buffers["hidden_states"]