@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ce01f3066
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Performance] Optimize `get_seqs` (#7051)

Woosuk Kwon committed Aug 1, 2024 at 18:29 UTC 6ce01f30667bbae33f112152e07a3b66b841078f
3 files changed +22 -22
vllm/core/block_manager_v1.py
+1 -1
@@ -700,5 +700,5 @@ class BlockSpaceManagerV1(BlockSpaceManager):
700
701 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
702 if self.enable_caching:
703 - for seq in seq_group.seqs_dict.values():
703 + for seq in seq_group.get_seqs():
704 self.compute_full_blocks_in_seq(seq)
vllm/sequence.py
+20 -20
@@ -444,6 +444,7 @@ class SequenceGroup:
444 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
445 ) -> None:
446 self.request_id = request_id
447 + self.seqs = seqs
448 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
449 self.sampling_params = sampling_params
450 self.metrics = RequestMetrics(arrival_time=arrival_time,
@@ -458,25 +459,24 @@ class SequenceGroup:
459 self.prompt_adapter_request = prompt_adapter_request
460 self.encoder_seq = encoder_seq
461 self.trace_headers = trace_headers
461 - self._first_seq = next(iter(self.seqs_dict.values()))
462
463 @property
464 def prompt(self) -> Optional[str]:
465 # All sequences in the group should have the same prompt.
466 # We use the prompt of an arbitrary sequence.
467 - return self._first_seq.prompt
467 + return self.seqs[0].prompt
468
469 @property
470 def prompt_token_ids(self) -> List[int]:
471 # All sequences in the group should have the same prompt.
472 # We use the prompt of an arbitrary sequence.
473 - return self._first_seq.prompt_token_ids
473 + return self.seqs[0].prompt_token_ids
474
475 @property
476 def multi_modal_data(self) -> "MultiModalDataDict":
477 # All sequences in the group should have the same multi-modal data.
478 # We use the multi-modal data of an arbitrary sequence.
479 - return self._first_seq.multi_modal_data
479 + return self.seqs[0].multi_modal_data
480
481 @property
482 def lora_int_id(self) -> int:
@@ -512,7 +512,7 @@ class SequenceGroup:
512 # in TPOT, rather than recalculating TTFT (since from the )
513 # POV of the user, there is simply a long generation delay.
514 if (self.metrics.first_token_time is None
515 - and self.get_seqs()[0].get_output_len() == 1):
515 + and self.seqs[0].get_output_len() == 1):
516 self.metrics.first_token_time = time
517
518 def maybe_set_first_scheduled_time(self, time: float) -> None:
@@ -548,9 +548,9 @@ class SequenceGroup:
548 self,
549 status: Optional[SequenceStatus] = None,
550 ) -> List[Sequence]:
551 - return list(self.seqs_dict.values()) if status is None else [
552 - seq for seq in self.seqs_dict.values() if seq.status == status
553 - ]
551 + if status is None:
552 + return self.seqs
553 + return [seq for seq in self.seqs if seq.status == status]
554
555 def is_encoder_decoder(self) -> bool:
556 return self.encoder_seq is not None
@@ -559,22 +559,20 @@ class SequenceGroup:
559 return self.encoder_seq
560
561 def get_unfinished_seqs(self) -> List[Sequence]:
562 - return [
563 - seq for seq in self.seqs_dict.values() if not seq.is_finished()
564 - ]
562 + return [seq for seq in self.seqs if not seq.is_finished()]
563
564 def get_finished_seqs(self) -> List[Sequence]:
567 - return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
565 + return [seq for seq in self.seqs if seq.is_finished()]
566
567 def update_num_computed_tokens(self, num_new_computed_tokens: int):
568 """Update number of tokens computed so far."""
571 - for seq in self.seqs_dict.values():
569 + for seq in self.seqs:
570 if not seq.is_finished():
571 seq.data.update_num_computed_tokens(num_new_computed_tokens)
572
573 def get_num_uncomputed_tokens(self) -> int:
574 num_uncomputed_tokens = 0
577 - for seq in self.get_seqs():
575 + for seq in self.seqs:
576 if not seq.is_finished():
577 num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
578 return num_uncomputed_tokens
@@ -583,7 +581,7 @@ class SequenceGroup:
581 # Optimization. We don't need to call get_seqs if we don't need to
582 # filter by states.
583 if status is None:
586 - return len(self.seqs_dict)
584 + return len(self.seqs)
585
586 return len(self.get_seqs(status))
587
@@ -602,23 +600,25 @@ class SequenceGroup:
600 if seq.seq_id in self.seqs_dict:
601 raise ValueError(f"Sequence {seq.seq_id} already exists.")
602 self.seqs_dict[seq.seq_id] = seq
603 + self.seqs.append(seq)
604
605 def remove(self, seq_id: int) -> None:
607 - if seq_id not in self.seqs_dict:
606 + seq = self.seqs_dict.pop(seq_id, None)
607 + if seq is None:
608 raise ValueError(f"Sequence {seq_id} not found.")
609 - del self.seqs_dict[seq_id]
609 + self.seqs.remove(seq)
610
611 def is_finished(self) -> bool:
612 - return all(seq.is_finished() for seq in self.get_seqs())
612 + return all(seq.is_finished() for seq in self.seqs)
613
614 def is_prefill(self) -> bool:
615 # Every sequence should be in the same stage.
616 - return self.get_seqs()[0].is_prefill()
616 + return self.seqs[0].is_prefill()
617
618 def __repr__(self) -> str:
619 return (f"SequenceGroup(request_id={self.request_id}, "
620 f"sampling_params={self.sampling_params}, "
621 - f"num_seqs={len(self.seqs_dict)})")
621 + f"num_seqs={len(self.seqs)})")
622
623
624 class SequenceGroupMetadata:
vllm/transformers_utils/detokenizer.py
+1 -1
@@ -40,7 +40,7 @@ class Detokenizer:
40 assert prms is not None
41
42 # We can pick any sequence for the prompt.
43 - seq = next(iter(seq_group.seqs_dict.values()))
43 + seq = seq_group.get_seqs()[0]
44 # Only prompt, without the generated token.
45 all_token_ids = seq.get_token_ids()
46 prompt_token_ids = all_token_ids[:-1]