A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Performance] Optimize `get_seqs` (#7051)
Woosuk Kwon committed
Aug 1, 2024 at 18:29 UTC
6ce01f30667bbae33f112152e07a3b66b841078f
3 files changed
+22
-22
vllm/core/block_manager_v1.py
+1
-1
@@ -700,5 +700,5 @@ class BlockSpaceManagerV1(BlockSpaceManager):
700
701
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
702
if self.enable_caching:
703
- for seq in seq_group.seqs_dict.values():
703
+ for seq in seq_group.get_seqs():
704
self.compute_full_blocks_in_seq(seq)
vllm/sequence.py
+20
-20
@@ -444,6 +444,7 @@ class SequenceGroup:
444
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
445
) -> None:
446
self.request_id = request_id
447
+ self.seqs = seqs
448
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
449
self.sampling_params = sampling_params
450
self.metrics = RequestMetrics(arrival_time=arrival_time,
@@ -458,25 +459,24 @@ class SequenceGroup:
459
self.prompt_adapter_request = prompt_adapter_request
460
self.encoder_seq = encoder_seq
461
self.trace_headers = trace_headers
461
- self._first_seq = next(iter(self.seqs_dict.values()))
462
463
@property
464
def prompt(self) -> Optional[str]:
465
# All sequences in the group should have the same prompt.
466
# We use the prompt of an arbitrary sequence.
467
- return self._first_seq.prompt
467
+ return self.seqs[0].prompt
468
469
@property
470
def prompt_token_ids(self) -> List[int]:
471
# All sequences in the group should have the same prompt.
472
# We use the prompt of an arbitrary sequence.
473
- return self._first_seq.prompt_token_ids
473
+ return self.seqs[0].prompt_token_ids
474
475
@property
476
def multi_modal_data(self) -> "MultiModalDataDict":
477
# All sequences in the group should have the same multi-modal data.
478
# We use the multi-modal data of an arbitrary sequence.
479
- return self._first_seq.multi_modal_data
479
+ return self.seqs[0].multi_modal_data
480
481
@property
482
def lora_int_id(self) -> int:
@@ -512,7 +512,7 @@ class SequenceGroup:
512
# in TPOT, rather than recalculating TTFT (since from the )
513
# POV of the user, there is simply a long generation delay.
514
if (self.metrics.first_token_time is None
515
- and self.get_seqs()[0].get_output_len() == 1):
515
+ and self.seqs[0].get_output_len() == 1):
516
self.metrics.first_token_time = time
517
518
def maybe_set_first_scheduled_time(self, time: float) -> None:
@@ -548,9 +548,9 @@ class SequenceGroup:
548
self,
549
status: Optional[SequenceStatus] = None,
550
) -> List[Sequence]:
551
- return list(self.seqs_dict.values()) if status is None else [
552
- seq for seq in self.seqs_dict.values() if seq.status == status
553
- ]
551
+ if status is None:
552
+ return self.seqs
553
+ return [seq for seq in self.seqs if seq.status == status]
554
555
def is_encoder_decoder(self) -> bool:
556
return self.encoder_seq is not None
@@ -559,22 +559,20 @@ class SequenceGroup:
559
return self.encoder_seq
560
561
def get_unfinished_seqs(self) -> List[Sequence]:
562
- return [
563
- seq for seq in self.seqs_dict.values() if not seq.is_finished()
564
- ]
562
+ return [seq for seq in self.seqs if not seq.is_finished()]
563
564
def get_finished_seqs(self) -> List[Sequence]:
567
- return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
565
+ return [seq for seq in self.seqs if seq.is_finished()]
566
567
def update_num_computed_tokens(self, num_new_computed_tokens: int):
568
"""Update number of tokens computed so far."""
571
- for seq in self.seqs_dict.values():
569
+ for seq in self.seqs:
570
if not seq.is_finished():
571
seq.data.update_num_computed_tokens(num_new_computed_tokens)
572
573
def get_num_uncomputed_tokens(self) -> int:
574
num_uncomputed_tokens = 0
577
- for seq in self.get_seqs():
575
+ for seq in self.seqs:
576
if not seq.is_finished():
577
num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
578
return num_uncomputed_tokens
@@ -583,7 +581,7 @@ class SequenceGroup:
581
# Optimization. We don't need to call get_seqs if we don't need to
582
# filter by states.
583
if status is None:
586
- return len(self.seqs_dict)
584
+ return len(self.seqs)
585
586
return len(self.get_seqs(status))
587
@@ -602,23 +600,25 @@ class SequenceGroup:
600
if seq.seq_id in self.seqs_dict:
601
raise ValueError(f"Sequence {seq.seq_id} already exists.")
602
self.seqs_dict[seq.seq_id] = seq
603
+ self.seqs.append(seq)
604
605
def remove(self, seq_id: int) -> None:
607
- if seq_id not in self.seqs_dict:
606
+ seq = self.seqs_dict.pop(seq_id, None)
607
+ if seq is None:
608
raise ValueError(f"Sequence {seq_id} not found.")
609
- del self.seqs_dict[seq_id]
609
+ self.seqs.remove(seq)
610
611
def is_finished(self) -> bool:
612
- return all(seq.is_finished() for seq in self.get_seqs())
612
+ return all(seq.is_finished() for seq in self.seqs)
613
614
def is_prefill(self) -> bool:
615
# Every sequence should be in the same stage.
616
- return self.get_seqs()[0].is_prefill()
616
+ return self.seqs[0].is_prefill()
617
618
def __repr__(self) -> str:
619
return (f"SequenceGroup(request_id={self.request_id}, "
620
f"sampling_params={self.sampling_params}, "
621
- f"num_seqs={len(self.seqs_dict)})")
621
+ f"num_seqs={len(self.seqs)})")
622
623
624
class SequenceGroupMetadata:
vllm/transformers_utils/detokenizer.py
+1
-1
@@ -40,7 +40,7 @@ class Detokenizer:
40
assert prms is not None
41
42
# We can pick any sequence for the prompt.
43
- seq = next(iter(seq_group.seqs_dict.values()))
43
+ seq = seq_group.get_seqs()[0]
44
# Only prompt, without the generated token.
45
all_token_ids = seq.get_token_ids()
46
prompt_token_ids = all_token_ids[:-1]