@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18fd4a8331
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Multi-sequence broken (#11898)

Signed-off-by: Andy Lo <andy@mistral.ai>

Andy Lo committed Jan 21, 2025 at 19:51 UTC 18fd4a83316868747def6e7e1e2a6caebf8b8ace
3 files changed +59 -39
tests/samplers/test_seeded_generate.py
+6 -1
@@ -31,7 +31,7 @@ def test_random_sample_with_seed(
31
32 sampling_params = SamplingParams(
33 # Parameters to ensure sufficient randomness
34 - temperature=2.0,
34 + temperature=3.0,
35 top_p=min(random.random() + 0.3, 1),
36 top_k=random.randint(5, 20),
37 n=random.randint(1, 10),
@@ -75,3 +75,8 @@ def test_random_sample_with_seed(
75 # verify requests with the same seed match
76 assert outputs[1] == outputs[4]
77 assert outputs[2] == outputs[5]
78 +
79 + # verify generations within the same parallel sampling group differ
80 + for output in outputs:
81 + for sub_output_a, sub_output_b in combinations(output, 2):
82 + assert sub_output_a != sub_output_b
vllm/outputs.py
+1 -1
@@ -172,9 +172,9 @@ class RequestOutput:
172 if seq_group.request_id in seq_id_to_seq_group:
173 group: SequenceGroupBase = seq_id_to_seq_group[
174 seq_group.request_id]
175 + assembled_seq_group = group.maybe_assemble_group(seq_group)
176 if finished:
177 group.finish_seq(seq_group)
177 - assembled_seq_group = group.maybe_assemble_group(seq_group)
178 if assembled_seq_group is None:
179 return None
180 return cls.from_seq_group(assembled_seq_group, use_cache,
vllm/sequence.py
+52 -37
@@ -815,7 +815,9 @@ class SequenceGroup:
815 def get_max_num_running_seqs(self) -> int:
816 """The maximum number of sequences running in parallel in the remaining
817 lifetime of the request."""
818 - return 0 if self.first_seq.is_finished() else 1
818 + if self.is_single_seq:
819 + return 0 if self.first_seq.is_finished() else 1
820 + return self.num_seqs() - self.num_finished_seqs()
821
822 def get_seqs(
823 self,
@@ -824,7 +826,10 @@ class SequenceGroup:
826 if status is None:
827 return self.seqs
828
827 - return self.seqs if self.first_seq.status == status else []
829 + if self.is_single_seq:
830 + return self.seqs if self.first_seq.status == status else []
831 +
832 + return [seq for seq in self.seqs if seq.status == status]
833
834 def is_encoder_decoder(self) -> bool:
835 return self.encoder_seq is not None
@@ -833,19 +838,22 @@ class SequenceGroup:
838 return self.encoder_seq
839
840 def get_finished_seqs(self) -> List[Sequence]:
836 - return self.seqs if self.first_seq.is_finished() else []
841 + if self.is_single_seq:
842 + return self.seqs if self.first_seq.is_finished() else []
843 +
844 + return [seq for seq in self.seqs if seq.is_finished()]
845
846 def update_num_computed_tokens(self, num_new_computed_tokens: int):
847 """Update number of tokens computed so far."""
840 - seq = self.first_seq
841 - if not seq.is_finished():
842 - seq.data.update_num_computed_tokens(num_new_computed_tokens)
848 + for seq in self.seqs:
849 + if not seq.is_finished():
850 + seq.data.update_num_computed_tokens(num_new_computed_tokens)
851
852 def get_num_uncomputed_tokens(self) -> int:
853 num_uncomputed_tokens = 0
846 - seq = self.first_seq
847 - if not seq.is_finished():
848 - num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
854 + for seq in self.seqs:
855 + if not seq.is_finished():
856 + num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
857 return num_uncomputed_tokens
858
859 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
@@ -860,10 +868,14 @@ class SequenceGroup:
868 return len(self.get_seqs(status))
869
870 def num_finished_seqs(self) -> int:
863 - return 1 if self.first_seq.is_finished() else 0
871 + if self.is_single_seq:
872 + return 1 if self.seqs[0].is_finished() else 0
873 + return len(self.get_finished_seqs())
874
875 def is_finished(self) -> bool:
866 - return self.first_seq.is_finished()
876 + if self.is_single_seq:
877 + return self.first_seq.is_finished()
878 + return all(seq.is_finished() for seq in self.seqs)
879
880 def is_prefill(self) -> bool:
881 return self.first_seq.is_prefill()
@@ -1391,13 +1403,15 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1403 @staticmethod
1404 def add_request(request_id: str, engine, params, **kwargs):
1405 original_params = params
1394 - params = original_params.clone()
1395 - params.n = 1
1406 group = ParallelSampleSequenceGroup(request_id)
1407 seqs = []
1408 for i in range(original_params.n):
1409 request_id_i = f"{request_id}_parallel_sample_{i}"
1410 group.seq_id_to_index[request_id_i] = i
1411 + params = copy.deepcopy(original_params)
1412 + params.n = 1
1413 + if params.seed is not None:
1414 + params.seed += i
1415 seq_group = engine._add_processed_request(
1416 request_id_i,
1417 params=params,
@@ -1432,33 +1446,34 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1446 self, seq_group: SequenceGroup) -> Optional[SequenceGroup]:
1447
1448 # in the streaming mode, we will return the assembled sequence
1435 - # for the first sequence, and then return None for the rest of
1436 - # sequences
1449 + # for the first remaining sequence, and then return None for the
1450 + # rest of sequences
1451 if self.streaming:
1438 - if self.seq_id_to_index[seq_group.request_id] == 0:
1452 + first_remaining_id = next(iter(self.to_be_finished))
1453 + if seq_group.request_id == first_remaining_id:
1454 return self.assembled_seq_group
1455 return None
1456
1457 # in the non-streaming mode, we will return the assembled sequence
1443 - # once after all sequences finish, and then return None for the
1458 + # when the last sequences finishes, and then return None for the
1459 # rest of the time
1445 -
1446 - if len(self.to_be_finished) > 0:
1447 - return None
1448 -
1449 - assert self.assembled_seq_group is not None
1450 - params = self.assembled_seq_group.sampling_params
1451 - assert isinstance(params, SamplingParams)
1452 - if not self.output_produced:
1453 - self.output_produced = True
1454 - if params._real_n is not None:
1455 - # Get the top-n sequences.
1456 - n = params._real_n or params.n
1457 - seqs = self.assembled_seq_group.seqs
1458 - sorting_key = lambda seq: seq.get_cumulative_logprob()
1459 - sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
1460 - top_n_seqs = sorted_seqs[:n]
1461 - self.assembled_seq_group.seqs = top_n_seqs
1462 - return self.assembled_seq_group
1463 - if self.output_produced:
1464 - return None
1460 + if (len(self.to_be_finished) == 1
1461 + and seq_group.request_id in self.to_be_finished
1462 + and seq_group.is_finished()):
1463 + assert self.assembled_seq_group is not None
1464 + params = self.assembled_seq_group.sampling_params
1465 + assert isinstance(params, SamplingParams)
1466 + if not self.output_produced:
1467 + self.output_produced = True
1468 + if params._real_n is not None:
1469 + # Get the top-n sequences.
1470 + n = params._real_n or params.n
1471 + seqs = self.assembled_seq_group.seqs
1472 + sorting_key = lambda seq: seq.get_cumulative_logprob()
1473 + sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
1474 + top_n_seqs = sorted_seqs[:n]
1475 + self.assembled_seq_group.seqs = top_n_seqs
1476 + return self.assembled_seq_group
1477 + if self.output_produced:
1478 + return None
1479 + return None