A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Multi-sequence broken (#11898)
Signed-off-by: Andy Lo <andy@mistral.ai>
Andy Lo committed
Jan 21, 2025 at 19:51 UTC
18fd4a83316868747def6e7e1e2a6caebf8b8ace
3 files changed
+59
-39
tests/samplers/test_seeded_generate.py
+6
-1
@@ -31,7 +31,7 @@ def test_random_sample_with_seed(
31
32
sampling_params = SamplingParams(
33
# Parameters to ensure sufficient randomness
34
- temperature=2.0,
34
+ temperature=3.0,
35
top_p=min(random.random() + 0.3, 1),
36
top_k=random.randint(5, 20),
37
n=random.randint(1, 10),
@@ -75,3 +75,8 @@ def test_random_sample_with_seed(
75
# verify requests with the same seed match
76
assert outputs[1] == outputs[4]
77
assert outputs[2] == outputs[5]
78
+
79
+ # verify generations within the same parallel sampling group differ
80
+ for output in outputs:
81
+ for sub_output_a, sub_output_b in combinations(output, 2):
82
+ assert sub_output_a != sub_output_b
vllm/outputs.py
+1
-1
@@ -172,9 +172,9 @@ class RequestOutput:
172
if seq_group.request_id in seq_id_to_seq_group:
173
group: SequenceGroupBase = seq_id_to_seq_group[
174
seq_group.request_id]
175
+ assembled_seq_group = group.maybe_assemble_group(seq_group)
176
if finished:
177
group.finish_seq(seq_group)
177
- assembled_seq_group = group.maybe_assemble_group(seq_group)
178
if assembled_seq_group is None:
179
return None
180
return cls.from_seq_group(assembled_seq_group, use_cache,
vllm/sequence.py
+52
-37
@@ -815,7 +815,9 @@ class SequenceGroup:
815
def get_max_num_running_seqs(self) -> int:
816
"""The maximum number of sequences running in parallel in the remaining
817
lifetime of the request."""
818
- return 0 if self.first_seq.is_finished() else 1
818
+ if self.is_single_seq:
819
+ return 0 if self.first_seq.is_finished() else 1
820
+ return self.num_seqs() - self.num_finished_seqs()
821
822
def get_seqs(
823
self,
@@ -824,7 +826,10 @@ class SequenceGroup:
826
if status is None:
827
return self.seqs
828
827
- return self.seqs if self.first_seq.status == status else []
829
+ if self.is_single_seq:
830
+ return self.seqs if self.first_seq.status == status else []
831
+
832
+ return [seq for seq in self.seqs if seq.status == status]
833
834
def is_encoder_decoder(self) -> bool:
835
return self.encoder_seq is not None
@@ -833,19 +838,22 @@ class SequenceGroup:
838
return self.encoder_seq
839
840
def get_finished_seqs(self) -> List[Sequence]:
836
- return self.seqs if self.first_seq.is_finished() else []
841
+ if self.is_single_seq:
842
+ return self.seqs if self.first_seq.is_finished() else []
843
+
844
+ return [seq for seq in self.seqs if seq.is_finished()]
845
846
def update_num_computed_tokens(self, num_new_computed_tokens: int):
847
"""Update number of tokens computed so far."""
840
- seq = self.first_seq
841
- if not seq.is_finished():
842
- seq.data.update_num_computed_tokens(num_new_computed_tokens)
848
+ for seq in self.seqs:
849
+ if not seq.is_finished():
850
+ seq.data.update_num_computed_tokens(num_new_computed_tokens)
851
852
def get_num_uncomputed_tokens(self) -> int:
853
num_uncomputed_tokens = 0
846
- seq = self.first_seq
847
- if not seq.is_finished():
848
- num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
854
+ for seq in self.seqs:
855
+ if not seq.is_finished():
856
+ num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
857
return num_uncomputed_tokens
858
859
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
@@ -860,10 +868,14 @@ class SequenceGroup:
868
return len(self.get_seqs(status))
869
870
def num_finished_seqs(self) -> int:
863
- return 1 if self.first_seq.is_finished() else 0
871
+ if self.is_single_seq:
872
+ return 1 if self.seqs[0].is_finished() else 0
873
+ return len(self.get_finished_seqs())
874
875
def is_finished(self) -> bool:
866
- return self.first_seq.is_finished()
876
+ if self.is_single_seq:
877
+ return self.first_seq.is_finished()
878
+ return all(seq.is_finished() for seq in self.seqs)
879
880
def is_prefill(self) -> bool:
881
return self.first_seq.is_prefill()
@@ -1391,13 +1403,15 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1403
@staticmethod
1404
def add_request(request_id: str, engine, params, **kwargs):
1405
original_params = params
1394
- params = original_params.clone()
1395
- params.n = 1
1406
group = ParallelSampleSequenceGroup(request_id)
1407
seqs = []
1408
for i in range(original_params.n):
1409
request_id_i = f"{request_id}_parallel_sample_{i}"
1410
group.seq_id_to_index[request_id_i] = i
1411
+ params = copy.deepcopy(original_params)
1412
+ params.n = 1
1413
+ if params.seed is not None:
1414
+ params.seed += i
1415
seq_group = engine._add_processed_request(
1416
request_id_i,
1417
params=params,
@@ -1432,33 +1446,34 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1446
self, seq_group: SequenceGroup) -> Optional[SequenceGroup]:
1447
1448
# in the streaming mode, we will return the assembled sequence
1435
- # for the first sequence, and then return None for the rest of
1436
- # sequences
1449
+ # for the first remaining sequence, and then return None for the
1450
+ # rest of sequences
1451
if self.streaming:
1438
- if self.seq_id_to_index[seq_group.request_id] == 0:
1452
+ first_remaining_id = next(iter(self.to_be_finished))
1453
+ if seq_group.request_id == first_remaining_id:
1454
return self.assembled_seq_group
1455
return None
1456
1457
# in the non-streaming mode, we will return the assembled sequence
1443
- # once after all sequences finish, and then return None for the
1458
+ # when the last sequences finishes, and then return None for the
1459
# rest of the time
1445
-
1446
- if len(self.to_be_finished) > 0:
1447
- return None
1448
-
1449
- assert self.assembled_seq_group is not None
1450
- params = self.assembled_seq_group.sampling_params
1451
- assert isinstance(params, SamplingParams)
1452
- if not self.output_produced:
1453
- self.output_produced = True
1454
- if params._real_n is not None:
1455
- # Get the top-n sequences.
1456
- n = params._real_n or params.n
1457
- seqs = self.assembled_seq_group.seqs
1458
- sorting_key = lambda seq: seq.get_cumulative_logprob()
1459
- sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
1460
- top_n_seqs = sorted_seqs[:n]
1461
- self.assembled_seq_group.seqs = top_n_seqs
1462
- return self.assembled_seq_group
1463
- if self.output_produced:
1464
- return None
1460
+ if (len(self.to_be_finished) == 1
1461
+ and seq_group.request_id in self.to_be_finished
1462
+ and seq_group.is_finished()):
1463
+ assert self.assembled_seq_group is not None
1464
+ params = self.assembled_seq_group.sampling_params
1465
+ assert isinstance(params, SamplingParams)
1466
+ if not self.output_produced:
1467
+ self.output_produced = True
1468
+ if params._real_n is not None:
1469
+ # Get the top-n sequences.
1470
+ n = params._real_n or params.n
1471
+ seqs = self.assembled_seq_group.seqs
1472
+ sorting_key = lambda seq: seq.get_cumulative_logprob()
1473
+ sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
1474
+ top_n_seqs = sorted_seqs[:n]
1475
+ self.assembled_seq_group.seqs = top_n_seqs
1476
+ return self.assembled_seq_group
1477
+ if self.output_produced:
1478
+ return None
1479
+ return None