A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[core] simplify seq group code (#9569)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
youkaichao committed
Oct 24, 2024 at 00:16 UTC
4fdc581f9e5740ba10b16ebf8a4c467e65bb9822
6 files changed
+62
-566
tests/core/test_chunked_prefill_scheduler.py
-153
@@ -4,7 +4,6 @@ from unittest.mock import MagicMock
4
import pytest # noqa
5
6
from vllm.config import CacheConfig, SchedulerConfig
7
-from vllm.core.interfaces import AllocStatus
7
from vllm.core.scheduler import Scheduler
8
from vllm.sequence import Logprob, SequenceGroup
9
@@ -347,158 +346,6 @@ def test_prompt_limit_exceed():
346
assert out.ignored_seq_groups[0] == seq_group
347
348
350
-def test_swap():
351
- """Verify swapping works with chunked prefill requests"""
352
- block_size = 4
353
- max_seqs = 30
354
- max_model_len = 200
355
- max_num_batched_tokens = 30
356
- scheduler_config = SchedulerConfig(
357
- "generate",
358
- max_num_batched_tokens,
359
- max_seqs,
360
- max_model_len,
361
- enable_chunked_prefill=True,
362
- )
363
- cache_config = CacheConfig(block_size, 1.0, 1, "auto")
364
- cache_config.num_cpu_blocks = 16
365
- cache_config.num_gpu_blocks = 16
366
- scheduler = Scheduler(scheduler_config, cache_config, None)
367
-
368
- _, seq_group = create_dummy_prompt("1",
369
- prompt_length=60,
370
- best_of=2,
371
- block_size=block_size)
372
- scheduler.add_seq_group(seq_group)
373
- _, out = schedule_and_update_computed_tokens(scheduler)
374
- # The request is chunked.
375
- # prefill scheduled now.
376
- assert len(out.scheduled_seq_groups) == 1
377
- assert out.num_prefill_groups == 1
378
- assert seq_group.is_prefill()
379
- assert out.num_batched_tokens == max_num_batched_tokens
380
-
381
- # The last request should be swapped out.
382
- scheduler.block_manager.can_append_slots = MagicMock()
383
-
384
- def cannot_append_second_group(seq_group, num_lookahead_slots):
385
- return seq_group.request_id != "1"
386
-
387
- scheduler.block_manager.can_append_slots.side_effect = (
388
- cannot_append_second_group)
389
-
390
- # The running prefill is now swapped.
391
- _, out = schedule_and_update_computed_tokens(scheduler)
392
- assert len(out.scheduled_seq_groups) == 0
393
- assert out.num_batched_tokens == 0
394
- assert out.blocks_to_swap_out != []
395
- assert out.blocks_to_swap_in == []
396
-
397
- # Add 1 more task. Swap should be prioritized over new prefill.
398
- _, seq_group = create_dummy_prompt("2", prompt_length=60)
399
- scheduler.add_seq_group(seq_group)
400
- _, out = schedule_and_update_computed_tokens(scheduler)
401
- assert len(out.scheduled_seq_groups) == 1
402
- # 3 decodes. It is swapped in.
403
- assert out.num_batched_tokens == 30
404
- assert out.blocks_to_swap_in != []
405
- assert out.blocks_to_swap_out == []
406
-
407
-
408
-def test_running_prefill_prioritized_over_swap():
409
- block_size = 4
410
- max_seqs = 30
411
- max_model_len = 200
412
- max_num_batched_tokens = 30
413
- scheduler_config = SchedulerConfig(
414
- "generate",
415
- max_num_batched_tokens,
416
- max_seqs,
417
- max_model_len,
418
- enable_chunked_prefill=True,
419
- )
420
- cache_config = CacheConfig(block_size, 1.0, 1, "auto")
421
- cache_config.num_cpu_blocks = 32
422
- cache_config.num_gpu_blocks = 32
423
- scheduler = Scheduler(scheduler_config, cache_config, None)
424
-
425
- _, seq_group = create_dummy_prompt("1",
426
- prompt_length=60,
427
- best_of=2,
428
- block_size=block_size)
429
- scheduler.add_seq_group(seq_group)
430
- _, out = schedule_and_update_computed_tokens(scheduler)
431
- # The request is chunked.
432
- # prefill scheduled now.
433
- assert len(out.scheduled_seq_groups) == 1
434
- assert out.num_prefill_groups == 1
435
- assert seq_group.is_prefill()
436
- assert out.num_batched_tokens == max_num_batched_tokens
437
-
438
- # The request should be swapped out.
439
- scheduler.block_manager.can_append_slots = MagicMock()
440
-
441
- def cannot_append_second_group(seq_group, num_lookahead_slots):
442
- return seq_group.request_id != "1"
443
-
444
- scheduler.block_manager.can_append_slots.side_effect = (
445
- cannot_append_second_group)
446
-
447
- # The running prefill is now swapped.
448
- _, out = schedule_and_update_computed_tokens(scheduler)
449
- assert len(out.scheduled_seq_groups) == 0
450
- assert out.num_batched_tokens == 0
451
- assert out.blocks_to_swap_out != []
452
- assert out.blocks_to_swap_in == []
453
-
454
- # Add 1 more task. Swap is not possible, so prefill is running.
455
- scheduler.block_manager.can_swap_in = MagicMock()
456
- scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
457
-
458
- _, seq_group2 = create_dummy_prompt("2",
459
- prompt_length=60,
460
- block_size=block_size)
461
- scheduler.add_seq_group(seq_group2)
462
- _, out = schedule_and_update_computed_tokens(scheduler)
463
- assert len(out.scheduled_seq_groups) == 1
464
- # 3 decodes. It is swapped in.
465
- assert out.num_batched_tokens == 30
466
- assert out.blocks_to_swap_in == []
467
- assert out.blocks_to_swap_out == []
468
- assert out.scheduled_seq_groups[0].seq_group == seq_group2
469
-
470
- # Now although swap is possible, running prefill is prioritized.
471
- scheduler.block_manager.can_swap_in.return_value = AllocStatus.OK
472
- _, out = schedule_and_update_computed_tokens(scheduler)
473
- assert len(out.scheduled_seq_groups) == 1
474
- # 3 decodes. It is swapped in.
475
- assert out.num_batched_tokens == 30
476
- assert out.blocks_to_swap_in == []
477
- assert out.blocks_to_swap_out == []
478
- assert not seq_group2.is_prefill()
479
- assert out.scheduled_seq_groups[0].seq_group == seq_group2
480
- append_new_token(seq_group2, 1)
481
-
482
- # Decoding is prioritized.
483
- _, out = schedule_and_update_computed_tokens(scheduler)
484
- assert len(out.scheduled_seq_groups) == 1
485
- # 3 decodes. It is swapped in.
486
- assert out.num_batched_tokens == 1
487
- assert out.blocks_to_swap_in == []
488
- assert out.blocks_to_swap_out == []
489
- assert not seq_group2.is_prefill()
490
- assert out.scheduled_seq_groups[0].seq_group == seq_group2
491
- append_new_token(seq_group2, 1)
492
-
493
- # Since we abort the sequence group, we can finally swap.
494
- scheduler.abort_seq_group(seq_group2.request_id)
495
- _, out = schedule_and_update_computed_tokens(scheduler)
496
- assert len(out.scheduled_seq_groups) == 1
497
- assert out.num_batched_tokens == 30
498
- assert out.blocks_to_swap_in != []
499
- assert out.blocks_to_swap_out == []
500
-
501
-
349
def test_chunked_prefill_preempt():
350
"""Verify preempt works with chunked prefill requests"""
351
block_size = 4
tests/core/test_scheduler.py
+1
-203
@@ -10,7 +10,7 @@ from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10
from vllm.core.interfaces import AllocStatus
11
from vllm.core.scheduler import Scheduler, SchedulingBudget
12
from vllm.lora.request import LoRARequest
13
-from vllm.sequence import SequenceGroup, SequenceStatus
13
+from vllm.sequence import SequenceGroup
14
15
from .utils import (append_new_token, append_new_token_seq_group,
16
create_dummy_prompt, get_sequence_groups,
@@ -296,55 +296,6 @@ def test_scheduler_delay_factor():
296
append_new_token(out, 1)
297
298
299
-def test_swapped_out_prioritized():
300
- block_size = 4
301
- scheduler = initialize_scheduler(max_num_seqs=6,
302
- block_size=block_size,
303
- num_cpu_blocks=64,
304
- num_gpu_blocks=64)
305
- # best_of=2 * 3 == 6 sequences.
306
- for i in range(3):
307
- _, seq_group = create_dummy_prompt(str(i),
308
- prompt_length=60,
309
- best_of=2,
310
- block_size=block_size)
311
- scheduler.add_seq_group(seq_group)
312
- seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
313
- # prefill scheduled now.
314
- assert len(out.scheduled_seq_groups) == 3
315
- append_new_token(out, 1)
316
-
317
- # The last request should be swapped out.
318
- scheduler.block_manager.can_append_slots = MagicMock()
319
-
320
- def cannot_append_second_group(seq_group, num_lookahead_slots):
321
- return seq_group.request_id != "2"
322
-
323
- scheduler.block_manager.can_append_slots.side_effect = (
324
- cannot_append_second_group)
325
-
326
- seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
327
- assert len(out.scheduled_seq_groups) == 2
328
- assert out.num_batched_tokens == 2
329
- assert out.blocks_to_swap_out != []
330
- assert out.blocks_to_swap_in == []
331
- append_new_token(out, 1)
332
-
333
- # Add 1 more task. Swap should be prioritized over prefill.
334
- _, seq_group = create_dummy_prompt(str(i),
335
- prompt_length=60,
336
- best_of=2,
337
- block_size=block_size)
338
- scheduler.add_seq_group(seq_group)
339
- seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
340
- append_new_token(out, 1)
341
- assert len(out.scheduled_seq_groups) == 3
342
- # 3 decodes. It is swapped in.
343
- assert out.num_batched_tokens == 3
344
- assert out.blocks_to_swap_in != []
345
- assert out.blocks_to_swap_out == []
346
-
347
-
299
def initialize_scheduler(
300
*,
301
max_num_seqs=1000,
@@ -646,60 +597,6 @@ def test_decode_schedule_preempted():
597
assert output.blocks_to_copy == []
598
599
649
-def test_decode_swap_beam_search():
650
- """
651
- Test best_of > 1 swap out blocks
652
- """
653
- block_size = 4
654
- scheduler = initialize_scheduler(block_size=block_size,
655
- num_gpu_blocks=64,
656
- num_cpu_blocks=64)
657
- curr_loras = None
658
- budget = create_token_budget()
659
- for i in range(3):
660
- _, seq_group = create_dummy_prompt(str(i),
661
- prompt_length=60,
662
- best_of=2,
663
- block_size=block_size)
664
- scheduler._allocate_and_set_running(seq_group)
665
- scheduler._add_seq_group_to_running(seq_group)
666
- append_new_token_seq_group(60, seq_group, 1)
667
- budget.add_num_seqs(seq_group.request_id,
668
- seq_group.get_max_num_running_seqs())
669
- budget.add_num_batched_tokens(
670
- seq_group.request_id, seq_group.num_seqs(SequenceStatus.RUNNING))
671
-
672
- # The last request should be swapped out.
673
- scheduler.block_manager.can_append_slots = MagicMock()
674
-
675
- def cannot_append_second_group(seq_group, num_lookahead_slots):
676
- return seq_group.request_id != "2"
677
-
678
- scheduler.block_manager.can_append_slots.side_effect = (
679
- cannot_append_second_group)
680
- scheduler.block_manager.swap_out = MagicMock()
681
- expected_swap_mapping = [("5", "7")]
682
- scheduler.block_manager.swap_out.return_value = expected_swap_mapping
683
-
684
- output = scheduler._schedule_running(budget, curr_loras)
685
- remainig_running = scheduler.running
686
- assert len(remainig_running) == 0
687
- assert len(output.decode_seq_groups) == 2
688
- assert len(output.prefill_seq_groups) == 0
689
- assert output.decode_seq_groups[0].seq_group.request_id == "0"
690
- assert output.decode_seq_groups[1].seq_group.request_id == "1"
691
- assert len(output.preempted) == 0
692
- assert len(output.swapped_out) == 1
693
- # Budget should refledct preempted requests.
694
- assert budget.num_batched_tokens == 2
695
- # since there are 2 sequences, 2 should be subtracted.
696
- assert budget.num_curr_seqs == 4
697
- # Both should be preempted, not swapped.
698
- assert output.blocks_to_swap_out == expected_swap_mapping
699
- # Nothing is copied.
700
- assert output.blocks_to_copy == []
701
-
702
-
600
def test_schedule_decode_blocks_to_copy_update():
601
"""
602
Verify blocks_to_copy is updated.
@@ -736,105 +633,6 @@ def test_schedule_decode_blocks_to_copy_update():
633
assert output.blocks_to_copy == [(2, 3)]
634
635
739
-def test_schedule_swapped_simple():
740
- block_size = 4
741
- scheduler = initialize_scheduler(block_size=block_size)
742
- curr_loras = None
743
- blocks_to_swap_out: List[Tuple[int, int]] = []
744
- _, seq_group = create_dummy_prompt("1",
745
- prompt_length=4,
746
- best_of=2,
747
- block_size=block_size)
748
- scheduler._allocate_and_set_running(seq_group)
749
- append_new_token_seq_group(4, seq_group, 1)
750
- scheduler._swap_out(seq_group, blocks_to_swap_out)
751
- scheduler._add_seq_group_to_swapped(seq_group)
752
-
753
- budget = create_token_budget()
754
- output = scheduler._schedule_swapped(budget, curr_loras)
755
- remaining_swapped = scheduler.swapped
756
- assert len(remaining_swapped) == 0
757
- assert budget.num_batched_tokens == 1
758
- assert budget.num_curr_seqs == 2
759
- assert len(output.decode_seq_groups) == 1
760
- assert len(output.prefill_seq_groups) == 0
761
- # swap in is the reverse of swap out
762
- blocks_to_swap_in_reverse = []
763
- for swapin, swapout in output.blocks_to_swap_in:
764
- blocks_to_swap_in_reverse.append((swapout, swapin))
765
- assert blocks_to_swap_out == blocks_to_swap_in_reverse
766
-
767
-
768
-def test_schedule_swapped_max_token_budget():
769
- block_size = 4
770
- scheduler = initialize_scheduler(block_size=block_size,
771
- num_cpu_blocks=32,
772
- num_gpu_blocks=32)
773
- curr_loras = None
774
- blocks_to_swap_out: List[Tuple[int, int]] = []
775
- for i in range(2):
776
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
777
- scheduler._allocate_and_set_running(seq_group)
778
- append_new_token_seq_group(60, seq_group, 1)
779
- scheduler._swap_out(seq_group, blocks_to_swap_out)
780
- scheduler._add_seq_group_to_swapped(seq_group)
781
-
782
- budget = create_token_budget(token_budget=1)
783
- output = scheduler._schedule_swapped(budget, curr_loras)
784
- remaining_swapped = scheduler.swapped
785
- assert len(remaining_swapped) == 1
786
- assert budget.num_batched_tokens == 1
787
- assert budget.num_curr_seqs == 2
788
- assert len(output.decode_seq_groups) == 1
789
- assert len(output.prefill_seq_groups) == 0
790
-
791
- # Verify num_batched_tokens are respected.
792
- budget = create_token_budget(token_budget=1)
793
- add_token_budget(budget, 1, 0)
794
- output = scheduler._schedule_swapped(budget, curr_loras)
795
- remaining_swapped = scheduler.swapped
796
- assert len(remaining_swapped) == 1
797
- assert budget.num_batched_tokens == 1
798
- assert budget.num_curr_seqs == 0
799
- assert len(output.decode_seq_groups) == 0
800
- assert len(output.prefill_seq_groups) == 0
801
-
802
-
803
-def test_schedule_swapped_max_seqs():
804
- block_size = 4
805
- scheduler = initialize_scheduler(block_size=block_size,
806
- num_cpu_blocks=64,
807
- num_gpu_blocks=64)
808
- curr_loras = None
809
- blocks_to_swap_out: List[Tuple[int, int]] = []
810
- for i in range(4):
811
- _, seq_group = create_dummy_prompt(str(i),
812
- prompt_length=60,
813
- block_size=4)
814
- scheduler._allocate_and_set_running(seq_group)
815
- append_new_token_seq_group(60, seq_group, 1)
816
- scheduler._swap_out(seq_group, blocks_to_swap_out)
817
- scheduler._add_seq_group_to_swapped(seq_group)
818
-
819
- budget = create_token_budget(max_num_seqs=2)
820
- output = scheduler._schedule_swapped(budget, curr_loras)
821
- remaining_swapped = scheduler.swapped
822
- assert len(remaining_swapped) == 2
823
- assert budget.num_batched_tokens == 2
824
- assert budget.num_curr_seqs == 2
825
- assert len(output.decode_seq_groups) == 2
826
- assert len(output.prefill_seq_groups) == 0
827
-
828
- # Verify num_curr_seqs are respected.
829
- output = scheduler._schedule_swapped(budget, curr_loras)
830
- remaining_swapped = scheduler.swapped
831
- assert len(remaining_swapped) == 2
832
- assert budget.num_batched_tokens == 2
833
- assert budget.num_curr_seqs == 2
834
- assert len(output.decode_seq_groups) == 0
835
- assert len(output.prefill_seq_groups) == 0
836
-
837
-
636
def test_schedule_swapped_max_loras():
637
block_size = 4
638
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
vllm/core/scheduler.py
+1
-1
@@ -290,7 +290,7 @@ def scheduler_running_outputs_builder():
290
291
292
def scheduled_seq_group_builder():
293
- return ScheduledSequenceGroup(SequenceGroup("", [], -1),
293
+ return ScheduledSequenceGroup(SequenceGroup.__new__(SequenceGroup),
294
token_chunk_size=0)
295
# return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
296
vllm/engine/llm_engine.py
+19
-21
@@ -647,10 +647,24 @@ class LLMEngine:
647
prompt_adapter_request: Optional[PromptAdapterRequest],
648
trace_headers: Optional[Mapping[str, str]] = None,
649
priority: int = 0,
650
- ) -> SequenceGroup:
650
+ ) -> Optional[SequenceGroup]:
651
"""Add a processed request to the engine's request pool.
652
return the created sequence group.
653
"""
654
+ if isinstance(params, SamplingParams) and params.n > 1:
655
+ ParallelSampleSequenceGroup.add_request(
656
+ request_id,
657
+ self,
658
+ params,
659
+ processed_inputs=processed_inputs,
660
+ arrival_time=arrival_time,
661
+ lora_request=lora_request,
662
+ trace_headers=trace_headers,
663
+ prompt_adapter_request=prompt_adapter_request,
664
+ priority=priority,
665
+ )
666
+ return None
667
+
668
self._validate_model_inputs(processed_inputs)
669
# Create the sequences.
670
block_size = self.cache_config.block_size
@@ -721,7 +735,7 @@ class LLMEngine:
735
trace_headers: Optional[Mapping[str, str]] = None,
736
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
737
priority: int = 0,
724
- ) -> Optional[SequenceGroup]:
738
+ ) -> None:
739
...
740
741
@overload
@@ -735,7 +749,7 @@ class LLMEngine:
749
trace_headers: Optional[Mapping[str, str]] = None,
750
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
751
priority: int = 0,
738
- ) -> Optional[SequenceGroup]:
752
+ ) -> None:
753
...
754
755
@deprecate_kwargs(
@@ -754,7 +768,7 @@ class LLMEngine:
768
priority: int = 0,
769
*,
770
inputs: Optional[PromptType] = None, # DEPRECATED
757
- ) -> Optional[SequenceGroup]:
771
+ ) -> None:
772
"""Add a request to the engine's request pool.
773
774
The request is added to the request pool and will be processed by the
@@ -798,22 +812,6 @@ class LLMEngine:
812
>>> # continue the request processing
813
>>> ...
814
"""
801
-
802
- if isinstance(params, SamplingParams) and params.n > 1:
803
- ParallelSampleSequenceGroup.add_request(
804
- request_id,
805
- self,
806
- params,
807
- prompt=prompt,
808
- arrival_time=arrival_time,
809
- lora_request=lora_request,
810
- trace_headers=trace_headers,
811
- prompt_adapter_request=prompt_adapter_request,
812
- priority=priority,
813
- inputs=inputs,
814
- )
815
- return None
816
-
815
if inputs is not None:
816
prompt = inputs
817
assert prompt is not None and params is not None
@@ -844,7 +842,7 @@ class LLMEngine:
842
processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
843
"mm_processor_kwargs")
844
847
- return self._add_processed_request(
845
+ self._add_processed_request(
846
request_id=request_id,
847
processed_inputs=processed_inputs,
848
params=params,
vllm/engine/output_processor/single_step.py
+22
-105
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Tuple
1
+from typing import List
2
3
from vllm.config import SchedulerConfig
4
from vllm.core.scheduler import Scheduler
@@ -6,9 +6,8 @@ from vllm.engine.output_processor.interfaces import (
6
SequenceGroupOutputProcessor)
7
from vllm.engine.output_processor.stop_checker import StopChecker
8
from vllm.logger import init_logger
9
-from vllm.sequence import (CompletionSequenceGroupOutput, Sequence,
10
- SequenceGroup, SequenceGroupOutput, SequenceOutput,
11
- SequenceStatus)
9
+from vllm.sequence import (CompletionSequenceGroupOutput, SequenceGroup,
10
+ SequenceGroupOutput)
11
from vllm.transformers_utils.detokenizer import Detokenizer
12
from vllm.utils import Counter
13
@@ -114,104 +113,22 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
113
outputs: SequenceGroupOutput,
114
is_async: bool) -> None:
115
sampling_params = seq_group.sampling_params
117
- if sampling_params.n == 1:
118
- # only have one output sample
119
- sample = outputs.samples[0]
120
- # only have one sequence
121
- seq = seq_group.seqs[0]
122
- if not is_async:
123
- seq.append_token_id(sample.output_token, sample.logprobs)
124
- if sampling_params.detokenize and self.detokenizer:
125
- new_char_count = self.detokenizer.decode_sequence_inplace(
126
- seq, sampling_params)
127
- else:
128
- new_char_count = 0
129
- self.stop_checker.maybe_stop_sequence(
130
- seq,
131
- new_char_count,
132
- sampling_params,
133
- lora_req=seq_group.lora_request,
134
- )
135
- if seq.is_finished():
136
- for scheduler in self.scheduler:
137
- scheduler.free_seq(seq)
138
- return
139
-
140
- # TODO: Add support for async for beam search
141
- assert not is_async
142
-
143
- # Process samples
144
- samples = outputs.samples
145
- parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
146
- parent_child_dict: Dict[int, List[SequenceOutput]] = {
147
- parent_seq.seq_id: []
148
- for parent_seq in parent_seqs
149
- }
150
- for sample in samples:
151
- # Guard against a KeyError which can occur if the request was
152
- # aborted while the output was generated
153
- if (child_list :=
154
- parent_child_dict.get(sample.parent_seq_id)) is not None:
155
- child_list.append(sample)
156
- # List of (child, parent)
157
- child_seqs: List[Tuple[Sequence, Sequence]] = []
158
-
159
- # Process the child samples for each parent sequence
160
- for parent in parent_seqs:
161
- child_samples: List[SequenceOutput] = parent_child_dict[
162
- parent.seq_id]
163
- if len(child_samples) == 0:
164
- # This parent sequence has no children samples. Remove
165
- # the parent sequence from the sequence group since it will
166
- # not be used in the future iterations.
167
- parent.status = SequenceStatus.FINISHED_ABORTED
168
- seq_group.remove(parent.seq_id)
169
- for scheduler in self.scheduler:
170
- scheduler.free_seq(parent)
171
- continue
172
- # Fork the parent sequence if there are multiple child samples.
173
- for child_sample in child_samples[:-1]:
174
- new_child_seq_id: int = next(self.seq_counter)
175
- child = parent.fork(new_child_seq_id)
176
- child.append_token_id(child_sample.output_token,
177
- child_sample.logprobs)
178
- child_seqs.append((child, parent))
179
- # Continue the parent sequence for the last child sample.
180
- # We reuse the parent sequence here to reduce redundant memory
181
- # copies, especially when using non-beam search sampling methods.
182
- last_child_sample = child_samples[-1]
183
- parent.append_token_id(last_child_sample.output_token,
184
- last_child_sample.logprobs)
185
- child_seqs.append((parent, parent))
186
-
187
- for seq, _ in child_seqs:
188
- if sampling_params.detokenize and self.detokenizer:
189
- new_char_count = self.detokenizer.decode_sequence_inplace(
190
- seq, sampling_params)
191
- else:
192
- new_char_count = 0
193
- self.stop_checker.maybe_stop_sequence(
194
- seq,
195
- new_char_count,
196
- sampling_params,
197
- lora_req=seq_group.lora_request,
198
- )
199
-
200
- # For newly created child sequences, add them to the sequence group
201
- # and fork them in block manager if they are not finished.
202
- for seq, parent in child_seqs:
203
- if seq is not parent:
204
- seq_group.add(seq)
205
- if not seq.is_finished():
206
- for scheduler in self.scheduler:
207
- scheduler.fork_seq(parent, seq)
208
-
209
- # Free the finished and selected parent sequences' memory in block
210
- # manager. Keep them in the sequence group as candidate output.
211
- # NOTE: we need to fork the new sequences before freeing the
212
- # old sequences.
213
- for seq, parent in child_seqs:
214
- if seq is parent and seq.is_finished():
215
- for scheduler in self.scheduler:
216
- scheduler.free_seq(seq)
217
- return
116
+
117
+ sample = outputs.samples[0]
118
+ seq = seq_group.first_seq
119
+ if not is_async:
120
+ seq.append_token_id(sample.output_token, sample.logprobs)
121
+ if sampling_params.detokenize and self.detokenizer:
122
+ new_char_count = self.detokenizer.decode_sequence_inplace(
123
+ seq, sampling_params)
124
+ else:
125
+ new_char_count = 0
126
+ self.stop_checker.maybe_stop_sequence(
127
+ seq,
128
+ new_char_count,
129
+ sampling_params,
130
+ lora_req=seq_group.lora_request,
131
+ )
132
+ if seq.is_finished():
133
+ for scheduler in self.scheduler:
134
+ scheduler.free_seq(seq)
vllm/sequence.py
+19
-83
@@ -681,6 +681,7 @@ class SequenceGroup:
681
) -> None:
682
self.request_id = request_id
683
self.seqs = seqs
684
+ self.first_seq = seqs[0]
685
self.arrival_time = arrival_time
686
self.is_single_seq = len(seqs) == 1
687
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -705,15 +706,11 @@ class SequenceGroup:
706
707
@property
708
def prompt(self) -> Optional[str]:
708
- # All sequences in the group should have the same prompt.
709
- # We use the prompt of an arbitrary sequence.
710
- return self.seqs[0].prompt
709
+ return self.first_seq.prompt
710
711
@property
712
def prompt_token_ids(self) -> List[int]:
714
- # All sequences in the group should have the same prompt.
715
- # We use the prompt of an arbitrary sequence.
716
- return self.seqs[0].prompt_token_ids
713
+ return self.first_seq.prompt_token_ids
714
715
@property
716
def encoder_prompt(self) -> Optional[str]:
@@ -733,17 +730,11 @@ class SequenceGroup:
730
731
@property
732
def multi_modal_data(self) -> "MultiModalDataDict":
736
- # All sequences in the group should have the same multi-modal data.
737
- # We use the multi-modal data of an arbitrary sequence.
738
- return self.seqs[0].multi_modal_data
733
+ return self.first_seq.multi_modal_data
734
735
@property
736
def mm_processor_kwargs(self) -> Dict[str, Any]:
742
- # As with multi-modal data, all sequences in the group should have the
743
- # same processor kwargs (i.e., mm_processor_kwargs are optionally
744
- # provided per request; note that are independent of whether the model
745
- # decoder-only or an encoder-decoder).
746
- return self.seqs[0].mm_processor_kwargs
737
+ return self.first_seq.mm_processor_kwargs
738
739
@property
740
def lora_int_id(self) -> int:
@@ -808,7 +799,7 @@ class SequenceGroup:
799
# in TPOT, rather than recalculating TTFT (since from the )
800
# POV of the user, there is simply a long generation delay.
801
if (self.metrics.first_token_time is None
811
- and self.seqs[0].get_output_len() == 1):
802
+ and self.first_seq.get_output_len() == 1):
803
self.metrics.first_token_time = time
804
805
def maybe_set_first_scheduled_time(self, time: float) -> None:
@@ -825,18 +816,7 @@ class SequenceGroup:
816
def get_max_num_running_seqs(self) -> int:
817
"""The maximum number of sequences running in parallel in the remaining
818
lifetime of the request."""
828
- if self.sampling_params:
829
- n = self.sampling_params.n
830
- assert isinstance(n, int)
831
- if n > self.num_seqs():
832
- # At prompt stage, the sequence group is not yet filled up
833
- # and only have one sequence running. However, in the
834
- # generation stage, we will have `n` sequences
835
- # running.
836
- return n
837
- # At sampling stages, return the number of actual sequences
838
- # that are not finished yet.
839
- return self.num_unfinished_seqs()
819
+ return 0 if self.first_seq.is_finished() else 1
820
821
def get_seqs(
822
self,
@@ -845,10 +825,7 @@ class SequenceGroup:
825
if status is None:
826
return self.seqs
827
848
- if self.is_single_seq:
849
- return self.seqs if self.seqs[0].status == status else []
850
-
851
- return [seq for seq in self.seqs if seq.status == status]
828
+ return self.seqs if self.first_seq.status == status else []
829
830
def is_encoder_decoder(self) -> bool:
831
return self.encoder_seq is not None
@@ -856,29 +833,20 @@ class SequenceGroup:
833
def get_encoder_seq(self) -> Optional[Sequence]:
834
return self.encoder_seq
835
859
- def get_unfinished_seqs(self) -> List[Sequence]:
860
- if self.is_single_seq:
861
- return self.seqs if not self.seqs[0].is_finished() else []
862
-
863
- return [seq for seq in self.seqs if not seq.is_finished()]
864
-
836
def get_finished_seqs(self) -> List[Sequence]:
866
- if self.is_single_seq:
867
- return self.seqs if self.seqs[0].is_finished() else []
868
-
869
- return [seq for seq in self.seqs if seq.is_finished()]
837
+ return self.seqs if self.first_seq.is_finished() else []
838
839
def update_num_computed_tokens(self, num_new_computed_tokens: int):
840
"""Update number of tokens computed so far."""
873
- for seq in self.seqs:
874
- if not seq.is_finished():
875
- seq.data.update_num_computed_tokens(num_new_computed_tokens)
841
+ seq = self.first_seq
842
+ if not seq.is_finished():
843
+ seq.data.update_num_computed_tokens(num_new_computed_tokens)
844
845
def get_num_uncomputed_tokens(self) -> int:
846
num_uncomputed_tokens = 0
879
- for seq in self.seqs:
880
- if not seq.is_finished():
881
- num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
847
+ seq = self.first_seq
848
+ if not seq.is_finished():
849
+ num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
850
return num_uncomputed_tokens
851
852
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
@@ -892,46 +860,14 @@ class SequenceGroup:
860
861
return len(self.get_seqs(status))
862
895
- def num_unfinished_seqs(self) -> int:
896
- if self.is_single_seq:
897
- return 1 if not self.seqs[0].is_finished() else 0
898
-
899
- return len(self.get_unfinished_seqs())
900
-
863
def num_finished_seqs(self) -> int:
902
- if self.is_single_seq:
903
- return 1 if self.seqs[0].is_finished() else 0
904
-
905
- return len(self.get_finished_seqs())
906
-
907
- def find(self, seq_id: int) -> Sequence:
908
- if seq_id not in self.seqs_dict:
909
- raise ValueError(f"Sequence {seq_id} not found.")
910
- return self.seqs_dict[seq_id]
911
-
912
- def add(self, seq: Sequence) -> None:
913
- if seq.seq_id in self.seqs_dict:
914
- raise ValueError(f"Sequence {seq.seq_id} already exists.")
915
- self.seqs_dict[seq.seq_id] = seq
916
- self.seqs.append(seq)
917
- self.is_single_seq = len(self.seqs) == 1
918
-
919
- def remove(self, seq_id: int) -> None:
920
- seq = self.seqs_dict.pop(seq_id, None)
921
- if seq is None:
922
- raise ValueError(f"Sequence {seq_id} not found.")
923
- self.seqs.remove(seq)
924
- self.is_single_seq = len(self.seqs) == 1
864
+ return 1 if self.first_seq.is_finished() else 0
865
866
def is_finished(self) -> bool:
927
- if self.is_single_seq:
928
- return self.seqs[0].is_finished()
929
-
930
- return all(seq.is_finished() for seq in self.seqs)
867
+ return self.first_seq.is_finished()
868
869
def is_prefill(self) -> bool:
933
- # Every sequence should be in the same stage.
934
- return self.seqs[0].is_prefill()
870
+ return self.first_seq.is_prefill()
871
872
def __repr__(self) -> str:
873
return (f"SequenceGroup(request_id={self.request_id}, "
@@ -1455,7 +1391,7 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1391
for i in range(original_params.n):
1392
request_id_i = f"{request_id}_parallel_sample_{i}"
1393
group.seq_id_to_index[request_id_i] = i
1458
- seq_group = engine.add_request(
1394
+ seq_group = engine._add_processed_request(
1395
request_id_i,
1396
params=params,
1397
**kwargs,