@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4fdc581f9e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[core] simplify seq group code (#9569)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

youkaichao committed Oct 24, 2024 at 00:16 UTC 4fdc581f9e5740ba10b16ebf8a4c467e65bb9822
6 files changed +62 -566
tests/core/test_chunked_prefill_scheduler.py
-153
@@ -4,7 +4,6 @@ from unittest.mock import MagicMock
4 import pytest # noqa
5
6 from vllm.config import CacheConfig, SchedulerConfig
7 -from vllm.core.interfaces import AllocStatus
7 from vllm.core.scheduler import Scheduler
8 from vllm.sequence import Logprob, SequenceGroup
9
@@ -347,158 +346,6 @@ def test_prompt_limit_exceed():
346 assert out.ignored_seq_groups[0] == seq_group
347
348
350 -def test_swap():
351 - """Verify swapping works with chunked prefill requests"""
352 - block_size = 4
353 - max_seqs = 30
354 - max_model_len = 200
355 - max_num_batched_tokens = 30
356 - scheduler_config = SchedulerConfig(
357 - "generate",
358 - max_num_batched_tokens,
359 - max_seqs,
360 - max_model_len,
361 - enable_chunked_prefill=True,
362 - )
363 - cache_config = CacheConfig(block_size, 1.0, 1, "auto")
364 - cache_config.num_cpu_blocks = 16
365 - cache_config.num_gpu_blocks = 16
366 - scheduler = Scheduler(scheduler_config, cache_config, None)
367 -
368 - _, seq_group = create_dummy_prompt("1",
369 - prompt_length=60,
370 - best_of=2,
371 - block_size=block_size)
372 - scheduler.add_seq_group(seq_group)
373 - _, out = schedule_and_update_computed_tokens(scheduler)
374 - # The request is chunked.
375 - # prefill scheduled now.
376 - assert len(out.scheduled_seq_groups) == 1
377 - assert out.num_prefill_groups == 1
378 - assert seq_group.is_prefill()
379 - assert out.num_batched_tokens == max_num_batched_tokens
380 -
381 - # The last request should be swapped out.
382 - scheduler.block_manager.can_append_slots = MagicMock()
383 -
384 - def cannot_append_second_group(seq_group, num_lookahead_slots):
385 - return seq_group.request_id != "1"
386 -
387 - scheduler.block_manager.can_append_slots.side_effect = (
388 - cannot_append_second_group)
389 -
390 - # The running prefill is now swapped.
391 - _, out = schedule_and_update_computed_tokens(scheduler)
392 - assert len(out.scheduled_seq_groups) == 0
393 - assert out.num_batched_tokens == 0
394 - assert out.blocks_to_swap_out != []
395 - assert out.blocks_to_swap_in == []
396 -
397 - # Add 1 more task. Swap should be prioritized over new prefill.
398 - _, seq_group = create_dummy_prompt("2", prompt_length=60)
399 - scheduler.add_seq_group(seq_group)
400 - _, out = schedule_and_update_computed_tokens(scheduler)
401 - assert len(out.scheduled_seq_groups) == 1
402 - # 3 decodes. It is swapped in.
403 - assert out.num_batched_tokens == 30
404 - assert out.blocks_to_swap_in != []
405 - assert out.blocks_to_swap_out == []
406 -
407 -
408 -def test_running_prefill_prioritized_over_swap():
409 - block_size = 4
410 - max_seqs = 30
411 - max_model_len = 200
412 - max_num_batched_tokens = 30
413 - scheduler_config = SchedulerConfig(
414 - "generate",
415 - max_num_batched_tokens,
416 - max_seqs,
417 - max_model_len,
418 - enable_chunked_prefill=True,
419 - )
420 - cache_config = CacheConfig(block_size, 1.0, 1, "auto")
421 - cache_config.num_cpu_blocks = 32
422 - cache_config.num_gpu_blocks = 32
423 - scheduler = Scheduler(scheduler_config, cache_config, None)
424 -
425 - _, seq_group = create_dummy_prompt("1",
426 - prompt_length=60,
427 - best_of=2,
428 - block_size=block_size)
429 - scheduler.add_seq_group(seq_group)
430 - _, out = schedule_and_update_computed_tokens(scheduler)
431 - # The request is chunked.
432 - # prefill scheduled now.
433 - assert len(out.scheduled_seq_groups) == 1
434 - assert out.num_prefill_groups == 1
435 - assert seq_group.is_prefill()
436 - assert out.num_batched_tokens == max_num_batched_tokens
437 -
438 - # The request should be swapped out.
439 - scheduler.block_manager.can_append_slots = MagicMock()
440 -
441 - def cannot_append_second_group(seq_group, num_lookahead_slots):
442 - return seq_group.request_id != "1"
443 -
444 - scheduler.block_manager.can_append_slots.side_effect = (
445 - cannot_append_second_group)
446 -
447 - # The running prefill is now swapped.
448 - _, out = schedule_and_update_computed_tokens(scheduler)
449 - assert len(out.scheduled_seq_groups) == 0
450 - assert out.num_batched_tokens == 0
451 - assert out.blocks_to_swap_out != []
452 - assert out.blocks_to_swap_in == []
453 -
454 - # Add 1 more task. Swap is not possible, so prefill is running.
455 - scheduler.block_manager.can_swap_in = MagicMock()
456 - scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
457 -
458 - _, seq_group2 = create_dummy_prompt("2",
459 - prompt_length=60,
460 - block_size=block_size)
461 - scheduler.add_seq_group(seq_group2)
462 - _, out = schedule_and_update_computed_tokens(scheduler)
463 - assert len(out.scheduled_seq_groups) == 1
464 - # 3 decodes. It is swapped in.
465 - assert out.num_batched_tokens == 30
466 - assert out.blocks_to_swap_in == []
467 - assert out.blocks_to_swap_out == []
468 - assert out.scheduled_seq_groups[0].seq_group == seq_group2
469 -
470 - # Now although swap is possible, running prefill is prioritized.
471 - scheduler.block_manager.can_swap_in.return_value = AllocStatus.OK
472 - _, out = schedule_and_update_computed_tokens(scheduler)
473 - assert len(out.scheduled_seq_groups) == 1
474 - # 3 decodes. It is swapped in.
475 - assert out.num_batched_tokens == 30
476 - assert out.blocks_to_swap_in == []
477 - assert out.blocks_to_swap_out == []
478 - assert not seq_group2.is_prefill()
479 - assert out.scheduled_seq_groups[0].seq_group == seq_group2
480 - append_new_token(seq_group2, 1)
481 -
482 - # Decoding is prioritized.
483 - _, out = schedule_and_update_computed_tokens(scheduler)
484 - assert len(out.scheduled_seq_groups) == 1
485 - # 3 decodes. It is swapped in.
486 - assert out.num_batched_tokens == 1
487 - assert out.blocks_to_swap_in == []
488 - assert out.blocks_to_swap_out == []
489 - assert not seq_group2.is_prefill()
490 - assert out.scheduled_seq_groups[0].seq_group == seq_group2
491 - append_new_token(seq_group2, 1)
492 -
493 - # Since we abort the sequence group, we can finally swap.
494 - scheduler.abort_seq_group(seq_group2.request_id)
495 - _, out = schedule_and_update_computed_tokens(scheduler)
496 - assert len(out.scheduled_seq_groups) == 1
497 - assert out.num_batched_tokens == 30
498 - assert out.blocks_to_swap_in != []
499 - assert out.blocks_to_swap_out == []
500 -
501 -
349 def test_chunked_prefill_preempt():
350 """Verify preempt works with chunked prefill requests"""
351 block_size = 4
tests/core/test_scheduler.py
+1 -203
@@ -10,7 +10,7 @@ from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10 from vllm.core.interfaces import AllocStatus
11 from vllm.core.scheduler import Scheduler, SchedulingBudget
12 from vllm.lora.request import LoRARequest
13 -from vllm.sequence import SequenceGroup, SequenceStatus
13 +from vllm.sequence import SequenceGroup
14
15 from .utils import (append_new_token, append_new_token_seq_group,
16 create_dummy_prompt, get_sequence_groups,
@@ -296,55 +296,6 @@ def test_scheduler_delay_factor():
296 append_new_token(out, 1)
297
298
299 -def test_swapped_out_prioritized():
300 - block_size = 4
301 - scheduler = initialize_scheduler(max_num_seqs=6,
302 - block_size=block_size,
303 - num_cpu_blocks=64,
304 - num_gpu_blocks=64)
305 - # best_of=2 * 3 == 6 sequences.
306 - for i in range(3):
307 - _, seq_group = create_dummy_prompt(str(i),
308 - prompt_length=60,
309 - best_of=2,
310 - block_size=block_size)
311 - scheduler.add_seq_group(seq_group)
312 - seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
313 - # prefill scheduled now.
314 - assert len(out.scheduled_seq_groups) == 3
315 - append_new_token(out, 1)
316 -
317 - # The last request should be swapped out.
318 - scheduler.block_manager.can_append_slots = MagicMock()
319 -
320 - def cannot_append_second_group(seq_group, num_lookahead_slots):
321 - return seq_group.request_id != "2"
322 -
323 - scheduler.block_manager.can_append_slots.side_effect = (
324 - cannot_append_second_group)
325 -
326 - seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
327 - assert len(out.scheduled_seq_groups) == 2
328 - assert out.num_batched_tokens == 2
329 - assert out.blocks_to_swap_out != []
330 - assert out.blocks_to_swap_in == []
331 - append_new_token(out, 1)
332 -
333 - # Add 1 more task. Swap should be prioritized over prefill.
334 - _, seq_group = create_dummy_prompt(str(i),
335 - prompt_length=60,
336 - best_of=2,
337 - block_size=block_size)
338 - scheduler.add_seq_group(seq_group)
339 - seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
340 - append_new_token(out, 1)
341 - assert len(out.scheduled_seq_groups) == 3
342 - # 3 decodes. It is swapped in.
343 - assert out.num_batched_tokens == 3
344 - assert out.blocks_to_swap_in != []
345 - assert out.blocks_to_swap_out == []
346 -
347 -
299 def initialize_scheduler(
300 *,
301 max_num_seqs=1000,
@@ -646,60 +597,6 @@ def test_decode_schedule_preempted():
597 assert output.blocks_to_copy == []
598
599
649 -def test_decode_swap_beam_search():
650 - """
651 - Test best_of > 1 swap out blocks
652 - """
653 - block_size = 4
654 - scheduler = initialize_scheduler(block_size=block_size,
655 - num_gpu_blocks=64,
656 - num_cpu_blocks=64)
657 - curr_loras = None
658 - budget = create_token_budget()
659 - for i in range(3):
660 - _, seq_group = create_dummy_prompt(str(i),
661 - prompt_length=60,
662 - best_of=2,
663 - block_size=block_size)
664 - scheduler._allocate_and_set_running(seq_group)
665 - scheduler._add_seq_group_to_running(seq_group)
666 - append_new_token_seq_group(60, seq_group, 1)
667 - budget.add_num_seqs(seq_group.request_id,
668 - seq_group.get_max_num_running_seqs())
669 - budget.add_num_batched_tokens(
670 - seq_group.request_id, seq_group.num_seqs(SequenceStatus.RUNNING))
671 -
672 - # The last request should be swapped out.
673 - scheduler.block_manager.can_append_slots = MagicMock()
674 -
675 - def cannot_append_second_group(seq_group, num_lookahead_slots):
676 - return seq_group.request_id != "2"
677 -
678 - scheduler.block_manager.can_append_slots.side_effect = (
679 - cannot_append_second_group)
680 - scheduler.block_manager.swap_out = MagicMock()
681 - expected_swap_mapping = [("5", "7")]
682 - scheduler.block_manager.swap_out.return_value = expected_swap_mapping
683 -
684 - output = scheduler._schedule_running(budget, curr_loras)
685 - remainig_running = scheduler.running
686 - assert len(remainig_running) == 0
687 - assert len(output.decode_seq_groups) == 2
688 - assert len(output.prefill_seq_groups) == 0
689 - assert output.decode_seq_groups[0].seq_group.request_id == "0"
690 - assert output.decode_seq_groups[1].seq_group.request_id == "1"
691 - assert len(output.preempted) == 0
692 - assert len(output.swapped_out) == 1
693 - # Budget should refledct preempted requests.
694 - assert budget.num_batched_tokens == 2
695 - # since there are 2 sequences, 2 should be subtracted.
696 - assert budget.num_curr_seqs == 4
697 - # Both should be preempted, not swapped.
698 - assert output.blocks_to_swap_out == expected_swap_mapping
699 - # Nothing is copied.
700 - assert output.blocks_to_copy == []
701 -
702 -
600 def test_schedule_decode_blocks_to_copy_update():
601 """
602 Verify blocks_to_copy is updated.
@@ -736,105 +633,6 @@ def test_schedule_decode_blocks_to_copy_update():
633 assert output.blocks_to_copy == [(2, 3)]
634
635
739 -def test_schedule_swapped_simple():
740 - block_size = 4
741 - scheduler = initialize_scheduler(block_size=block_size)
742 - curr_loras = None
743 - blocks_to_swap_out: List[Tuple[int, int]] = []
744 - _, seq_group = create_dummy_prompt("1",
745 - prompt_length=4,
746 - best_of=2,
747 - block_size=block_size)
748 - scheduler._allocate_and_set_running(seq_group)
749 - append_new_token_seq_group(4, seq_group, 1)
750 - scheduler._swap_out(seq_group, blocks_to_swap_out)
751 - scheduler._add_seq_group_to_swapped(seq_group)
752 -
753 - budget = create_token_budget()
754 - output = scheduler._schedule_swapped(budget, curr_loras)
755 - remaining_swapped = scheduler.swapped
756 - assert len(remaining_swapped) == 0
757 - assert budget.num_batched_tokens == 1
758 - assert budget.num_curr_seqs == 2
759 - assert len(output.decode_seq_groups) == 1
760 - assert len(output.prefill_seq_groups) == 0
761 - # swap in is the reverse of swap out
762 - blocks_to_swap_in_reverse = []
763 - for swapin, swapout in output.blocks_to_swap_in:
764 - blocks_to_swap_in_reverse.append((swapout, swapin))
765 - assert blocks_to_swap_out == blocks_to_swap_in_reverse
766 -
767 -
768 -def test_schedule_swapped_max_token_budget():
769 - block_size = 4
770 - scheduler = initialize_scheduler(block_size=block_size,
771 - num_cpu_blocks=32,
772 - num_gpu_blocks=32)
773 - curr_loras = None
774 - blocks_to_swap_out: List[Tuple[int, int]] = []
775 - for i in range(2):
776 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
777 - scheduler._allocate_and_set_running(seq_group)
778 - append_new_token_seq_group(60, seq_group, 1)
779 - scheduler._swap_out(seq_group, blocks_to_swap_out)
780 - scheduler._add_seq_group_to_swapped(seq_group)
781 -
782 - budget = create_token_budget(token_budget=1)
783 - output = scheduler._schedule_swapped(budget, curr_loras)
784 - remaining_swapped = scheduler.swapped
785 - assert len(remaining_swapped) == 1
786 - assert budget.num_batched_tokens == 1
787 - assert budget.num_curr_seqs == 2
788 - assert len(output.decode_seq_groups) == 1
789 - assert len(output.prefill_seq_groups) == 0
790 -
791 - # Verify num_batched_tokens are respected.
792 - budget = create_token_budget(token_budget=1)
793 - add_token_budget(budget, 1, 0)
794 - output = scheduler._schedule_swapped(budget, curr_loras)
795 - remaining_swapped = scheduler.swapped
796 - assert len(remaining_swapped) == 1
797 - assert budget.num_batched_tokens == 1
798 - assert budget.num_curr_seqs == 0
799 - assert len(output.decode_seq_groups) == 0
800 - assert len(output.prefill_seq_groups) == 0
801 -
802 -
803 -def test_schedule_swapped_max_seqs():
804 - block_size = 4
805 - scheduler = initialize_scheduler(block_size=block_size,
806 - num_cpu_blocks=64,
807 - num_gpu_blocks=64)
808 - curr_loras = None
809 - blocks_to_swap_out: List[Tuple[int, int]] = []
810 - for i in range(4):
811 - _, seq_group = create_dummy_prompt(str(i),
812 - prompt_length=60,
813 - block_size=4)
814 - scheduler._allocate_and_set_running(seq_group)
815 - append_new_token_seq_group(60, seq_group, 1)
816 - scheduler._swap_out(seq_group, blocks_to_swap_out)
817 - scheduler._add_seq_group_to_swapped(seq_group)
818 -
819 - budget = create_token_budget(max_num_seqs=2)
820 - output = scheduler._schedule_swapped(budget, curr_loras)
821 - remaining_swapped = scheduler.swapped
822 - assert len(remaining_swapped) == 2
823 - assert budget.num_batched_tokens == 2
824 - assert budget.num_curr_seqs == 2
825 - assert len(output.decode_seq_groups) == 2
826 - assert len(output.prefill_seq_groups) == 0
827 -
828 - # Verify num_curr_seqs are respected.
829 - output = scheduler._schedule_swapped(budget, curr_loras)
830 - remaining_swapped = scheduler.swapped
831 - assert len(remaining_swapped) == 2
832 - assert budget.num_batched_tokens == 2
833 - assert budget.num_curr_seqs == 2
834 - assert len(output.decode_seq_groups) == 0
835 - assert len(output.prefill_seq_groups) == 0
836 -
837 -
636 def test_schedule_swapped_max_loras():
637 block_size = 4
638 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
vllm/core/scheduler.py
+1 -1
@@ -290,7 +290,7 @@ def scheduler_running_outputs_builder():
290
291
292 def scheduled_seq_group_builder():
293 - return ScheduledSequenceGroup(SequenceGroup("", [], -1),
293 + return ScheduledSequenceGroup(SequenceGroup.__new__(SequenceGroup),
294 token_chunk_size=0)
295 # return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
296
vllm/engine/llm_engine.py
+19 -21
@@ -647,10 +647,24 @@ class LLMEngine:
647 prompt_adapter_request: Optional[PromptAdapterRequest],
648 trace_headers: Optional[Mapping[str, str]] = None,
649 priority: int = 0,
650 - ) -> SequenceGroup:
650 + ) -> Optional[SequenceGroup]:
651 """Add a processed request to the engine's request pool.
652 return the created sequence group.
653 """
654 + if isinstance(params, SamplingParams) and params.n > 1:
655 + ParallelSampleSequenceGroup.add_request(
656 + request_id,
657 + self,
658 + params,
659 + processed_inputs=processed_inputs,
660 + arrival_time=arrival_time,
661 + lora_request=lora_request,
662 + trace_headers=trace_headers,
663 + prompt_adapter_request=prompt_adapter_request,
664 + priority=priority,
665 + )
666 + return None
667 +
668 self._validate_model_inputs(processed_inputs)
669 # Create the sequences.
670 block_size = self.cache_config.block_size
@@ -721,7 +735,7 @@ class LLMEngine:
735 trace_headers: Optional[Mapping[str, str]] = None,
736 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
737 priority: int = 0,
724 - ) -> Optional[SequenceGroup]:
738 + ) -> None:
739 ...
740
741 @overload
@@ -735,7 +749,7 @@ class LLMEngine:
749 trace_headers: Optional[Mapping[str, str]] = None,
750 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
751 priority: int = 0,
738 - ) -> Optional[SequenceGroup]:
752 + ) -> None:
753 ...
754
755 @deprecate_kwargs(
@@ -754,7 +768,7 @@ class LLMEngine:
768 priority: int = 0,
769 *,
770 inputs: Optional[PromptType] = None, # DEPRECATED
757 - ) -> Optional[SequenceGroup]:
771 + ) -> None:
772 """Add a request to the engine's request pool.
773
774 The request is added to the request pool and will be processed by the
@@ -798,22 +812,6 @@ class LLMEngine:
812 >>> # continue the request processing
813 >>> ...
814 """
801 -
802 - if isinstance(params, SamplingParams) and params.n > 1:
803 - ParallelSampleSequenceGroup.add_request(
804 - request_id,
805 - self,
806 - params,
807 - prompt=prompt,
808 - arrival_time=arrival_time,
809 - lora_request=lora_request,
810 - trace_headers=trace_headers,
811 - prompt_adapter_request=prompt_adapter_request,
812 - priority=priority,
813 - inputs=inputs,
814 - )
815 - return None
816 -
815 if inputs is not None:
816 prompt = inputs
817 assert prompt is not None and params is not None
@@ -844,7 +842,7 @@ class LLMEngine:
842 processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
843 "mm_processor_kwargs")
844
847 - return self._add_processed_request(
845 + self._add_processed_request(
846 request_id=request_id,
847 processed_inputs=processed_inputs,
848 params=params,
vllm/engine/output_processor/single_step.py
+22 -105
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Tuple
1 +from typing import List
2
3 from vllm.config import SchedulerConfig
4 from vllm.core.scheduler import Scheduler
@@ -6,9 +6,8 @@ from vllm.engine.output_processor.interfaces import (
6 SequenceGroupOutputProcessor)
7 from vllm.engine.output_processor.stop_checker import StopChecker
8 from vllm.logger import init_logger
9 -from vllm.sequence import (CompletionSequenceGroupOutput, Sequence,
10 - SequenceGroup, SequenceGroupOutput, SequenceOutput,
11 - SequenceStatus)
9 +from vllm.sequence import (CompletionSequenceGroupOutput, SequenceGroup,
10 + SequenceGroupOutput)
11 from vllm.transformers_utils.detokenizer import Detokenizer
12 from vllm.utils import Counter
13
@@ -114,104 +113,22 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
113 outputs: SequenceGroupOutput,
114 is_async: bool) -> None:
115 sampling_params = seq_group.sampling_params
117 - if sampling_params.n == 1:
118 - # only have one output sample
119 - sample = outputs.samples[0]
120 - # only have one sequence
121 - seq = seq_group.seqs[0]
122 - if not is_async:
123 - seq.append_token_id(sample.output_token, sample.logprobs)
124 - if sampling_params.detokenize and self.detokenizer:
125 - new_char_count = self.detokenizer.decode_sequence_inplace(
126 - seq, sampling_params)
127 - else:
128 - new_char_count = 0
129 - self.stop_checker.maybe_stop_sequence(
130 - seq,
131 - new_char_count,
132 - sampling_params,
133 - lora_req=seq_group.lora_request,
134 - )
135 - if seq.is_finished():
136 - for scheduler in self.scheduler:
137 - scheduler.free_seq(seq)
138 - return
139 -
140 - # TODO: Add support for async for beam search
141 - assert not is_async
142 -
143 - # Process samples
144 - samples = outputs.samples
145 - parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
146 - parent_child_dict: Dict[int, List[SequenceOutput]] = {
147 - parent_seq.seq_id: []
148 - for parent_seq in parent_seqs
149 - }
150 - for sample in samples:
151 - # Guard against a KeyError which can occur if the request was
152 - # aborted while the output was generated
153 - if (child_list :=
154 - parent_child_dict.get(sample.parent_seq_id)) is not None:
155 - child_list.append(sample)
156 - # List of (child, parent)
157 - child_seqs: List[Tuple[Sequence, Sequence]] = []
158 -
159 - # Process the child samples for each parent sequence
160 - for parent in parent_seqs:
161 - child_samples: List[SequenceOutput] = parent_child_dict[
162 - parent.seq_id]
163 - if len(child_samples) == 0:
164 - # This parent sequence has no children samples. Remove
165 - # the parent sequence from the sequence group since it will
166 - # not be used in the future iterations.
167 - parent.status = SequenceStatus.FINISHED_ABORTED
168 - seq_group.remove(parent.seq_id)
169 - for scheduler in self.scheduler:
170 - scheduler.free_seq(parent)
171 - continue
172 - # Fork the parent sequence if there are multiple child samples.
173 - for child_sample in child_samples[:-1]:
174 - new_child_seq_id: int = next(self.seq_counter)
175 - child = parent.fork(new_child_seq_id)
176 - child.append_token_id(child_sample.output_token,
177 - child_sample.logprobs)
178 - child_seqs.append((child, parent))
179 - # Continue the parent sequence for the last child sample.
180 - # We reuse the parent sequence here to reduce redundant memory
181 - # copies, especially when using non-beam search sampling methods.
182 - last_child_sample = child_samples[-1]
183 - parent.append_token_id(last_child_sample.output_token,
184 - last_child_sample.logprobs)
185 - child_seqs.append((parent, parent))
186 -
187 - for seq, _ in child_seqs:
188 - if sampling_params.detokenize and self.detokenizer:
189 - new_char_count = self.detokenizer.decode_sequence_inplace(
190 - seq, sampling_params)
191 - else:
192 - new_char_count = 0
193 - self.stop_checker.maybe_stop_sequence(
194 - seq,
195 - new_char_count,
196 - sampling_params,
197 - lora_req=seq_group.lora_request,
198 - )
199 -
200 - # For newly created child sequences, add them to the sequence group
201 - # and fork them in block manager if they are not finished.
202 - for seq, parent in child_seqs:
203 - if seq is not parent:
204 - seq_group.add(seq)
205 - if not seq.is_finished():
206 - for scheduler in self.scheduler:
207 - scheduler.fork_seq(parent, seq)
208 -
209 - # Free the finished and selected parent sequences' memory in block
210 - # manager. Keep them in the sequence group as candidate output.
211 - # NOTE: we need to fork the new sequences before freeing the
212 - # old sequences.
213 - for seq, parent in child_seqs:
214 - if seq is parent and seq.is_finished():
215 - for scheduler in self.scheduler:
216 - scheduler.free_seq(seq)
217 - return
116 +
117 + sample = outputs.samples[0]
118 + seq = seq_group.first_seq
119 + if not is_async:
120 + seq.append_token_id(sample.output_token, sample.logprobs)
121 + if sampling_params.detokenize and self.detokenizer:
122 + new_char_count = self.detokenizer.decode_sequence_inplace(
123 + seq, sampling_params)
124 + else:
125 + new_char_count = 0
126 + self.stop_checker.maybe_stop_sequence(
127 + seq,
128 + new_char_count,
129 + sampling_params,
130 + lora_req=seq_group.lora_request,
131 + )
132 + if seq.is_finished():
133 + for scheduler in self.scheduler:
134 + scheduler.free_seq(seq)
vllm/sequence.py
+19 -83
@@ -681,6 +681,7 @@ class SequenceGroup:
681 ) -> None:
682 self.request_id = request_id
683 self.seqs = seqs
684 + self.first_seq = seqs[0]
685 self.arrival_time = arrival_time
686 self.is_single_seq = len(seqs) == 1
687 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -705,15 +706,11 @@ class SequenceGroup:
706
707 @property
708 def prompt(self) -> Optional[str]:
708 - # All sequences in the group should have the same prompt.
709 - # We use the prompt of an arbitrary sequence.
710 - return self.seqs[0].prompt
709 + return self.first_seq.prompt
710
711 @property
712 def prompt_token_ids(self) -> List[int]:
714 - # All sequences in the group should have the same prompt.
715 - # We use the prompt of an arbitrary sequence.
716 - return self.seqs[0].prompt_token_ids
713 + return self.first_seq.prompt_token_ids
714
715 @property
716 def encoder_prompt(self) -> Optional[str]:
@@ -733,17 +730,11 @@ class SequenceGroup:
730
731 @property
732 def multi_modal_data(self) -> "MultiModalDataDict":
736 - # All sequences in the group should have the same multi-modal data.
737 - # We use the multi-modal data of an arbitrary sequence.
738 - return self.seqs[0].multi_modal_data
733 + return self.first_seq.multi_modal_data
734
735 @property
736 def mm_processor_kwargs(self) -> Dict[str, Any]:
742 - # As with multi-modal data, all sequences in the group should have the
743 - # same processor kwargs (i.e., mm_processor_kwargs are optionally
744 - # provided per request; note that are independent of whether the model
745 - # decoder-only or an encoder-decoder).
746 - return self.seqs[0].mm_processor_kwargs
737 + return self.first_seq.mm_processor_kwargs
738
739 @property
740 def lora_int_id(self) -> int:
@@ -808,7 +799,7 @@ class SequenceGroup:
799 # in TPOT, rather than recalculating TTFT (since from the )
800 # POV of the user, there is simply a long generation delay.
801 if (self.metrics.first_token_time is None
811 - and self.seqs[0].get_output_len() == 1):
802 + and self.first_seq.get_output_len() == 1):
803 self.metrics.first_token_time = time
804
805 def maybe_set_first_scheduled_time(self, time: float) -> None:
@@ -825,18 +816,7 @@ class SequenceGroup:
816 def get_max_num_running_seqs(self) -> int:
817 """The maximum number of sequences running in parallel in the remaining
818 lifetime of the request."""
828 - if self.sampling_params:
829 - n = self.sampling_params.n
830 - assert isinstance(n, int)
831 - if n > self.num_seqs():
832 - # At prompt stage, the sequence group is not yet filled up
833 - # and only have one sequence running. However, in the
834 - # generation stage, we will have `n` sequences
835 - # running.
836 - return n
837 - # At sampling stages, return the number of actual sequences
838 - # that are not finished yet.
839 - return self.num_unfinished_seqs()
819 + return 0 if self.first_seq.is_finished() else 1
820
821 def get_seqs(
822 self,
@@ -845,10 +825,7 @@ class SequenceGroup:
825 if status is None:
826 return self.seqs
827
848 - if self.is_single_seq:
849 - return self.seqs if self.seqs[0].status == status else []
850 -
851 - return [seq for seq in self.seqs if seq.status == status]
828 + return self.seqs if self.first_seq.status == status else []
829
830 def is_encoder_decoder(self) -> bool:
831 return self.encoder_seq is not None
@@ -856,29 +833,20 @@ class SequenceGroup:
833 def get_encoder_seq(self) -> Optional[Sequence]:
834 return self.encoder_seq
835
859 - def get_unfinished_seqs(self) -> List[Sequence]:
860 - if self.is_single_seq:
861 - return self.seqs if not self.seqs[0].is_finished() else []
862 -
863 - return [seq for seq in self.seqs if not seq.is_finished()]
864 -
836 def get_finished_seqs(self) -> List[Sequence]:
866 - if self.is_single_seq:
867 - return self.seqs if self.seqs[0].is_finished() else []
868 -
869 - return [seq for seq in self.seqs if seq.is_finished()]
837 + return self.seqs if self.first_seq.is_finished() else []
838
839 def update_num_computed_tokens(self, num_new_computed_tokens: int):
840 """Update number of tokens computed so far."""
873 - for seq in self.seqs:
874 - if not seq.is_finished():
875 - seq.data.update_num_computed_tokens(num_new_computed_tokens)
841 + seq = self.first_seq
842 + if not seq.is_finished():
843 + seq.data.update_num_computed_tokens(num_new_computed_tokens)
844
845 def get_num_uncomputed_tokens(self) -> int:
846 num_uncomputed_tokens = 0
879 - for seq in self.seqs:
880 - if not seq.is_finished():
881 - num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
847 + seq = self.first_seq
848 + if not seq.is_finished():
849 + num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
850 return num_uncomputed_tokens
851
852 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
@@ -892,46 +860,14 @@ class SequenceGroup:
860
861 return len(self.get_seqs(status))
862
895 - def num_unfinished_seqs(self) -> int:
896 - if self.is_single_seq:
897 - return 1 if not self.seqs[0].is_finished() else 0
898 -
899 - return len(self.get_unfinished_seqs())
900 -
863 def num_finished_seqs(self) -> int:
902 - if self.is_single_seq:
903 - return 1 if self.seqs[0].is_finished() else 0
904 -
905 - return len(self.get_finished_seqs())
906 -
907 - def find(self, seq_id: int) -> Sequence:
908 - if seq_id not in self.seqs_dict:
909 - raise ValueError(f"Sequence {seq_id} not found.")
910 - return self.seqs_dict[seq_id]
911 -
912 - def add(self, seq: Sequence) -> None:
913 - if seq.seq_id in self.seqs_dict:
914 - raise ValueError(f"Sequence {seq.seq_id} already exists.")
915 - self.seqs_dict[seq.seq_id] = seq
916 - self.seqs.append(seq)
917 - self.is_single_seq = len(self.seqs) == 1
918 -
919 - def remove(self, seq_id: int) -> None:
920 - seq = self.seqs_dict.pop(seq_id, None)
921 - if seq is None:
922 - raise ValueError(f"Sequence {seq_id} not found.")
923 - self.seqs.remove(seq)
924 - self.is_single_seq = len(self.seqs) == 1
864 + return 1 if self.first_seq.is_finished() else 0
865
866 def is_finished(self) -> bool:
927 - if self.is_single_seq:
928 - return self.seqs[0].is_finished()
929 -
930 - return all(seq.is_finished() for seq in self.seqs)
867 + return self.first_seq.is_finished()
868
869 def is_prefill(self) -> bool:
933 - # Every sequence should be in the same stage.
934 - return self.seqs[0].is_prefill()
870 + return self.first_seq.is_prefill()
871
872 def __repr__(self) -> str:
873 return (f"SequenceGroup(request_id={self.request_id}, "
@@ -1455,7 +1391,7 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1391 for i in range(original_params.n):
1392 request_id_i = f"{request_id}_parallel_sample_{i}"
1393 group.seq_id_to_index[request_id_i] = i
1458 - seq_group = engine.add_request(
1394 + seq_group = engine._add_processed_request(
1395 request_id_i,
1396 params=params,
1397 **kwargs,