A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Minor] Move scheduler outputs to a separate file (#13062)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Feb 10, 2025 at 18:10 UTC
2ff4857678044407a959398178a7a04a9530919a
4 files changed
+113
-89
vllm/v1/core/scheduler.py
+3
-86
@@ -1,26 +1,20 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from collections import deque
4
-from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
6
- Tuple, Union)
4
+from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
5
6
from vllm.config import CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig
7
from vllm.logger import init_logger
10
-from vllm.lora.request import LoRARequest
11
-from vllm.sampling_params import SamplingParams
8
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
9
compute_encoder_budget)
10
from vllm.v1.core.kv_cache_manager import KVCacheManager
11
+from vllm.v1.core.scheduler_output import (CachedRequestData, NewRequestData,
12
+ SchedulerOutput)
13
from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
14
from vllm.v1.metrics.stats import SchedulerStats
15
from vllm.v1.outputs import ModelRunnerOutput
16
from vllm.v1.request import Request, RequestStatus
17
20
-if TYPE_CHECKING:
21
- from vllm.multimodal import MultiModalKwargs
22
- from vllm.multimodal.base import PlaceholderRange
23
-
18
logger = init_logger(__name__)
19
20
@@ -600,80 +594,3 @@ class Scheduler:
594
num_waiting_reqs=len(self.waiting),
595
gpu_cache_usage=self.kv_cache_manager.usage,
596
)
603
-
604
-
605
-@dataclass
606
-class NewRequestData:
607
-
608
- req_id: str
609
- prompt_token_ids: List[int]
610
- prompt: Optional[str]
611
- mm_inputs: List["MultiModalKwargs"]
612
- mm_hashes: List[str]
613
- mm_positions: List["PlaceholderRange"]
614
- sampling_params: SamplingParams
615
- block_ids: List[int]
616
- num_computed_tokens: int
617
- lora_request: Optional[LoRARequest]
618
-
619
- @classmethod
620
- def from_request(
621
- cls,
622
- request: Request,
623
- block_ids: List[int],
624
- num_computed_tokens: int,
625
- ) -> "NewRequestData":
626
- return cls(
627
- req_id=request.request_id,
628
- prompt_token_ids=request.prompt_token_ids,
629
- prompt=request.prompt,
630
- mm_inputs=request.mm_inputs,
631
- mm_hashes=request.mm_hashes,
632
- mm_positions=request.mm_positions,
633
- sampling_params=request.sampling_params,
634
- block_ids=block_ids,
635
- num_computed_tokens=num_computed_tokens,
636
- lora_request=request.lora_request,
637
- )
638
-
639
-
640
-@dataclass
641
-class CachedRequestData:
642
-
643
- req_id: str
644
- # If resumed_from_preemption is False, new_block_ids will be appended to
645
- # the request's block IDs. If True, new_block_ids will be used as the
646
- # request's block IDs instead of appending to the existing block IDs.
647
- resumed_from_preemption: bool
648
- new_block_ids: List[int]
649
- num_computed_tokens: int
650
-
651
- @classmethod
652
- def from_request(
653
- cls,
654
- request: Request,
655
- resumed_from_preemption: bool,
656
- new_block_ids: List[int],
657
- num_computed_tokens: int,
658
- ) -> "CachedRequestData":
659
- return cls(
660
- req_id=request.request_id,
661
- resumed_from_preemption=resumed_from_preemption,
662
- new_block_ids=new_block_ids,
663
- num_computed_tokens=num_computed_tokens,
664
- )
665
-
666
-
667
-@dataclass
668
-class SchedulerOutput:
669
-
670
- scheduled_new_reqs: List[NewRequestData]
671
- scheduled_cached_reqs: List[CachedRequestData]
672
-
673
- num_scheduled_tokens: Dict[str, int]
674
- total_num_scheduled_tokens: int
675
- scheduled_encoder_inputs: Dict[str, List[int]]
676
- num_common_prefix_blocks: int
677
-
678
- finished_req_ids: Set[str]
679
- free_encoder_input_ids: List[Tuple[str, int]]
vllm/v1/core/scheduler_output.py
new
+108
@@ -0,0 +1,108 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from dataclasses import dataclass
4
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
5
+
6
+if TYPE_CHECKING:
7
+ from vllm.lora.request import LoRARequest
8
+ from vllm.multimodal import MultiModalKwargs
9
+ from vllm.multimodal.base import PlaceholderRange
10
+ from vllm.sampling_params import SamplingParams
11
+ from vllm.v1.request import Request
12
+
13
+
14
+@dataclass
15
+class NewRequestData:
16
+
17
+ req_id: str
18
+ prompt_token_ids: List[int]
19
+ prompt: Optional[str]
20
+ mm_inputs: List["MultiModalKwargs"]
21
+ mm_hashes: List[str]
22
+ mm_positions: List["PlaceholderRange"]
23
+ sampling_params: "SamplingParams"
24
+ block_ids: List[int]
25
+ num_computed_tokens: int
26
+ lora_request: Optional["LoRARequest"]
27
+
28
+ @classmethod
29
+ def from_request(
30
+ cls,
31
+ request: "Request",
32
+ block_ids: List[int],
33
+ num_computed_tokens: int,
34
+ ) -> "NewRequestData":
35
+ return cls(
36
+ req_id=request.request_id,
37
+ prompt_token_ids=request.prompt_token_ids,
38
+ prompt=request.prompt,
39
+ mm_inputs=request.mm_inputs,
40
+ mm_hashes=request.mm_hashes,
41
+ mm_positions=request.mm_positions,
42
+ sampling_params=request.sampling_params,
43
+ block_ids=block_ids,
44
+ num_computed_tokens=num_computed_tokens,
45
+ lora_request=request.lora_request,
46
+ )
47
+
48
+
49
+@dataclass
50
+class CachedRequestData:
51
+
52
+ req_id: str
53
+ # If resumed_from_preemption is False, new_block_ids will be appended to
54
+ # the request's block IDs. If True, new_block_ids will be used as the
55
+ # request's block IDs instead of appending to the existing block IDs.
56
+ resumed_from_preemption: bool
57
+ new_block_ids: List[int]
58
+ num_computed_tokens: int
59
+
60
+ @classmethod
61
+ def from_request(
62
+ cls,
63
+ request: "Request",
64
+ resumed_from_preemption: bool,
65
+ new_block_ids: List[int],
66
+ num_computed_tokens: int,
67
+ ) -> "CachedRequestData":
68
+ return cls(
69
+ req_id=request.request_id,
70
+ resumed_from_preemption=resumed_from_preemption,
71
+ new_block_ids=new_block_ids,
72
+ num_computed_tokens=num_computed_tokens,
73
+ )
74
+
75
+
76
+@dataclass
77
+class SchedulerOutput:
78
+
79
+ # List of the requests that are scheduled for the first time.
80
+ # We cache the request's data in each worker process, so that we don't
81
+ # need to re-send it every scheduling step.
82
+ scheduled_new_reqs: List[NewRequestData]
83
+ # List of the requests that have been scheduled before.
84
+ # Since the request's data is already cached in the worker processes,
85
+ # we only send the diff to minimize the communication cost.
86
+ scheduled_cached_reqs: List[CachedRequestData]
87
+
88
+ # req_id -> num_scheduled_tokens
89
+ # Number of tokens scheduled for each request.
90
+ num_scheduled_tokens: Dict[str, int]
91
+ # Total number of tokens scheduled for all requests.
92
+ # Equal to sum(num_scheduled_tokens.values())
93
+ total_num_scheduled_tokens: int
94
+ # req_id -> encoder input indices that need processing.
95
+ # E.g., if a request has [0, 1], it could mean the vision encoder needs
96
+ # to process that the request's 0-th and 1-th images in the current step.
97
+ scheduled_encoder_inputs: Dict[str, List[int]]
98
+ # Number of common prefix blocks for all requests.
99
+ # This can be used for cascade attention.
100
+ num_common_prefix_blocks: int
101
+
102
+ # Request IDs that are finished in between the previous and the current
103
+ # steps. This is used to notify the workers about the finished requests
104
+ # so that they can free the cached states for those requests.
105
+ finished_req_ids: Set[str]
106
+ # List of (req_id, encoder_input_index) tuples.
107
+ # Used to free the encoder cache.
108
+ free_encoder_input_ids: List[Tuple[str, int]]
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -36,7 +36,7 @@ from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
36
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
37
38
if TYPE_CHECKING:
39
- from vllm.v1.core.scheduler import SchedulerOutput
39
+ from vllm.v1.core.scheduler_output import SchedulerOutput
40
41
logger = init_logger(__name__)
42
vllm/v1/worker/gpu_worker.py
+1
-2
@@ -18,7 +18,6 @@ from vllm.logger import init_logger
18
from vllm.model_executor import set_random_seed
19
from vllm.platforms import current_platform
20
from vllm.utils import GiB_bytes
21
-from vllm.v1.core.scheduler import SchedulerOutput
21
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
22
from vllm.v1.outputs import ModelRunnerOutput
23
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -26,7 +25,7 @@ from vllm.v1.worker.gpu_model_runner import GPUModelRunner
25
logger = init_logger(__name__)
26
27
if TYPE_CHECKING:
29
- from vllm.v1.core.scheduler import SchedulerOutput
28
+ from vllm.v1.core.scheduler_output import SchedulerOutput
29
30
31
class Worker: