A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[V1][Minor] Minor code cleanup for scheduling metrics (#14800)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Mar 14, 2025 at 01:21 UTC
c77620d22d43daa7e0440e6267cbdd83f849ac64
2 files changed
+23
-40
vllm/v1/core/scheduler.py
+10
-25
@@ -15,8 +15,8 @@ from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
15
from vllm.v1.core.kv_cache_manager import KVCacheManager
16
from vllm.v1.core.scheduler_output import (CachedRequestData, NewRequestData,
17
SchedulerOutput)
18
-from vllm.v1.engine import (EngineCoreEvent, EngineCoreEventType,
19
- EngineCoreOutput, EngineCoreOutputs)
18
+from vllm.v1.engine import (EngineCoreEventType, EngineCoreOutput,
19
+ EngineCoreOutputs)
20
from vllm.v1.metrics.stats import SchedulerStats
21
from vllm.v1.outputs import ModelRunnerOutput
22
from vllm.v1.request import Request, RequestStatus
@@ -178,7 +178,9 @@ class Scheduler:
178
self.kv_cache_manager.free(preempted_req)
179
preempted_req.status = RequestStatus.PREEMPTED
180
preempted_req.num_computed_tokens = 0
181
- self.request_preempted(preempted_req, scheduled_timestamp)
181
+ if self.log_stats:
182
+ preempted_req.record_event(
183
+ EngineCoreEventType.PREEMPTED, scheduled_timestamp)
184
185
self.waiting.appendleft(preempted_req)
186
preempted_reqs.append(preempted_req)
@@ -320,7 +322,9 @@ class Scheduler:
322
req_index += 1
323
self.running.append(request)
324
self.scheduled_req_ids.add(request.request_id)
323
- self.request_scheduled(request, scheduled_timestamp)
325
+ if self.log_stats:
326
+ request.record_event(EngineCoreEventType.SCHEDULED,
327
+ scheduled_timestamp)
328
if request.status == RequestStatus.WAITING:
329
scheduled_new_reqs.append(request)
330
elif request.status == RequestStatus.PREEMPTED:
@@ -666,7 +670,8 @@ class Scheduler:
670
def add_request(self, request: Request) -> None:
671
self.waiting.append(request)
672
self.requests[request.request_id] = request
669
- self.request_queued(request)
673
+ if self.log_stats:
674
+ request.record_event(EngineCoreEventType.QUEUED)
675
676
def finish_requests(
677
self,
@@ -728,26 +733,6 @@ class Scheduler:
733
def reset_prefix_cache(self) -> bool:
734
return self.kv_cache_manager.reset_prefix_cache()
735
731
- def request_queued(self, request: Request):
732
- if not self.log_stats:
733
- return
734
- request.events.append(
735
- EngineCoreEvent.new_event(EngineCoreEventType.QUEUED))
736
-
737
- def request_scheduled(self, request: Request, timestamp: float):
738
- if not self.log_stats:
739
- return
740
- request.events.append(
741
- EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
742
- timestamp))
743
-
744
- def request_preempted(self, request: Request, timestamp: float):
745
- if not self.log_stats:
746
- return
747
- request.events.append(
748
- EngineCoreEvent.new_event(EngineCoreEventType.PREEMPTED,
749
- timestamp))
750
-
736
def make_stats(self) -> Optional[SchedulerStats]:
737
if not self.log_stats:
738
return None
vllm/v1/request.py
+13
-15
@@ -88,21 +88,6 @@ class Request:
88
sampling_params=request.sampling_params),
89
)
90
91
- def queued(self, timestamp: Optional[float] = None) -> None:
92
- self.events.append(
93
- EngineCoreEvent.new_event(EngineCoreEventType.QUEUED, timestamp))
94
-
95
- def scheduled(self, timestamp: Optional[float] = None) -> None:
96
- self.events.append(
97
- EngineCoreEvent.new_event(EngineCoreEventType.SCHEDULED,
98
- timestamp))
99
-
100
- def take_events(self) -> Optional[list[EngineCoreEvent]]:
101
- if not self.events:
102
- return None
103
- events, self.events = self.events, []
104
- return events
105
-
91
def append_output_token_ids(
92
self,
93
token_ids: Union[int, list[int]],
@@ -146,6 +131,19 @@ class Request:
131
def use_structured_output(self) -> bool:
132
return self.sampling_params.guided_decoding is not None
133
134
+ def record_event(
135
+ self,
136
+ event_type: EngineCoreEventType,
137
+ timestamp: Optional[float] = None,
138
+ ) -> None:
139
+ self.events.append(EngineCoreEvent.new_event(event_type, timestamp))
140
+
141
+ def take_events(self) -> Optional[list[EngineCoreEvent]]:
142
+ if not self.events:
143
+ return None
144
+ events, self.events = self.events, []
145
+ return events
146
+
147
148
class RequestStatus(enum.IntEnum):
149
"""Status of a request."""