@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc32bc73aa

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 22:15 UTC

[V1][Metrics] Implement vllm:lora_requests_info metric (#13504)

Mark McLoughlin committed Feb 25, 2025 at 04:01 UTC bc32bc73aad076849ac88565cff745b01b17d89c

3 files changed +121 -10

vllm/v1/engine/output_processor.py

+19 -4

@@ -11,7 +11,8 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup

11 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason

12 from vllm.v1.engine.detokenizer import IncrementalDetokenizer

13 from vllm.v1.engine.logprobs import LogprobsProcessor

14 -from vllm.v1.metrics.stats import IterationStats, RequestStateStats

14 +from vllm.v1.metrics.stats import (IterationStats, LoRARequestStates,

15 + RequestStateStats)

18 @dataclass

@@ -26,6 +27,7 @@ class RequestState:

27 def __init__(

28 self,

29 request_id: str,

30 + lora_name: Optional[str],

31 output_kind: RequestOutputKind,

32 prompt: Optional[str],

33 prompt_token_ids: List[int],

@@ -36,6 +38,7 @@ class RequestState:

38 log_stats: bool,

39 ):

40 self.request_id = request_id

41 + self.lora_name = lora_name

42 self.output_kind = output_kind

43 self.prompt = prompt

44 self.prompt_token_ids = prompt_token_ids

@@ -58,6 +61,8 @@ class RequestState:

61 ) -> "RequestState":

62 return cls(

63 request_id=request.request_id,

64 + lora_name=(request.lora_request.name

65 + if request.lora_request is not None else None),

66 output_kind=request.sampling_params.output_kind,

67 prompt=request.prompt,

68 prompt_token_ids=request.prompt_token_ids,

@@ -86,6 +91,7 @@ class OutputProcessor:

91 self.log_stats = log_stats

92 self.tokenizer = tokenizer

93 self.request_states: Dict[str, RequestState] = {}

94 + self.lora_states = LoRARequestStates()

96 def is_request_active(self, request_id: str) -> bool:

97 return request_id in self.request_states

@@ -101,7 +107,9 @@ class OutputProcessor:

107 request_ids: List[str],

108 ) -> None:

109 for request_id in request_ids:

104 - self.request_states.pop(request_id, None)

110 + req_state = self.request_states.pop(request_id, None)

111 + if req_state is not None:

112 + self.lora_states.abort_request(req_state)

113

114 def add_request(

115 self,

@@ -112,11 +120,13 @@ class OutputProcessor:

120 if request_id in self.request_states:

121 raise ValueError(f"Request id {request_id} already running.")

122

115 - self.request_states[request_id] = RequestState.from_new_request(

123 + req_state = RequestState.from_new_request(

124 tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),

125 request=request,

126 queue=queue,

127 log_stats=self.log_stats)

128 + self.request_states[request_id] = req_state

129 + self.lora_states.add_request(req_state)

130

131 def process_outputs(

132 self,

@@ -214,6 +224,8 @@ class OutputProcessor:

224 finish_reason,

225 iteration_stats)

226

227 + self.lora_states.update_iteration_stats(iteration_stats)

228 +

229 return OutputProcessorOutput(

230 request_outputs=request_outputs,

231 reqs_to_abort=reqs_to_abort,

@@ -226,13 +238,15 @@ class OutputProcessor:

238 if iteration_stats is None:

239 return

240

241 + lora_stats = self.lora_states.get_stats(req_state)

242 +

243 assert engine_core_timestamp is not None

244 assert req_state.stats is not None

245 iteration_stats.update_from_output(engine_core_output,

246 engine_core_timestamp,

247 req_state.is_prefilling,

248 req_state.prompt_len,

235 - req_state.stats)

249 + req_state.stats, lora_stats)

250

251 def _update_stats_from_finished(self, req_state: RequestState,

252 request_output: RequestOutput,

@@ -246,6 +260,7 @@ class OutputProcessor:

260 iteration_stats.update_from_finished_request(finish_reason,

261 request_output,

262 req_state.stats)

263 + self.lora_states.finish_request(req_state)

264

265 @staticmethod

266 def _make_request_output(

vllm/v1/metrics/loggers.py

+30 -1

@@ -2,7 +2,7 @@

3 import time

4 from abc import ABC, abstractmethod

5 -from typing import Dict, List

5 +from typing import Dict, List, Optional

7 import numpy as np

8 import prometheus_client

@@ -233,6 +233,22 @@ class PrometheusStatLogger(StatLoggerBase):

233 buckets=request_latency_buckets,

234 labelnames=labelnames).labels(*labelvalues)

235

236 + self.gauge_lora_info: Optional[prometheus_client.Gauge] = None

237 + if vllm_config.lora_config is not None:

238 + self.labelname_max_lora = "max_lora"

239 + self.labelname_waiting_lora_adapters = "waiting_lora_adapters"

240 + self.labelname_running_lora_adapters = "running_lora_adapters"

241 + self.max_lora = vllm_config.lora_config.max_loras

242 + self.gauge_lora_info = \

243 + prometheus_client.Gauge(

244 + name="vllm:lora_requests_info",

245 + documentation="Running stats on lora requests.",

246 + labelnames=[

247 + self.labelname_max_lora,

248 + self.labelname_waiting_lora_adapters,

249 + self.labelname_running_lora_adapters,

250 + ])

251 +

252 self.log_metrics_info("cache_config", vllm_config.cache_config)

253

254 def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):

@@ -295,6 +311,19 @@ class PrometheusStatLogger(StatLoggerBase):

311 for prefill_time in iteration_stats.prefill_times_iter:

312 self.histogram_prefill_time_request.observe(prefill_time)

313

314 + if self.gauge_lora_info is not None:

315 + running_lora_adapters = \

316 + ",".join(iteration_stats.running_lora_adapters.keys())

317 + waiting_lora_adapters = \

318 + ",".join(iteration_stats.waiting_lora_adapters.keys())

319 + lora_info_labels = {

320 + self.labelname_running_lora_adapters: running_lora_adapters,

321 + self.labelname_waiting_lora_adapters: waiting_lora_adapters,

322 + self.labelname_max_lora: self.max_lora,

323 + }

324 + self.gauge_lora_info.labels(**lora_info_labels)\

325 + .set_to_current_time()

326 +

327 @staticmethod

328 def _unregister_vllm_metrics():

329 # Unregister any existing vLLM collectors (for CI/CD

vllm/v1/metrics/stats.py

+72 -5

@@ -2,11 +2,12 @@

3 import time

4 from dataclasses import dataclass, field

5 -from typing import TYPE_CHECKING, List

5 +from typing import TYPE_CHECKING, Dict, List, Optional, Set

7 if TYPE_CHECKING:

8 from vllm.outputs import RequestOutput

9 from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason

10 + from vllm.v1.output_processor import RequestState

13 @dataclass

@@ -36,6 +37,12 @@ class SchedulerStats:

37 default_factory=PrefixCacheStats)

40 +@dataclass

41 +class LoRAStats:

42 + waiting_requests: Set[str] = field(default_factory=set)

43 + running_requests: Set[str] = field(default_factory=set)

44 +

45 +

46 @dataclass

47 class RequestStateStats:

48 """Stats that need to be tracked across delta updates."""

@@ -76,6 +83,8 @@ class IterationStats:

83 self.time_per_output_tokens_iter: List[float] = []

84 self.queue_times_iter: List[float] = []

85 self.prefill_times_iter: List[float] = []

86 + self.waiting_lora_adapters: Dict[str, int] = {}

87 + self.running_lora_adapters: Dict[str, int] = {}

89 def _time_since(self, start: float) -> float:

90 """Calculate an interval relative to this iteration's timestamp."""

@@ -83,7 +92,8 @@ class IterationStats:

93 def update_from_output(self, output: "EngineCoreOutput",

94 engine_core_timestamp: float, is_prefilling: bool,

86 - prompt_len: int, req_stats: RequestStateStats):

95 + prompt_len: int, req_stats: RequestStateStats,

96 + lora_stats: Optional[LoRAStats]):

97 num_new_generation_tokens = len(output.new_token_ids)

99 self.num_generation_tokens += num_new_generation_tokens

@@ -105,7 +115,8 @@ class IterationStats:

115

116 # Process request-level engine core events

117 if output.events is not None:

108 - self.update_from_events(output.events, is_prefilling, req_stats)

118 + self.update_from_events(output.request_id, output.events,

119 + is_prefilling, req_stats, lora_stats)

120

121 # Process the batch-level "new tokens" engine core event

122 if is_prefilling:

@@ -123,17 +134,21 @@ class IterationStats:

134 if num_new_generation_tokens > 0:

135 req_stats.last_token_ts = engine_core_timestamp

136

126 - def update_from_events(self, events: List["EngineCoreEvent"],

127 - is_prefilling: bool, req_stats: RequestStateStats):

137 + def update_from_events(self, req_id: str, events: List["EngineCoreEvent"],

138 + is_prefilling: bool, req_stats: RequestStateStats,

139 + lora_stats: Optional[LoRAStats]):

140 # Avoid circular dependency

141 from vllm.v1.engine import EngineCoreEventType

142 for event in events:

143 if event.type == EngineCoreEventType.QUEUED:

144 req_stats.queued_ts = event.timestamp

145 + if lora_stats is not None:

146 + lora_stats.waiting_requests.add(req_id)

147 elif event.type == EngineCoreEventType.SCHEDULED:

148 queued_interval = event.timestamp - req_stats.queued_ts

149 self.queue_times_iter.append(queued_interval)

150 req_stats.scheduled_ts = event.timestamp

151 + LoRARequestStates.scheduled_request(lora_stats, req_id)

152

153 def update_from_finished_request(self, finish_reason: "FinishReason",

154 request_output: "RequestOutput",

@@ -151,3 +166,55 @@ class IterationStats:

166 inference_time=inference_time,

167 decode_time=decode_time)

168 self.finished_requests.append(finished_req)

169 +

170 +

171 +class LoRARequestStates:

172 + """Per-LoRA request state stats."""

173 +

174 + def __init__(self):

175 + self.lora_name_to_stats: Dict[str, LoRAStats] = {}

176 +

177 + def get_stats(self, req_state: 'RequestState') -> Optional[LoRAStats]:

178 + if req_state.lora_name is None:

179 + return None

180 + if req_state.lora_name not in self.lora_name_to_stats:

181 + self.lora_name_to_stats[req_state.lora_name] = LoRAStats()

182 + return self.lora_name_to_stats[req_state.lora_name]

183 +

184 + def add_request(self, req_state: 'RequestState'):

185 + if (lora_stats := self.get_stats(req_state)) is not None:

186 + lora_stats.waiting_requests.add(req_state.request_id)

187 +

188 + def finish_request(self, req_state: 'RequestState'):

189 + if req_state.lora_name is None:

190 + return

191 + lora_stats = self.lora_name_to_stats[req_state.lora_name]

192 + lora_stats.running_requests.remove(req_state.request_id)

193 +

194 + def abort_request(self, req_state: 'RequestState'):

195 + if req_state.lora_name is None:

196 + return

197 + lora_stats = self.lora_name_to_stats[req_state.lora_name]

198 + lora_stats.waiting_requests.discard(req_state.request_id)

199 + lora_stats.running_requests.discard(req_state.request_id)

200 +

201 + # Break the pattern for this lifecycle methods so we can

202 + # call this from IterationStats.update_from_events()

203 + @staticmethod

204 + def scheduled_request(lora_stats: Optional[LoRAStats], request_id: str):

205 + if lora_stats is None:

206 + return

207 + lora_stats.waiting_requests.remove(request_id)

208 + lora_stats.running_requests.add(request_id)

209 +

210 + def update_iteration_stats(self,

211 + iteration_stats: Optional[IterationStats]):

212 + if iteration_stats is None:

213 + return

214 + for lora_name, stats in self.lora_name_to_stats.items():

215 + if stats.waiting_requests:

216 + iteration_stats.waiting_lora_adapters[lora_name] = \

217 + len(stats.waiting_requests)

218 + if stats.running_requests:

219 + iteration_stats.running_lora_adapters[lora_name] = \

220 + len(stats.running_requests)