@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae122b1cbd

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[WIP][[V1][Metrics] Implement max_num_generation_tokens, request_params_n, and request_params_max_tokens metrics (#14055)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Mar 3, 2025 at 19:04 UTC ae122b1cbde96c871fb74611363e04eecfbcce03

5 files changed +111 -2

tests/entrypoints/openai/test_metrics.py

@@ -239,6 +239,12 @@ EXPECTED_METRICS_V1 = [

239 "vllm:request_generation_tokens_sum",

240 "vllm:request_generation_tokens_bucket",

241 "vllm:request_generation_tokens_count",

242 + "vllm:request_params_n_sum",

243 + "vllm:request_params_n_bucket",

244 + "vllm:request_params_n_count",

245 + "vllm:request_params_max_tokens_sum",

246 + "vllm:request_params_max_tokens_bucket",

247 + "vllm:request_params_max_tokens_count",

248 "vllm:time_to_first_token_seconds_sum",

249 "vllm:time_to_first_token_seconds_bucket",

250 "vllm:time_to_first_token_seconds_count",

vllm/v1/engine/output_processor.py

+13

@@ -36,6 +36,7 @@ class RequestState:

36 prompt_token_ids: list[int],

37 logprobs_processor: LogprobsProcessor,

38 detokenizer: IncrementalDetokenizer,

39 + max_tokens_param: Optional[int],

40 arrival_time: float,

41 queue: Optional[asyncio.Queue[RequestOutput]],

42 log_stats: bool,

@@ -50,6 +51,7 @@ class RequestState:

51 self.prompt_len = len(prompt_token_ids)

52 self.logprobs_processor = logprobs_processor

53 self.detokenizer = detokenizer

54 + self.max_tokens_param = max_tokens_param

55 self.is_prefilling = True

56 self.queue = queue

@@ -83,6 +85,8 @@ class RequestState:

85 tokenizer=tokenizer,

86 request=request,

87 ),

88 + max_tokens_param=(request.sampling_params.max_tokens if

89 + request.sampling_params is not None else None),

90 arrival_time=request.arrival_time,

91 queue=queue,

92 log_stats=log_stats,

@@ -198,6 +202,8 @@ class OutputProcessor:

202 req_state = self.request_states.pop(request_id, None)

203 if req_state is not None:

204 self.lora_states.abort_request(req_state)

205 + if req_state.parent_req is not None:

206 + req_state.parent_req.finish_child_request(request_id)

207

208 def add_request(

209 self,

@@ -310,6 +316,8 @@ class OutputProcessor:

316 # If req not finished in EngineCore, but Detokenizer

317 # detected stop string, abort needed in EngineCore.

318 reqs_to_abort.append(req_id)

319 + if req_state.parent_req is not None:

320 + req_state.parent_req.finish_child_request(req_id)

321

322 # Track per-request stats

323 self._update_stats_from_finished(req_state, finish_reason,

@@ -350,5 +358,10 @@ class OutputProcessor:

358 iteration_stats.update_from_finished_request(

359 finish_reason=finish_reason,

360 num_prompt_tokens=len(req_state.prompt_token_ids),

361 + max_tokens_param=req_state.max_tokens_param,

362 req_stats=req_state.stats)

363 self.lora_states.finish_request(req_state)

364 +

365 + ParentRequest.observe_finished_request(

366 + req_state.parent_req, iteration_stats,

367 + req_state.stats.num_generation_tokens)

vllm/v1/engine/parallel_sampling.py

+37 -2

@@ -6,6 +6,7 @@ from typing import Callable, Optional, Union

6 from vllm.outputs import CompletionOutput, RequestOutput

7 from vllm.pooling_params import PoolingParams

8 from vllm.sampling_params import SamplingParams

9 +from vllm.v1.metrics.stats import IterationStats

12 class ParentRequest:

@@ -18,9 +19,15 @@ class ParentRequest:

19 request_id: str

20 sampling_params: SamplingParams

22 + # To track the completion of child requests

23 + child_requests: set[str]

24 +

25 # To aggregate child completions when not streaming

26 output_aggregator: Optional[RequestOutput]

28 + # To find the max number of generated tokens across all children

29 + max_num_generation_tokens: int

30 +

31 # To efficiently obtain child sampling params

32 cached_child_sampling_params: Optional[SamplingParams]

@@ -29,7 +36,9 @@ class ParentRequest:

36 self.request_id = request_id

37 self.sampling_params = sampling_params

39 + self.child_requests = set()

40 self.output_aggregator = None

41 + self.max_num_generation_tokens = 0

42 self.cached_child_sampling_params = None

44 @classmethod

@@ -82,8 +91,12 @@ class ParentRequest:

91 Returns:

92 (request ID, sampling_params) tuple

93 """

85 - return (f"{index}_{self.request_id}",

86 - self._get_child_sampling_params(index))

94 + child_req_id = f"{index}_{self.request_id}"

95 + self.child_requests.add(child_req_id)

96 + return (child_req_id, self._get_child_sampling_params(index))

97 +

98 + def finish_child_request(self, req_id: str):

99 + self.child_requests.remove(req_id)

100

101 @property

102 def n(self) -> int:

@@ -117,3 +130,25 @@ class ParentRequest:

130 request_output.outputs = sorted(request_output.outputs,

131 key=lambda x: x.index)

132 return request_output

133 +

134 + def observe_num_generation_tokens(self, num_generation_tokens: int):

135 + self.max_num_generation_tokens = max(num_generation_tokens,

136 + self.max_num_generation_tokens)

137 + return self.max_num_generation_tokens

138 +

139 + @staticmethod

140 + def observe_finished_request(parent_req: Optional['ParentRequest'],

141 + iteration_stats: IterationStats,

142 + num_generation_tokens: int):

143 +

144 + n_param = parent_req.n if parent_req is not None else 1

145 +

146 + if parent_req is not None:

147 + num_generation_tokens = parent_req.observe_num_generation_tokens(

148 + num_generation_tokens)

149 +

150 + # Child requests finished, we can now record to iteration stats

151 + if parent_req is None or not parent_req.child_requests:

152 + iteration_stats.max_num_generation_tokens_iter.append(

153 + num_generation_tokens)

154 + iteration_stats.n_params_iter.append(n_param)

vllm/v1/metrics/loggers.py

+50

@@ -106,6 +106,9 @@ class PrometheusStatLogger(StatLoggerBase):

106

107 max_model_len = vllm_config.model_config.max_model_len

108

109 + #

110 + # Scheduler state

111 + #

112 self.gauge_scheduler_running = prometheus_client.Gauge(

113 name="vllm:num_requests_running",

114 documentation="Number of requests in model execution batches.",

@@ -116,6 +119,9 @@ class PrometheusStatLogger(StatLoggerBase):

119 documentation="Number of requests waiting to be processed.",

120 labelnames=labelnames).labels(*labelvalues)

121

122 + #

123 + # GPU cache

124 + #

125 self.gauge_gpu_cache_usage = prometheus_client.Gauge(

126 name="vllm:gpu_cache_usage_perc",

127 documentation="GPU KV-cache usage. 1 means 100 percent usage.",

@@ -133,6 +139,9 @@ class PrometheusStatLogger(StatLoggerBase):

139 "GPU prefix cache hits, in terms of number of cached blocks.",

140 labelnames=labelnames).labels(*labelvalues)

141

142 + #

143 + # Counters

144 + #

145 self.counter_num_preempted_reqs = prometheus_client.Counter(

146 name="vllm:num_preemptions_total",

147 documentation="Cumulative number of preemption from the engine.",

@@ -159,6 +168,9 @@ class PrometheusStatLogger(StatLoggerBase):

168 reason] = counter_request_success_base.labels(*(labelvalues +

169 [str(reason)]))

170

171 + #

172 + # Histograms of counts

173 + #

174 self.histogram_num_prompt_tokens_request = \

175 prometheus_client.Histogram(

176 name="vllm:request_prompt_tokens",

@@ -180,6 +192,31 @@ class PrometheusStatLogger(StatLoggerBase):

192 buckets=build_cudagraph_buckets(vllm_config),

193 labelnames=labelnames).labels(*labelvalues)

194

195 + self.histogram_max_num_generation_tokens_request = \

196 + prometheus_client.Histogram(

197 + name="vllm:request_max_num_generation_tokens",

198 + documentation=

199 + "Histogram of maximum number of requested generation tokens.",

200 + buckets=build_1_2_5_buckets(max_model_len),

201 + labelnames=labelnames).labels(*labelvalues)

202 +

203 + self.histogram_n_request = \

204 + prometheus_client.Histogram(

205 + name="vllm:request_params_n",

206 + documentation="Histogram of the n request parameter.",

207 + buckets=[1, 2, 5, 10, 20],

208 + labelnames=labelnames).labels(*labelvalues)

209 +

210 + self.histogram_max_tokens_request = \

211 + prometheus_client.Histogram(

212 + name="vllm:request_params_max_tokens",

213 + documentation="Histogram of the max_tokens request parameter.",

214 + buckets=build_1_2_5_buckets(max_model_len),

215 + labelnames=labelnames).labels(*labelvalues)

216 +

217 + #

218 + # Histogram of timing intervals

219 + #

220 self.histogram_time_to_first_token = \

221 prometheus_client.Histogram(

222 name="vllm:time_to_first_token_seconds",

@@ -239,6 +276,9 @@ class PrometheusStatLogger(StatLoggerBase):

276 buckets=request_latency_buckets,

277 labelnames=labelnames).labels(*labelvalues)

278

279 + #

280 + # LoRA metrics

281 + #

282 self.gauge_lora_info: Optional[prometheus_client.Gauge] = None

283 if vllm_config.lora_config is not None:

284 self.labelname_max_lora = "max_lora"

@@ -255,6 +295,9 @@ class PrometheusStatLogger(StatLoggerBase):

295 self.labelname_running_lora_adapters,

296 ])

297

298 + #

299 + # Cache config info metric

300 + #

301 self.log_metrics_info("cache_config", vllm_config.cache_config)

302

303 def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):

@@ -296,6 +339,11 @@ class PrometheusStatLogger(StatLoggerBase):

339 iteration_stats.num_prompt_tokens + \

340 iteration_stats.num_generation_tokens)

341

342 + for max_gen_tokens in iteration_stats.max_num_generation_tokens_iter:

343 + self.histogram_max_num_generation_tokens_request.observe(

344 + max_gen_tokens)

345 + for n_param in iteration_stats.n_params_iter:

346 + self.histogram_n_request.observe(n_param)

347 for ttft in iteration_stats.time_to_first_tokens_iter:

348 self.histogram_time_to_first_token.observe(ttft)

349 for tpot in iteration_stats.time_per_output_tokens_iter:

@@ -317,6 +365,8 @@ class PrometheusStatLogger(StatLoggerBase):

365 finished_request.num_prompt_tokens)

366 self.histogram_num_generation_tokens_request.observe(

367 finished_request.num_generation_tokens)

368 + self.histogram_max_tokens_request.observe(

369 + finished_request.max_tokens_param)

370

371 if self.gauge_lora_info is not None:

372 running_lora_adapters = \

vllm/v1/metrics/stats.py

@@ -66,6 +66,7 @@ class FinishedRequestStats:

66 e2e_latency: float = 0.0

67 num_prompt_tokens: int = 0

68 num_generation_tokens: int = 0

69 + max_tokens_param: Optional[int] = None

70 queued_time: float = 0.0

71 prefill_time: float = 0.0

72 inference_time: float = 0.0

@@ -81,6 +82,8 @@ class IterationStats:

82 self.num_prompt_tokens = 0

83 self.num_preempted_reqs = 0

84 self.finished_requests: list[FinishedRequestStats] = []

85 + self.max_num_generation_tokens_iter: list[int] = []

86 + self.n_params_iter: list[int] = []

87 self.time_to_first_tokens_iter: list[float] = []

88 self.time_per_output_tokens_iter: list[float] = []

89 self.waiting_lora_adapters: dict[str, int] = {}

@@ -150,6 +153,7 @@ class IterationStats:

153

154 def update_from_finished_request(self, finish_reason: "FinishReason",

155 num_prompt_tokens: int,

156 + max_tokens_param: Optional[int],

157 req_stats: RequestStateStats):

158 e2e_latency = self._time_since(req_stats.arrival_time)

159

@@ -173,6 +177,7 @@ class IterationStats:

177 e2e_latency=e2e_latency,

178 num_prompt_tokens=num_prompt_tokens,

179 num_generation_tokens=req_stats.num_generation_tokens,

180 + max_tokens_param=max_tokens_param,

181 queued_time=queued_time,

182 prefill_time=prefill_time,

183 inference_time=inference_time,