@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e15601789b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 18:15 UTC

[Feature]: Add corrupted request metric to V1 metrics system. (#27306)

Signed-off-by: atalhens <sneh.lata@nutanix.com>

Snehlata committed Nov 6, 2025 at 03:15 UTC e15601789be803c8e27a7806b1b6ec8924f20b03

6 files changed +51 -13

tests/v1/metrics/test_stats.py

+2 -1

@@ -18,6 +18,7 @@ def test_iteration_stats_repr():

18 "time_to_first_tokens_iter=[], "

19 "inter_token_latencies_iter=[], "

20 "waiting_lora_adapters={}, "

21 - "running_lora_adapters={})"

21 + "running_lora_adapters={}, "

22 + "num_corrupted_reqs=0)"

23 )

24 assert repr(iteration_stats) == expected_repr

vllm/v1/core/sched/scheduler.py

+1 -1

@@ -1023,6 +1023,7 @@ class Scheduler(SchedulerInterface):

1023 kv_transfer_params=kv_transfer_params,

1024 trace_headers=request.trace_headers,

1025 num_cached_tokens=request.num_cached_tokens,

1026 + num_nans_in_logits=request.num_nans_in_logits,

1027 )

1028 )

1029 else:

@@ -1259,7 +1260,6 @@ class Scheduler(SchedulerInterface):

1260 prefix_cache_stats=prefix_cache_stats,

1261 connector_prefix_cache_stats=connector_prefix_cache_stats,

1262 spec_decoding_stats=spec_decoding_stats,

1262 - num_corrupted_reqs=sum(req.is_output_corrupted for req in self.running),

1263 kv_connector_stats=kv_connector_stats.data if kv_connector_stats else None,

1264 )

1265

vllm/v1/engine/__init__.py

@@ -122,6 +122,10 @@ class EngineCoreOutput(

122 # The number of tokens with prefix cache hits.

123 num_cached_tokens: int = 0

124

125 + # The number of NaNs in logits.

126 + # A value greater than 0 indicates that the output is corrupted.

127 + num_nans_in_logits: int = 0

128 +

129 @property

130 def finished(self) -> bool:

131 return self.finish_reason is not None

vllm/v1/metrics/loggers.py

+24 -5

@@ -9,6 +9,7 @@ from typing import TypeAlias

10 from prometheus_client import Counter, Gauge, Histogram

12 +import vllm.envs as envs

13 from vllm.config import SupportsMetricsInfo, VllmConfig

14 from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (

15 KVConnectorLogging,

@@ -116,11 +117,13 @@ class LoggingStatLogger(StatLoggerBase):

117 # Tracked stats over current local logging interval.

118 self.num_prompt_tokens: int = 0

119 self.num_generation_tokens: int = 0

120 + self.num_corrupted_reqs: int = 0

121

122 def _track_iteration_stats(self, iteration_stats: IterationStats):

123 # Save tracked stats for token counters.

124 self.num_prompt_tokens += iteration_stats.num_prompt_tokens

125 self.num_generation_tokens += iteration_stats.num_generation_tokens

126 + self.num_corrupted_reqs += iteration_stats.num_corrupted_reqs

127

128 def _get_throughput(self, tracked_stats: int, now: float) -> float:

129 # Compute summary metrics for tracked stats

@@ -204,6 +207,10 @@ class LoggingStatLogger(StatLoggerBase):

207 self.last_scheduler_stats.kv_cache_usage * 100,

208 self.prefix_caching_metrics.hit_rate * 100,

209 ]

210 +

211 + if envs.VLLM_COMPUTE_NANS_IN_LOGITS:

212 + log_parts.append("Corrupted: %d reqs")

213 + log_args.append(self.num_corrupted_reqs)

214 if not self.connector_prefix_caching_metrics.empty:

215 log_parts.append("External prefix cache hit rate: %.1f%%")

216 log_args.append(self.connector_prefix_caching_metrics.hit_rate * 100)

@@ -275,9 +282,6 @@ class AggregatedLoggingStatLogger(LoggingStatLogger, AggregateStatLoggerBase):

282 self.last_scheduler_stats.num_running_reqs += (

283 last_scheduler_stats.num_running_reqs

284 )

278 - self.last_scheduler_stats.num_corrupted_reqs += (

279 - last_scheduler_stats.num_corrupted_reqs

280 - )

285 self.last_scheduler_stats.kv_cache_usage += (

286 last_scheduler_stats.kv_cache_usage

287 )

@@ -481,6 +485,19 @@ class PrometheusStatLogger(AggregateStatLoggerBase):

485 gauge_kv_cache_usage, engine_indexes, model_name

486 )

487

488 + if envs.VLLM_COMPUTE_NANS_IN_LOGITS:

489 + counter_corrupted_requests = self._counter_cls(

490 + name="vllm:corrupted_requests",

491 + documentation=(

492 + "Corrupted requests, in terms of total number of requests "

493 + "with NaNs in logits."

494 + ),

495 + labelnames=labelnames,

496 + )

497 + self.counter_corrupted_requests = make_per_engine(

498 + counter_corrupted_requests, engine_indexes, model_name

499 + )

500 +

501 counter_prefix_cache_queries = self._counter_cls(

502 name="vllm:prefix_cache_queries",

503 documentation=(

@@ -933,7 +950,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):

950 self.gauge_scheduler_waiting[engine_idx].set(

951 scheduler_stats.num_waiting_reqs

952 )

936 -

953 if self.show_hidden_metrics:

954 self.gauge_gpu_cache_usage[engine_idx].set(

955 scheduler_stats.kv_cache_usage

@@ -979,7 +995,10 @@ class PrometheusStatLogger(AggregateStatLoggerBase):

995

996 if iteration_stats is None:

997 return

982 -

998 + if envs.VLLM_COMPUTE_NANS_IN_LOGITS:

999 + self.counter_corrupted_requests[engine_idx].inc(

1000 + iteration_stats.num_corrupted_reqs

1001 + )

1002 self.counter_num_preempted_reqs[engine_idx].inc(

1003 iteration_stats.num_preempted_reqs

1004 )

vllm/v1/metrics/stats.py

+20 -2

@@ -6,6 +6,7 @@ from collections import deque

6 from dataclasses import dataclass, field

7 from typing import TYPE_CHECKING, Any

9 +import vllm.envs as envs

10 from vllm.v1.spec_decode.metrics import SpecDecodingStats

12 if TYPE_CHECKING:

@@ -169,8 +170,6 @@ class SchedulerStats:

170 spec_decoding_stats: SpecDecodingStats | None = None

171 kv_connector_stats: dict[str, Any] | None = None

172

172 - num_corrupted_reqs: int = 0

173 -

173

174 @dataclass

175 class LoRAStats:

@@ -196,6 +195,9 @@ class RequestStateStats:

195 # first token latency

196 first_token_latency: float = 0.0

197

198 + # Track if this request is corrupted (NaNs in logits)

199 + is_corrupted: bool = False

200 +

201

202 @dataclass

203 class FinishedRequestStats:

@@ -211,6 +213,7 @@ class FinishedRequestStats:

213 inference_time: float = 0.0

214 decode_time: float = 0.0

215 mean_time_per_output_token: float = 0.0

216 + is_corrupted: bool = False

217

218

219 class IterationStats:

@@ -228,6 +231,7 @@ class IterationStats:

231 self.inter_token_latencies_iter: list[float] = []

232 self.waiting_lora_adapters: dict[str, int] = {}

233 self.running_lora_adapters: dict[str, int] = {}

234 + self.num_corrupted_reqs: int = 0

235

236 def __repr__(self) -> str:

237 field_to_value_str = ", ".join(f"{k}={v}" for k, v in vars(self).items())

@@ -258,6 +262,15 @@ class IterationStats:

262

263 req_stats.num_generation_tokens += num_new_generation_tokens

264

265 + # Track if this request is corrupted (only check once per request)

266 + # Early exit if already marked as corrupted to avoid redundant checks

267 + if (

268 + envs.VLLM_COMPUTE_NANS_IN_LOGITS

269 + and not req_stats.is_corrupted

270 + and output.num_nans_in_logits > 0

271 + ):

272 + req_stats.is_corrupted = True

273 +

274 # Process request-level engine core events

275 if output.events is not None:

276 self.update_from_events(

@@ -339,9 +352,14 @@ class IterationStats:

352 inference_time=inference_time,

353 decode_time=decode_time,

354 mean_time_per_output_token=mean_time_per_output_token,

355 + is_corrupted=req_stats.is_corrupted,

356 )

357 self.finished_requests.append(finished_req)

358

359 + # Count corrupted requests when they finish (only once per request)

360 + if req_stats.is_corrupted:

361 + self.num_corrupted_reqs += 1

362 +

363

364 class LoRARequestStates:

365 """Per-LoRA request state stats."""

vllm/v1/request.py

-4

@@ -168,10 +168,6 @@ class Request:

168 def use_structured_output(self) -> bool:

169 return self.structured_output_request is not None

170

171 - @property

172 - def is_output_corrupted(self) -> bool:

173 - return self.num_nans_in_logits > 0

174 -

171 @property

172 def num_tokens(self) -> int:

173 return len(self._all_token_ids)