[Bugfix] Redact credentials from benchmark logs (#56662)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Taneem Ibrahim committed
Sep 14, 2026 at 17:58 UTC
8e08cef46ebeef501a1b1593fd7da77909afe4ae
3 files changed
+23
-4
benchmarks/multi_turn/benchmark_serving_multi_turn.py
+4
-1
@@ -32,8 +32,11 @@ from bench_dataset import (
32
from bench_utils import TEXT_SEPARATOR, Color, logger
33
from transformers import AutoTokenizer # type: ignore
34
35
+from vllm.benchmarks.lib.utils import redact_sensitive_namespace
36
+
37
NUM_TOKENS_FROM_DATASET = 0
38
TERM_SIGNAL = None
39
+_SENSITIVE_ARG_FIELDS = ("api_key", "header")
40
41
42
class ConversationSampling(str, Enum):
@@ -1548,7 +1551,7 @@ async def main() -> None:
1551
1552
args = parser.parse_args()
1553
1551
- logger.info(args)
1554
+ logger.info(redact_sensitive_namespace(args, _SENSITIVE_ARG_FIELDS))
1555
1556
logger.info(f"{Color.GREEN}Input parameters:{Color.RESET}")
1557
logger.info(f"url={args.url}")
vllm/benchmarks/lib/utils.py
+11
@@ -9,6 +9,17 @@ from contextlib import contextmanager
9
from typing import Any
10
11
12
+def redact_sensitive_namespace(
13
+ args: argparse.Namespace, fields: tuple[str, ...]
14
+) -> argparse.Namespace:
15
+ """Return a copy of CLI arguments with selected values redacted."""
16
+ redacted_args = argparse.Namespace(**vars(args))
17
+ for field in fields:
18
+ if getattr(redacted_args, field, None) is not None:
19
+ setattr(redacted_args, field, "***")
20
+ return redacted_args
21
+
22
+
23
def extract_field(
24
args: argparse.Namespace, extra_info: dict[str, Any], field_name: str
25
) -> str:
vllm/benchmarks/serve.py
+8
-3
@@ -51,13 +51,18 @@ from vllm.benchmarks.lib.endpoint_request_func import (
51
RequestFuncOutput,
52
)
53
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
54
-from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
54
+from vllm.benchmarks.lib.utils import (
55
+ convert_to_pytorch_benchmark_format,
56
+ redact_sensitive_namespace,
57
+ write_to_json,
58
+)
59
from vllm.tokenizers import TokenizerLike, get_tokenizer
60
from vllm.utils.argparse_utils import FlexibleArgumentParser
61
from vllm.utils.gc_utils import freeze_gc_heap
62
from vllm.utils.network_utils import join_host_port
63
64
MILLISECONDS_TO_SECONDS_CONVERSION = 1000
65
+_SENSITIVE_ARG_FIELDS = ("header",)
66
67
68
def _merge_overrides(base: dict | None, override: dict | None) -> dict | None:
@@ -1536,7 +1541,7 @@ def save_to_pytorch_benchmark_format(
1541
# later if needed
1542
ignored_metrics = ["ttfts", "itls", "generated_texts", "errors"]
1543
pt_records = convert_to_pytorch_benchmark_format(
1539
- args=args,
1544
+ args=redact_sensitive_namespace(args, _SENSITIVE_ARG_FIELDS),
1545
metrics={k: [results[k]] for k in metrics if k in results},
1546
extra_info={
1547
k: results[k]
@@ -2013,7 +2018,7 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
2018
2019
2020
async def main_async(args: argparse.Namespace) -> dict[str, Any]:
2016
- print(args)
2021
+ print(redact_sensitive_namespace(args, _SENSITIVE_ARG_FIELDS))
2022
if args.max_concurrency is not None and args.max_concurrency <= 0:
2023
raise ValueError("--max-concurrency must be greater than 0")
2024