Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | """Best-effort secret redaction for free-form telemetry text.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import re |
| 6 | import typing as t |
| 7 | from collections.abc import Mapping |
| 8 | |
| 9 | _REDACTED = "[REDACTED]" |
| 10 | _MAX_STRUCTURED_REDACTION_DEPTH = 32 |
| 11 | _MAX_STRUCTURED_REDACTION_NODES = 10_000 |
| 12 | _URL_QUERY = re.compile(r"(\bhttps?://[^\s?#'\"]+)\?[^\s'\"]*", re.IGNORECASE) |
| 13 | _AUTHORIZATION = re.compile(r"\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE) |
| 14 | _SECRET_KEY_PATTERN = ( |
| 15 | r"authorization|auth|api[-_]?key|apikey|x-api-key|access[-_]?token|" |
| 16 | r"refresh[-_]?token|client[-_]?secret|secret|password|passwd|pwd" |
| 17 | ) |
| 18 | _SECRET_KEY_WITH_PREFIX_PATTERN = rf"(?:[A-Za-z0-9]+_)*(?:{_SECRET_KEY_PATTERN})" |
| 19 | _SECRET_KEY = re.compile(_SECRET_KEY_WITH_PREFIX_PATTERN, re.IGNORECASE) |
| 20 | # Quoted keys are matched as a unit so a key-like word at the end of an error |
| 21 | # string cannot treat the string's closing quote as the start of a secret value. |
| 22 | _QUOTED_SECRET_KEY_PREFIX = ( |
| 23 | rf"([\"'])({_SECRET_KEY_WITH_PREFIX_PATTERN})\1(\s*[:=]+\s*)" |
| 24 | ) |
| 25 | _BARE_SECRET_KEY_PREFIX = rf"(?<![A-Za-z0-9\"'])({_SECRET_KEY_PATTERN})\b(\s*[:=]+\s*)" |
| 26 | _ESCAPED_SECRET_KEY_PREFIX = ( |
| 27 | rf"(?<![A-Za-z0-9])({_SECRET_KEY_PATTERN})\b((?:\\[\"'])?\s*[:=]+\s*)" |
| 28 | ) |
| 29 | _QUOTED_KEY_DOUBLE_QUOTED_VALUE = re.compile( |
| 30 | rf'{_QUOTED_SECRET_KEY_PREFIX}"(?:\\.|[^"\\\r\n])*"', re.IGNORECASE |
| 31 | ) |
| 32 | _QUOTED_KEY_SINGLE_QUOTED_VALUE = re.compile( |
| 33 | rf"{_QUOTED_SECRET_KEY_PREFIX}'(?:\\.|[^'\\\r\n])*'", re.IGNORECASE |
| 34 | ) |
| 35 | _BARE_KEY_DOUBLE_QUOTED_VALUE = re.compile( |
| 36 | rf'{_BARE_SECRET_KEY_PREFIX}"((?:\\.|[^"\\\r\n])*)"', |
| 37 | re.IGNORECASE, |
| 38 | ) |
| 39 | _BARE_KEY_SINGLE_QUOTED_VALUE = re.compile( |
| 40 | rf"{_BARE_SECRET_KEY_PREFIX}'((?:\\.|[^'\\\r\n])*)'", |
| 41 | re.IGNORECASE, |
| 42 | ) |
| 43 | _BARE_KEY_ESCAPED_DOUBLE_QUOTED_VALUE = re.compile( |
| 44 | rf'{_ESCAPED_SECRET_KEY_PREFIX}\\"((?:\\.|[^"\\\r\n])*)\\"', |
| 45 | re.IGNORECASE, |
| 46 | ) |
| 47 | _BARE_KEY_ESCAPED_SINGLE_QUOTED_VALUE = re.compile( |
| 48 | rf"{_ESCAPED_SECRET_KEY_PREFIX}\\'((?:\\.|[^'\\\r\n])*)\\'", |
| 49 | re.IGNORECASE, |
| 50 | ) |
| 51 | _SECRET_PAIR_PREFIX = rf"(?<![A-Za-z0-9])({_SECRET_KEY_PATTERN})\b([\"']?\s*[:=]+\s*)" |
| 52 | _SECRET_PAIR_UNQUOTED = re.compile( |
| 53 | rf"{_SECRET_PAIR_PREFIX}(?!\\[\"'])([^\s\"',}}&]+)", re.IGNORECASE |
| 54 | ) |
| 55 | _COMPLETES_DOUBLE_QUOTED_FIELD = re.compile(r'(?:\\.|[^"\\\r\n])*"\s*[:=]') |
| 56 | _COMPLETES_SINGLE_QUOTED_FIELD = re.compile(r"(?:\\.|[^'\\\r\n])*'\s*[:=]") |
| 57 | _CONTAINS_QUOTED_FIELD = re.compile(r"(?:^|[,{\s])([\"'])[^\"'\\\r\n]+\1\s*[:=]\s*$") |
| 58 | |
| 59 | |
| 60 | def _active_quotes_at(value: str, offsets: set[int]) -> dict[int, str | None]: |
| 61 | active_quotes: dict[int, str | None] = {} |
| 62 | quote: str | None = None |
| 63 | escaped = False |
| 64 | last_offset = max(offsets) |
| 65 | |
| 66 | for position, character in enumerate(value): |
| 67 | if position in offsets: |
| 68 | active_quotes[position] = quote |
| 69 | if position == last_offset: |
| 70 | break |
| 71 | if character in "\r\n": |
| 72 | quote = None |
| 73 | escaped = False |
| 74 | continue |
| 75 | previous = value[position - 1] if position > 0 else "" |
| 76 | if previous in "\r\n": |
| 77 | previous = "" |
| 78 | following = value[position + 1] if position + 1 < len(value) else "" |
| 79 | if quote is None: |
| 80 | if character == '"' or ( |
| 81 | character == "'" and not (previous.isalnum() or previous == "_") |
| 82 | ): |
| 83 | quote = character |
| 84 | continue |
| 85 | if escaped: |
| 86 | escaped = False |
| 87 | elif character == "\\": |
| 88 | escaped = True |
| 89 | elif character == quote: |
| 90 | if ( |
| 91 | quote == "'" |
| 92 | and (previous.isalnum() or previous == "_") |
| 93 | and (following.isalnum() or following == "_") |
| 94 | ): |
| 95 | continue |
| 96 | quote = None |
| 97 | |
| 98 | return active_quotes |
| 99 | |
| 100 | |
| 101 | def _redact_bare_quoted_value(value: str, pattern: re.Pattern[str], quote: str) -> str: |
| 102 | matches = list(pattern.finditer(value)) |
| 103 | if not matches: |
| 104 | return value |
| 105 | |
| 106 | active_quotes = _active_quotes_at(value, {match.start() for match in matches}) |
| 107 | completes_field = ( |
| 108 | _COMPLETES_DOUBLE_QUOTED_FIELD |
| 109 | if quote == '"' |
| 110 | else _COMPLETES_SINGLE_QUOTED_FIELD |
| 111 | ) |
| 112 | parts: list[str] = [] |
| 113 | cursor = 0 |
| 114 | for match in matches: |
| 115 | parts.append(value[cursor : match.start()]) |
| 116 | if active_quotes[match.start()] == quote and ( |
| 117 | completes_field.match(value, match.end()) |
| 118 | or _CONTAINS_QUOTED_FIELD.search(match.group(3)) |
| 119 | ): |
| 120 | parts.append(match.group(0)) |
| 121 | else: |
| 122 | parts.append(f"{match.group(1)}{match.group(2)}{quote}{_REDACTED}{quote}") |
| 123 | cursor = match.end() |
| 124 | |
| 125 | parts.append(value[cursor:]) |
| 126 | return "".join(parts) |
| 127 | |
| 128 | |
| 129 | def redact_sensitive_text(value: str) -> str: |
| 130 | """Remove common URL, authorization, and key-value secret shapes.""" |
| 131 | value = _URL_QUERY.sub(rf"\1?{_REDACTED}", value) |
| 132 | value = _AUTHORIZATION.sub(rf"\1 {_REDACTED}", value) |
| 133 | value = _QUOTED_KEY_DOUBLE_QUOTED_VALUE.sub(rf'\1\2\1\3"{_REDACTED}"', value) |
| 134 | value = _QUOTED_KEY_SINGLE_QUOTED_VALUE.sub(rf"\1\2\1\3'{_REDACTED}'", value) |
| 135 | value = _BARE_KEY_ESCAPED_DOUBLE_QUOTED_VALUE.sub(rf'\1\2\\"{_REDACTED}\\"', value) |
| 136 | value = _BARE_KEY_ESCAPED_SINGLE_QUOTED_VALUE.sub(rf"\1\2\\'{_REDACTED}\\'", value) |
| 137 | value = _redact_bare_quoted_value(value, _BARE_KEY_DOUBLE_QUOTED_VALUE, '"') |
| 138 | value = _redact_bare_quoted_value(value, _BARE_KEY_SINGLE_QUOTED_VALUE, "'") |
| 139 | return _SECRET_PAIR_UNQUOTED.sub(rf"\1\2{_REDACTED}", value) |
| 140 | |
| 141 | |
| 142 | def redact_sensitive_value(value: t.Any) -> t.Any: |
| 143 | """Redact secrets in structured logging metadata without changing safe scalar types.""" |
| 144 | active_containers: set[int] = set() |
| 145 | remaining_nodes = _MAX_STRUCTURED_REDACTION_NODES |
| 146 | |
| 147 | def redact(item: t.Any, depth: int) -> t.Any: |
| 148 | nonlocal remaining_nodes |
| 149 | if depth >= _MAX_STRUCTURED_REDACTION_DEPTH or remaining_nodes <= 0: |
| 150 | return _REDACTED |
| 151 | remaining_nodes -= 1 |
| 152 | |
| 153 | if isinstance(item, (Mapping, list, tuple)): |
| 154 | identity = id(item) |
| 155 | if identity in active_containers: |
| 156 | return _REDACTED |
| 157 | active_containers.add(identity) |
| 158 | try: |
| 159 | if isinstance(item, Mapping): |
| 160 | redacted_mapping: dict[t.Any, t.Any] = {} |
| 161 | for key, nested in item.items(): |
| 162 | if remaining_nodes <= 0: |
| 163 | break |
| 164 | if isinstance(key, str): |
| 165 | redacted_key: t.Any = redact_sensitive_text(key) |
| 166 | elif isinstance(key, (int, float, bool, type(None))): |
| 167 | redacted_key = key |
| 168 | else: |
| 169 | redacted_key = _REDACTED |
| 170 | if isinstance(key, str) and _SECRET_KEY.fullmatch(key): |
| 171 | remaining_nodes -= 1 |
| 172 | redacted_mapping[redacted_key] = _REDACTED |
| 173 | else: |
| 174 | redacted_mapping[redacted_key] = redact(nested, depth + 1) |
| 175 | return redacted_mapping |
| 176 | |
| 177 | redacted_items: list[t.Any] = [] |
| 178 | fields = getattr(item, "_fields", ()) if isinstance(item, tuple) else () |
| 179 | for index, nested in enumerate(item): |
| 180 | if remaining_nodes <= 0: |
| 181 | return _REDACTED |
| 182 | if ( |
| 183 | index < len(fields) |
| 184 | and isinstance(fields[index], str) |
| 185 | and _SECRET_KEY.fullmatch(fields[index]) |
| 186 | ): |
| 187 | remaining_nodes -= 1 |
| 188 | redacted_items.append(_REDACTED) |
| 189 | else: |
| 190 | redacted_items.append(redact(nested, depth + 1)) |
| 191 | if isinstance(item, list): |
| 192 | if type(item) is list: |
| 193 | return redacted_items |
| 194 | try: |
| 195 | return type(item)(redacted_items) |
| 196 | except Exception: |
| 197 | return redacted_items |
| 198 | if hasattr(item, "_fields"): |
| 199 | try: |
| 200 | return type(item)(*redacted_items) |
| 201 | except Exception: |
| 202 | pass |
| 203 | return tuple(redacted_items) |
| 204 | finally: |
| 205 | active_containers.remove(identity) |
| 206 | |
| 207 | if isinstance(item, str): |
| 208 | return redact_sensitive_text(item) |
| 209 | if isinstance(item, (int, float, bool, type(None))): |
| 210 | return item |
| 211 | if isinstance(item, (bytes, bytearray, memoryview)): |
| 212 | return _REDACTED |
| 213 | return _REDACTED |
| 214 | |
| 215 | return redact(value, 0) |