Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
py 215 lines 8.49 KB
Raw
1 """Best-effort secret redaction for free-form telemetry text."""
2
3 from __future__ import annotations
4
5 import re
6 import typing as t
7 from collections.abc import Mapping
8
9 _REDACTED = "[REDACTED]"
10 _MAX_STRUCTURED_REDACTION_DEPTH = 32
11 _MAX_STRUCTURED_REDACTION_NODES = 10_000
12 _URL_QUERY = re.compile(r"(\bhttps?://[^\s?#'\"]+)\?[^\s'\"]*", re.IGNORECASE)
13 _AUTHORIZATION = re.compile(r"\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
14 _SECRET_KEY_PATTERN = (
15 r"authorization|auth|api[-_]?key|apikey|x-api-key|access[-_]?token|"
16 r"refresh[-_]?token|client[-_]?secret|secret|password|passwd|pwd"
17 )
18 _SECRET_KEY_WITH_PREFIX_PATTERN = rf"(?:[A-Za-z0-9]+_)*(?:{_SECRET_KEY_PATTERN})"
19 _SECRET_KEY = re.compile(_SECRET_KEY_WITH_PREFIX_PATTERN, re.IGNORECASE)
20 # Quoted keys are matched as a unit so a key-like word at the end of an error
21 # string cannot treat the string's closing quote as the start of a secret value.
22 _QUOTED_SECRET_KEY_PREFIX = (
23 rf"([\"'])({_SECRET_KEY_WITH_PREFIX_PATTERN})\1(\s*[:=]+\s*)"
24 )
25 _BARE_SECRET_KEY_PREFIX = rf"(?<![A-Za-z0-9\"'])({_SECRET_KEY_PATTERN})\b(\s*[:=]+\s*)"
26 _ESCAPED_SECRET_KEY_PREFIX = (
27 rf"(?<![A-Za-z0-9])({_SECRET_KEY_PATTERN})\b((?:\\[\"'])?\s*[:=]+\s*)"
28 )
29 _QUOTED_KEY_DOUBLE_QUOTED_VALUE = re.compile(
30 rf'{_QUOTED_SECRET_KEY_PREFIX}"(?:\\.|[^"\\\r\n])*"', re.IGNORECASE
31 )
32 _QUOTED_KEY_SINGLE_QUOTED_VALUE = re.compile(
33 rf"{_QUOTED_SECRET_KEY_PREFIX}'(?:\\.|[^'\\\r\n])*'", re.IGNORECASE
34 )
35 _BARE_KEY_DOUBLE_QUOTED_VALUE = re.compile(
36 rf'{_BARE_SECRET_KEY_PREFIX}"((?:\\.|[^"\\\r\n])*)"',
37 re.IGNORECASE,
38 )
39 _BARE_KEY_SINGLE_QUOTED_VALUE = re.compile(
40 rf"{_BARE_SECRET_KEY_PREFIX}'((?:\\.|[^'\\\r\n])*)'",
41 re.IGNORECASE,
42 )
43 _BARE_KEY_ESCAPED_DOUBLE_QUOTED_VALUE = re.compile(
44 rf'{_ESCAPED_SECRET_KEY_PREFIX}\\"((?:\\.|[^"\\\r\n])*)\\"',
45 re.IGNORECASE,
46 )
47 _BARE_KEY_ESCAPED_SINGLE_QUOTED_VALUE = re.compile(
48 rf"{_ESCAPED_SECRET_KEY_PREFIX}\\'((?:\\.|[^'\\\r\n])*)\\'",
49 re.IGNORECASE,
50 )
51 _SECRET_PAIR_PREFIX = rf"(?<![A-Za-z0-9])({_SECRET_KEY_PATTERN})\b([\"']?\s*[:=]+\s*)"
52 _SECRET_PAIR_UNQUOTED = re.compile(
53 rf"{_SECRET_PAIR_PREFIX}(?!\\[\"'])([^\s\"',}}&]+)", re.IGNORECASE
54 )
55 _COMPLETES_DOUBLE_QUOTED_FIELD = re.compile(r'(?:\\.|[^"\\\r\n])*"\s*[:=]')
56 _COMPLETES_SINGLE_QUOTED_FIELD = re.compile(r"(?:\\.|[^'\\\r\n])*'\s*[:=]")
57 _CONTAINS_QUOTED_FIELD = re.compile(r"(?:^|[,{\s])([\"'])[^\"'\\\r\n]+\1\s*[:=]\s*$")
58
59
60 def _active_quotes_at(value: str, offsets: set[int]) -> dict[int, str | None]:
61 active_quotes: dict[int, str | None] = {}
62 quote: str | None = None
63 escaped = False
64 last_offset = max(offsets)
65
66 for position, character in enumerate(value):
67 if position in offsets:
68 active_quotes[position] = quote
69 if position == last_offset:
70 break
71 if character in "\r\n":
72 quote = None
73 escaped = False
74 continue
75 previous = value[position - 1] if position > 0 else ""
76 if previous in "\r\n":
77 previous = ""
78 following = value[position + 1] if position + 1 < len(value) else ""
79 if quote is None:
80 if character == '"' or (
81 character == "'" and not (previous.isalnum() or previous == "_")
82 ):
83 quote = character
84 continue
85 if escaped:
86 escaped = False
87 elif character == "\\":
88 escaped = True
89 elif character == quote:
90 if (
91 quote == "'"
92 and (previous.isalnum() or previous == "_")
93 and (following.isalnum() or following == "_")
94 ):
95 continue
96 quote = None
97
98 return active_quotes
99
100
101 def _redact_bare_quoted_value(value: str, pattern: re.Pattern[str], quote: str) -> str:
102 matches = list(pattern.finditer(value))
103 if not matches:
104 return value
105
106 active_quotes = _active_quotes_at(value, {match.start() for match in matches})
107 completes_field = (
108 _COMPLETES_DOUBLE_QUOTED_FIELD
109 if quote == '"'
110 else _COMPLETES_SINGLE_QUOTED_FIELD
111 )
112 parts: list[str] = []
113 cursor = 0
114 for match in matches:
115 parts.append(value[cursor : match.start()])
116 if active_quotes[match.start()] == quote and (
117 completes_field.match(value, match.end())
118 or _CONTAINS_QUOTED_FIELD.search(match.group(3))
119 ):
120 parts.append(match.group(0))
121 else:
122 parts.append(f"{match.group(1)}{match.group(2)}{quote}{_REDACTED}{quote}")
123 cursor = match.end()
124
125 parts.append(value[cursor:])
126 return "".join(parts)
127
128
129 def redact_sensitive_text(value: str) -> str:
130 """Remove common URL, authorization, and key-value secret shapes."""
131 value = _URL_QUERY.sub(rf"\1?{_REDACTED}", value)
132 value = _AUTHORIZATION.sub(rf"\1 {_REDACTED}", value)
133 value = _QUOTED_KEY_DOUBLE_QUOTED_VALUE.sub(rf'\1\2\1\3"{_REDACTED}"', value)
134 value = _QUOTED_KEY_SINGLE_QUOTED_VALUE.sub(rf"\1\2\1\3'{_REDACTED}'", value)
135 value = _BARE_KEY_ESCAPED_DOUBLE_QUOTED_VALUE.sub(rf'\1\2\\"{_REDACTED}\\"', value)
136 value = _BARE_KEY_ESCAPED_SINGLE_QUOTED_VALUE.sub(rf"\1\2\\'{_REDACTED}\\'", value)
137 value = _redact_bare_quoted_value(value, _BARE_KEY_DOUBLE_QUOTED_VALUE, '"')
138 value = _redact_bare_quoted_value(value, _BARE_KEY_SINGLE_QUOTED_VALUE, "'")
139 return _SECRET_PAIR_UNQUOTED.sub(rf"\1\2{_REDACTED}", value)
140
141
142 def redact_sensitive_value(value: t.Any) -> t.Any:
143 """Redact secrets in structured logging metadata without changing safe scalar types."""
144 active_containers: set[int] = set()
145 remaining_nodes = _MAX_STRUCTURED_REDACTION_NODES
146
147 def redact(item: t.Any, depth: int) -> t.Any:
148 nonlocal remaining_nodes
149 if depth >= _MAX_STRUCTURED_REDACTION_DEPTH or remaining_nodes <= 0:
150 return _REDACTED
151 remaining_nodes -= 1
152
153 if isinstance(item, (Mapping, list, tuple)):
154 identity = id(item)
155 if identity in active_containers:
156 return _REDACTED
157 active_containers.add(identity)
158 try:
159 if isinstance(item, Mapping):
160 redacted_mapping: dict[t.Any, t.Any] = {}
161 for key, nested in item.items():
162 if remaining_nodes <= 0:
163 break
164 if isinstance(key, str):
165 redacted_key: t.Any = redact_sensitive_text(key)
166 elif isinstance(key, (int, float, bool, type(None))):
167 redacted_key = key
168 else:
169 redacted_key = _REDACTED
170 if isinstance(key, str) and _SECRET_KEY.fullmatch(key):
171 remaining_nodes -= 1
172 redacted_mapping[redacted_key] = _REDACTED
173 else:
174 redacted_mapping[redacted_key] = redact(nested, depth + 1)
175 return redacted_mapping
176
177 redacted_items: list[t.Any] = []
178 fields = getattr(item, "_fields", ()) if isinstance(item, tuple) else ()
179 for index, nested in enumerate(item):
180 if remaining_nodes <= 0:
181 return _REDACTED
182 if (
183 index < len(fields)
184 and isinstance(fields[index], str)
185 and _SECRET_KEY.fullmatch(fields[index])
186 ):
187 remaining_nodes -= 1
188 redacted_items.append(_REDACTED)
189 else:
190 redacted_items.append(redact(nested, depth + 1))
191 if isinstance(item, list):
192 if type(item) is list:
193 return redacted_items
194 try:
195 return type(item)(redacted_items)
196 except Exception:
197 return redacted_items
198 if hasattr(item, "_fields"):
199 try:
200 return type(item)(*redacted_items)
201 except Exception:
202 pass
203 return tuple(redacted_items)
204 finally:
205 active_containers.remove(identity)
206
207 if isinstance(item, str):
208 return redact_sensitive_text(item)
209 if isinstance(item, (int, float, bool, type(None))):
210 return item
211 if isinstance(item, (bytes, bytearray, memoryview)):
212 return _REDACTED
213 return _REDACTED
214
215 return redact(value, 0)