Log to windows (#18688)
* split netdata logger into multiple files - no acctual code changes * move around some more code * base for implementing windows events logging * fix for the last commit * working logging to windows events, but not pretty yet * fix compilation on linux * added scripts for compiling the resource file and importing the manifest * added validation that the provider is available * working manifest for ETW (Event Tracing for Windows) * compile the messages dll with msys tools * handle wevents configuration * when starting under clion, do not start as service * unify conversion to utf16 * fix bug in windows-events.plugin that was incorrectly not processing right the publishers that do not have a UUID * enable wevents as default logging for all methods, under windows * log to windows using EventCreate.exe for the messages * do not log all the fields * added log-forwarder to spawn-server-windows * fix last character being cut-off when converting from utf-16 * updated info * updated any_to_utf16() to be always consistent * added utf16_to_utf8() * external plugins inherit windows events * fix wrong log source * fix spawn server logs * log to multiple event log sources * generate custom messages dll for event viewer - working * removed debugging code * cleanup log forwarder entries from the thread, to avoid bad file descriptor in poll() * .mc and its manifest are automatically generated * sanitizers should not remove trailing underscores * use the resources dll for the netdata directory; set the default maxSize to windows events * do not set customer flag on event ids; use the same naming for channels and providers * work to unify manifest and resources * netdata now logs using ETW * implemented etw and wel logging in netdata * minor changes * updated windows installer to install the manifest * do not install etw if the manifest is not there * allow loggings to WEL and ETW at the same time * fix the installer conditions * fix nsi * detect ci paths for sys utils * enable ETW is CI * better integration of spawn server with logger * use script to find SDK path * use auto-discovery of sdk and visual studio * fix overlapping link.exe with msys; do not escape percentage when it is not followed by a number; added more documentation about windows * debug info for path * fixes compilation scripts * ETW and WEL are always required on Windows * in progress for supporting full text search queries * find mvc versions * improve find-sdk-path.sh * fix the script once again * fetch event data for full text search * fix script again * fix script, yes again * fts using event data * code renames and cleanup for clarity * update documentation * full text search switches plugin to load everything synchronously * full text search using the individual event data fields, without using XML * close all idle provider handles after 5 mins * added EventsAPI field * supported exposing all system fields; started documentation about windows events plugin * avoid crash because of unitialized memory * remove debugging * do not add qualifiers and version when they are zero * updated docs * copy the manifest too * rework on installing manifest and dll * completed documentation * work on windows-events sources list * fix windows installer logic * removed unecessary include * added image to documentation
Costa Tsaousis committed
Oct 15, 2024 at 09:04 UTC
f6141cc4f3e4bc0cfb6f854ada4dd9f265285836
72 files changed
+8026
-3447
CMakeLists.txt
+108
-9
@@ -670,10 +670,10 @@ set(LIBNETDATA_FILES
670
src/libnetdata/libnetdata.h
671
src/libnetdata/locks/locks.c
672
src/libnetdata/locks/locks.h
673
- src/libnetdata/log/journal.c
674
- src/libnetdata/log/journal.h
675
- src/libnetdata/log/log.c
676
- src/libnetdata/log/log.h
673
+ src/libnetdata/log/systemd-journal-helpers.c
674
+ src/libnetdata/log/systemd-journal-helpers.h
675
+ src/libnetdata/log/nd_log.c
676
+ src/libnetdata/log/nd_log.h
677
src/libnetdata/os/os.c
678
src/libnetdata/os/os.h
679
src/libnetdata/simple_hashtable.h
@@ -822,6 +822,25 @@ set(LIBNETDATA_FILES
822
src/libnetdata/sanitizers/sanitizers-functions.h
823
src/libnetdata/sanitizers/sanitizers-pluginsd.c
824
src/libnetdata/sanitizers/sanitizers-pluginsd.h
825
+ src/libnetdata/log/nd_log-internals.c
826
+ src/libnetdata/log/nd_log-internals.h
827
+ src/libnetdata/log/nd_log_limit.c
828
+ src/libnetdata/log/nd_log_limit.h
829
+ src/libnetdata/log/nd_log-config.c
830
+ src/libnetdata/log/nd_log-init.c
831
+ src/libnetdata/log/nd_log-to-syslog.c
832
+ src/libnetdata/log/nd_log-to-systemd-journal.c
833
+ src/libnetdata/log/nd_log-annotators.c
834
+ src/libnetdata/log/nd_log-field-formatters.c
835
+ src/libnetdata/log/nd_log-format-logfmt.c
836
+ src/libnetdata/log/nd_log-format-json.c
837
+ src/libnetdata/log/nd_log-to-file.c
838
+ src/libnetdata/log/nd_log-to-windows-events.c
839
+ src/libnetdata/string/utf8.c
840
+ src/libnetdata/spawn_server/log-forwarder.c
841
+ src/libnetdata/spawn_server/log-forwarder.h
842
+ src/libnetdata/log/nd_log-common.h
843
+ src/libnetdata/log/nd_log-to-windows-common.h
844
)
845
846
if(ENABLE_PLUGIN_EBPF)
@@ -1439,12 +1458,13 @@ set(WINDOWS_EVENTS_PLUGIN_FILES
1458
src/collectors/windows-events.plugin/windows-events-unicode.h
1459
src/collectors/windows-events.plugin/windows-events-xml.c
1460
src/collectors/windows-events.plugin/windows-events-xml.h
1442
- src/collectors/windows-events.plugin/windows-events-publishers.c
1443
- src/collectors/windows-events.plugin/windows-events-publishers.h
1461
+ src/collectors/windows-events.plugin/windows-events-providers.c
1462
+ src/collectors/windows-events.plugin/windows-events-providers.h
1463
src/collectors/windows-events.plugin/windows-events-fields-cache.c
1464
src/collectors/windows-events.plugin/windows-events-fields-cache.h
1465
src/collectors/windows-events.plugin/windows-events-query-builder.c
1466
src/collectors/windows-events.plugin/windows-events-query-builder.h
1467
+ src/collectors/windows-events.plugin/windows-events-query-evt-variant.c
1468
)
1469
1470
set(WINDOWS_PLUGIN_FILES
@@ -1746,10 +1766,90 @@ target_include_directories(libnetdata BEFORE PUBLIC ${CONFIG_H_DIR} ${CMAKE_SOUR
1766
target_link_libraries(libnetdata PUBLIC
1767
"$<$<NOT:$<BOOL:${HAVE_BUILTIN_ATOMICS}>>:atomic>"
1768
"$<$<OR:$<BOOL:${OS_LINUX}>,$<BOOL:${OS_FREEBSD}>>:pthread;rt>"
1749
- "$<$<BOOL:${OS_WINDOWS}>:kernel32;advapi32;winmm;rpcrt4;bcrypt>"
1769
+ "$<$<BOOL:${OS_WINDOWS}>:kernel32;advapi32;winmm;rpcrt4;bcrypt;wevtapi>"
1770
"$<$<BOOL:${LINK_LIBM}>:m>"
1771
"${SYSTEMD_LDFLAGS}")
1772
1773
+if(OS_WINDOWS)
1774
+ set(HAVE_ETW True)
1775
+ set(HAVE_WEL True)
1776
+
1777
+ # Output the results for debugging purposes
1778
+ message(STATUS "Have Event Tracing for Windows (ETW): ${HAVE_ETW}")
1779
+ message(STATUS "Have Windows Event Log (WEL): ${HAVE_WEL}")
1780
+
1781
+ if(HAVE_WEL OR HAVE_ETW)
1782
+ # Define the source and generated file paths
1783
+ set(WEVT_GEN_SRC_H_FILE "${CMAKE_SOURCE_DIR}/src/libnetdata/log/nd_log-to-windows-common.h")
1784
+ set(WEVT_GEN_SRC_C_FILE "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_mc_generate.c")
1785
+ set(WEVT_GEN_BIN_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_mc_generate")
1786
+
1787
+ set(WEVT_BUILD_SCRIPT "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_compile.sh")
1788
+
1789
+ set(WEVT_MC_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.mc")
1790
+ set(WEVT_MAN_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_manifest.xml")
1791
+ set(WEVT_RC_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.rc")
1792
+ set(WEVT_MC_H_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.h")
1793
+ set(WEVT_MAN_H_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_manifest.h")
1794
+ set(WEVT_RES_OBJECT "${CMAKE_BINARY_DIR}/wevt_netdata_res.o")
1795
+
1796
+ set(WEVT_DLL_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.dll")
1797
+ set(WEVT_ETW_INSTALL_SCRIPT "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_install.bat")
1798
+
1799
+ # we compile ${WEVT_GEN_BIN_FILE}, which generates the manifest, the .mc,
1800
+ # and the headers required for compiling libnetdata/logs
1801
+
1802
+ if(HAVE_ETW)
1803
+ # ETW method also supports WEL
1804
+ # but it requires Microsoft tools mc, rc, and link
1805
+ add_custom_command(
1806
+ OUTPUT "${WEVT_MC_H_FILE}" "${WEVT_MAN_H_FILE}" "${WEVT_DLL_FILE}"
1807
+ COMMAND "${CMAKE_C_COMPILER}" -o "${WEVT_GEN_BIN_FILE}" "${WEVT_GEN_SRC_C_FILE}"
1808
+ COMMAND "${WEVT_GEN_BIN_FILE}" >"${WEVT_MC_FILE}"
1809
+ COMMAND "${WEVT_GEN_BIN_FILE}" --manifest >"${WEVT_MAN_FILE}"
1810
+ COMMAND "${WEVT_BUILD_SCRIPT}" "${CMAKE_SOURCE_DIR}/src/libnetdata/log" "${CMAKE_BINARY_DIR}"
1811
+ DEPENDS "${WEVT_GEN_SRC_C_FILE}" "${WEVT_GEN_SRC_H_FILE}"
1812
+ COMMENT "Compiling ${WEVT_MC_FILE} to generate ${WEVT_MC_H_FILE} and ${WEVT_DLL_FILE}"
1813
+ )
1814
+ else()
1815
+ # WEL method can be built with windmc, windres and the normal linker
1816
+ add_custom_command(
1817
+ OUTPUT "${WEVT_MC_H_FILE}" "${WEVT_DLL_FILE}"
1818
+ COMMAND "${CMAKE_C_COMPILER}" -o "${WEVT_GEN_BIN_FILE}" "${WEVT_GEN_SRC_C_FILE}"
1819
+ COMMAND "${WEVT_GEN_BIN_FILE}" >"${WEVT_MC_FILE}"
1820
+ COMMAND "${WEVT_GEN_BIN_FILE}" --manifest >"${WEVT_MAN_FILE}"
1821
+ COMMAND windmc -r "${CMAKE_BINARY_DIR}" -h "${CMAKE_BINARY_DIR}" ${WEVT_MC_FILE}
1822
+ COMMAND echo "1 2004" "wevt_netdata_manifest.xml" >> "${WEVT_RC_FILE}"
1823
+ COMMAND windres ${WEVT_RC_FILE} -o ${WEVT_RES_OBJECT}
1824
+ COMMAND ${CMAKE_LINKER} -dll --entry 0 -nostdlib -o ${WEVT_DLL_FILE} ${WEVT_RES_OBJECT}
1825
+ DEPENDS "${WEVT_GEN_SRC_C_FILE}" "${WEVT_GEN_SRC_H_FILE}"
1826
+ COMMENT "Compiling ${WEVT_MC_FILE} to generate ${WEVT_MC_H_FILE} and ${WEVT_DLL_FILE}"
1827
+ )
1828
+ endif()
1829
+
1830
+ # Create a custom target for the DLL
1831
+ add_custom_target(wevt_netdata ALL DEPENDS ${WEVT_DLL_FILE})
1832
+
1833
+ set_source_files_properties(src/libnetdata/log/nd_log-to-windows-events.c PROPERTIES
1834
+ OBJECT_DEPENDS "${WEVT_MC_H_FILE}")
1835
+
1836
+ if(HAVE_ETW)
1837
+ set_source_files_properties(src/libnetdata/log/nd_log-to-windows-events.c PROPERTIES
1838
+ OBJECT_DEPENDS "${WEVT_MAN_H_FILE}")
1839
+
1840
+ install(FILES "${WEVT_DLL_FILE}" "${WEVT_MAN_FILE}" "${WEVT_ETW_INSTALL_SCRIPT}"
1841
+ COMPONENT wevt_netdata_dll
1842
+ DESTINATION "${BINDIR}")
1843
+ else()
1844
+ # do not install the manifest in this case
1845
+ # the nsi installer will skip registering the ETW publisher
1846
+ install(FILES "${WEVT_DLL_FILE}"
1847
+ COMPONENT wevt_netdata_dll
1848
+ DESTINATION "${BINDIR}")
1849
+ endif()
1850
+ endif()
1851
+endif()
1852
+
1853
# ebpf
1854
if(ENABLE_PLUGIN_EBPF)
1855
netdata_add_libbpf_to_target(libnetdata)
@@ -1925,8 +2025,7 @@ if(ENABLE_PLUGIN_APPS)
2025
add_executable(apps.plugin ${APPS_PLUGIN_FILES})
2026
2027
target_link_libraries(apps.plugin libnetdata ${CAP_LIBRARIES}
1928
- "$<$<BOOL:${OS_WINDOWS}>:Version>"
1929
- "$<$<BOOL:${OS_WINDOWS}>:ntdll>")
2028
+ "$<$<BOOL:${OS_WINDOWS}>:Version;ntdll>")
2029
2030
target_include_directories(apps.plugin PRIVATE ${CAP_INCLUDE_DIRS})
2031
target_compile_options(apps.plugin PRIVATE ${CAP_CFLAGS_OTHER})
packaging/cmake/config.cmake.h.in
+2
@@ -169,6 +169,8 @@
169
#cmakedefine HAVE_LIBYAML
170
#cmakedefine HAVE_LIBMNL
171
172
+#cmakedefine HAVE_WEL
173
+#cmakedefine HAVE_ETW
174
#cmakedefine RUN_UNDER_CLION
175
176
// /* Enable GNU extensions on systems that have them. */
packaging/utils/compile-and-run-windows.sh
renamed
+8
-3
@@ -70,15 +70,20 @@ then
70
${NULL}
71
fi
72
73
+ninja -v -C "${build}" || ninja -v -C "${build}" -j 1
74
+
75
+echo "Stopping service Netdata"
76
+sc stop "Netdata" || echo "Failed"
77
+
78
ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1
79
80
+# register the event log publisher
81
+cmd.exe //c "$(cygpath -w -a "/opt/netdata/usr/bin/wevt_netdata_install.bat")"
82
+
83
#echo
84
#echo "Compile with:"
85
#echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
86
79
-echo "Stopping service Netdata"
80
-sc stop "Netdata" || echo "Failed"
81
-
87
echo "starting netdata..."
88
# enable JIT debug with gdb
89
export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"
packaging/windows/clion-msys-msys-environment.bat
+3
-1
@@ -13,7 +13,9 @@ set MSYSTEM=MSYS
13
:: go exists only mingw64 / ucrt64 / etc, not under msys profile
14
set GOROOT=C:\msys64\mingw64
15
16
-set PATH="%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin"
16
+set "PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
17
+set "PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
18
+set "PATH=%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin"
19
::set PKG_CONFIG_EXECUTABLE=C:\msys64\mingw64\bin\pkg-config.exe
20
::set CMAKE_C_COMPILER=C:\msys64\mingw64\bin\gcc.exe
21
::set CMAKE_CC_COMPILER=C:\msys64\mingw64\bin\g++.exe
packaging/windows/find-sdk-path.sh
new
+217
@@ -0,0 +1,217 @@
1
+#!/bin/bash
2
+
3
+# Function to output the path in Windows format (convert from MSYS2/Unix format using cygpath)
4
+convert_to_windows_format() {
5
+ cygpath -w -a "$1"
6
+}
7
+
8
+# Function to display help message
9
+display_help() {
10
+ echo "Usage: $0 [-s|--sdk] [-v|--visualstudio] [-w|--windows] [--help]"
11
+ echo
12
+ echo "Options:"
13
+ echo " -s, --sdk Search for tools in the Windows SDK."
14
+ echo " -v, --visualstudio Search for tools in Visual Studio."
15
+ echo " -w, --windows Output the path in Windows format (using cygpath)."
16
+ echo " --help Display this help message."
17
+ exit 0
18
+}
19
+
20
+# Function to find tools in the Windows SDK
21
+find_sdk_tools() {
22
+ sdk_base_path="/c/Program Files (x86)/Windows Kits/10/bin"
23
+
24
+ if [ ! -d "$sdk_base_path" ]; then
25
+ echo "ERROR: SDK base path \"$sdk_base_path\" does not exist. No SDK installations found." >&2
26
+ echo "$system_root"
27
+ return 1
28
+ fi
29
+
30
+ echo "SDK base path exists: \"$sdk_base_path\"" >&2
31
+
32
+ # Find all SDK versions
33
+ sdk_versions=($(ls "$sdk_base_path" | tr ' ' '\n' | grep -E "^[0-9]+\..*$"))
34
+ echo "Found SDK versions: ${sdk_versions[*]}" >&2
35
+
36
+ if [ ${#sdk_versions[@]} -eq 0 ]; then
37
+ echo "ERROR: No valid Windows SDK versions found in \"$sdk_base_path\"." >&2
38
+ echo "$system_root"
39
+ return 1
40
+ fi
41
+
42
+ # Sort versions and pick the latest
43
+ sorted_versions=$(printf '%s\n' "${sdk_versions[@]}" | sort -V)
44
+ latest_sdk_version=$(echo "$sorted_versions" | tail -n 1)
45
+ sdk_tool_path="$sdk_base_path/$latest_sdk_version/x64"
46
+
47
+ echo "Latest SDK version: \"$latest_sdk_version\"" >&2
48
+
49
+ if [ ! -d "$sdk_tool_path" ]; then
50
+ echo "ERROR: Tool path \"$sdk_tool_path\" does not exist." >&2
51
+ echo "$system_root"
52
+ return 1
53
+ fi
54
+
55
+ # Check if required tools exist
56
+ tools=("mc.exe" "rc.exe")
57
+ for tool in "${tools[@]}"; do
58
+ if [ ! -f "$sdk_tool_path/$tool" ]; then
59
+ echo "ERROR: $tool not found in \"$sdk_tool_path\"" >&2
60
+ echo "$system_root"
61
+ return 1
62
+ else
63
+ echo "$tool found in \"$sdk_tool_path\"" >&2
64
+ fi
65
+ done
66
+
67
+ echo >&2
68
+ echo "DONE: All required tools found in \"$sdk_tool_path\"" >&2
69
+ echo >&2
70
+
71
+ echo "$sdk_tool_path"
72
+}
73
+
74
+# Function to find tools in Visual Studio
75
+find_visual_studio_tools() {
76
+ studio_base_path="/c/Program Files/Microsoft Visual Studio/2022"
77
+ echo "Checking for Visual Studio installations in: \"$studio_base_path\"" >&2
78
+
79
+ if [ ! -d "$studio_base_path" ]; then
80
+ echo "ERROR: Visual Studio base path \"$studio_base_path\" does not exist. No Visual Studio installations found." >&2
81
+ echo "$system_root"
82
+ return 1
83
+ fi
84
+
85
+ # Visual Studio editions we want to check
86
+ editions=("Enterprise" "Professional" "Community")
87
+ available_editions=()
88
+
89
+ # Loop through each edition and check for tools
90
+ for edition in "${editions[@]}"; do
91
+ edition_path="$studio_base_path/$edition/VC/Tools/MSVC"
92
+ if [ -d "$edition_path" ]; then
93
+ available_editions+=("$edition")
94
+ echo "Checking edition: $edition in $studio_base_path" >&2
95
+
96
+ # Find all MSVC versions and sort them
97
+ msvc_versions=($(ls "$edition_path" | tr ' ' '\n' | grep -E "^[0-9]+\..*$"))
98
+ echo "Found MSVC versions in $edition: ${msvc_versions[*]}" >&2
99
+
100
+ if [ ${#msvc_versions[@]} -gt 0 ]; then
101
+ sorted_versions=$(printf '%s\n' "${msvc_versions[@]}" | sort -V)
102
+ latest_msvc_version=$(echo "${sorted_versions[@]}" | tail -n 1)
103
+ vs_tool_path="$edition_path/$latest_msvc_version/bin/Hostx64/x64"
104
+
105
+ echo "Latest MSVC version: \"$latest_msvc_version\" in $edition" >&2
106
+
107
+ if [ ! -d "$vs_tool_path" ]; then
108
+ echo "WARNING: Tool path \"$vs_tool_path\" does not exist." >&2
109
+ continue
110
+ fi
111
+
112
+ # Check if required tools exist
113
+ tools=("link.exe")
114
+ missing_tool=0
115
+
116
+ for tool in "${tools[@]}"; do
117
+ if [ ! -f "$vs_tool_path/$tool" ]; then
118
+ echo "WARNING: $tool not found in \"$vs_tool_path\" for $edition" >&2
119
+ missing_tool=1
120
+ else
121
+ echo "$tool found in \"$vs_tool_path\"" >&2
122
+ fi
123
+ done
124
+
125
+ if [ $missing_tool -eq 0 ]; then
126
+ echo >&2
127
+ echo "All required tools found in \"$vs_tool_path\"" >&2
128
+ echo >&2
129
+
130
+ echo "$vs_tool_path"
131
+ return 0
132
+ else
133
+ echo "WARNING: skipping edition '$edition', directory does not exist." >&2
134
+ fi
135
+ else
136
+ echo "WARNING: skipping edition '$edition', MSVC directory does not exist." >&2
137
+ fi
138
+ else
139
+ echo "WARNING: skipping edition '$edition', directory does not exist." >&2
140
+ fi
141
+ done
142
+
143
+ echo "ERROR: No valid Visual Studio editions found in \"$studio_base_path\"." >&2
144
+ echo "$system_root"
145
+ return 1
146
+}
147
+
148
+# Parse options using getopt
149
+TEMP=$(getopt -o svwh --long sdk,visualstudio,windows,help -- "$@")
150
+if [ $? != 0 ]; then
151
+ echo "ERROR: Invalid options provided." >&2
152
+ exit 1
153
+fi
154
+
155
+eval set -- "$TEMP"
156
+
157
+search_mode="sdk"
158
+windows_format=0
159
+system_root="/usr/bin"
160
+
161
+# Process getopt options
162
+while true; do
163
+ case "$1" in
164
+ -s|--sdk)
165
+ search_mode="sdk"
166
+ shift
167
+ ;;
168
+ -v|--visualstudio)
169
+ search_mode="visualstudio"
170
+ shift
171
+ ;;
172
+ -w|--windows)
173
+ system_root="%SYSTEMROOT%"
174
+ windows_format=1
175
+ shift
176
+ ;;
177
+ --help|-h)
178
+ display_help
179
+ ;;
180
+ --)
181
+ shift
182
+ break
183
+ ;;
184
+ *)
185
+ echo "ERROR: Invalid option: $1" >&2
186
+ exit 1
187
+ ;;
188
+ esac
189
+done
190
+
191
+# Ensure that one of --sdk or --visualstudio is selected
192
+if [ -z "$search_mode" ]; then
193
+ echo "ERROR: You must specify either --sdk or --visualstudio." >&2
194
+ display_help
195
+fi
196
+
197
+# Determine which function to call based on the search mode
198
+if [ "$search_mode" = "sdk" ]; then
199
+ tool_path=$(find_sdk_tools)
200
+else
201
+ tool_path=$(find_visual_studio_tools)
202
+fi
203
+
204
+# If a valid path is found, output it
205
+if [ "$tool_path" != "$system_root" ]; then
206
+ if [ "$windows_format" -eq 1 ]; then
207
+ windows_tool_path=$(convert_to_windows_format "$tool_path")
208
+ echo "$windows_tool_path"
209
+ else
210
+ echo "$tool_path"
211
+ fi
212
+else
213
+ echo "$system_root"
214
+ exit 1
215
+fi
216
+
217
+exit 0
packaging/windows/installer.nsi
+93
-10
@@ -257,29 +257,97 @@ Function NetdataUninstallRegistry
257
end:
258
FunctionEnd
259
260
+Function InstallDLL
261
+ ; Check if certutil is available
262
+ nsExec::ExecToStack 'where certutil'
263
+ Pop $R0
264
+ StrCmp $R0 "" NoCertUtil FoundCertUtil
265
+
266
+ NoCertUtil:
267
+ DetailPrint "certutil not found, assuming files are different."
268
+ Goto CopyDLL
269
+
270
+ FoundCertUtil:
271
+ ; Calculate hash of the existing DLL
272
+ nsExec::ExecToStack 'certutil -hashfile "$SYSDIR\wevt_netdata.dll" MD5'
273
+ Pop $R0
274
+
275
+ ; Calculate hash of the new DLL
276
+ nsExec::ExecToStack 'certutil -hashfile "$INSTDIR\usr\bin\wevt_netdata.dll" MD5'
277
+ Pop $R1
278
+
279
+ StrCmp $R0 $R1 SetPermissions
280
+
281
+ CopyDLL:
282
+ ClearErrors
283
+ CopyFiles /SILENT "$INSTDIR\usr\bin\wevt_netdata.dll" "$SYSDIR"
284
+ IfErrors RetryPrompt SetPermissions
285
+
286
+ RetryPrompt:
287
+ MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Failed to copy wevt_netdata.dll probably because it is in use. Please close the Event Viewer (or other Event Log applications) and press Retry."
288
+ StrCmp $R0 IDRETRY CopyDLL
289
+ StrCmp $R0 IDCANCEL ExitInstall
290
+
291
+ Goto End
292
+
293
+ SetPermissions:
294
+ nsExec::ExecToLog 'icacls "$SYSDIR\wevt_netdata.dll" /grant "NT SERVICE\EventLog":R'
295
+ Goto End
296
+
297
+ ExitInstall:
298
+ Abort
299
+
300
+ End:
301
+FunctionEnd
302
+
303
+Function InstallManifest
304
+ IfFileExists "$INSTDIR\usr\bin\wevt_netdata_manifest.xml" CopyManifest End
305
+
306
+ CopyManifest:
307
+ ClearErrors
308
+ CopyFiles /SILENT "$INSTDIR\usr\bin\wevt_netdata_manifest.xml" "$SYSDIR"
309
+ IfErrors RetryPrompt InstallManifest
310
+
311
+ RetryPrompt:
312
+ MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Failed to copy wevt_netdata_manifest.xml."
313
+ StrCmp $R0 IDRETRY CopyManifest
314
+ StrCmp $R0 IDCANCEL ExitInstall
315
+
316
+ InstallManifest:
317
+ nsExec::ExecToLog 'wevtutil im "$SYSDIR\wevt_netdata_manifest.xml" "/mf:$SYSDIR\wevt_netdata.dll" "/rf:$SYSDIR\wevt_netdata.dll"'
318
+ Goto End
319
+
320
+ ExitInstall:
321
+ Abort
322
+
323
+ End:
324
+FunctionEnd
325
+
326
Section "Install Netdata"
261
- SetOutPath $INSTDIR
262
- SetCompress off
327
+ SetOutPath $INSTDIR
328
+ SetCompress off
329
264
- File /r "C:\msys64\opt\netdata\*.*"
330
+ File /r "C:\msys64\opt\netdata\*.*"
331
266
- ClearErrors
332
+ ClearErrors
333
nsExec::ExecToLog '$SYSDIR\sc.exe create Netdata binPath= "$INSTDIR\usr\bin\netdata.exe" start= delayed-auto'
334
pop $0
335
${If} $0 != 0
270
- DetailPrint "Warning: Failed to create Netdata service."
336
+ DetailPrint "Warning: Failed to create Netdata service."
337
${EndIf}
338
273
- ClearErrors
339
+ ClearErrors
340
nsExec::ExecToLog '$SYSDIR\sc.exe description Netdata "Real-time system monitoring service"'
341
pop $0
342
${If} $0 != 0
277
- DetailPrint "Warning: Failed to add Netdata service description."
343
+ DetailPrint "Warning: Failed to add Netdata service description."
344
${EndIf}
345
346
WriteUninstaller "$INSTDIR\Uninstall.exe"
347
348
Call NetdataUninstallRegistry
349
+ Call InstallDLL
350
+ Call InstallManifest
351
352
StrLen $0 $cloudToken
353
StrLen $1 $cloudRooms
@@ -325,9 +393,24 @@ Section "Uninstall"
393
DetailPrint "Warning: Failed to delete Netdata service."
394
${EndIf}
395
328
- # https://nsis.sourceforge.io/Reference/RMDir
329
- RMDir /r /REBOOTOK "$INSTDIR"
396
+ ; Check if the manifest exists before uninstalling it
397
+ IfFileExists "$SYSDIR\wevt_netdata_manifest.xml" ManifestExistsForUninstall ManifestNotExistsForUninstall
398
+
399
+ManifestExistsForUninstall:
400
+ nsExec::ExecToLog 'wevtutil um "$SYSDIR\wevt_netdata_manifest.xml"'
401
+ pop $0
402
+ ${If} $0 != 0
403
+ DetailPrint "Warning: Failed to uninstall the event manifest."
404
+ ${EndIf}
405
+ Goto DoneUninstall
406
+
407
+ManifestNotExistsForUninstall:
408
+ DetailPrint "Manifest not found, skipping manifest uninstall."
409
+
410
+DoneUninstall:
411
+
412
+ ; Remove files
413
+ RMDir /r /REBOOTOK "$INSTDIR"
414
415
DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Netdata"
416
SectionEnd
333
-
src/collectors/apps.plugin/apps_os_windows.c
+2
-6
@@ -523,14 +523,10 @@ static char *wchar_to_utf8(WCHAR *s) {
523
524
static char *ansi_to_utf8(LPCSTR str) {
525
static __thread WCHAR unicode[PATH_MAX];
526
- static __thread int unicode_size = sizeof(unicode) / sizeof(*unicode);
526
527
// Step 1: Convert ANSI string (LPSTR) to wide string (UTF-16)
529
- int wideLength = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0);
530
- if (wideLength == 0 || wideLength > unicode_size)
531
- return NULL;
532
-
533
- MultiByteToWideChar(CP_ACP, 0, str, -1, unicode, wideLength);
528
+ size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1);
529
+ if (!count) return NULL;
530
531
return wchar_to_utf8(unicode);
532
}
src/collectors/apps.plugin/apps_pid.c
+4
-8
@@ -327,15 +327,11 @@ static bool is_filename(const char *s) {
327
(*s == '/' && s[1] == '/' && isalpha((uint8_t)s[2]) && s[3] == '/')) { // windows native "//x/"
328
329
WCHAR ws[FILENAME_MAX];
330
- int wlen = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0);
331
- if (wlen <= 0 || (size_t)wlen > sizeof(ws) / sizeof(*ws)) {
332
- return false; // Failed to convert UTF-8 to UTF-16
330
+ if(utf8_to_utf16(ws, _countof(ws), s, -1) > 0) {
331
+ DWORD attributes = GetFileAttributesW(ws);
332
+ if (attributes != INVALID_FILE_ATTRIBUTES)
333
+ return true;
334
}
334
-
335
- MultiByteToWideChar(CP_UTF8, 0, s, -1, ws, wlen);
336
- DWORD attributes = GetFileAttributesW(ws);
337
- if (attributes != INVALID_FILE_ATTRIBUTES)
338
- return true;
335
}
336
#endif
337
src/collectors/systemd-journal.plugin/systemd-journal.c
+1
@@ -85,6 +85,7 @@ struct lqs_extension {
85
#define LQS_SOURCE_TYPE SD_JOURNAL_FILE_SOURCE_TYPE
86
#define LQS_SOURCE_TYPE_ALL SDJF_ALL
87
#define LQS_SOURCE_TYPE_NONE SDJF_NONE
88
+#define LQS_PARAMETER_SOURCE_NAME "Journal Sources" // this is how it is shown to users
89
#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) get_internal_source_type(value)
90
#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) available_journal_file_sources_to_json_array(wb)
91
#include "libnetdata/facets/logs_query_status.h"
src/collectors/windows-events.plugin/README.md
new
+289
@@ -0,0 +1,289 @@
1
+# Windows Events plugin
2
+
3
+[KEY FEATURES](#key-features) | [EVENTS SOURCES](#events-sources) | [EVENT FIELDS](#event-fields) |
4
+[PLAY MODE](#play-mode) | [FULL TEXT SEARCH](#full-text-search) | [PERFORMANCE](#query-performance) |
5
+[CONFIGURATION](#configuration-and-maintenance) | [FAQ](#faq)
6
+
7
+The Windows Events plugin by Netdata makes viewing, exploring and analyzing Windows Events simple and
8
+efficient.
9
+
10
+
11
+
12
+## Key features
13
+
14
+- Supports **Windows Event Logs (WEL)**.
15
+- Supports **Event Tracing for Windows (ETW)** and **TraceLogging (TL)**, when events are routed to Event Log.
16
+- Allows filtering on all System Events fields.
17
+- Allows **full text search** (`grep`) on all System and User fields.
18
+- Provides a **histogram** for log entries over time, with a break down per field-value, for any System Event field and any
19
+ time-frame.
20
+- Supports coloring log entries based on severity.
21
+- In PLAY mode it "tails" all the Events, showing new log entries immediately after they are received.
22
+
23
+### Prerequisites
24
+
25
+`windows-events.plugin` is a Netdata Function Plugin.
26
+
27
+To protect your privacy, as with all Netdata Functions, a free Netdata Cloud user account is required to access it.
28
+For more information check [this discussion](https://github.com/netdata/netdata/discussions/16136).
29
+
30
+## Events Sources
31
+
32
+The plugin automatically detects all the available channels and offers a list of "Event Channels".
33
+
34
+By default, it aggregates events from all event channels, providing a unified systems view of all events.
35
+
36
+> To improve query performance, we recommend selecting the relevant event channels, before doing more
37
+> analysis on the events.
38
+
39
+In the list of events channels, several shortcuts are added, aggregating events according to various attributes:
40
+
41
+- `All`, aggregates events from all available channels. This provides a holistic view of all events in the system.
42
+- `All-Admin`, `All-Operational`, `All-Analytic` and `All-Debug` aggregates events from channels marked `Admin`, `Operational`, `Analytic` and `Debug`, respectively.
43
+- `All-Windows`, aggregates events from `Application`, `Security`, `System` and `Setup`.
44
+- `All-Enabled` and `All-Disabled` aggregates events from channels depending on their status.
45
+- `All-Forwarded` aggregates events from channels owned by `Microsoft-Windows-EventCollector`.
46
+- `All-Classic` aggregates events from channels using the Classic Event Log API.
47
+- `All-Of-X`, where `X` is a provider name, is offered for all providers having more than a channel.
48
+- `All-In-X`, where `X` is `Backup-Mode`, `Overwrite-Mode`, `StopWhenFull-Mode` and `RetainAndBackup-Mode`, aggregate events based on their channel retention policy.
49
+
50
+Channels that are configured but are not queryable, and channels that do not have any events in them, are automatically excluded from the channels list.
51
+
52
+## Event Fields
53
+
54
+Windows Events are structured with both system-defined fields and user-defined fields.
55
+The Windows Events plugin primarily works with the system-defined fields, which are consistently available
56
+across all event types.
57
+
58
+### System-defined fields
59
+
60
+The system-defined fields are:
61
+
62
+1. **EventRecordID**
63
+ A unique, sequential identifier for the event within the channel. This ID increases as new events are logged.
64
+
65
+2. **Version**
66
+ The version of the event, indicating possible structural changes or updates to the event definition.
67
+
68
+ Netdata adds this field automatically when it is not zero.
69
+
70
+3. **Level**
71
+ The severity or importance of the event. Levels can include:
72
+ - 0: LogAlways (reserved)
73
+ - 1: Critical
74
+ - 2: Error
75
+ - 3: Warning
76
+ - 4: Informational
77
+ - 5: Verbose
78
+
79
+ Additionally, applications may define their own levels.
80
+
81
+ Netdata provides 2 fields: `Level` and `LevelID` for the text and numeric representation of it.
82
+
83
+4. **Opcode**
84
+ The action or state within a provider when the event was logged.
85
+
86
+ Netdata provides 2 fields: `Opcode` and `OpcodeID` for the text and numeric representation of it.
87
+
88
+5. **EventID**
89
+ This identifies the event template, linking it to a specific message or event type. Event IDs are provider-specific.
90
+
91
+6. **Task**
92
+ Defines a higher-level categorization or logical grouping for the event, often related to a specific function within the application or provider.
93
+
94
+ Netdata provides 2 fields: `Task` and `TaskID` for the text and numeric representation of it.
95
+
96
+7. **Qualifiers**
97
+ Provides additional detail for interpreting the event and is often specific to the event source.
98
+
99
+ Netdata adds this field automatically when it is not zero.
100
+
101
+8. **ProcessID**
102
+ The ID of the process that generated the event, useful for pinpointing the source of the event within the system.
103
+
104
+9. **ThreadID**
105
+ The ID of the thread within the process that generated the event, which helps in more detailed debugging scenarios.
106
+
107
+10. **Keywords**
108
+ A categorization field that can be used for event filtering. Keywords are bit flags that represent categories or purposes of the event, providing additional context.
109
+
110
+ Netdata provides 2 fields: `Keywords` and `keywordsID` for the text and numeric representation of it.
111
+
112
+11. **Provider**
113
+ The unique identifier (GUID) of the event provider. This is essential for knowing which application or system component generated the event.
114
+
115
+ Netdata provides 2 fields: `Provider` and `ProviderGUID` for its name and GUID of it.
116
+
117
+12. **ActivityID**
118
+ A GUID that correlates events generated as part of the same operation or transaction, helping to track activities across different components or stages.
119
+
120
+ Netdata adds this field automatically when it is not zero.
121
+
122
+13. **RelatedActivityID**
123
+ A GUID that links related operations or transactions, allowing for tracing complex workflows where one event triggers or relates to another.
124
+
125
+ Netdata adds this field automatically when it is not zero.
126
+
127
+14. **Timestamp**
128
+ The timestamp when the event was created. This provides precise timing information about when the event occurred.
129
+
130
+15. **User**
131
+ The system user who logged this event.
132
+
133
+ Netdata provides 3 fields: `UserAccount`, `UserDomain` and `UserSID`.
134
+
135
+### User-defined fields
136
+Each event log entry can include up to 100 user-defined fields (per event-id).
137
+
138
+Unfortunately, accessing these fields is significantly slower, to a level that is not practical to do so
139
+when there are more than few thousand log entries to explore. So, Netdata presents them
140
+with lazy loading.
141
+
142
+This prevents Netdata for offering filtering for user-defined fields, although Netdata does support
143
+full text search on user-defined field values.
144
+
145
+### Event fields as columns in the table
146
+
147
+The system fields mentioned above are offered as columns on the UI. Use the gear button above the table to
148
+select visible columns.
149
+
150
+### Event fields as filters
151
+
152
+The plugin presents the system fields as filters for the query, with counters for each of the possible values
153
+for the field. This list can be used to quickly check which fields and values are available for the entire
154
+time-frame of the query, across multiple providers and channels.
155
+
156
+### Event fields as histogram sources
157
+
158
+The histogram can be based on any of the system fields that are available as filters. This allows you to
159
+visualize the distribution of events over time based on different criteria such as Level, Provider, or EventID.
160
+
161
+## PLAY mode
162
+
163
+The PLAY mode in this plugin allows real-time monitoring of new events as they are added to the Windows Event
164
+Log. This feature works by continuously querying for new events and updating the display.
165
+
166
+## Full-text search
167
+
168
+The plugin supports searching for text within all system and user fields of the events. This means that while
169
+user-defined fields are not directly filterable, they are searchable through the full-text search feature.
170
+
171
+Keep in mind that query performance is slower while doing full text search, mainly because the plugin
172
+needs to ask from the system to provide all the user fields values.
173
+
174
+## Query performance
175
+
176
+The plugin is optimized to work efficiently with Event Logs. It uses several layers of caching and
177
+similar techniques to offload as much work as possible from the system, offering quick responses even when
178
+hundreds of thousands of events are within the visible timeframe.
179
+
180
+To achieve this level of efficiency, the plugin:
181
+
182
+- pre-loads ETW providers' manifests for resolving numeric Levels, Opcodes, Tasks and Keywords to text.
183
+- caches number to text maps for Levels, Opcodes, Tasks and Keywords per provider for WEL providers.
184
+- caches user SID to account and domain maps.
185
+- lazy loads the "expensive" event Message and XML, so that the system is queried only for the visible events.
186
+
187
+For Full Text Search:
188
+
189
+- requests only the Message and the values of the user-fields from the system, avoiding the "expensive" XML call (which is still lazy-loaded).
190
+
191
+The result is a system that is highly efficient for working with moderate volumes (hundreds of thousands) of events.
192
+
193
+## Configuration and maintenance
194
+
195
+This Netdata plugin does not require any specific configuration. It automatically detects available event logs
196
+on the system.
197
+
198
+## FAQ
199
+
200
+### Can I use this plugin on event centralization servers?
201
+
202
+Yes. You can centralize your Windows Events using Windows Event Forwarding (WEF) or other event collection
203
+mechanisms, and then install Netdata on this events centralization server to explore the events of all your
204
+infrastructure.
205
+
206
+This plugin will automatically provide multi-node views of your events and also give you the ability to
207
+combine the events of multiple servers, as you see fit.
208
+
209
+### Can I use this plugin from a parent Netdata?
210
+
211
+Yes. When your nodes are connected to a Netdata parent, all their functions are available via the parent's UI.
212
+So, from the parent UI, you can access the functions of all your nodes.
213
+
214
+Keep in mind that to protect your privacy, in order to access Netdata functions, you need a free Netdata Cloud
215
+account.
216
+
217
+### Is any of my data exposed to Netdata Cloud from this plugin?
218
+
219
+No. When you access the agent directly, none of your data passes through Netdata Cloud. You need a free Netdata
220
+Cloud account only to verify your identity and enable the use of Netdata Functions. Once this is done, all the
221
+data flow directly from your Netdata agent to your web browser.
222
+
223
+When you access Netdata via https://app.netdata.cloud, your data travel via Netdata Cloud, but they are not stored
224
+in Netdata Cloud. This is to allow you access your Netdata agents from anywhere. All communication from/to
225
+Netdata Cloud is encrypted.
226
+
227
+### What are the different types of event logs supported by this plugin?
228
+
229
+The plugin supports all the kinds of event logs currently supported by the Windows Event Viewer:
230
+
231
+- Windows Event Logs (WEL): The traditional event logging system in Windows.
232
+- Event Tracing for Windows (ETW): A more detailed and efficient event tracing system.
233
+- TraceLogging (TL): An extension of ETW that simplifies the process of adding events to your code.
234
+
235
+The plugin can access all of these when they are routed to the Windows Event Log.
236
+
237
+### How does this plugin handle user-defined fields in Windows Events?
238
+
239
+User-defined fields are not directly exposed as table columns or filters in the plugin interface. However,
240
+they are included in the XML representation of each event, which can be viewed in the info sidebar when
241
+clicking on an event entry. Additionally, the full-text search feature does search through these
242
+user-defined fields, allowing you to find specific information even if it's not in the main system fields.
243
+
244
+### Can I use this plugin to monitor real-time events?
245
+
246
+Yes, the plugin supports a PLAY mode that allows you to monitor events in real-time. When activated, it
247
+continuously updates to show new events as they are logged, similar to the "tail" functionality in
248
+Unix-like systems.
249
+
250
+### How does the plugin handle large volumes of events?
251
+
252
+The plugin is designed to handle moderate volumes of events (hundreds of thousands of events) efficiently.
253
+
254
+It is in our roadmap to port the `systemd-journal` sampling techniques to it, for working with very large
255
+datasets to provide quick responses while still giving accurate representations of the data. However, for
256
+the best performance, we recommend querying smaller time frames or using more specific filters when dealing
257
+with extremely large event volumes.
258
+
259
+### Can I use this plugin to analyze events from multiple servers?
260
+
261
+Yes, if you have set up Windows Event Forwarding (WEF) or another method of centralizing your Windows Events,
262
+you can use this plugin on the central server to analyze events from multiple sources. The plugin will
263
+automatically detect the available event sources.
264
+
265
+### How does the histogram feature work in this plugin?
266
+
267
+The histogram feature provides a visual representation of event frequency over time. You can base the
268
+histogram on any of the system fields available as filters (such as Level, Provider, or EventID). This
269
+allows you to quickly identify patterns or anomalies in your event logs.
270
+
271
+### Is it possible to export or share the results from this plugin?
272
+
273
+While the plugin doesn't have a direct export feature, you can use browser-based methods to save or share
274
+the results. This could include taking screenshots, using browser print/save as PDF functionality, or
275
+copying data from the table view. For more advanced data export needs, you might need to use the Windows
276
+Event Log API directly or other Windows administrative tools.
277
+
278
+### How often does the plugin update its data?
279
+
280
+The plugin updates its data in real-time when in PLAY mode. In normal mode, it refreshes data based on the
281
+query you've submitted. The plugin is designed to provide the most up-to-date information available in the
282
+Windows Event Logs at the time of the query.
283
+
284
+## TODO
285
+
286
+1. Support Sampling, so that the plugin can respond faster even on very busy systems (millions of events visible).
287
+2. Support exploring events from live Tracing sessions.
288
+3. Support exploring events in saved Event Trace Log files (`.etl` files).
289
+4. Support exploring events in saved Event Logs files (`.evtx` files).
src/collectors/windows-events.plugin/windows-events-fields-cache.c
+5
-5
@@ -53,16 +53,16 @@ void field_cache_init(void) {
53
static inline bool should_zero_provider(WEVT_FIELD_TYPE type, uint64_t value) {
54
switch(type) {
55
case WEVT_FIELD_TYPE_LEVEL:
56
- return !is_valid_publisher_level(value, true);
56
+ return !is_valid_provider_level(value, true);
57
58
- case WEVT_FIELD_TYPE_KEYWORDS:
59
- return !is_valid_publisher_keywords(value, true);
58
+ case WEVT_FIELD_TYPE_KEYWORD:
59
+ return !is_valid_provider_keyword(value, true);
60
61
case WEVT_FIELD_TYPE_OPCODE:
62
- return !is_valid_publisher_opcode(value, true);
62
+ return !is_valid_provider_opcode(value, true);
63
64
case WEVT_FIELD_TYPE_TASK:
65
- return !is_valid_publisher_task(value, true);
65
+ return !is_valid_provider_task(value, true);
66
67
default:
68
return false;
src/collectors/windows-events.plugin/windows-events-fields-cache.h
+1
-1
@@ -8,7 +8,7 @@
8
typedef enum __attribute__((packed)) {
9
WEVT_FIELD_TYPE_LEVEL = 0,
10
WEVT_FIELD_TYPE_OPCODE,
11
- WEVT_FIELD_TYPE_KEYWORDS,
11
+ WEVT_FIELD_TYPE_KEYWORD,
12
WEVT_FIELD_TYPE_TASK,
13
14
// terminator
src/collectors/windows-events.plugin/windows-events-providers.c
renamed
+206
-106
@@ -1,17 +1,19 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
-#include "windows-events-publishers.h"
3
+#include "windows-events-providers.h"
4
5
-#define MAX_OPEN_HANDLES_PER_PUBLISHER 5
5
+#define MAX_OPEN_HANDLES_PER_PROVIDER 5
6
7
-struct publisher;
7
+struct provider;
8
9
// typedef as PROVIDER_META_HANDLE in include file
10
struct provider_meta_handle {
11
pid_t owner; // the owner of the handle, or zero
12
uint32_t locks; // the number of locks the owner has on this handle
13
EVT_HANDLE hMetadata; // the handle
14
- struct publisher *publisher; // a pointer back to the publisher
14
+ struct provider *provider; // a pointer back to the provider
15
+
16
+ usec_t created_monotonic_ut; // the monotonic timestamp this handle was created
17
18
// double linked list
19
PROVIDER_META_HANDLE *prev;
@@ -32,78 +34,156 @@ struct provider_list {
34
struct provider_data *array; // the array of entries, sorted (for binary search)
35
};
36
35
-typedef struct publisher {
37
+typedef struct provider_key {
38
ND_UUID uuid; // the Provider GUID
37
- const char *name; // the Provider name (UTF-8)
39
+ DWORD len; // the length of the Provider Name
40
+ const wchar_t *wname; // the Provider wide-string Name (UTF-16)
41
+} PROVIDER_KEY;
42
+
43
+typedef struct provider {
44
+ PROVIDER_KEY key;
45
+ const char *name; // the Provider Name (UTF-8)
46
uint32_t total_handles; // the number of handles allocated
47
uint32_t available_handles; // the number of available handles
48
uint32_t deleted_handles; // the number of deleted handles
49
PROVIDER_META_HANDLE *handles; // a double linked list of all the handles
50
43
- struct provider_list keywords;
51
+ WEVT_PROVIDER_PLATFORM platform;
52
+
53
+ struct provider_list keyword;
54
struct provider_list tasks;
55
struct provider_list opcodes;
56
struct provider_list levels;
47
-} PUBLISHER;
48
-
49
-// A hashtable implementation for publishers
50
-// using the provider GUID as key and PUBLISHER as value
51
-#define SIMPLE_HASHTABLE_NAME _PROVIDER_GUID
52
-#define SIMPLE_HASHTABLE_VALUE_TYPE PUBLISHER
53
-#define SIMPLE_HASHTABLE_KEY_TYPE ND_UUID
54
-#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION publisher_value_to_key
55
-#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION publisher_cache_compar
57
+} PROVIDER;
58
+
59
+// A hashtable implementation for Providers
60
+// using the Provider GUID as key and PROVIDER as value
61
+#define SIMPLE_HASHTABLE_NAME _PROVIDER
62
+#define SIMPLE_HASHTABLE_VALUE_TYPE PROVIDER
63
+#define SIMPLE_HASHTABLE_KEY_TYPE PROVIDER_KEY
64
+#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION provider_value_to_key
65
+#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION provider_cache_compar
66
#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
67
#include "libnetdata/simple_hashtable.h"
68
69
static struct {
70
SPINLOCK spinlock;
61
- uint32_t total_publishers;
71
+ uint32_t total_providers;
72
uint32_t total_handles;
73
uint32_t deleted_handles;
64
- struct simple_hashtable_PROVIDER_GUID hashtable;
65
- ARAL *aral_publishers;
74
+ struct simple_hashtable_PROVIDER hashtable;
75
+ ARAL *aral_providers;
76
ARAL *aral_handles;
77
} pbc = {
78
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
79
};
80
71
-static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
81
+static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
82
+
83
+const char *provider_get_name(PROVIDER_META_HANDLE *p) {
84
+ return (p && p->provider && p->provider->name) ? p->provider->name : "__UNKNOWN PROVIDER__";
85
+}
86
+
87
+ND_UUID provider_get_uuid(PROVIDER_META_HANDLE *p) {
88
+ return (p && p->provider) ? p->provider->key.uuid : UUID_ZERO;
89
+}
90
73
-static inline ND_UUID *publisher_value_to_key(PUBLISHER *p) {
74
- return &p->uuid;
91
+static inline PROVIDER_KEY *provider_value_to_key(PROVIDER *p) {
92
+ return &p->key;
93
}
94
77
-static inline bool publisher_cache_compar(ND_UUID *a, ND_UUID *b) {
78
- return UUIDeq(*a, *b);
95
+static inline bool provider_cache_compar(PROVIDER_KEY *a, PROVIDER_KEY *b) {
96
+ return a->len == b->len && UUIDeq(a->uuid, b->uuid) && memcmp(a->wname, b->wname, a->len) == 0;
97
}
98
81
-void publisher_cache_init(void) {
82
- simple_hashtable_init_PROVIDER_GUID(&pbc.hashtable, 100000);
83
- pbc.aral_publishers = aral_create("wevt_publishers", sizeof(PUBLISHER), 0, 4096, NULL, NULL, NULL, false, true);
99
+void provider_cache_init(void) {
100
+ simple_hashtable_init_PROVIDER(&pbc.hashtable, 100000);
101
+ pbc.aral_providers = aral_create("wevt_providers", sizeof(PROVIDER), 0, 4096, NULL, NULL, NULL, false, true);
102
pbc.aral_handles = aral_create("wevt_handles", sizeof(PROVIDER_META_HANDLE), 0, 4096, NULL, NULL, NULL, false, true);
103
}
104
87
-PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
88
- if(!providerName || !providerName[0] || UUIDiszero(uuid))
105
+static bool provider_property_get(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
106
+ DWORD bufferUsed = 0;
107
+
108
+ if(!EvtGetPublisherMetadataProperty(h->hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
109
+ DWORD status = GetLastError();
110
+ if (status != ERROR_INSUFFICIENT_BUFFER) {
111
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
112
+ goto cleanup;
113
+ }
114
+ }
115
+
116
+ wevt_variant_resize(content, bufferUsed);
117
+ if (!EvtGetPublisherMetadataProperty(h->hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
118
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
119
+ goto cleanup;
120
+ }
121
+
122
+ return true;
123
+
124
+cleanup:
125
+ return false;
126
+}
127
+
128
+static bool provider_string_property_exists(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
129
+ if(!provider_property_get(h, content, property_id))
130
+ return false;
131
+
132
+ if(content->data->Type != EvtVarTypeString)
133
+ return false;
134
+
135
+ if(!content->data->StringVal[0])
136
+ return false;
137
+
138
+ return true;
139
+}
140
+
141
+static void provider_detect_platform(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content) {
142
+ if(UUIDiszero(h->provider->key.uuid))
143
+ h->provider->platform = WEVT_PLATFORM_WEL;
144
+ else if(h->hMetadata) {
145
+ if (provider_string_property_exists(h, content, EvtPublisherMetadataMessageFilePath) ||
146
+ provider_string_property_exists(h, content, EvtPublisherMetadataResourceFilePath) ||
147
+ provider_string_property_exists(h, content, EvtPublisherMetadataParameterFilePath))
148
+ h->provider->platform = WEVT_PLATFORM_ETW;
149
+ else
150
+ // The provider cannot be opened, does not have any resource files (message, resource, parameter)
151
+ h->provider->platform = WEVT_PLATFORM_TL;
152
+ }
153
+ else h->provider->platform = WEVT_PLATFORM_ETW;
154
+}
155
+
156
+WEVT_PROVIDER_PLATFORM provider_get_platform(PROVIDER_META_HANDLE *p) {
157
+ return p->provider->platform;
158
+}
159
+
160
+PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName) {
161
+ if(!providerName || !providerName[0])
162
return NULL;
163
91
- // XXH64_hash_t hash = XXH3_64bits(&uuid, sizeof(uuid));
92
- uint64_t hash = uuid.parts.low64 + uuid.parts.hig64;
164
+ PROVIDER_KEY key = {
165
+ .uuid = uuid,
166
+ .len = wcslen(providerName),
167
+ .wname = providerName,
168
+ };
169
+ XXH64_hash_t hash = XXH3_64bits(providerName, wcslen(key.wname) * sizeof(*key.wname));
170
171
spinlock_lock(&pbc.spinlock);
172
96
- SIMPLE_HASHTABLE_SLOT_PROVIDER_GUID *slot =
97
- simple_hashtable_get_slot_PROVIDER_GUID(&pbc.hashtable, hash, &uuid, true);
173
+ SIMPLE_HASHTABLE_SLOT_PROVIDER *slot =
174
+ simple_hashtable_get_slot_PROVIDER(&pbc.hashtable, hash, &key, true);
175
176
bool load_it = false;
100
- PUBLISHER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
177
+ PROVIDER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
178
if(!p) {
102
- p = aral_callocz(pbc.aral_publishers);
103
- p->uuid = uuid;
104
- simple_hashtable_set_slot_PROVIDER_GUID(&pbc.hashtable, slot, hash, p);
179
+ p = aral_callocz(pbc.aral_providers);
180
+ p->key.uuid = key.uuid;
181
+ p->key.len = key.len;
182
+ p->key.wname = wcsdup(key.wname);
183
+ p->name = strdupz(provider2utf8(key.wname));
184
+ simple_hashtable_set_slot_PROVIDER(&pbc.hashtable, slot, hash, p);
185
load_it = true;
106
- pbc.total_publishers++;
186
+ pbc.total_providers++;
187
}
188
189
pid_t me = gettid_cached();
@@ -117,7 +197,8 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
197
198
if(!h) {
199
h = aral_callocz(pbc.aral_handles);
120
- h->publisher = p;
200
+ h->provider = p;
201
+ h->created_monotonic_ut = now_monotonic_usec();
202
h->hMetadata = EvtOpenPublisherMetadata(
203
NULL, // Local machine
204
providerName, // Provider name
@@ -147,10 +228,11 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
228
WEVT_VARIANT property = { 0 };
229
TXT_UNICODE unicode = { 0 };
230
150
- publisher_load_list(h, &content, &property, &unicode, &p->keywords, EvtPublisherMetadataKeywords);
151
- publisher_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
152
- publisher_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
153
- publisher_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
231
+ provider_detect_platform(h, &content);
232
+ provider_load_list(h, &content, &property, &unicode, &p->keyword, EvtPublisherMetadataKeywords);
233
+ provider_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
234
+ provider_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
235
+ provider_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
236
237
txt_unicode_cleanup(&unicode);
238
wevt_variant_cleanup(&content);
@@ -162,42 +244,72 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
244
return h;
245
}
246
165
-EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h) {
247
+EVT_HANDLE provider_handle(PROVIDER_META_HANDLE *h) {
248
return h ? h->hMetadata : NULL;
249
}
250
169
-PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h) {
251
+PROVIDER_META_HANDLE *provider_dup(PROVIDER_META_HANDLE *h) {
252
if(h) h->locks++;
253
return h;
254
}
255
174
-void publisher_release(PROVIDER_META_HANDLE *h) {
256
+static void provider_meta_handle_delete(PROVIDER_META_HANDLE *h) {
257
+ PROVIDER *p = h->provider;
258
+
259
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
260
+
261
+ if(h->hMetadata)
262
+ EvtClose(h->hMetadata);
263
+
264
+ aral_freez(pbc.aral_handles, h);
265
+
266
+ fatal_assert(pbc.total_handles && p->total_handles && p->available_handles);
267
+
268
+ pbc.total_handles--;
269
+ p->total_handles--;
270
+
271
+ pbc.deleted_handles++;
272
+ p->deleted_handles++;
273
+
274
+ p->available_handles--;
275
+}
276
+
277
+void providers_release_unused_handles(void) {
278
+ usec_t now_ut = now_monotonic_usec();
279
+
280
+ spinlock_lock(&pbc.spinlock);
281
+ for(size_t i = 0; i < pbc.hashtable.size ; i++) {
282
+ SIMPLE_HASHTABLE_SLOT_PROVIDER *slot = &pbc.hashtable.hashtable[i];
283
+ PROVIDER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
284
+ if(!p) continue;
285
+
286
+ PROVIDER_META_HANDLE *h = p->handles;
287
+ while(h) {
288
+ PROVIDER_META_HANDLE *next = h->next;
289
+
290
+ if(!h->locks && (now_ut - h->created_monotonic_ut) >= WINDOWS_EVENTS_RELEASE_IDLE_PROVIDER_HANDLES_TIME_UT)
291
+ provider_meta_handle_delete(h);
292
+
293
+ h = next;
294
+ }
295
+ }
296
+ spinlock_unlock(&pbc.spinlock);
297
+}
298
+
299
+void provider_release(PROVIDER_META_HANDLE *h) {
300
if(!h) return;
301
pid_t me = gettid_cached();
302
fatal_assert(h->owner == me);
303
fatal_assert(h->locks > 0);
304
if(--h->locks == 0) {
180
- PUBLISHER *p = h->publisher;
305
+ PROVIDER *p = h->provider;
306
307
spinlock_lock(&pbc.spinlock);
308
h->owner = 0;
309
185
- if(++p->available_handles > MAX_OPEN_HANDLES_PER_PUBLISHER) {
186
- // there are multiple handles on this publisher
187
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
188
-
189
- if(h->hMetadata)
190
- EvtClose(h->hMetadata);
191
-
192
- aral_freez(pbc.aral_handles, h);
193
-
194
- pbc.total_handles--;
195
- p->total_handles--;
196
-
197
- pbc.deleted_handles++;
198
- p->deleted_handles++;
199
-
200
- p->available_handles--;
310
+ if(++p->available_handles > MAX_OPEN_HANDLES_PER_PROVIDER) {
311
+ // there are too many idle handles on this provider
312
+ provider_meta_handle_delete(h);
313
}
314
else if(h->next) {
315
// it is not the last, put it at the end
@@ -210,7 +322,7 @@ void publisher_release(PROVIDER_META_HANDLE *h) {
322
}
323
324
// --------------------------------------------------------------------------------------------------------------------
213
-// load publisher lists
325
+// load provider lists
326
327
static bool wevt_get_property_from_array(WEVT_VARIANT *property, EVT_HANDLE handle, DWORD dwIndex, EVT_PUBLISHER_METADATA_PROPERTY_ID PropertyId) {
328
DWORD used = 0;
@@ -253,7 +365,7 @@ static int compare_ascending(const void *a, const void *b) {
365
// return 0;
366
//}
367
256
-static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
368
+static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
369
if(!h || !h->hMetadata) return;
370
371
EVT_PUBLISHER_METADATA_PROPERTY_ID name_id, message_id, value_id;
@@ -268,7 +380,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
380
value_id = EvtPublisherMetadataLevelValue;
381
value_bits = 32;
382
compare_func = compare_ascending;
271
- is_valid = is_valid_publisher_level;
383
+ is_valid = is_valid_provider_level;
384
break;
385
386
case EvtPublisherMetadataOpcodes:
@@ -276,7 +388,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
388
message_id = EvtPublisherMetadataOpcodeMessageID;
389
value_id = EvtPublisherMetadataOpcodeValue;
390
value_bits = 32;
279
- is_valid = is_valid_publisher_opcode;
391
+ is_valid = is_valid_provider_opcode;
392
compare_func = compare_ascending;
393
break;
394
@@ -285,7 +397,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
397
message_id = EvtPublisherMetadataTaskMessageID;
398
value_id = EvtPublisherMetadataTaskValue;
399
value_bits = 32;
288
- is_valid = is_valid_publisher_task;
400
+ is_valid = is_valid_provider_task;
401
compare_func = compare_ascending;
402
break;
403
@@ -294,7 +406,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
406
message_id = EvtPublisherMetadataKeywordMessageID;
407
value_id = EvtPublisherMetadataKeywordValue;
408
value_bits = 64;
297
- is_valid = is_valid_publisher_keywords;
409
+ is_valid = is_valid_provider_keyword;
410
compare_func = NULL;
411
break;
412
@@ -305,23 +417,11 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
417
418
EVT_HANDLE hMetadata = h->hMetadata;
419
EVT_HANDLE hArray = NULL;
308
- DWORD bufferUsed = 0;
420
DWORD itemCount = 0;
421
422
// Get the metadata array for the list (e.g., opcodes, tasks, or levels)
312
- if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
313
- DWORD status = GetLastError();
314
- if (status != ERROR_INSUFFICIENT_BUFFER) {
315
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
316
- goto cleanup;
317
- }
318
- }
319
-
320
- wevt_variant_resize(content, bufferUsed);
321
- if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
322
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
423
+ if(!provider_property_get(h, content, property_id))
424
goto cleanup;
324
- }
425
426
// Get the number of items (e.g., levels, tasks, or opcodes)
427
hArray = content->data->EvtHandleVal;
@@ -375,7 +475,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
475
uint32_t messageID = wevt_field_get_uint32(property->data);
476
477
if (messageID != (uint32_t)-1) {
378
- if (wevt_get_message_unicode(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) {
478
+ if (EvtFormatMessage_utf16(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) {
479
size_t len;
480
d->name = unicode2utf8_strdupz(dst->data, &len);
481
d->len = len;
@@ -414,7 +514,7 @@ cleanup:
514
// lookup functions
515
516
// lookup bitmap metdata (returns a comma separated list of strings)
417
-static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
517
+static bool provider_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
518
if(!(value & l->mask) || !l->total || !l->array || l->exceeds_data_type)
519
return false;
520
@@ -445,7 +545,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
545
546
memcpy(&dst->data[dst->used], s, slen);
547
dst->used += slen;
448
- dst->src = TXT_SOURCE_PUBLISHER;
548
+ dst->src = TXT_SOURCE_PROVIDER;
549
added++;
550
}
551
}
@@ -460,7 +560,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
560
}
561
562
//// lookup a single value (returns its string)
463
-//static bool publisher_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
563
+//static bool provider_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
564
// if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
565
// return false;
566
//
@@ -477,7 +577,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
577
//
578
// memcpy(dst->data, s, slen);
579
// dst->used = slen;
480
-// dst->src = TXT_SOURCE_PUBLISHER;
580
+// dst->src = TXT_SOURCE_PROVIDER;
581
//
582
// break;
583
// }
@@ -493,11 +593,11 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
593
// return (dst->used > 0);
594
//}
595
496
-static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
596
+static bool provider_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
597
if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
598
return false;
599
500
- // if(l->total < 3) return publisher_value_metadata_linear(dst, l, value);
600
+ // if(l->total < 3) return provider_value_metadata_linear(dst, l, value);
601
602
dst->used = 0;
603
@@ -519,7 +619,7 @@ static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uin
619
memcpy(dst->data, s, slen);
620
dst->used = slen;
621
dst->data[dst->used++] = 0;
522
- dst->src = TXT_SOURCE_PUBLISHER;
622
+ dst->src = TXT_SOURCE_PROVIDER;
623
}
624
break;
625
}
@@ -539,38 +639,38 @@ static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uin
639
// --------------------------------------------------------------------------------------------------------------------
640
// public API to lookup metadata
641
542
-bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h) {
543
- return h && !h->publisher->keywords.exceeds_data_type;
642
+bool provider_keyword_cacheable(PROVIDER_META_HANDLE *h) {
643
+ return h && !h->provider->keyword.exceeds_data_type;
644
}
645
546
-bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h) {
547
- return h && !h->publisher->tasks.exceeds_data_type;
646
+bool provider_tasks_cacheable(PROVIDER_META_HANDLE *h) {
647
+ return h && !h->provider->tasks.exceeds_data_type;
648
}
649
550
-bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h) {
551
- return h && !h->publisher->levels.exceeds_data_type;
650
+bool is_useful_provider_for_levels(PROVIDER_META_HANDLE *h) {
651
+ return h && !h->provider->levels.exceeds_data_type;
652
}
653
554
-bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
555
- return h && !h->publisher->opcodes.exceeds_data_type;
654
+bool provider_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
655
+ return h && !h->provider->opcodes.exceeds_data_type;
656
}
657
558
-bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
658
+bool provider_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
659
if(!h) return false;
560
- return publisher_bitmap_metadata(dst, &h->publisher->keywords, value);
660
+ return provider_bitmap_metadata(dst, &h->provider->keyword, value);
661
}
662
563
-bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
663
+bool provider_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
664
if(!h) return false;
565
- return publisher_value_metadata(dst, &h->publisher->levels, value);
665
+ return provider_value_metadata(dst, &h->provider->levels, value);
666
}
667
568
-bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
668
+bool provider_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
669
if(!h) return false;
570
- return publisher_value_metadata(dst, &h->publisher->tasks, value);
670
+ return provider_value_metadata(dst, &h->provider->tasks, value);
671
}
672
573
-bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
673
+bool provider_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
674
if(!h) return false;
575
- return publisher_value_metadata(dst, &h->publisher->opcodes, value);
675
+ return provider_value_metadata(dst, &h->provider->opcodes, value);
676
}
src/collectors/windows-events.plugin/windows-events-providers.h
new
+41
@@ -0,0 +1,41 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_WINDOWS_EVENTS_PROVIDERS_H
4
+#define NETDATA_WINDOWS_EVENTS_PROVIDERS_H
5
+
6
+typedef enum __attribute__((packed)) {
7
+ WEVT_PLATFORM_UNKNOWN = 0,
8
+ WEVT_PLATFORM_WEL,
9
+ WEVT_PLATFORM_ETW,
10
+ WEVT_PLATFORM_TL,
11
+} WEVT_PROVIDER_PLATFORM;
12
+
13
+#include "windows-events.h"
14
+
15
+struct provider_meta_handle;
16
+typedef struct provider_meta_handle PROVIDER_META_HANDLE;
17
+
18
+PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName);
19
+void provider_release(PROVIDER_META_HANDLE *h);
20
+EVT_HANDLE provider_handle(PROVIDER_META_HANDLE *h);
21
+PROVIDER_META_HANDLE *provider_dup(PROVIDER_META_HANDLE *h);
22
+
23
+void providers_release_unused_handles(void);
24
+
25
+const char *provider_get_name(PROVIDER_META_HANDLE *p);
26
+ND_UUID provider_get_uuid(PROVIDER_META_HANDLE *p);
27
+
28
+void provider_cache_init(void);
29
+
30
+bool provider_keyword_cacheable(PROVIDER_META_HANDLE *h);
31
+bool provider_tasks_cacheable(PROVIDER_META_HANDLE *h);
32
+bool is_useful_provider_for_levels(PROVIDER_META_HANDLE *h);
33
+bool provider_opcodes_cacheable(PROVIDER_META_HANDLE *h);
34
+
35
+bool provider_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
36
+bool provider_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
37
+bool provider_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
38
+bool provider_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
39
+WEVT_PROVIDER_PLATFORM provider_get_platform(PROVIDER_META_HANDLE *p);
40
+
41
+#endif //NETDATA_WINDOWS_EVENTS_PROVIDERS_H
src/collectors/windows-events.plugin/windows-events-publishers.h
deleted
-28
@@ -1,28 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
4
-#define NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
5
-
6
-#include "windows-events.h"
7
-
8
-struct provider_meta_handle;
9
-typedef struct provider_meta_handle PROVIDER_META_HANDLE;
10
-
11
-PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName);
12
-void publisher_release(PROVIDER_META_HANDLE *h);
13
-EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h);
14
-PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h);
15
-
16
-void publisher_cache_init(void);
17
-
18
-bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h);
19
-bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h);
20
-bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h);
21
-bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h);
22
-
23
-bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
24
-bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
25
-bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
26
-bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
27
-
28
-#endif //NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
src/collectors/windows-events.plugin/windows-events-query-evt-variant.c
new
+353
@@ -0,0 +1,353 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "windows-events.h"
4
+#include <sddl.h> // For SID string conversion
5
+
6
+// Function to append the separator if the buffer is not empty
7
+static inline void append_separator_if_needed(BUFFER *b, const char *separator) {
8
+ if (buffer_strlen(b) > 0 && separator != NULL)
9
+ buffer_strcat(b, separator);
10
+}
11
+
12
+// Helper function to convert UTF16 strings to UTF8 and append to the buffer
13
+static inline void append_utf16(BUFFER *b, LPCWSTR utf16Str, const char *separator) {
14
+ if (!utf16Str || !*utf16Str) return;
15
+
16
+ append_separator_if_needed(b, separator);
17
+
18
+ size_t remaining = b->size - b->len;
19
+ if(remaining < 128) {
20
+ buffer_need_bytes(b, 128);
21
+ remaining = b->size - b->len;
22
+ }
23
+
24
+ size_t used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1);
25
+ if(used >= remaining) {
26
+ // oops, we need to resize
27
+ size_t needed = utf16_to_utf8(NULL, 0, utf16Str, -1); // find the size needed
28
+ buffer_need_bytes(b, needed);
29
+ remaining = b->size - b->len;
30
+ used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1);
31
+ }
32
+
33
+ if(used) {
34
+ b->len += used - 1;
35
+
36
+ internal_fatal(buffer_strlen(b) != strlen(buffer_tostring(b)),
37
+ "Buffer length mismatch.");
38
+ }
39
+}
40
+
41
+// Function to append binary data to the buffer
42
+static inline void append_binary(BUFFER *b, PBYTE data, DWORD size, const char *separator) {
43
+ if (data == NULL || size == 0) return;
44
+
45
+ append_separator_if_needed(b, separator);
46
+
47
+ buffer_need_bytes(b, size * 4);
48
+ for (DWORD i = 0; i < size; i++) {
49
+ uint8_t value = data[i];
50
+ b->buffer[b->len++] = hex_digits[(value & 0xf0) >> 4];
51
+ b->buffer[b->len++] = hex_digits[(value & 0x0f)];
52
+ }
53
+}
54
+
55
+// Function to append size_t to the buffer
56
+static inline void append_size_t(BUFFER *b, size_t size, const char *separator) {
57
+ append_separator_if_needed(b, separator);
58
+ buffer_print_uint64(b, size);
59
+}
60
+
61
+// Function to append HexInt32 in hexadecimal format
62
+static inline void append_uint32_hex(BUFFER *b, UINT32 n, const char *separator) {
63
+ append_separator_if_needed(b, separator);
64
+ buffer_print_uint64_hex(b, n);
65
+}
66
+
67
+// Function to append HexInt64 in hexadecimal format
68
+static inline void append_uint64_hex(BUFFER *b, UINT64 n, const char *separator) {
69
+ append_separator_if_needed(b, separator);
70
+ buffer_print_uint64_hex(b, n);
71
+}
72
+
73
+// Function to append various data types to the buffer
74
+static inline void append_uint64(BUFFER *b, UINT64 n, const char *separator) {
75
+ append_separator_if_needed(b, separator);
76
+ buffer_print_uint64(b, n);
77
+}
78
+
79
+static inline void append_int64(BUFFER *b, INT64 n, const char *separator) {
80
+ append_separator_if_needed(b, separator);
81
+ buffer_print_int64(b, n);
82
+}
83
+
84
+static inline void append_double(BUFFER *b, double n, const char *separator) {
85
+ append_separator_if_needed(b, separator);
86
+ buffer_print_netdata_double(b, n);
87
+}
88
+
89
+static inline void append_guid(BUFFER *b, GUID *guid, const char *separator) {
90
+ fatal_assert(sizeof(GUID) == sizeof(nd_uuid_t));
91
+
92
+ append_separator_if_needed(b, separator);
93
+
94
+ ND_UUID *uuid = (ND_UUID *)guid;
95
+ buffer_need_bytes(b, UUID_STR_LEN);
96
+ uuid_unparse_lower(uuid->uuid, &b->buffer[b->len]);
97
+ b->len += UUID_STR_LEN - 1;
98
+
99
+ internal_fatal(buffer_strlen(b) != strlen(buffer_tostring(b)),
100
+ "Buffer length mismatch.");
101
+}
102
+
103
+static inline void append_systime(BUFFER *b, SYSTEMTIME *st, const char *separator) {
104
+ append_separator_if_needed(b, separator);
105
+ buffer_sprintf(b, "%04d-%02d-%02d %02d:%02d:%02d",
106
+ st->wYear, st->wMonth, st->wDay, st->wHour, st->wMinute, st->wSecond);
107
+}
108
+
109
+static inline void append_filetime(BUFFER *b, FILETIME *ft, const char *separator) {
110
+ SYSTEMTIME st;
111
+ if (FileTimeToSystemTime(ft, &st))
112
+ append_systime(b, &st, separator);
113
+}
114
+
115
+static inline void append_sid(BUFFER *b, PSID sid, const char *separator) {
116
+ buffer_sid_to_sid_str_and_name(sid, b, separator);
117
+}
118
+
119
+static inline void append_sbyte(BUFFER *b, INT8 n, const char *separator) {
120
+ append_separator_if_needed(b, separator);
121
+ buffer_print_int64(b, n);
122
+}
123
+
124
+static inline void append_byte(BUFFER *b, UINT8 n, const char *separator) {
125
+ append_separator_if_needed(b, separator);
126
+ buffer_print_uint64(b, n);
127
+}
128
+
129
+static inline void append_int16(BUFFER *b, INT16 n, const char *separator) {
130
+ append_separator_if_needed(b, separator);
131
+ buffer_print_int64(b, n);
132
+}
133
+
134
+static inline void append_uint16(BUFFER *b, UINT16 n, const char *separator) {
135
+ append_separator_if_needed(b, separator);
136
+ buffer_print_uint64(b, n);
137
+}
138
+
139
+static inline void append_int32(BUFFER *b, INT32 n, const char *separator) {
140
+ append_separator_if_needed(b, separator);
141
+ buffer_print_int64(b, n);
142
+}
143
+
144
+static inline void append_uint32(BUFFER *b, UINT32 n, const char *separator) {
145
+ append_separator_if_needed(b, separator);
146
+ buffer_print_uint64(b, n);
147
+}
148
+
149
+// Function to append EVT_HANDLE to the buffer
150
+static inline void append_evt_handle(BUFFER *b, EVT_HANDLE h, const char *separator) {
151
+ append_separator_if_needed(b, separator);
152
+ buffer_print_uint64_hex(b, (uintptr_t)h);
153
+}
154
+
155
+// Function to append XML data (UTF-16) to the buffer
156
+static inline void append_evt_xml(BUFFER *b, LPCWSTR xmlData, const char *separator) {
157
+ append_utf16(b, xmlData, separator); // XML data is essentially UTF-16 string
158
+}
159
+
160
+void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator) {
161
+ if(ev->Type == EvtVarTypeNull) return;
162
+
163
+ if (ev->Type & EVT_VARIANT_TYPE_ARRAY) {
164
+ for (DWORD i = 0; i < ev->Count; i++) {
165
+ switch (ev->Type & EVT_VARIANT_TYPE_MASK) {
166
+ case EvtVarTypeString:
167
+ append_utf16(b, ev->StringArr[i], separator);
168
+ break;
169
+
170
+ case EvtVarTypeAnsiString:
171
+ if (ev->AnsiStringArr[i] != NULL) {
172
+ append_utf16(b, (LPCWSTR)ev->AnsiStringArr[i], separator);
173
+ }
174
+ break;
175
+
176
+ case EvtVarTypeSByte:
177
+ append_sbyte(b, ev->SByteArr[i], separator);
178
+ break;
179
+
180
+ case EvtVarTypeByte:
181
+ append_byte(b, ev->ByteArr[i], separator);
182
+ break;
183
+
184
+ case EvtVarTypeInt16:
185
+ append_int16(b, ev->Int16Arr[i], separator);
186
+ break;
187
+
188
+ case EvtVarTypeUInt16:
189
+ append_uint16(b, ev->UInt16Arr[i], separator);
190
+ break;
191
+
192
+ case EvtVarTypeInt32:
193
+ append_int32(b, ev->Int32Arr[i], separator);
194
+ break;
195
+
196
+ case EvtVarTypeUInt32:
197
+ append_uint32(b, ev->UInt32Arr[i], separator);
198
+ break;
199
+
200
+ case EvtVarTypeInt64:
201
+ append_int64(b, ev->Int64Arr[i], separator);
202
+ break;
203
+
204
+ case EvtVarTypeUInt64:
205
+ append_uint64(b, ev->UInt64Arr[i], separator);
206
+ break;
207
+
208
+ case EvtVarTypeSingle:
209
+ append_double(b, ev->SingleArr[i], separator);
210
+ break;
211
+
212
+ case EvtVarTypeDouble:
213
+ append_double(b, ev->DoubleArr[i], separator);
214
+ break;
215
+
216
+ case EvtVarTypeGuid:
217
+ append_guid(b, &ev->GuidArr[i], separator);
218
+ break;
219
+
220
+ case EvtVarTypeFileTime:
221
+ append_filetime(b, &ev->FileTimeArr[i], separator);
222
+ break;
223
+
224
+ case EvtVarTypeSysTime:
225
+ append_systime(b, &ev->SysTimeArr[i], separator);
226
+ break;
227
+
228
+ case EvtVarTypeSid:
229
+ append_sid(b, ev->SidArr[i], separator);
230
+ break;
231
+
232
+ case EvtVarTypeBinary:
233
+ append_binary(b, ev->BinaryVal, ev->Count, separator);
234
+ break;
235
+
236
+ case EvtVarTypeSizeT:
237
+ append_size_t(b, ev->SizeTArr[i], separator);
238
+ break;
239
+
240
+ case EvtVarTypeHexInt32:
241
+ append_uint32_hex(b, ev->UInt32Arr[i], separator);
242
+ break;
243
+
244
+ case EvtVarTypeHexInt64:
245
+ append_uint64_hex(b, ev->UInt64Arr[i], separator);
246
+ break;
247
+
248
+ case EvtVarTypeEvtHandle:
249
+ append_evt_handle(b, ev->EvtHandleVal, separator);
250
+ break;
251
+
252
+ case EvtVarTypeEvtXml:
253
+ append_evt_xml(b, ev->XmlValArr[i], separator);
254
+ break;
255
+
256
+ default:
257
+ // Skip unknown array types
258
+ break;
259
+ }
260
+ }
261
+ } else {
262
+ switch (ev->Type & EVT_VARIANT_TYPE_MASK) {
263
+ case EvtVarTypeNull:
264
+ // Do nothing for null types
265
+ break;
266
+
267
+ case EvtVarTypeString:
268
+ append_utf16(b, ev->StringVal, separator);
269
+ break;
270
+
271
+ case EvtVarTypeAnsiString:
272
+ append_utf16(b, (LPCWSTR)ev->AnsiStringVal, separator);
273
+ break;
274
+
275
+ case EvtVarTypeSByte:
276
+ append_sbyte(b, ev->SByteVal, separator);
277
+ break;
278
+
279
+ case EvtVarTypeByte:
280
+ append_byte(b, ev->ByteVal, separator);
281
+ break;
282
+
283
+ case EvtVarTypeInt16:
284
+ append_int16(b, ev->Int16Val, separator);
285
+ break;
286
+
287
+ case EvtVarTypeUInt16:
288
+ append_uint16(b, ev->UInt16Val, separator);
289
+ break;
290
+
291
+ case EvtVarTypeInt32:
292
+ append_int32(b, ev->Int32Val, separator);
293
+ break;
294
+
295
+ case EvtVarTypeUInt32:
296
+ append_uint32(b, ev->UInt32Val, separator);
297
+ break;
298
+
299
+ case EvtVarTypeInt64:
300
+ append_int64(b, ev->Int64Val, separator);
301
+ break;
302
+
303
+ case EvtVarTypeUInt64:
304
+ append_uint64(b, ev->UInt64Val, separator);
305
+ break;
306
+
307
+ case EvtVarTypeSingle:
308
+ append_double(b, ev->SingleVal, separator);
309
+ break;
310
+
311
+ case EvtVarTypeDouble:
312
+ append_double(b, ev->DoubleVal, separator);
313
+ break;
314
+
315
+ case EvtVarTypeBoolean:
316
+ append_separator_if_needed(b, separator);
317
+ buffer_strcat(b, ev->BooleanVal ? "true" : "false");
318
+ break;
319
+
320
+ case EvtVarTypeGuid:
321
+ append_guid(b, ev->GuidVal, separator);
322
+ break;
323
+
324
+ case EvtVarTypeBinary:
325
+ append_binary(b, ev->BinaryVal, ev->Count, separator);
326
+ break;
327
+
328
+ case EvtVarTypeSizeT:
329
+ append_size_t(b, ev->SizeTVal, separator);
330
+ break;
331
+
332
+ case EvtVarTypeHexInt32:
333
+ append_uint32_hex(b, ev->UInt32Val, separator);
334
+ break;
335
+
336
+ case EvtVarTypeHexInt64:
337
+ append_uint64_hex(b, ev->UInt64Val, separator);
338
+ break;
339
+
340
+ case EvtVarTypeEvtHandle:
341
+ append_evt_handle(b, ev->EvtHandleVal, separator);
342
+ break;
343
+
344
+ case EvtVarTypeEvtXml:
345
+ append_evt_xml(b, ev->XmlVal, separator);
346
+ break;
347
+
348
+ default:
349
+ // Skip unknown types
350
+ break;
351
+ }
352
+ }
353
+}
src/collectors/windows-events.plugin/windows-events-query.c
+247
-180
@@ -2,48 +2,13 @@
2
3
#include "windows-events.h"
4
5
+static void wevt_event_done(WEVT_LOG *log);
6
+
7
static uint64_t wevt_log_file_size(const wchar_t *channel);
8
7
-#define FIELD_RECORD_NUMBER (0)
8
-#define FIELD_EVENT_ID (1)
9
-#define FIELD_LEVEL (2)
10
-#define FIELD_OPCODE (3)
11
-#define FIELD_KEYWORDS (4)
12
-#define FIELD_VERSION (5)
13
-#define FIELD_TASK (6)
14
-#define FIELD_PROCESS_ID (7)
15
-#define FIELD_THREAD_ID (8)
16
-#define FIELD_TIME_CREATED (9)
17
-#define FIELD_CHANNEL (10)
18
-#define FIELD_COMPUTER_NAME (11)
19
-#define FIELD_PROVIDER_NAME (12)
20
-#define FIELD_EVENT_SOURCE_NAME (13)
21
-#define FIELD_PROVIDER_GUID (14)
22
-#define FIELD_CORRELATION_ACTIVITY_ID (15)
23
-#define FIELD_USER_ID (16)
24
-
25
-// These are the fields we extract from the logs
26
-static const wchar_t *RENDER_ITEMS[] = {
27
- L"/Event/System/EventRecordID",
28
- L"/Event/System/EventID",
29
- L"/Event/System/Level",
30
- L"/Event/System/Opcode",
31
- L"/Event/System/Keywords",
32
- L"/Event/System/Version",
33
- L"/Event/System/Task",
34
- L"/Event/System/Execution/@ProcessID",
35
- L"/Event/System/Execution/@ThreadID",
36
- L"/Event/System/TimeCreated/@SystemTime",
37
- L"/Event/System/Channel",
38
- L"/Event/System/Computer",
39
- L"/Event/System/Provider/@Name",
40
- L"/Event/System/Provider/@EventSourceName",
41
- L"/Event/System/Provider/@Guid",
42
- L"/Event/System/Correlation/@ActivityID",
43
- L"/Event/System/Security/@UserID",
44
-};
45
-
46
-static const char *wevt_extended_status(void) {
9
+// --------------------------------------------------------------------------------------------------------------------
10
+
11
+static const char *EvtGetExtendedStatus_utf8(void) {
12
static __thread wchar_t wbuf[4096];
13
static __thread char buf[4096];
14
DWORD wbuf_used = 0;
@@ -62,7 +27,9 @@ static const char *wevt_extended_status(void) {
27
return buf;
28
}
29
65
-bool wevt_get_message_unicode(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
30
+// --------------------------------------------------------------------------------------------------------------------
31
+
32
+bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
33
dst->used = 0;
34
35
DWORD size = 0;
@@ -107,56 +74,44 @@ cleanup:
74
return false;
75
}
76
110
-static bool wevt_get_field_from_events_log(
111
- WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent,
112
- TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) {
77
+static bool EvtFormatMessage_utf8(
78
+ TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent,
79
+ TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) {
80
81
dst->src = TXT_SOURCE_EVENT_LOG;
82
116
- if(wevt_get_message_unicode(&log->ops.unicode, publisher_handle(p), hEvent, 0, flags))
117
- return wevt_str_unicode_to_utf8(dst, &log->ops.unicode);
83
+ if(EvtFormatMessage_utf16(tmp, provider_handle(p), hEvent, 0, flags))
84
+ return wevt_str_unicode_to_utf8(dst, tmp);
85
86
wevt_utf8_empty(dst);
87
return false;
88
}
89
123
-bool wevt_get_event_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
124
- return wevt_get_field_from_events_log(log, p, hEvent, dst, EvtFormatMessageEvent);
90
+bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
91
+ return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageEvent);
92
}
93
127
-bool wevt_get_xml_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
128
- return wevt_get_field_from_events_log(log, p, hEvent, dst, EvtFormatMessageXml);
94
+bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
95
+ return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageXml);
96
}
97
131
-static inline void wevt_event_done(WEVT_LOG *log) {
132
- if (log->publisher) {
133
- publisher_release(log->publisher);
134
- log->publisher = NULL;
135
- }
136
-
137
- if (log->hEvent) {
138
- EvtClose(log->hEvent);
139
- log->hEvent = NULL;
140
- }
141
-
142
- log->ops.level.src = TXT_SOURCE_UNKNOWN;
143
- log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
144
- log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
145
- log->ops.task.src = TXT_SOURCE_UNKNOWN;
146
-}
98
+// --------------------------------------------------------------------------------------------------------------------
99
100
static void wevt_get_field_from_cache(
149
- WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
150
- TXT_UTF8 *dst, const ND_UUID *provider,
151
- WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
101
+ WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
102
+ TXT_UTF8 *dst, const ND_UUID *provider,
103
+ WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
104
105
if (field_cache_get(cache_type, provider, value, dst))
106
return;
107
156
- wevt_get_field_from_events_log(log, h, log->hEvent, dst, flags);
108
+ EvtFormatMessage_utf8(&log->ops.unicode, h, log->hEvent, dst, flags);
109
field_cache_set(cache_type, provider, value, dst);
110
}
111
112
+// --------------------------------------------------------------------------------------------------------------------
113
+// Level
114
+
115
#define SET_LEN_AND_RETURN(constant) *len = sizeof(constant) - 1; return constant
116
117
static inline const char *wevt_level_hardcoded(uint64_t level, size_t *len) {
@@ -179,9 +134,9 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
134
135
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageLevel;
136
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_LEVEL;
182
- bool is_publisher = is_valid_publisher_level(value, true);
137
+ bool is_provider = is_valid_provider_level(value, true);
138
184
- if(!is_publisher) {
139
+ if(!is_provider) {
140
size_t len;
141
const char *hardcoded = wevt_level_hardcoded(value, &len);
142
if(hardcoded) {
@@ -189,12 +144,12 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
144
dst->src = TXT_SOURCE_HARDCODED;
145
}
146
else {
192
- // since this is not a publisher value
147
+ // since this is not a provider value
148
// we expect to get the system description of it
149
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
150
}
151
}
197
- else if (!publisher_get_level(dst, h, value)) {
152
+ else if (!provider_get_level(dst, h, value)) {
153
// not found in the manifest, get it from the cache
154
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
155
}
@@ -203,6 +158,9 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
158
dst, WEVT_PREFIX_LEVEL, sizeof(WEVT_PREFIX_LEVEL) - 1, ev->level);
159
}
160
161
+// --------------------------------------------------------------------------------------------------------------------
162
+// Opcode
163
+
164
static inline const char *wevt_opcode_hardcoded(uint64_t opcode, size_t *len) {
165
switch(opcode) {
166
case WEVT_OPCODE_INFO: SET_LEN_AND_RETURN(WEVT_OPCODE_NAME_INFO);
@@ -228,9 +186,9 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
186
187
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageOpcode;
188
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_OPCODE;
231
- bool is_publisher = is_valid_publisher_opcode(value, true);
189
+ bool is_provider = is_valid_provider_opcode(value, true);
190
233
- if(!is_publisher) {
191
+ if(!is_provider) {
192
size_t len;
193
const char *hardcoded = wevt_opcode_hardcoded(value, &len);
194
if(hardcoded) {
@@ -238,12 +196,12 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
196
dst->src = TXT_SOURCE_HARDCODED;
197
}
198
else {
241
- // since this is not a publisher value
199
+ // since this is not a provider value
200
// we expect to get the system description of it
201
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
202
}
203
}
246
- else if (!publisher_get_opcode(dst, h, value)) {
204
+ else if (!provider_get_opcode(dst, h, value)) {
205
// not found in the manifest, get it from the cache
206
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
207
}
@@ -252,6 +210,9 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
210
dst, WEVT_PREFIX_OPCODE, sizeof(WEVT_PREFIX_OPCODE) - 1, ev->opcode);
211
}
212
213
+// --------------------------------------------------------------------------------------------------------------------
214
+// Task
215
+
216
static const char *wevt_task_hardcoded(uint64_t task, size_t *len) {
217
switch(task) {
218
case WEVT_TASK_NONE: SET_LEN_AND_RETURN(WEVT_TASK_NAME_NONE);
@@ -267,9 +228,9 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
228
229
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageTask;
230
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_TASK;
270
- bool is_publisher = is_valid_publisher_task(value, true);
231
+ bool is_provider = is_valid_provider_task(value, true);
232
272
- if(!is_publisher) {
233
+ if(!is_provider) {
234
size_t len;
235
const char *hardcoded = wevt_task_hardcoded(value, &len);
236
if(hardcoded) {
@@ -277,12 +238,12 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
238
dst->src = TXT_SOURCE_HARDCODED;
239
}
240
else {
280
- // since this is not a publisher value
241
+ // since this is not a provider value
242
// we expect to get the system description of it
243
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
244
}
245
}
285
- else if (!publisher_get_task(dst, h, value)) {
246
+ else if (!provider_get_task(dst, h, value)) {
247
// not found in the manifest, get it from the cache
248
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
249
}
@@ -291,9 +252,12 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
252
dst, WEVT_PREFIX_TASK, sizeof(WEVT_PREFIX_TASK) - 1, ev->task);
253
}
254
255
+// --------------------------------------------------------------------------------------------------------------------
256
+// Keyword
257
+
258
#define SET_BITS(msk, txt) { .mask = msk, .name = txt, .len = sizeof(txt) - 1, }
259
296
-static uint64_t wevt_keywords_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
260
+static uint64_t wevt_keyword_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
261
struct {
262
uint64_t mask;
263
const char *name;
@@ -324,7 +288,7 @@ static uint64_t wevt_keywords_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
288
return value & 0x0000FFFFFFFFFFFF;
289
}
290
327
-static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
291
+static void wevt_get_keyword(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
292
TXT_UTF8 *dst = &log->ops.keywords;
293
294
if(ev->keywords == WEVT_KEYWORD_NONE) {
@@ -332,18 +296,18 @@ static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDL
296
dst->src = TXT_SOURCE_HARDCODED;
297
}
298
335
- uint64_t value = wevt_keywords_handle_reserved(ev->keywords, dst);
299
+ uint64_t value = wevt_keyword_handle_reserved(ev->keywords, dst);
300
301
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageKeyword;
338
- WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORDS;
302
+ WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORD;
303
304
if(!value && dst->used <= 1) {
305
// no hardcoded info in the buffer, make it None
306
txt_utf8_set(dst, WEVT_KEYWORD_NAME_NONE, sizeof(WEVT_KEYWORD_NAME_NONE) - 1);
307
dst->src = TXT_SOURCE_HARDCODED;
308
}
345
- else if (value && !publisher_get_keywords(dst, h, value) && dst->used <= 1) {
346
- // the publisher did not provide any info and the description is still empty.
309
+ else if (value && !provider_get_keywords(dst, h, value) && dst->used <= 1) {
310
+ // the provider did not provide any info and the description is still empty.
311
// the system returns 1 keyword, the highest bit, not a list
312
// so, when we call the system, we pass the original value (ev->keywords)
313
wevt_get_field_from_cache(log, ev->keywords, h, dst, &ev->provider, cache_type, flags);
@@ -353,57 +317,86 @@ static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDL
317
dst, WEVT_PREFIX_KEYWORDS, sizeof(WEVT_PREFIX_KEYWORDS) - 1, ev->keywords);
318
}
319
356
-bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
357
- bool ret = false;
320
+// --------------------------------------------------------------------------------------------------------------------
321
+// Fetching Events
322
359
- // obtain the information from selected events
323
+static inline bool wEvtRender(WEVT_LOG *log, EVT_HANDLE context, WEVT_VARIANT *raw) {
324
DWORD bytes_used = 0, property_count = 0;
361
- if (!EvtRender(log->hRenderContext, log->hEvent, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
325
+ if (!EvtRender(context, log->hEvent, EvtRenderEventValues, raw->size, raw->data, &bytes_used, &property_count)) {
326
// information exceeds the allocated space
327
if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
364
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, hRenderContext: 0x%lx, hEvent: 0x%lx, content: 0x%lx, size: %zu, extended info: %s",
365
- (uintptr_t)log->hRenderContext, (uintptr_t)log->hEvent, (uintptr_t)log->ops.content.data, log->ops.content.size, wevt_extended_status());
366
- goto cleanup;
328
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
329
+ "EvtRender() failed, hRenderSystemContext: 0x%lx, hEvent: 0x%lx, content: 0x%lx, size: %u, extended info: %s",
330
+ (uintptr_t)context, (uintptr_t)log->hEvent, (uintptr_t)raw->data, raw->size,
331
+ EvtGetExtendedStatus_utf8());
332
+ return false;
333
}
334
369
- wevt_variant_resize(&log->ops.content, bytes_used);
370
- if (!EvtRender(log->hRenderContext, log->hEvent, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
371
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, after bytes_used increase, extended info: %s",
372
- wevt_extended_status());
373
- goto cleanup;
335
+ wevt_variant_resize(raw, bytes_used);
336
+ if (!EvtRender(context, log->hEvent, EvtRenderEventValues, raw->size, raw->data, &bytes_used, &property_count)) {
337
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
338
+ "EvtRender() failed, after bytes_used increase, extended info: %s",
339
+ EvtGetExtendedStatus_utf8());
340
+ return false;
341
}
342
}
376
- log->ops.content.used = bytes_used;
377
-
378
- EVT_VARIANT *content = log->ops.content.data;
343
+ raw->used = bytes_used;
344
+ raw->count = property_count;
345
380
- ev->id = wevt_field_get_uint64(&content[FIELD_RECORD_NUMBER]);
381
- ev->event_id = wevt_field_get_uint16(&content[FIELD_EVENT_ID]);
382
- ev->level = wevt_field_get_uint8(&content[FIELD_LEVEL]);
383
- ev->opcode = wevt_field_get_uint8(&content[FIELD_OPCODE]);
384
- ev->keywords = wevt_field_get_uint64_hex(&content[FIELD_KEYWORDS]);
385
- ev->version = wevt_field_get_uint8(&content[FIELD_VERSION]);
386
- ev->task = wevt_field_get_uint16(&content[FIELD_TASK]);
387
- ev->process_id = wevt_field_get_uint32(&content[FIELD_PROCESS_ID]);
388
- ev->thread_id = wevt_field_get_uint32(&content[FIELD_THREAD_ID]);
389
- ev->created_ns = wevt_field_get_filetime_to_ns(&content[FIELD_TIME_CREATED]);
346
+ return true;
347
+}
348
391
- if(full) {
392
- wevt_field_get_string_utf8(&content[FIELD_CHANNEL], &log->ops.channel);
393
- wevt_field_get_string_utf8(&content[FIELD_COMPUTER_NAME], &log->ops.computer);
394
- wevt_field_get_string_utf8(&content[FIELD_PROVIDER_NAME], &log->ops.provider);
395
- wevt_field_get_string_utf8(&content[FIELD_EVENT_SOURCE_NAME], &log->ops.source);
396
- wevt_get_uuid_by_type(&content[FIELD_PROVIDER_GUID], &ev->provider);
397
- wevt_get_uuid_by_type(&content[FIELD_CORRELATION_ACTIVITY_ID], &ev->correlation_activity_id);
398
- wevt_field_get_sid(&content[FIELD_USER_ID], &log->ops.user);
349
+static bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev) {
350
+ bool ret = false;
351
400
- PROVIDER_META_HANDLE *h = log->publisher =
401
- publisher_get(ev->provider, content[FIELD_PROVIDER_NAME].StringVal);
352
+ if(!wEvtRender(log, log->hRenderSystemContext, &log->ops.raw.system))
353
+ goto cleanup;
354
403
- wevt_get_level(log, ev, h);
404
- wevt_get_task(log, ev, h);
405
- wevt_get_opcode(log, ev, h);
406
- wevt_get_keywords(log, ev, h);
355
+ EVT_VARIANT *content = log->ops.raw.system.data;
356
+
357
+ ev->id = wevt_field_get_uint64(&content[EvtSystemEventRecordId]);
358
+ ev->event_id = wevt_field_get_uint16(&content[EvtSystemEventID]);
359
+ ev->level = wevt_field_get_uint8(&content[EvtSystemLevel]);
360
+ ev->opcode = wevt_field_get_uint8(&content[EvtSystemOpcode]);
361
+ ev->keywords = wevt_field_get_uint64_hex(&content[EvtSystemKeywords]);
362
+ ev->version = wevt_field_get_uint8(&content[EvtSystemVersion]);
363
+ ev->task = wevt_field_get_uint16(&content[EvtSystemTask]);
364
+ ev->qualifiers = wevt_field_get_uint16(&content[EvtSystemQualifiers]);
365
+ ev->process_id = wevt_field_get_uint32(&content[EvtSystemProcessID]);
366
+ ev->thread_id = wevt_field_get_uint32(&content[EvtSystemThreadID]);
367
+ ev->created_ns = wevt_field_get_filetime_to_ns(&content[EvtSystemTimeCreated]);
368
+
369
+ if(log->type & WEVT_QUERY_EXTENDED) {
370
+ wevt_field_get_string_utf8(&content[EvtSystemChannel], &log->ops.channel);
371
+ wevt_field_get_string_utf8(&content[EvtSystemComputer], &log->ops.computer);
372
+ wevt_field_get_string_utf8(&content[EvtSystemProviderName], &log->ops.provider);
373
+ wevt_get_uuid_by_type(&content[EvtSystemProviderGuid], &ev->provider);
374
+ wevt_get_uuid_by_type(&content[EvtSystemActivityID], &ev->activity_id);
375
+ wevt_get_uuid_by_type(&content[EvtSystemRelatedActivityID], &ev->related_activity_id);
376
+ wevt_field_get_sid(&content[EvtSystemUserID], &log->ops.account, &log->ops.domain, &log->ops.sid);
377
+
378
+ PROVIDER_META_HANDLE *p = log->provider =
379
+ provider_get(ev->provider, content[EvtSystemProviderName].StringVal);
380
+
381
+ ev->platform = provider_get_platform(p);
382
+
383
+ wevt_get_level(log, ev, p);
384
+ wevt_get_task(log, ev, p);
385
+ wevt_get_opcode(log, ev, p);
386
+ wevt_get_keyword(log, ev, p);
387
+
388
+ if(log->type & WEVT_QUERY_EVENT_DATA && wEvtRender(log, log->hRenderUserContext, &log->ops.raw.user)) {
389
+#if (ON_FTS_PRELOAD_MESSAGE == 1)
390
+ EvtFormatMessage_Event_utf8(&log->ops.unicode, log->provider, log->hEvent, &log->ops.event);
391
+#endif
392
+#if (ON_FTS_PRELOAD_XML == 1)
393
+ EvtFormatMessage_Xml_utf8(&log->ops.unicode, log->provider, log->hEvent, &log->ops.xml);
394
+#endif
395
+#if (ON_FTS_PRELOAD_EVENT_DATA == 1)
396
+ for(size_t i = 0; i < log->ops.raw.user.count ;i++)
397
+ evt_variant_to_buffer(log->ops.event_data, &log->ops.raw.user.data[i], " ||| ");
398
+#endif
399
+ }
400
}
401
402
ret = true;
@@ -412,11 +405,11 @@ cleanup:
405
return ret;
406
}
407
415
-bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
416
- DWORD size = full ? BATCH_NEXT_EVENT : 1;
408
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev) {
409
+ DWORD size = (log->type & WEVT_QUERY_EXTENDED) ? BATCH_NEXT_EVENT : 1;
410
DWORD max_failures = 10;
411
419
- fatal_assert(log && log->hQuery && log->hRenderContext);
412
+ fatal_assert(log && log->hQuery && log->hRenderSystemContext);
413
414
while(max_failures > 0) {
415
if (log->batch.used >= log->batch.size) {
@@ -433,7 +426,7 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
426
if(size == 1) {
427
nd_log(NDLS_COLLECTORS, NDLP_ERR,
428
"EvtNext() failed, hQuery: 0x%lx, size: %zu, extended info: %s",
436
- (uintptr_t)log->hQuery, (size_t)size, wevt_extended_status());
429
+ (uintptr_t)log->hQuery, (size_t)size, EvtGetExtendedStatus_utf8());
430
return false;
431
}
432
@@ -455,7 +448,7 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
448
log->batch.hEvents[log->batch.used] = NULL;
449
log->batch.used++;
450
458
- if(wevt_get_next_event_one(log, ev, full))
451
+ if(wevt_get_next_event_one(log, ev))
452
return true;
453
else {
454
log->query_stats.failed_count++;
@@ -467,6 +460,69 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
460
return false;
461
}
462
463
+static void wevt_event_done(WEVT_LOG *log) {
464
+ if (log->provider) {
465
+ provider_release(log->provider);
466
+ log->provider = NULL;
467
+ }
468
+
469
+ if (log->hEvent) {
470
+ EvtClose(log->hEvent);
471
+ log->hEvent = NULL;
472
+ }
473
+
474
+ log->ops.channel.src = TXT_SOURCE_UNKNOWN;
475
+ log->ops.provider.src = TXT_SOURCE_UNKNOWN;
476
+ log->ops.computer.src = TXT_SOURCE_UNKNOWN;
477
+ log->ops.account.src = TXT_SOURCE_UNKNOWN;
478
+ log->ops.domain.src = TXT_SOURCE_UNKNOWN;
479
+ log->ops.sid.src = TXT_SOURCE_UNKNOWN;
480
+
481
+ log->ops.event.src = TXT_SOURCE_UNKNOWN;
482
+ log->ops.level.src = TXT_SOURCE_UNKNOWN;
483
+ log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
484
+ log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
485
+ log->ops.task.src = TXT_SOURCE_UNKNOWN;
486
+ log->ops.xml.src = TXT_SOURCE_UNKNOWN;
487
+
488
+ log->ops.channel.used = 0;
489
+ log->ops.provider.used = 0;
490
+ log->ops.computer.used = 0;
491
+ log->ops.account.used = 0;
492
+ log->ops.domain.used = 0;
493
+ log->ops.sid.used = 0;
494
+
495
+ log->ops.event.used = 0;
496
+ log->ops.level.used = 0;
497
+ log->ops.keywords.used = 0;
498
+ log->ops.opcode.used = 0;
499
+ log->ops.task.used = 0;
500
+ log->ops.xml.used = 0;
501
+
502
+ if(log->ops.event_data)
503
+ log->ops.event_data->len = 0;
504
+}
505
+
506
+// --------------------------------------------------------------------------------------------------------------------
507
+// Query management
508
+
509
+bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
510
+ wevt_query_done(log);
511
+ log->log_stats.queries_count++;
512
+
513
+ EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
514
+ if (!hQuery) {
515
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
516
+ query2utf8(query), EvtGetExtendedStatus_utf8());
517
+
518
+ log->log_stats.queries_failed++;
519
+ return false;
520
+ }
521
+
522
+ log->hQuery = hQuery;
523
+ return true;
524
+}
525
+
526
void wevt_query_done(WEVT_LOG *log) {
527
// close the last working hEvent
528
wevt_event_done(log);
@@ -490,19 +546,59 @@ void wevt_query_done(WEVT_LOG *log) {
546
log->query_stats.failed_count = 0;
547
}
548
549
+// --------------------------------------------------------------------------------------------------------------------
550
+// Log management
551
+
552
+WEVT_LOG *wevt_openlog6(WEVT_QUERY_TYPE type) {
553
+ WEVT_LOG *log = callocz(1, sizeof(*log));
554
+ log->type = type;
555
+
556
+ // create the system render
557
+ log->hRenderSystemContext = EvtCreateRenderContext(0, NULL, EvtRenderContextSystem);
558
+ if (!log->hRenderSystemContext) {
559
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
560
+ "EvtCreateRenderContext() on system context failed, extended info: %s",
561
+ EvtGetExtendedStatus_utf8());
562
+ goto cleanup;
563
+ }
564
+
565
+ if(type & WEVT_QUERY_EVENT_DATA) {
566
+ log->hRenderUserContext = EvtCreateRenderContext(0, NULL, EvtRenderContextUser);
567
+ if (!log->hRenderUserContext) {
568
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
569
+ "EvtCreateRenderContext failed, on user context failed, extended info: %s",
570
+ EvtGetExtendedStatus_utf8());
571
+ goto cleanup;
572
+ }
573
+
574
+ log->ops.event_data = buffer_create(4096, NULL);
575
+ }
576
+
577
+ return log;
578
+
579
+cleanup:
580
+ wevt_closelog6(log);
581
+ return NULL;
582
+}
583
+
584
void wevt_closelog6(WEVT_LOG *log) {
585
wevt_query_done(log);
586
496
- if (log->hRenderContext)
497
- EvtClose(log->hRenderContext);
587
+ if (log->hRenderSystemContext)
588
+ EvtClose(log->hRenderSystemContext);
589
+
590
+ if (log->hRenderUserContext)
591
+ EvtClose(log->hRenderUserContext);
592
499
- wevt_variant_cleanup(&log->ops.content);
593
+ wevt_variant_cleanup(&log->ops.raw.system);
594
+ wevt_variant_cleanup(&log->ops.raw.user);
595
txt_unicode_cleanup(&log->ops.unicode);
596
txt_utf8_cleanup(&log->ops.channel);
597
txt_utf8_cleanup(&log->ops.provider);
503
- txt_utf8_cleanup(&log->ops.source);
598
txt_utf8_cleanup(&log->ops.computer);
505
- txt_utf8_cleanup(&log->ops.user);
599
+ txt_utf8_cleanup(&log->ops.account);
600
+ txt_utf8_cleanup(&log->ops.domain);
601
+ txt_utf8_cleanup(&log->ops.sid);
602
603
txt_utf8_cleanup(&log->ops.event);
604
txt_utf8_cleanup(&log->ops.level);
@@ -510,9 +606,15 @@ void wevt_closelog6(WEVT_LOG *log) {
606
txt_utf8_cleanup(&log->ops.opcode);
607
txt_utf8_cleanup(&log->ops.task);
608
txt_utf8_cleanup(&log->ops.xml);
609
+
610
+ buffer_free(log->ops.event_data);
611
+
612
freez(log);
613
}
614
615
+// --------------------------------------------------------------------------------------------------------------------
616
+// Retention
617
+
618
bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t *query, EVT_RETENTION *retention) {
619
bool ret = false;
620
@@ -525,15 +627,15 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
627
if (!log->hQuery) {
628
if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
629
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, cannot get retention, extended info: %s",
528
- channel2utf8(channel), wevt_extended_status());
630
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
631
else
632
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, cannot get retention, extended info: %s",
531
- channel2utf8(channel), wevt_extended_status());
633
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
634
635
goto cleanup;
636
}
637
536
- if (!wevt_get_next_event(log, &retention->first_event, false))
638
+ if (!wevt_get_next_event(log, &retention->first_event))
639
goto cleanup;
640
641
if (!retention->first_event.id) {
@@ -548,15 +650,15 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
650
if (!log->hQuery) {
651
if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
652
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, extended info: %s",
551
- channel2utf8(channel), wevt_extended_status());
653
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
654
else
655
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, extended info: %s",
554
- channel2utf8(channel), wevt_extended_status());
656
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
657
658
goto cleanup;
659
}
660
559
- if (!wevt_get_next_event(log, &retention->last_event, false) || retention->last_event.id == 0) {
661
+ if (!wevt_get_next_event(log, &retention->last_event) || retention->last_event.id == 0) {
662
// no data in eventlog
663
retention->last_event = retention->first_event;
664
}
@@ -582,24 +684,6 @@ cleanup:
684
return ret;
685
}
686
585
-WEVT_LOG *wevt_openlog6(void) {
586
- size_t RENDER_ITEMS_count = (sizeof(RENDER_ITEMS) / sizeof(const wchar_t *));
587
-
588
- WEVT_LOG *log = callocz(1, sizeof(*log));
589
-
590
- // create the system render
591
- log->hRenderContext = EvtCreateRenderContext(RENDER_ITEMS_count, RENDER_ITEMS, EvtRenderContextValues);
592
- if (!log->hRenderContext) {
593
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtCreateRenderContext failed, extended info: %s", wevt_extended_status());
594
- freez(log);
595
- log = NULL;
596
- goto cleanup;
597
- }
598
-
599
-cleanup:
600
- return log;
601
-}
602
-
687
static uint64_t wevt_log_file_size(const wchar_t *channel) {
688
EVT_HANDLE hLog = NULL;
689
EVT_VARIANT evtVariant;
@@ -610,14 +694,14 @@ static uint64_t wevt_log_file_size(const wchar_t *channel) {
694
hLog = EvtOpenLog(NULL, channel, EvtOpenChannelPath);
695
if (!hLog) {
696
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtOpenLog() on channel '%s' failed, extended info: %s",
613
- channel2utf8(channel), wevt_extended_status());
697
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
698
goto cleanup;
699
}
700
701
// Get the file size of the log
702
if (!EvtGetLogInfo(hLog, EvtLogFileSize, sizeof(evtVariant), &evtVariant, &bufferUsed)) {
703
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetLogInfo() on channel '%s' failed, extended info: %s",
620
- channel2utf8(channel), wevt_extended_status());
704
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
705
goto cleanup;
706
}
707
@@ -630,20 +714,3 @@ cleanup:
714
715
return file_size;
716
}
633
-
634
-bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
635
- wevt_query_done(log);
636
- log->log_stats.queries_count++;
637
-
638
- EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
639
- if (!hQuery) {
640
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
641
- query2utf8(query), wevt_extended_status());
642
-
643
- log->log_stats.queries_failed++;
644
- return false;
645
- }
646
-
647
- log->hQuery = hQuery;
648
- return true;
649
-}
src/collectors/windows-events.plugin/windows-events-query.h
+65
-34
@@ -4,6 +4,7 @@
4
#define NETDATA_WINDOWS_EVENTS_QUERY_H
5
6
#include "libnetdata/libnetdata.h"
7
+#include "windows-events.h"
8
9
#define BATCH_NEXT_EVENT 500
10
@@ -11,23 +12,27 @@ typedef struct wevt_event {
12
uint64_t id; // EventRecordId (unique and sequential per channel)
13
uint8_t version;
14
uint8_t level; // The severity of event
14
- uint8_t opcode; // we receive this as 8bit, but publishers use 32bit
15
+ uint8_t opcode; // we receive this as 8bit, but providers use 32bit
16
uint16_t event_id; // This is the template that defines the message to be shown
17
uint16_t task;
18
+ uint16_t qualifiers;
19
uint32_t process_id;
20
uint32_t thread_id;
21
uint64_t keywords; // Categorization of the event
22
ND_UUID provider;
21
- ND_UUID correlation_activity_id;
23
+ ND_UUID activity_id;
24
+ ND_UUID related_activity_id;
25
nsec_t created_ns;
26
+ WEVT_PROVIDER_PLATFORM platform;
27
} WEVT_EVENT;
28
29
#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
30
31
typedef struct {
32
EVT_VARIANT *data;
29
- size_t size;
30
- size_t used;
33
+ DWORD size;
34
+ DWORD used;
35
+ DWORD count;
36
} WEVT_VARIANT;
37
38
typedef struct {
@@ -41,6 +46,16 @@ typedef struct {
46
47
struct provider_meta_handle;
48
49
+typedef enum __attribute__((packed)) {
50
+ WEVT_QUERY_BASIC = (1 << 0),
51
+ WEVT_QUERY_EXTENDED = (1 << 1),
52
+ WEVT_QUERY_EVENT_DATA = (1 << 2),
53
+} WEVT_QUERY_TYPE;
54
+
55
+#define WEVT_QUERY_RETENTION WEVT_QUERY_BASIC
56
+#define WEVT_QUERY_NORMAL (WEVT_QUERY_BASIC | WEVT_QUERY_EXTENDED)
57
+#define WEVT_QUERY_FTS (WEVT_QUERY_BASIC | WEVT_QUERY_EXTENDED | WEVT_QUERY_EVENT_DATA)
58
+
59
typedef struct wevt_log {
60
struct {
61
DWORD size;
@@ -50,13 +65,19 @@ typedef struct wevt_log {
65
66
EVT_HANDLE hEvent;
67
EVT_HANDLE hQuery;
53
- EVT_HANDLE hRenderContext;
54
- struct provider_meta_handle *publisher;
68
+ EVT_HANDLE hRenderSystemContext;
69
+ EVT_HANDLE hRenderUserContext;
70
+ struct provider_meta_handle *provider;
71
+
72
+ WEVT_QUERY_TYPE type;
73
74
struct {
57
- // temp buffer used for rendering event log messages
58
- // never use directly
59
- WEVT_VARIANT content;
75
+ struct {
76
+ // temp buffer used for rendering event log messages
77
+ // never use directly
78
+ WEVT_VARIANT system;
79
+ WEVT_VARIANT user;
80
+ } raw;
81
82
// temp buffer used for fetching and converting UNICODE and UTF-8
83
// every string operation overwrites it, multiple times per event log entry
@@ -75,16 +96,19 @@ typedef struct wevt_log {
96
97
TXT_UTF8 channel;
98
TXT_UTF8 provider;
78
- TXT_UTF8 source;
99
TXT_UTF8 computer;
80
- TXT_UTF8 user;
100
+ TXT_UTF8 account;
101
+ TXT_UTF8 domain;
102
+ TXT_UTF8 sid;
103
82
- TXT_UTF8 event;
104
+ TXT_UTF8 event; // the message to be shown to the user
105
TXT_UTF8 level;
106
TXT_UTF8 keywords;
107
TXT_UTF8 opcode;
108
TXT_UTF8 task;
109
TXT_UTF8 xml;
110
+
111
+ BUFFER *event_data;
112
} ops;
113
114
struct {
@@ -102,7 +126,7 @@ typedef struct wevt_log {
126
127
} WEVT_LOG;
128
105
-WEVT_LOG *wevt_openlog6(void);
129
+WEVT_LOG *wevt_openlog6(WEVT_QUERY_TYPE type);
130
void wevt_closelog6(WEVT_LOG *log);
131
132
bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t *query, EVT_RETENTION *retention);
@@ -110,12 +134,14 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
134
bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction);
135
void wevt_query_done(WEVT_LOG *log);
136
113
-bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
137
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev);
138
+
139
+bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
140
115
-bool wevt_get_message_unicode(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
141
+bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
142
+bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
143
117
-bool wevt_get_event_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
118
-bool wevt_get_xml_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
144
+void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator);
145
146
static inline void wevt_variant_cleanup(WEVT_VARIANT *v) {
147
freez(v->data);
@@ -130,6 +156,10 @@ static inline void wevt_variant_resize(WEVT_VARIANT *v, size_t required_size) {
156
v->data = mallocz(v->size);
157
}
158
159
+static inline void wevt_variant_count_from_used(WEVT_VARIANT *v) {
160
+ v->count = v->used / sizeof(*v->data);
161
+}
162
+
163
static inline uint8_t wevt_field_get_uint8(EVT_VARIANT *ev) {
164
if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull)
165
return 0;
@@ -180,16 +210,17 @@ static inline bool wevt_field_get_string_utf8(EVT_VARIANT *ev, TXT_UTF8 *dst) {
210
return wevt_str_wchar_to_utf8(dst, ev->StringVal, -1);
211
}
212
183
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
184
-
185
-static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst) {
213
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str);
214
+static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) {
215
if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
187
- wevt_utf8_empty(dst);
216
+ wevt_utf8_empty(dst_account);
217
+ wevt_utf8_empty(dst_domain);
218
+ wevt_utf8_empty(dst_sid_str);
219
return false;
220
}
221
222
fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeSid);
192
- return wevt_convert_user_id_to_name(ev->SidVal, dst);
223
+ return wevt_convert_user_id_to_name(ev->SidVal, dst_account, dst_domain, dst_sid_str);
224
}
225
226
static inline uint64_t wevt_field_get_filetime_to_ns(EVT_VARIANT *ev) {
@@ -222,42 +253,42 @@ static inline bool wevt_get_uuid_by_type(EVT_VARIANT *ev, ND_UUID *dst) {
253
}
254
255
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-severity-levels
225
-static inline bool is_valid_publisher_level(uint64_t level, bool strict) {
256
+static inline bool is_valid_provider_level(uint64_t level, bool strict) {
257
if(strict)
227
- // when checking if the name is publisher independent
258
+ // when checking if the name is provider independent
259
return level >= 16 && level <= 255;
260
else
230
- // when checking acceptable values in publisher manifests
261
+ // when checking acceptable values in provider manifests
262
return level <= 255;
263
}
264
265
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
235
-static inline bool is_valid_publisher_opcode(uint64_t opcode, bool strict) {
266
+static inline bool is_valid_provider_opcode(uint64_t opcode, bool strict) {
267
if(strict)
237
- // when checking if the name is publisher independent
268
+ // when checking if the name is provider independent
269
return opcode >= 10 && opcode <= 239;
270
else
240
- // when checking acceptable values in publisher manifests
271
+ // when checking acceptable values in provider manifests
272
return opcode <= 255;
273
}
274
275
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
245
-static inline bool is_valid_publisher_task(uint64_t task, bool strict) {
276
+static inline bool is_valid_provider_task(uint64_t task, bool strict) {
277
if(strict)
247
- // when checking if the name is publisher independent
278
+ // when checking if the name is provider independent
279
return task > 0 && task <= 0xFFFF;
280
else
250
- // when checking acceptable values in publisher manifests
281
+ // when checking acceptable values in provider manifests
282
return task <= 0xFFFF;
283
}
284
285
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-keywords-used-to-classify-types-of-events
255
-static inline bool is_valid_publisher_keywords(uint64_t keyword, bool strict) {
286
+static inline bool is_valid_provider_keyword(uint64_t keyword, bool strict) {
287
if(strict)
257
- // when checking if the name is publisher independent
288
+ // when checking if the name is provider independent
289
return keyword > 0 && keyword <= 0x0000FFFFFFFFFFFF;
290
else
260
- // when checking acceptable values in publisher manifests
291
+ // when checking acceptable values in provider manifests
292
return true;
293
}
294
src/collectors/windows-events.plugin/windows-events-sid.c
+100
-37
@@ -9,8 +9,18 @@ typedef struct {
9
} SID_KEY;
10
11
typedef struct {
12
- const char *user;
13
- size_t user_len;
12
+ // IMPORTANT:
13
+ // This is malloc'd ! You have to manually set fields to zero.
14
+
15
+ const char *account;
16
+ const char *domain;
17
+ const char *full;
18
+ const char *sid_str;
19
+
20
+ uint32_t account_len;
21
+ uint32_t domain_len;
22
+ uint32_t full_len;
23
+ uint32_t sid_str_len;
24
25
// this needs to be last, because of its variable size
26
SID_KEY key;
@@ -43,48 +53,46 @@ void sid_cache_init(void) {
53
simple_hashtable_init_SID(&sid_globals.hashtable, 100);
54
}
55
46
-static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
47
- if(found && found->user) {
48
- txt_utf8_resize(dst, found->user_len + 1, false);
49
- memcpy(dst->data, found->user, found->user_len + 1);
50
- dst->used = found->user_len + 1;
51
- return true;
52
- }
53
-
54
- txt_utf8_resize(dst, 1, false);
55
- dst->data[0] = '\0';
56
- dst->used = 1;
57
- return false;
58
-}
59
-
60
-static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
56
+static void lookup_user(SID_VALUE *sv) {
57
static __thread wchar_t account_unicode[256];
58
static __thread wchar_t domain_unicode[256];
59
+ static __thread char tmp[512 + 2];
60
+
61
DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
62
DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
63
SID_NAME_USE sid_type;
64
67
- txt_utf8_resize(dst, 1024, false);
68
-
69
- if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
70
- const char *user = account2utf8(account_unicode);
65
+ if (LookupAccountSidW(NULL, sv->key.sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
66
+ const char *account = account2utf8(account_unicode);
67
const char *domain = domain2utf8(domain_unicode);
72
- dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
68
+ snprintfz(tmp, sizeof(tmp), "%s\\%s", domain, account);
69
+ sv->domain = strdupz(domain); sv->domain_len = strlen(sv->domain);
70
+ sv->account = strdupz(account); sv->account_len = strlen(sv->account);
71
+ sv->full = strdupz(tmp); sv->full_len = strlen(sv->full);
72
}
73
else {
75
- wchar_t *sid_string = NULL;
76
- if (ConvertSidToStringSidW(sid, &sid_string)) {
77
- const char *user = account2utf8(sid_string);
78
- dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
79
- }
80
- else
81
- dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
74
+ sv->domain = NULL;
75
+ sv->account = NULL;
76
+ sv->full = NULL;
77
+ sv->domain_len = 0;
78
+ sv->account_len = 0;
79
+ sv->full_len = 0;
80
+ }
81
+
82
+ wchar_t *sid_string = NULL;
83
+ if (ConvertSidToStringSidW(sv->key.sid, &sid_string)) {
84
+ sv->sid_str = strdupz(account2utf8(sid_string));
85
+ sv->sid_str_len = strlen(sv->sid_str);
86
+ }
87
+ else {
88
+ sv->sid_str = NULL;
89
+ sv->sid_str_len = 0;
90
}
91
}
92
85
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
93
+static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) {
94
if(!sid || !IsValidSid(sid))
87
- return update_user(NULL, dst);
95
+ return NULL;
96
97
size_t size = GetLengthSid(sid);
98
@@ -98,21 +106,76 @@ bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
106
spinlock_lock(&sid_globals.spinlock);
107
SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
108
spinlock_unlock(&sid_globals.spinlock);
101
- if(found) return update_user(found, dst);
109
+ if(found) return found;
110
111
// allocate the SID_VALUE
112
found = mallocz(tmp_size);
113
memcpy(found, buf, tmp_size);
114
107
- // lookup the user
108
- lookup_user(sid, dst);
109
- found->user = strdupz(dst->data);
110
- found->user_len = dst->used - 1;
115
+ lookup_user(found);
116
117
// add it to the cache
118
spinlock_lock(&sid_globals.spinlock);
119
simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
120
spinlock_unlock(&sid_globals.spinlock);
121
117
- return update_user(found, dst);
122
+ return found;
123
+}
124
+
125
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) {
126
+ SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid);
127
+
128
+ if(found) {
129
+ if (found->account) {
130
+ txt_utf8_resize(dst_account, found->account_len + 1, false);
131
+ memcpy(dst_account->data, found->account, found->account_len + 1);
132
+ dst_account->used = found->account_len + 1;
133
+ }
134
+ else wevt_utf8_empty(dst_account);
135
+
136
+ if (found->domain) {
137
+ txt_utf8_resize(dst_domain, found->domain_len + 1, false);
138
+ memcpy(dst_domain->data, found->domain, found->domain_len + 1);
139
+ dst_domain->used = found->domain_len + 1;
140
+ }
141
+ else wevt_utf8_empty(dst_domain);
142
+
143
+ if (found->sid_str) {
144
+ txt_utf8_resize(dst_sid_str, found->sid_str_len + 1, false);
145
+ memcpy(dst_sid_str->data, found->sid_str, found->sid_str_len + 1);
146
+ dst_sid_str->used = found->sid_str_len + 1;
147
+ }
148
+ else wevt_utf8_empty(dst_sid_str);
149
+
150
+ return true;
151
+ }
152
+
153
+ wevt_utf8_empty(dst_account);
154
+ wevt_utf8_empty(dst_domain);
155
+ wevt_utf8_empty(dst_sid_str);
156
+ return false;
157
+}
158
+
159
+bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix) {
160
+ SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid);
161
+ size_t added = 0;
162
+
163
+ if(found) {
164
+ if (found->full) {
165
+ if (prefix && *prefix)
166
+ buffer_strcat(dst, prefix);
167
+
168
+ buffer_fast_strcat(dst, found->full, found->full_len);
169
+ added++;
170
+ }
171
+ if (found->sid_str) {
172
+ if (prefix && *prefix)
173
+ buffer_strcat(dst, prefix);
174
+
175
+ buffer_fast_strcat(dst, found->sid_str, found->sid_str_len);
176
+ added++;
177
+ }
178
+ }
179
+
180
+ return added > 0;
181
}
src/collectors/windows-events.plugin/windows-events-sid.h
+2
-1
@@ -6,7 +6,8 @@
6
#include "windows-events.h"
7
8
struct wevt_log;
9
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
9
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str);
10
+bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix);
11
void sid_cache_init(void);
12
13
#endif //NETDATA_WINDOWS_EVENTS_SID_H
src/collectors/windows-events.plugin/windows-events-sources.c
+207
-97
@@ -137,41 +137,32 @@
137
// }
138
//};
139
140
+ENUM_STR_MAP_DEFINE(WEVT_SOURCE_TYPE) = {
141
+ { .id = WEVTS_ALL, .name = WEVT_SOURCE_ALL_NAME },
142
+ { .id = WEVTS_ADMIN, .name = WEVT_SOURCE_ALL_ADMIN_NAME },
143
+ { .id = WEVTS_OPERATIONAL, .name = WEVT_SOURCE_ALL_OPERATIONAL_NAME },
144
+ { .id = WEVTS_ANALYTIC, .name = WEVT_SOURCE_ALL_ANALYTIC_NAME },
145
+ { .id = WEVTS_DEBUG, .name = WEVT_SOURCE_ALL_DEBUG_NAME },
146
+ { .id = WEVTS_WINDOWS, .name = WEVT_SOURCE_ALL_WINDOWS_NAME },
147
+ { .id = WEVTS_ENABLED, .name = WEVT_SOURCE_ALL_ENABLED_NAME },
148
+ { .id = WEVTS_DISABLED, .name = WEVT_SOURCE_ALL_DISABLED_NAME },
149
+ { .id = WEVTS_FORWARDED, .name = WEVT_SOURCE_ALL_FORWARDED_NAME },
150
+ { .id = WEVTS_CLASSIC, .name = WEVT_SOURCE_ALL_CLASSIC_NAME },
151
+ { .id = WEVTS_BACKUP_MODE, .name = WEVT_SOURCE_ALL_BACKUP_MODE_NAME },
152
+ { .id = WEVTS_OVERWRITE_MODE, .name = WEVT_SOURCE_ALL_OVERWRITE_MODE_NAME },
153
+ { .id = WEVTS_STOP_WHEN_FULL_MODE, .name = WEVT_SOURCE_ALL_STOP_WHEN_FULL_MODE_NAME },
154
+ { .id = WEVTS_RETAIN_AND_BACKUP_MODE, .name = WEVT_SOURCE_ALL_RETAIN_AND_BACKUP_MODE_NAME },
155
+
156
+ // terminator
157
+ { . id = 0, .name = NULL }
158
+};
159
+
160
+BITMAP_STR_DEFINE_FUNCTIONS(WEVT_SOURCE_TYPE, WEVTS_NONE, "");
161
+
162
DICTIONARY *wevt_sources = NULL;
163
DICTIONARY *used_hashes_registry = NULL;
164
static usec_t wevt_session = 0;
165
144
-WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
145
- if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
146
- return WEVTS_ALL;
147
-
148
- if(strcmp(value, WEVT_SOURCE_ALL_ADMIN_NAME) == 0)
149
- return WEVTS_ADMIN;
150
-
151
- if(strcmp(value, WEVT_SOURCE_ALL_OPERATIONAL_NAME) == 0)
152
- return WEVTS_OPERATIONAL;
153
-
154
- if(strcmp(value, WEVT_SOURCE_ALL_ANALYTIC_NAME) == 0)
155
- return WEVTS_ANALYTIC;
156
-
157
- if(strcmp(value, WEVT_SOURCE_ALL_DEBUG_NAME) == 0)
158
- return WEVTS_DEBUG;
159
-
160
- if(strcmp(value, WEVT_SOURCE_ALL_DIAGNOSTIC_NAME) == 0)
161
- return WEVTS_DIAGNOSTIC;
162
-
163
- if(strcmp(value, WEVT_SOURCE_ALL_TRACING_NAME) == 0)
164
- return WEVTS_TRACING;
165
-
166
- if(strcmp(value, WEVT_SOURCE_ALL_PERFORMANCE_NAME) == 0)
167
- return WEVTS_PERFORMANCE;
168
-
169
- if(strcmp(value, WEVT_SOURCE_ALL_WINDOWS_NAME) == 0)
170
- return WEVTS_WINDOWS;
171
-
172
- return WEVTS_NONE;
173
-}
174
-
166
void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
167
LOGS_QUERY_SOURCE *src = value;
168
freez((void *)src->fullname);
@@ -265,7 +256,14 @@ int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
256
257
// --------------------------------------------------------------------------------------------------------------------
258
259
+typedef enum {
260
+ wevt_source_type_internal,
261
+ wevt_source_type_provider,
262
+ wevt_source_type_channel,
263
+} wevt_source_type;
264
+
265
struct wevt_source {
266
+ wevt_source_type type;
267
usec_t first_ut;
268
usec_t last_ut;
269
size_t count;
@@ -279,6 +277,15 @@ static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry
277
278
const char *name = dictionary_acquired_item_name(item);
279
280
+ if(s->count == 1 && strncmp(name, WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX, sizeof(WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX) - 1) == 0)
281
+ // do not include "All-Of-X" when there is only 1 channel
282
+ return 0;
283
+
284
+ bool default_selected = (s->type == wevt_source_type_channel);
285
+ if(default_selected && (strcmp(name, "NetdataWEL") == 0 || strcmp(name, "Netdata/Access") == 0))
286
+ // do not select Netdata Access logs by default
287
+ default_selected = false;
288
+
289
buffer_json_add_array_item_object(wb);
290
{
291
char size_for_humans[128];
@@ -300,6 +307,7 @@ static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry
307
buffer_json_member_add_string(wb, "name", name);
308
buffer_json_member_add_string(wb, "pill", size_for_humans);
309
buffer_json_member_add_string(wb, "info", info);
310
+ buffer_json_member_add_boolean(wb, "default_selected", default_selected);
311
}
312
buffer_json_object_close(wb); // options object
313
@@ -337,61 +345,142 @@ void wevt_sources_to_json_array(BUFFER *wb) {
345
t.size = src->size;
346
t.entries = src->entries;
347
340
- dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
341
-
342
- if(src->source_type & WEVTS_ADMIN)
343
- dictionary_set(dict, WEVT_SOURCE_ALL_ADMIN_NAME, &t, sizeof(t));
344
-
345
- if(src->source_type & WEVTS_OPERATIONAL)
346
- dictionary_set(dict, WEVT_SOURCE_ALL_OPERATIONAL_NAME, &t, sizeof(t));
347
-
348
- if(src->source_type & WEVTS_ANALYTIC)
349
- dictionary_set(dict, WEVT_SOURCE_ALL_ANALYTIC_NAME, &t, sizeof(t));
350
-
351
- if(src->source_type & WEVTS_DEBUG)
352
- dictionary_set(dict, WEVT_SOURCE_ALL_DEBUG_NAME, &t, sizeof(t));
353
-
354
- if(src->source_type & WEVTS_DIAGNOSTIC)
355
- dictionary_set(dict, WEVT_SOURCE_ALL_DIAGNOSTIC_NAME, &t, sizeof(t));
356
-
357
- if(src->source_type & WEVTS_TRACING)
358
- dictionary_set(dict, WEVT_SOURCE_ALL_TRACING_NAME, &t, sizeof(t));
359
-
360
- if(src->source_type & WEVTS_PERFORMANCE)
361
- dictionary_set(dict, WEVT_SOURCE_ALL_PERFORMANCE_NAME, &t, sizeof(t));
348
+ src->source_type |= WEVTS_ALL;
349
+ t.type = wevt_source_type_internal;
350
+ for(size_t i = 0; WEVT_SOURCE_TYPE_names[i].name ;i++) {
351
+ if(src->source_type & WEVT_SOURCE_TYPE_names[i].id)
352
+ dictionary_set(dict, WEVT_SOURCE_TYPE_names[i].name, &t, sizeof(t));
353
+ }
354
363
- if(src->source_type & WEVTS_WINDOWS)
364
- dictionary_set(dict, WEVT_SOURCE_ALL_WINDOWS_NAME, &t, sizeof(t));
355
+ if(src->provider) {
356
+ t.type = wevt_source_type_provider;
357
+ dictionary_set(dict, string2str(src->provider), &t, sizeof(t));
358
+ }
359
366
- if(src->source)
360
+ if(src->source) {
361
+ t.type = wevt_source_type_channel;
362
dictionary_set(dict, string2str(src->source), &t, sizeof(t));
363
+ }
364
}
365
dfe_done(jf);
366
367
dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
368
}
369
374
-static bool check_and_remove_suffix(char *name, size_t len, const char *suffix) {
375
- char s[strlen(suffix) + 2];
376
- s[0] = '/';
377
- memcpy(&s[1], suffix, sizeof(s) - 1);
378
- size_t slen = sizeof(s) - 1;
370
+static bool ndEvtGetChannelConfigProperty(EVT_HANDLE hChannelConfig, WEVT_VARIANT *pr, EVT_CHANNEL_CONFIG_PROPERTY_ID id) {
371
+ if (!EvtGetChannelConfigProperty(hChannelConfig, id, 0, pr->size, pr->data, &pr->used)) {
372
+ DWORD status = GetLastError();
373
+ if (ERROR_INSUFFICIENT_BUFFER == status) {
374
+ wevt_variant_resize(pr, pr->used);
375
+ if(!EvtGetChannelConfigProperty(hChannelConfig, id, 0, pr->size, pr->data, &pr->used)) {
376
+ pr->used = 0;
377
+ pr->count = 0;
378
+ return false;
379
+ }
380
+ }
381
+ }
382
380
- if(slen + 1 >= len) return false;
383
+ wevt_variant_count_from_used(pr);
384
+ return true;
385
+}
386
382
- char *match = &name[len - slen];
383
- if(strcasecmp(match, s) == 0) {
384
- *match = '\0';
385
- return true;
387
+WEVT_SOURCE_TYPE categorize_channel(const wchar_t *channel_path, const char **provider, WEVT_VARIANT *property) {
388
+ EVT_HANDLE hChannelConfig = NULL;
389
+ WEVT_SOURCE_TYPE result = WEVTS_ALL;
390
+
391
+ // Open the channel configuration
392
+ hChannelConfig = EvtOpenChannelConfig(NULL, channel_path, 0);
393
+ if (!hChannelConfig)
394
+ goto cleanup;
395
+
396
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigType) &
397
+ property->count &&
398
+ property->data[0].Type == EvtVarTypeUInt32) {
399
+ switch (property->data[0].UInt32Val) {
400
+ case EvtChannelTypeAdmin:
401
+ result |= WEVTS_ADMIN;
402
+ break;
403
+
404
+ case EvtChannelTypeOperational:
405
+ result |= WEVTS_OPERATIONAL;
406
+ break;
407
+
408
+ case EvtChannelTypeAnalytic:
409
+ result |= WEVTS_ANALYTIC;
410
+ break;
411
+
412
+ case EvtChannelTypeDebug:
413
+ result |= WEVTS_DEBUG;
414
+ break;
415
+
416
+ default:
417
+ break;
418
+ }
419
}
420
388
- s[0] = '-';
389
- if(strcasecmp(match, s) == 0) {
390
- *match = '\0';
391
- return true;
421
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigClassicEventlog) &&
422
+ property->count &&
423
+ property->data[0].Type == EvtVarTypeBoolean &&
424
+ property->data[0].BooleanVal)
425
+ result |= WEVTS_CLASSIC;
426
+
427
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigOwningPublisher) &&
428
+ property->count &&
429
+ property->data[0].Type == EvtVarTypeString) {
430
+ *provider = provider2utf8(property->data[0].StringVal);
431
+ if(wcscasecmp(property->data[0].StringVal, L"Microsoft-Windows-EventCollector") == 0)
432
+ result |= WEVTS_FORWARDED;
433
+ }
434
+ else
435
+ *provider = NULL;
436
+
437
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigEnabled) &&
438
+ property->count &&
439
+ property->data[0].Type == EvtVarTypeBoolean) {
440
+ if(property->data[0].BooleanVal)
441
+ result |= WEVTS_ENABLED;
442
+ else
443
+ result |= WEVTS_DISABLED;
444
}
445
394
- return false;
446
+ bool got_retention = false;
447
+ bool retained = false;
448
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelLoggingConfigRetention) &&
449
+ property->count &&
450
+ property->data[0].Type == EvtVarTypeBoolean) {
451
+ got_retention = true;
452
+ retained = property->data[0].BooleanVal;
453
+ }
454
+
455
+ bool got_auto_backup = false;
456
+ bool auto_backup = false;
457
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelLoggingConfigAutoBackup) &&
458
+ property->count &&
459
+ property->data[0].Type == EvtVarTypeBoolean) {
460
+ got_auto_backup = true;
461
+ auto_backup = property->data[0].BooleanVal;
462
+ }
463
+
464
+ if(got_retention && got_auto_backup) {
465
+ if(!retained) {
466
+ if(auto_backup)
467
+ result |= WEVTS_BACKUP_MODE;
468
+ else
469
+ result |= WEVTS_OVERWRITE_MODE;
470
+ }
471
+ else {
472
+ if(auto_backup)
473
+ result |= WEVTS_STOP_WHEN_FULL_MODE;
474
+ else
475
+ result |= WEVTS_RETAIN_AND_BACKUP_MODE;
476
+ }
477
+ }
478
+
479
+cleanup:
480
+ if (hChannelConfig)
481
+ EvtClose(hChannelConfig);
482
+
483
+ return result;
484
}
485
486
void wevt_sources_scan(void) {
@@ -400,8 +489,9 @@ void wevt_sources_scan(void) {
489
EVT_HANDLE hChannelEnum = NULL;
490
491
if(spinlock_trylock(&spinlock)) {
403
- const usec_t now_monotonic_ut = now_monotonic_usec();
492
+ const usec_t started_ut = now_monotonic_usec();
493
494
+ WEVT_VARIANT property = { 0 };
495
DWORD dwChannelBufferSize = 0;
496
DWORD dwChannelBufferUsed = 0;
497
DWORD status = ERROR_SUCCESS;
@@ -414,7 +504,7 @@ void wevt_sources_scan(void) {
504
goto cleanup;
505
}
506
417
- WEVT_LOG *log = wevt_openlog6();
507
+ WEVT_LOG *log = wevt_openlog6(WEVT_QUERY_RETENTION);
508
if(!log) goto cleanup;
509
510
while (true) {
@@ -438,32 +528,26 @@ void wevt_sources_scan(void) {
528
if(!wevt_channel_retention(log, channel, NULL, &retention))
529
continue;
530
531
+ LOGS_QUERY_SOURCE *found = dictionary_get(wevt_sources, channel2utf8(channel));
532
+ if(found) {
533
+ // we just need to update its retention
534
+
535
+ found->last_scan_monotonic_ut = now_monotonic_usec();
536
+ found->msg_first_id = retention.first_event.id;
537
+ found->msg_last_id = retention.last_event.id;
538
+ found->msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC;
539
+ found->msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC;
540
+ found->size = retention.size_bytes;
541
+ continue;
542
+ }
543
+
544
const char *name = channel2utf8(channel);
545
const char *fullname = strdupz(name);
546
+ const char *provider;
547
444
- WEVT_SOURCE_TYPE sources = WEVTS_ALL;
445
- size_t len = strlen(fullname);
446
- if(check_and_remove_suffix((char *)name, len, "Admin"))
447
- sources |= WEVTS_ADMIN;
448
- else if(check_and_remove_suffix((char *)name, len, "Operational"))
449
- sources |= WEVTS_OPERATIONAL;
450
- else if(check_and_remove_suffix((char *)name, len, "Analytic"))
451
- sources |= WEVTS_ANALYTIC;
452
- else if(check_and_remove_suffix((char *)name, len, "Debug") ||
453
- check_and_remove_suffix((char *)name, len, "Verbose"))
454
- sources |= WEVTS_DEBUG;
455
- else if(check_and_remove_suffix((char *)name, len, "Diagnostic"))
456
- sources |= WEVTS_DIAGNOSTIC;
457
- else if(check_and_remove_suffix((char *)name, len, "Trace") ||
458
- check_and_remove_suffix((char *)name, len, "Tracing"))
459
- sources |= WEVTS_TRACING;
460
- else if(check_and_remove_suffix((char *)name, len, "Performance") ||
461
- check_and_remove_suffix((char *)name, len, "Perf"))
462
- sources |= WEVTS_PERFORMANCE;
463
-
548
+ WEVT_SOURCE_TYPE sources = categorize_channel(channel, &provider, &property);
549
char *slash = strchr(name, '/');
465
- if(slash)
466
- *slash = '\0';
550
+ if(slash) *slash = '\0';
551
552
if(strcasecmp(name, "Application") == 0)
553
sources |= WEVTS_WINDOWS;
@@ -485,9 +569,27 @@ void wevt_sources_scan(void) {
569
.msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
570
.size = retention.size_bytes,
571
.source_type = sources,
488
- .source = string_strdupz(name),
572
+ .source = string_strdupz(fullname),
573
};
574
575
+ if(strncmp(fullname, "Netdata", 7) == 0)
576
+ // WEL based providers of Netdata are named NetdataX
577
+ provider = "Netdata";
578
+
579
+ if(provider && *provider) {
580
+ char buf[sizeof(WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX) + strlen(provider)]; // sizeof() includes terminator
581
+ snprintf(buf, sizeof(buf), WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX "%s", provider);
582
+
583
+ if(trim_all(buf) != NULL) {
584
+ for (size_t i = 0; i < sizeof(buf) - 1; i++) {
585
+ // remove character that may interfere with our parsing
586
+ if (isspace((uint8_t) buf[i]) || buf[i] == '%' || buf[i] == '+' || buf[i] == '|' || buf[i] == ':')
587
+ buf[i] = '_';
588
+ }
589
+ src.provider = string_strdupz(buf);
590
+ }
591
+ }
592
+
593
dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
594
}
595
@@ -519,13 +621,21 @@ void wevt_sources_scan(void) {
621
LOGS_QUERY_SOURCE *src;
622
dfe_start_write(wevt_sources, src)
623
{
522
- if(src->last_scan_monotonic_ut < now_monotonic_ut)
624
+ if(src->last_scan_monotonic_ut < started_ut) {
625
+ src->msg_first_id = 0;
626
+ src->msg_last_id = 0;
627
+ src->msg_first_ut = 0;
628
+ src->msg_last_ut = 0;
629
+ src->size = 0;
630
dictionary_del(wevt_sources, src->fullname);
631
+ }
632
}
633
dfe_done(src);
634
dictionary_garbage_collect(wevt_sources);
635
636
spinlock_unlock(&spinlock);
637
+
638
+ wevt_variant_cleanup(&property);
639
}
640
641
cleanup:
src/collectors/windows-events.plugin/windows-events-sources.h
+35
-20
@@ -6,18 +6,42 @@
6
#include "libnetdata/libnetdata.h"
7
8
typedef enum {
9
- WEVTS_NONE = 0,
10
- WEVTS_ALL = (1 << 0),
11
- WEVTS_ADMIN = (1 << 1),
12
- WEVTS_OPERATIONAL = (1 << 2),
13
- WEVTS_ANALYTIC = (1 << 3),
14
- WEVTS_DEBUG = (1 << 4),
15
- WEVTS_DIAGNOSTIC = (1 << 5),
16
- WEVTS_TRACING = (1 << 6),
17
- WEVTS_PERFORMANCE = (1 << 7),
18
- WEVTS_WINDOWS = (1 << 8),
9
+ WEVTS_NONE = 0,
10
+ WEVTS_ALL = (1 << 0),
11
+ WEVTS_ADMIN = (1 << 1),
12
+ WEVTS_OPERATIONAL = (1 << 2),
13
+ WEVTS_ANALYTIC = (1 << 3),
14
+ WEVTS_DEBUG = (1 << 4),
15
+ WEVTS_WINDOWS = (1 << 5),
16
+ WEVTS_ENABLED = (1 << 6),
17
+ WEVTS_DISABLED = (1 << 7),
18
+ WEVTS_FORWARDED = (1 << 8),
19
+ WEVTS_CLASSIC = (1 << 9),
20
+ WEVTS_BACKUP_MODE = (1 << 10),
21
+ WEVTS_OVERWRITE_MODE = (1 << 11),
22
+ WEVTS_STOP_WHEN_FULL_MODE = (1 << 12),
23
+ WEVTS_RETAIN_AND_BACKUP_MODE = (1 << 13),
24
} WEVT_SOURCE_TYPE;
25
26
+BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(WEVT_SOURCE_TYPE)
27
+
28
+#define WEVT_SOURCE_ALL_NAME "All"
29
+#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
30
+#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
31
+#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
32
+#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
33
+#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
34
+#define WEVT_SOURCE_ALL_ENABLED_NAME "All-Enabled"
35
+#define WEVT_SOURCE_ALL_DISABLED_NAME "All-Disabled"
36
+#define WEVT_SOURCE_ALL_FORWARDED_NAME "All-Forwarded"
37
+#define WEVT_SOURCE_ALL_CLASSIC_NAME "All-Classic"
38
+#define WEVT_SOURCE_ALL_BACKUP_MODE_NAME "All-In-Backup-Mode"
39
+#define WEVT_SOURCE_ALL_OVERWRITE_MODE_NAME "All-In-Overwrite-Mode"
40
+#define WEVT_SOURCE_ALL_STOP_WHEN_FULL_MODE_NAME "All-In-StopWhenFull-Mode"
41
+#define WEVT_SOURCE_ALL_RETAIN_AND_BACKUP_MODE_NAME "All-In-RetainAndBackup-Mode"
42
+
43
+#define WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX "All-Of-"
44
+
45
typedef struct {
46
const char *fullname;
47
size_t fullname_len;
@@ -25,6 +49,7 @@ typedef struct {
49
const wchar_t *custom_query;
50
51
STRING *source;
52
+ STRING *provider;
53
WEVT_SOURCE_TYPE source_type;
54
usec_t msg_first_ut;
55
usec_t msg_last_ut;
@@ -40,16 +65,6 @@ typedef struct {
65
extern DICTIONARY *wevt_sources;
66
extern DICTIONARY *used_hashes_registry;
67
43
-#define WEVT_SOURCE_ALL_NAME "All"
44
-#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
45
-#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
46
-#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
47
-#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
48
-#define WEVT_SOURCE_ALL_DIAGNOSTIC_NAME "All-Diagnostic"
49
-#define WEVT_SOURCE_ALL_TRACING_NAME "All-Tracing"
50
-#define WEVT_SOURCE_ALL_PERFORMANCE_NAME "All-Performance"
51
-#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
52
-
68
void wevt_sources_init(void);
69
void wevt_sources_scan(void);
70
void buffer_json_wevt_versions(BUFFER *wb);
src/collectors/windows-events.plugin/windows-events-unicode.c
+8
-2
@@ -5,7 +5,7 @@
5
inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
6
if (src) {
7
// Convert from UTF-8 to wide char (UTF-16)
8
- if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
8
+ if (utf8_to_utf16(dst, dst_size, src, -1) == 0)
9
wcsncpy(dst, L"[failed conv.]", dst_size - 1);
10
}
11
else
@@ -41,7 +41,7 @@ char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len) {
41
42
wchar_t *channel2unicode(const char *utf8str) {
43
static __thread wchar_t buffer[1024];
44
- utf82unicode(buffer, sizeof(buffer) / sizeof(wchar_t), utf8str);
44
+ utf82unicode(buffer, _countof(buffer), utf8str);
45
return buffer;
46
}
47
@@ -69,6 +69,12 @@ char *query2utf8(const wchar_t *query) {
69
return buffer;
70
}
71
72
+char *provider2utf8(const wchar_t *provider) {
73
+ static __thread char buffer[256];
74
+ unicode2utf8(buffer, sizeof(buffer), provider);
75
+ return buffer;
76
+}
77
+
78
bool wevt_str_wchar_to_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len_with_null) {
79
if(!src || !src_len_with_null)
80
goto cleanup;
src/collectors/windows-events.plugin/windows-events-unicode.h
+2
-1
@@ -9,7 +9,7 @@
9
10
typedef enum __attribute__((packed)) {
11
TXT_SOURCE_UNKNOWN = 0,
12
- TXT_SOURCE_PUBLISHER,
12
+ TXT_SOURCE_PROVIDER,
13
TXT_SOURCE_FIELD_CACHE,
14
TXT_SOURCE_EVENT_LOG,
15
TXT_SOURCE_HARDCODED,
@@ -177,6 +177,7 @@ char *channel2utf8(const wchar_t *channel);
177
wchar_t *channel2unicode(const char *utf8str);
178
179
char *query2utf8(const wchar_t *query);
180
+char *provider2utf8(const wchar_t *provider);
181
182
char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len);
183
src/collectors/windows-events.plugin/windows-events.c
+218
-48
@@ -18,14 +18,17 @@ static bool plugin_should_exit = false;
18
#define WEVT_KEYS_INCLUDED_IN_FACETS \
19
"|" WEVT_FIELD_COMPUTER \
20
"|" WEVT_FIELD_PROVIDER \
21
- "|" WEVT_FIELD_SOURCE \
21
"|" WEVT_FIELD_LEVEL \
22
"|" WEVT_FIELD_KEYWORDS \
23
"|" WEVT_FIELD_OPCODE \
24
"|" WEVT_FIELD_TASK \
26
- "|" WEVT_FIELD_USER \
25
+ "|" WEVT_FIELD_ACCOUNT \
26
+ "|" WEVT_FIELD_DOMAIN \
27
+ "|" WEVT_FIELD_SID \
28
""
29
30
+#define query_has_fts(lqs) ((lqs)->rq.query != NULL)
31
+
32
static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
33
if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
34
nd_log(NDLS_COLLECTORS, NDLP_INFO, "Function has been cancelled");
@@ -66,9 +69,10 @@ FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused,
69
70
struct wevt_bin_data {
71
bool rendered;
72
+ WEVT_EVENT ev;
73
WEVT_LOG *log;
74
EVT_HANDLE hEvent;
71
- PROVIDER_META_HANDLE *publisher;
75
+ PROVIDER_META_HANDLE *provider;
76
};
77
78
static void wevt_cleanup_bin_data(void *data) {
@@ -77,21 +81,30 @@ static void wevt_cleanup_bin_data(void *data) {
81
if(d->hEvent)
82
EvtClose(d->hEvent);
83
80
- publisher_release(d->publisher);
84
+ provider_release(d->provider);
85
freez(d);
86
}
87
84
-static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev __maybe_unused) {
88
+static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev) {
89
struct wevt_bin_data *d = mallocz(sizeof(struct wevt_bin_data));
90
91
+#ifdef NETDATA_INTERNAL_CHECKS
92
+ internal_fatal(strcmp(log->ops.provider.data, provider_get_name(log->provider)) != 0,
93
+ "Provider name mismatch in data!");
94
+
95
+ internal_fatal(!UUIDeq(ev->provider, provider_get_uuid(log->provider)),
96
+ "Provider UUID mismatch in data!");
97
+#endif
98
+
99
+ d->ev = *ev;
100
d->log = log;
101
d->rendered = false;
102
103
// take the bookmark
104
d->hEvent = log->hEvent; log->hEvent = NULL;
105
93
- // dup the publisher
94
- d->publisher = publisher_dup(log->publisher);
106
+ // dup the provider
107
+ d->provider = provider_dup(log->provider);
108
109
facets_row_bin_data_set(facets, wevt_cleanup_bin_data, d);
110
}
@@ -99,12 +112,26 @@ static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets,
112
static void wevt_lazy_loading_event_and_xml(struct wevt_bin_data *d, FACET_ROW *row __maybe_unused) {
113
if(d->rendered) return;
114
102
- wevt_get_xml_utf8(d->log, d->publisher, d->hEvent, &d->log->ops.xml);
103
- wevt_get_event_utf8(d->log, d->publisher, d->hEvent, &d->log->ops.event);
115
+#ifdef NETDATA_INTERNAL_CHECKS
116
+ const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
117
+ internal_fatal(!provider_rkv || strcmp(buffer_tostring(provider_rkv->wb), provider_get_name(d->provider)) != 0,
118
+ "Provider of row does not match the bin data associated with it");
119
+
120
+ uint64_t event_record_id = UINT64_MAX;
121
+ const FACET_ROW_KEY_VALUE *event_record_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTRECORDID);
122
+ if(event_record_id_rkv)
123
+ event_record_id = str2uint64_t(buffer_tostring(event_record_id_rkv->wb), NULL);
124
+ internal_fatal(event_record_id != d->ev.id,
125
+ "Event Record ID of row does not match the bin data associated with it");
126
+#endif
127
+
128
+ // the message needs the xml
129
+ EvtFormatMessage_Xml_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.xml);
130
+ EvtFormatMessage_Event_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.event);
131
d->rendered = true;
132
}
133
107
-static void wevt_render_xml(
134
+static void wevt_lazy_load_xml(
135
FACETS *facets,
136
BUFFER *json_array,
137
FACET_ROW_KEY_VALUE *rkv __maybe_unused,
@@ -121,7 +148,7 @@ static void wevt_render_xml(
148
buffer_json_add_array_item_string(json_array, d->log->ops.xml.data);
149
}
150
124
-static void wevt_render_message(
151
+static void wevt_lazy_load_message(
152
FACETS *facets,
153
BUFFER *json_array,
154
FACET_ROW_KEY_VALUE *rkv __maybe_unused,
@@ -217,18 +244,22 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
244
245
facets_register_key_name(
246
facets, WEVT_FIELD_CHANNEL,
220
- FACET_KEY_OPTION_FTS);
247
+ rq->default_facet | FACET_KEY_OPTION_FTS);
248
249
facets_register_key_name(
250
facets, WEVT_FIELD_PROVIDER,
251
rq->default_facet | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
252
253
facets_register_key_name(
227
- facets, WEVT_FIELD_SOURCE,
254
+ facets, WEVT_FIELD_ACCOUNT,
255
rq->default_facet | FACET_KEY_OPTION_FTS);
256
257
facets_register_key_name(
231
- facets, WEVT_FIELD_USER,
258
+ facets, WEVT_FIELD_DOMAIN,
259
+ rq->default_facet | FACET_KEY_OPTION_FTS);
260
+
261
+ facets_register_key_name(
262
+ facets, WEVT_FIELD_SID,
263
rq->default_facet | FACET_KEY_OPTION_FTS);
264
265
facets_register_key_name(
@@ -236,6 +267,11 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
267
rq->default_facet |
268
FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
269
270
+ facets_register_key_name(
271
+ facets, WEVT_FIELD_EVENTS_API,
272
+ rq->default_facet |
273
+ FACET_KEY_OPTION_FTS);
274
+
275
facets_register_key_name(
276
facets, WEVT_FIELD_LEVEL,
277
rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_EXPANDED_FILTER);
@@ -277,14 +313,32 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
313
FACET_KEY_OPTION_NONE);
314
315
facets_register_dynamic_key_name(
280
- facets, WEVT_FIELD_MESSAGE,
316
+ facets,
317
+ WEVT_FIELD_MESSAGE,
318
FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
282
- wevt_render_message, NULL);
319
+ wevt_lazy_load_message,
320
+ NULL);
321
322
facets_register_dynamic_key_name(
285
- facets, WEVT_FIELD_XML,
286
- FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
287
- wevt_render_xml, NULL);
323
+ facets,
324
+ WEVT_FIELD_XML,
325
+ FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
326
+ wevt_lazy_load_xml,
327
+ NULL);
328
+
329
+ if(query_has_fts(lqs)) {
330
+ facets_register_key_name(
331
+ facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN,
332
+ FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
333
+
334
+ facets_register_key_name(
335
+ facets, WEVT_FIELD_EVENT_XML_HIDDEN,
336
+ FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
337
+
338
+ facets_register_key_name(
339
+ facets, WEVT_FIELD_EVENT_DATA_HIDDEN,
340
+ FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
341
+ }
342
343
#ifdef NETDATA_INTERNAL_CHECKS
344
facets_register_key_name(
@@ -315,8 +369,8 @@ static const char *source_to_str(TXT_UTF8 *txt) {
369
case TXT_SOURCE_EVENT_LOG:
370
return "event-log";
371
318
- case TXT_SOURCE_PUBLISHER:
319
- return "publisher";
372
+ case TXT_SOURCE_PROVIDER:
373
+ return "provider";
374
375
case TXT_SOURCE_FIELD_CACHE:
376
return "fields-cache";
@@ -327,34 +381,55 @@ static const char *source_to_str(TXT_UTF8 *txt) {
381
}
382
#endif
383
384
+static const char *events_api_to_str(WEVT_PROVIDER_PLATFORM platform) {
385
+ switch(platform) {
386
+ case WEVT_PLATFORM_WEL:
387
+ return "Windows Event Log";
388
+
389
+ case WEVT_PLATFORM_ETW:
390
+ return "Event Tracing for Windows";
391
+
392
+ case WEVT_PLATFORM_TL:
393
+ return "TraceLogging";
394
+
395
+ default:
396
+ return "Unknown";
397
+ }
398
+}
399
+
400
static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *ev) {
331
- size_t len, bytes = log->ops.content.used;
401
+ static __thread char uuid_str[UUID_STR_LEN];
402
333
- if(log->ops.provider.used > 1) {
334
- bytes += log->ops.provider.used * 2; // unicode is double
403
+ size_t len, bytes = log->ops.raw.system.used + log->ops.raw.user.used;
404
+
405
+ if(!UUIDiszero(ev->provider)) {
406
+ uuid_unparse_lower(ev->provider.uuid, uuid_str);
407
facets_add_key_value_length(
336
- facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
337
- log->ops.provider.data, log->ops.provider.used - 1);
408
+ facets, WEVT_FIELD_PROVIDER_GUID, sizeof(WEVT_FIELD_PROVIDER_GUID) - 1,
409
+ uuid_str, sizeof(uuid_str) - 1);
410
}
411
340
- if(log->ops.source.used > 1) {
341
- bytes += log->ops.source.used * 2;
412
+ if(!UUIDiszero(ev->activity_id)) {
413
+ uuid_unparse_lower(ev->activity_id.uuid, uuid_str);
414
facets_add_key_value_length(
343
- facets, WEVT_FIELD_SOURCE, sizeof(WEVT_FIELD_SOURCE) - 1,
344
- log->ops.source.data, log->ops.source.used - 1);
415
+ facets, WEVT_FIELD_ACTIVITY_ID, sizeof(WEVT_FIELD_ACTIVITY_ID) - 1,
416
+ uuid_str, sizeof(uuid_str) - 1);
417
}
418
347
- if(log->ops.channel.used > 1) {
348
- bytes += log->ops.channel.used * 2;
419
+ if(!UUIDiszero(ev->related_activity_id)) {
420
+ uuid_unparse_lower(ev->related_activity_id.uuid, uuid_str);
421
facets_add_key_value_length(
350
- facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
351
- log->ops.channel.data, log->ops.channel.used - 1);
422
+ facets, WEVT_FIELD_RELATED_ACTIVITY_ID, sizeof(WEVT_FIELD_RELATED_ACTIVITY_ID) - 1,
423
+ uuid_str, sizeof(uuid_str) - 1);
424
}
353
- else {
354
- bytes += src->fullname_len * 2;
425
+
426
+ if(ev->qualifiers) {
427
+ static __thread char qualifiers[UINT64_HEX_MAX_LENGTH];
428
+ len = print_uint64_hex(qualifiers, ev->qualifiers);
429
+ bytes += len;
430
facets_add_key_value_length(
356
- facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
357
- src->fullname, src->fullname_len);
431
+ facets, WEVT_FIELD_QUALIFIERS, sizeof(WEVT_FIELD_QUALIFIERS) - 1,
432
+ qualifiers, len);
433
}
434
435
{
@@ -366,6 +441,35 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
441
event_record_id_str, len);
442
}
443
444
+ if(ev->version) {
445
+ static __thread char version[UINT64_MAX_LENGTH];
446
+ len = print_uint64(version, ev->version);
447
+ bytes += len;
448
+ facets_add_key_value_length(
449
+ facets, WEVT_FIELD_VERSION, sizeof(WEVT_FIELD_VERSION) - 1,
450
+ version, len);
451
+ }
452
+
453
+ if(log->ops.provider.used > 1) {
454
+ bytes += log->ops.provider.used * 2; // unicode is double
455
+ facets_add_key_value_length(
456
+ facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
457
+ log->ops.provider.data, log->ops.provider.used - 1);
458
+ }
459
+
460
+ if(log->ops.channel.used > 1) {
461
+ bytes += log->ops.channel.used * 2;
462
+ facets_add_key_value_length(
463
+ facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
464
+ log->ops.channel.data, log->ops.channel.used - 1);
465
+ }
466
+ else {
467
+ bytes += src->fullname_len * 2;
468
+ facets_add_key_value_length(
469
+ facets, WEVT_FIELD_CHANNEL, sizeof(WEVT_FIELD_CHANNEL) - 1,
470
+ src->fullname, src->fullname_len);
471
+ }
472
+
473
if(log->ops.level.used > 1) {
474
bytes += log->ops.level.used * 2;
475
facets_add_key_value_length(
@@ -401,11 +505,28 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
505
log->ops.task.data, log->ops.task.used - 1);
506
}
507
404
- if(log->ops.user.used > 1) {
405
- bytes += log->ops.user.used * 2;
508
+ if(log->ops.account.used > 1) {
509
+ bytes += log->ops.account.used * 2;
510
+ facets_add_key_value_length(
511
+ facets,
512
+ WEVT_FIELD_ACCOUNT, sizeof(WEVT_FIELD_ACCOUNT) - 1,
513
+ log->ops.account.data, log->ops.account.used - 1);
514
+ }
515
+
516
+ if(log->ops.domain.used > 1) {
517
+ bytes += log->ops.domain.used * 2;
518
facets_add_key_value_length(
407
- facets, WEVT_FIELD_USER, sizeof(WEVT_FIELD_USER) - 1,
408
- log->ops.user.data, log->ops.user.used - 1);
519
+ facets,
520
+ WEVT_FIELD_DOMAIN, sizeof(WEVT_FIELD_DOMAIN) - 1,
521
+ log->ops.domain.data, log->ops.domain.used - 1);
522
+ }
523
+
524
+ if(log->ops.sid.used > 1) {
525
+ bytes += log->ops.sid.used * 2;
526
+ facets_add_key_value_length(
527
+ facets,
528
+ WEVT_FIELD_SID, sizeof(WEVT_FIELD_SID) - 1,
529
+ log->ops.sid.data, log->ops.sid.used - 1);
530
}
531
532
{
@@ -417,6 +538,12 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
538
event_id_str, len);
539
}
540
541
+ {
542
+ const char *s = events_api_to_str(ev->platform);
543
+ facets_add_key_value_length(
544
+ facets, WEVT_FIELD_EVENTS_API, sizeof(WEVT_FIELD_EVENTS_API) - 1, s, strlen(s));
545
+ }
546
+
547
if(ev->process_id) {
548
static __thread char process_id_str[UINT64_MAX_LENGTH];
549
len = print_uint64(process_id_str, ev->process_id);
@@ -467,6 +594,30 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
594
facets, WEVT_FIELD_TASK "ID", sizeof(WEVT_FIELD_TASK) + 2 - 1, str, len);
595
}
596
597
+ if(log->type & WEVT_QUERY_EVENT_DATA) {
598
+ // the query has full text-search
599
+ if(log->ops.event.used > 1) {
600
+ bytes += log->ops.event.used;
601
+ facets_add_key_value_length(
602
+ facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN, sizeof(WEVT_FIELD_EVENT_MESSAGE_HIDDEN) - 1,
603
+ log->ops.event.data, log->ops.event.used - 1);
604
+ }
605
+
606
+ if(log->ops.xml.used > 1) {
607
+ bytes += log->ops.xml.used;
608
+ facets_add_key_value_length(
609
+ facets, WEVT_FIELD_EVENT_XML_HIDDEN, sizeof(WEVT_FIELD_EVENT_XML_HIDDEN) - 1,
610
+ log->ops.xml.data, log->ops.xml.used - 1);
611
+ }
612
+
613
+ if(log->ops.event_data->len) {
614
+ bytes += log->ops.event_data->len;
615
+ facets_add_key_value_length(
616
+ facets, WEVT_FIELD_EVENT_DATA_HIDDEN, sizeof(WEVT_FIELD_EVENT_DATA_HIDDEN) - 1,
617
+ buffer_tostring(log->ops.event_data), buffer_strlen(log->ops.event_data));
618
+ }
619
+ }
620
+
621
wevt_facets_register_bin_data(log, facets, ev);
622
623
#ifdef NETDATA_INTERNAL_CHECKS
@@ -536,7 +687,7 @@ static WEVT_QUERY_STATUS wevt_query_backward(
687
688
facets_rows_begin(facets);
689
WEVT_EVENT e;
539
- while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
690
+ while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
691
usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
692
693
if(unlikely(!msg_ut)) {
@@ -650,7 +801,7 @@ static WEVT_QUERY_STATUS wevt_query_forward(
801
802
facets_rows_begin(facets);
803
WEVT_EVENT e;
653
- while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
804
+ while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
805
usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
806
807
if(unlikely(!msg_ut)) {
@@ -754,8 +905,20 @@ static WEVT_QUERY_STATUS wevt_query_one_channel(
905
}
906
907
static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
757
- if((lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) || (src->source_type & lqs->rq.source_type) ||
758
- (lqs->rq.sources && simple_pattern_matches(lqs->rq.sources, string2str(src->source)))) {
908
+ if(
909
+ // no source is requested
910
+ (lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) ||
911
+
912
+ // matches our internal source types
913
+ (src->source_type & lqs->rq.source_type) ||
914
+
915
+ // matches the source name
916
+ (lqs->rq.sources && src->source && simple_pattern_matches(lqs->rq.sources, string2str(src->source))) ||
917
+
918
+ // matches the provider (providers start with a special prefix to avoid mix and match)
919
+ (lqs->rq.sources && src->provider && simple_pattern_matches(lqs->rq.sources, string2str(src->provider)))
920
+
921
+ ) {
922
923
if(!src->msg_last_ut)
924
// the file is not scanned yet, or the timestamps have not been updated,
@@ -837,7 +1000,7 @@ static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs _
1000
usec_t ended_ut = started_ut;
1001
usec_t duration_ut, max_duration_ut = 0;
1002
840
- WEVT_LOG *log = wevt_openlog6();
1003
+ WEVT_LOG *log = wevt_openlog6(query_has_fts(lqs) ? WEVT_QUERY_FTS : WEVT_QUERY_NORMAL);
1004
if(!log) {
1005
// release the files
1006
for(size_t f = 0; f < files_used ;f++)
@@ -1135,7 +1298,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
1298
// initialization
1299
1300
wevt_sources_init();
1138
- publisher_cache_init();
1301
+ provider_cache_init();
1302
sid_cache_init();
1303
field_cache_init();
1304
@@ -1209,6 +1372,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
1372
const usec_t step_ut = 100 * USEC_PER_MS;
1373
usec_t send_newline_ut = 0;
1374
usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
1375
+ usec_t since_last_providers_release_ut = 0;
1376
const bool tty = isatty(fileno(stdout)) == 1;
1377
1378
heartbeat_t hb;
@@ -1220,7 +1384,13 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
1384
since_last_scan_ut = 0;
1385
}
1386
1387
+ if(since_last_providers_release_ut > WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT) {
1388
+ providers_release_unused_handles();
1389
+ since_last_providers_release_ut = 0;
1390
+ }
1391
+
1392
usec_t dt_ut = heartbeat_next(&hb, step_ut);
1393
+ since_last_providers_release_ut += dt_ut;
1394
since_last_scan_ut += dt_ut;
1395
send_newline_ut += dt_ut;
1396
src/collectors/windows-events.plugin/windows-events.h
+25
-5
@@ -128,10 +128,15 @@ typedef enum {
128
#include "windows-events-unicode.h"
129
#include "windows-events-sid.h"
130
#include "windows-events-xml.h"
131
-#include "windows-events-publishers.h"
131
+#include "windows-events-providers.h"
132
#include "windows-events-fields-cache.h"
133
#include "windows-events-query.h"
134
135
+// enable or disable preloading on full-text-search
136
+#define ON_FTS_PRELOAD_MESSAGE 1
137
+#define ON_FTS_PRELOAD_XML 0
138
+#define ON_FTS_PRELOAD_EVENT_DATA 1
139
+
140
#define WEVT_FUNCTION_DESCRIPTION "View, search and analyze the Microsoft Windows Events log."
141
#define WEVT_FUNCTION_NAME "windows-events"
142
@@ -139,26 +144,40 @@ typedef enum {
144
#define WINDOWS_EVENTS_DEFAULT_TIMEOUT 600
145
#define WINDOWS_EVENTS_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
146
#define WINDOWS_EVENTS_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
142
-
147
#define FUNCTION_PROGRESS_EVERY_ROWS (2000)
148
#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1000)
149
#define ANCHOR_DELTA_UT (10 * USEC_PER_SEC)
150
151
+// run providers release every 5 mins
152
+#define WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT (5 * 60 * USEC_PER_SEC)
153
+// release idle handles that are older than 5 mins
154
+#define WINDOWS_EVENTS_RELEASE_IDLE_PROVIDER_HANDLES_TIME_UT (5 * 60 * USEC_PER_SEC)
155
+
156
#define WEVT_FIELD_COMPUTER "Computer"
157
#define WEVT_FIELD_CHANNEL "Channel"
158
#define WEVT_FIELD_PROVIDER "Provider"
150
-#define WEVT_FIELD_SOURCE "Source"
159
+#define WEVT_FIELD_PROVIDER_GUID "ProviderGUID"
160
#define WEVT_FIELD_EVENTRECORDID "EventRecordID"
161
+#define WEVT_FIELD_VERSION "Version"
162
+#define WEVT_FIELD_QUALIFIERS "Qualifiers"
163
#define WEVT_FIELD_EVENTID "EventID"
164
#define WEVT_FIELD_LEVEL "Level"
165
#define WEVT_FIELD_KEYWORDS "Keywords"
166
#define WEVT_FIELD_OPCODE "Opcode"
156
-#define WEVT_FIELD_USER "User"
167
+#define WEVT_FIELD_ACCOUNT "UserAccount"
168
+#define WEVT_FIELD_DOMAIN "UserDomain"
169
+#define WEVT_FIELD_SID "UserSID"
170
#define WEVT_FIELD_TASK "Task"
171
#define WEVT_FIELD_PROCESSID "ProcessID"
172
#define WEVT_FIELD_THREADID "ThreadID"
173
+#define WEVT_FIELD_ACTIVITY_ID "ActivityID"
174
+#define WEVT_FIELD_RELATED_ACTIVITY_ID "RelatedActivityID"
175
#define WEVT_FIELD_XML "XML"
176
#define WEVT_FIELD_MESSAGE "Message"
177
+#define WEVT_FIELD_EVENTS_API "EventsAPI"
178
+#define WEVT_FIELD_EVENT_DATA_HIDDEN "__HIDDEN__EVENT__DATA__"
179
+#define WEVT_FIELD_EVENT_MESSAGE_HIDDEN "__HIDDEN__MESSAGE__DATA__"
180
+#define WEVT_FIELD_EVENT_XML_HIDDEN "__HIDDEN__XML__DATA__"
181
182
// functions needed by LQS
183
@@ -237,7 +256,8 @@ struct lqs_extension {
256
#define LQS_SOURCE_TYPE WEVT_SOURCE_TYPE
257
#define LQS_SOURCE_TYPE_ALL WEVTS_ALL
258
#define LQS_SOURCE_TYPE_NONE WEVTS_NONE
240
-#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) wevt_internal_source_type(value)
259
+#define LQS_PARAMETER_SOURCE_NAME "Event Channels" // this is how it is shown to users
260
+#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) WEVT_SOURCE_TYPE_2id_one(value)
261
#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) wevt_sources_to_json_array(wb)
262
#include "libnetdata/facets/logs_query_status.h"
263
src/daemon/buildinfo.c
+1
-1
@@ -1331,7 +1331,7 @@ char *get_value_from_key(char *buffer, char *key) {
1331
return s;
1332
}
1333
1334
-void get_install_type(char **install_type, char **prebuilt_arch, char **prebuilt_dist) {
1334
+void get_install_type(char **install_type, char **prebuilt_arch __maybe_unused, char **prebuilt_dist __maybe_unused) {
1335
#ifndef OS_WINDOWS
1336
char *install_type_filename;
1337
src/daemon/main.c
+33
-7
@@ -847,34 +847,60 @@ static void log_init(void) {
847
nd_log_set_priority_level(config_get(CONFIG_SECTION_LOGS, "level", netdata_log_level));
848
849
char filename[FILENAME_MAX + 1];
850
+ char* os_default_method = NULL;
851
+#if defined(OS_LINUX)
852
+ os_default_method = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */ ? "journal" : NULL;
853
+#elif defined(OS_WINDOWS)
854
+#if defined(HAVE_ETW)
855
+ os_default_method = "etw";
856
+#elif defined(HAVE_WEL)
857
+ os_default_method = "wel";
858
+#endif
859
+#endif
860
+
861
+#if defined(OS_WINDOWS)
862
+ // on windows, debug log goes to windows events
863
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
864
+#else
865
snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir);
866
+#endif
867
+
868
nd_log_set_user_settings(NDLS_DEBUG, config_get(CONFIG_SECTION_LOGS, "debug", filename));
869
853
- bool with_journal = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */;
854
- if(with_journal)
855
- snprintfz(filename, FILENAME_MAX, "journal");
870
+ if(os_default_method)
871
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
872
else
873
snprintfz(filename, FILENAME_MAX, "%s/daemon.log", netdata_configured_log_dir);
874
nd_log_set_user_settings(NDLS_DAEMON, config_get(CONFIG_SECTION_LOGS, "daemon", filename));
875
860
- if(with_journal)
861
- snprintfz(filename, FILENAME_MAX, "journal");
876
+ if(os_default_method)
877
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
878
else
879
snprintfz(filename, FILENAME_MAX, "%s/collector.log", netdata_configured_log_dir);
880
nd_log_set_user_settings(NDLS_COLLECTORS, config_get(CONFIG_SECTION_LOGS, "collector", filename));
881
882
+#if defined(OS_WINDOWS)
883
+ // on windows, access log goes to windows events
884
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
885
+#else
886
snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir);
887
+#endif
888
nd_log_set_user_settings(NDLS_ACCESS, config_get(CONFIG_SECTION_LOGS, "access", filename));
889
869
- if(with_journal)
870
- snprintfz(filename, FILENAME_MAX, "journal");
890
+ if(os_default_method)
891
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
892
else
893
snprintfz(filename, FILENAME_MAX, "%s/health.log", netdata_configured_log_dir);
894
nd_log_set_user_settings(NDLS_HEALTH, config_get(CONFIG_SECTION_LOGS, "health", filename));
895
896
aclklog_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "conversation log", CONFIG_BOOLEAN_NO);
897
if (aclklog_enabled) {
898
+#if defined(OS_WINDOWS)
899
+ // on windows, aclk log goes to windows events
900
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
901
+#else
902
snprintfz(filename, FILENAME_MAX, "%s/aclk.log", netdata_configured_log_dir);
903
+#endif
904
nd_log_set_user_settings(NDLS_ACLK, config_get(CONFIG_SECTION_CLOUD, "conversation log file", filename));
905
}
906
src/daemon/winsvc.cc
+4
@@ -219,7 +219,11 @@ static bool update_path() {
219
220
int main(int argc, char *argv[])
221
{
222
+#if defined(OS_WINDOWS) && defined(RUN_UNDER_CLION)
223
+ bool tty = true;
224
+#else
225
bool tty = isatty(fileno(stdin)) == 1;
226
+#endif
227
228
if (!update_path()) {
229
return 1;
src/libnetdata/buffer/buffer.h
+2
-2
@@ -521,13 +521,13 @@ static inline size_t print_int64(char *dst, int64_t value) {
521
522
#define UINT64_MAX_LENGTH (24) // 21 should be enough
523
static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
524
- buffer_need_bytes(wb, 50);
524
+ buffer_need_bytes(wb, UINT64_MAX_LENGTH);
525
wb->len += print_uint64(&wb->buffer[wb->len], value);
526
buffer_overflow_check(wb);
527
}
528
529
static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
530
- buffer_need_bytes(wb, 50);
530
+ buffer_need_bytes(wb, UINT64_MAX_LENGTH);
531
wb->len += print_int64(&wb->buffer[wb->len], value);
532
buffer_overflow_check(wb);
533
}
src/libnetdata/facets/facets.c
+41
-35
@@ -2363,8 +2363,8 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w
2363
2364
if(with_keys) {
2365
FACET_KEY *k;
2366
- foreach_key_in_facets(facets, k){
2367
- if (!k->values.enabled)
2366
+ foreach_key_in_facets(facets, k) {
2367
+ if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
2368
continue;
2369
2370
buffer_json_add_array_item_string(wb, facets_key_id(k));
@@ -2591,7 +2591,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2591
CLEAN_BUFFER *tb = buffer_create(0, NULL);
2592
FACET_KEY *k;
2593
foreach_key_in_facets(facets, k) {
2594
- if(!k->values.enabled)
2594
+ if(!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
2595
continue;
2596
2597
facets_sort_and_reorder_values(k);
@@ -2677,37 +2677,40 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2677
2678
FACET_KEY *k;
2679
foreach_key_in_facets(facets, k) {
2680
- RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
2681
- RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
2682
- RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
2683
-
2684
- if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) ||
2685
- ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) ||
2686
- simple_pattern_matches(facets->visible_keys, k->name))
2687
- options |= RRDF_FIELD_OPTS_VISIBLE;
2688
-
2689
- if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
2690
- options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
2691
-
2692
- if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
2693
- options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
2694
-
2695
- if (k->options & FACET_KEY_OPTION_PRETTY_XML)
2696
- transform = RRDF_FIELD_TRANSFORM_XML;
2697
-
2698
- const char *key_id = facets_key_id(k);
2699
-
2700
- buffer_rrdf_table_add_field(
2701
- wb, field_id++,
2702
- key_id, k->name ? k->name : key_id,
2703
- RRDF_FIELD_TYPE_STRING,
2704
- visual, transform, 0, NULL, NAN,
2705
- RRDF_FIELD_SORT_FIXED,
2706
- NULL,
2707
- RRDF_FIELD_SUMMARY_COUNT,
2708
- (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
2709
- options, FACET_VALUE_UNSET);
2710
- }
2680
+ if(k->options & FACET_KEY_OPTION_HIDDEN)
2681
+ continue;
2682
+
2683
+ RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
2684
+ RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
2685
+ RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
2686
+
2687
+ if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) ||
2688
+ ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) ||
2689
+ simple_pattern_matches(facets->visible_keys, k->name))
2690
+ options |= RRDF_FIELD_OPTS_VISIBLE;
2691
+
2692
+ if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
2693
+ options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
2694
+
2695
+ if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
2696
+ options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
2697
+
2698
+ if (k->options & FACET_KEY_OPTION_PRETTY_XML)
2699
+ transform = RRDF_FIELD_TRANSFORM_XML;
2700
+
2701
+ const char *key_id = facets_key_id(k);
2702
+
2703
+ buffer_rrdf_table_add_field(
2704
+ wb, field_id++,
2705
+ key_id, k->name ? k->name : key_id,
2706
+ RRDF_FIELD_TYPE_STRING,
2707
+ visual, transform, 0, NULL, NAN,
2708
+ RRDF_FIELD_SORT_FIXED,
2709
+ NULL,
2710
+ RRDF_FIELD_SUMMARY_COUNT,
2711
+ (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
2712
+ options, FACET_VALUE_UNSET);
2713
+ }
2714
foreach_key_in_facets_done(k);
2715
}
2716
buffer_json_object_close(wb); // columns
@@ -2744,6 +2747,9 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2747
2748
FACET_KEY *k;
2749
foreach_key_in_facets(facets, k) {
2750
+ if(k->options & FACET_KEY_OPTION_HIDDEN)
2751
+ continue;
2752
+
2753
FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
2754
2755
if(unlikely(k->dynamic.cb)) {
@@ -2786,7 +2792,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2792
{
2793
FACET_KEY *k;
2794
foreach_key_in_facets(facets, k) {
2789
- if (!k->values.enabled)
2795
+ if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
2796
continue;
2797
2798
if(unlikely(!first_histogram_hash))
src/libnetdata/facets/facets.h
+1
@@ -37,6 +37,7 @@ typedef enum __attribute__((packed)) {
37
FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 10), // when registering the transformation, do it only at the view, not on all data
38
FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 11), // the presentation should have this filter expanded by default
39
FACET_KEY_OPTION_PRETTY_XML = (1 << 12), // instruct the UI to parse this as an XML document
40
+ FACET_KEY_OPTION_HIDDEN = (1 << 13), // do not include this field in the response
41
} FACET_KEY_OPTIONS;
42
43
typedef enum __attribute__((packed)) {
src/libnetdata/facets/logs_query_status.h
-1
@@ -17,7 +17,6 @@
17
#define LQS_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"
18
#define LQS_PARAMETER_DATA_ONLY "data_only"
19
#define LQS_PARAMETER_SOURCE "__logs_sources" // this must never conflict with user fields
20
-#define LQS_PARAMETER_SOURCE_NAME "Logs Sources" // this is how it is shown to users
20
#define LQS_PARAMETER_INFO "info"
21
#define LQS_PARAMETER_SLICE "slice"
22
#define LQS_PARAMETER_DELTA "delta"
src/libnetdata/libnetdata.h
+2
-2
@@ -459,14 +459,14 @@ extern const char *netdata_configured_host_prefix;
459
#include "datetime/rfc3339.h"
460
#include "datetime/rfc7231.h"
461
#include "completion/completion.h"
462
-#include "log/log.h"
462
+#include "libnetdata/log/nd_log.h"
463
#include "spawn_server/spawn_server.h"
464
#include "spawn_server/spawn_popen.h"
465
#include "simple_pattern/simple_pattern.h"
466
#include "socket/security.h"
467
#include "socket/socket.h"
468
#include "config/appconfig.h"
469
-#include "log/journal.h"
469
+#include "log/systemd-journal-helpers.h"
470
#include "buffered_reader/buffered_reader.h"
471
#include "procfile/procfile.h"
472
#include "string/string.h"
src/libnetdata/log/README.md
+187
-63
@@ -17,14 +17,15 @@ For each log source, Netdata supports the following output methods:
17
18
- **off**, to disable this log source
19
- **journal**, to send the logs to systemd-journal.
20
+- **etw**, to send the logs to Event Tracing for Windows (ETW).
21
+- **wel**, to send the logs to the Windows Event Log (WEL).
22
- **syslog**, to send the logs to syslog.
23
- **system**, to send the output to `stderr` or `stdout` depending on the log source.
24
- **stdout**, to write the logs to Netdata's `stdout`.
25
- **stderr**, to write the logs to Netdata's `stderr`.
26
- **filename**, to send the logs to a file.
27
26
-For `daemon` and `collector` the default is `journal` when systemd-journal is available.
27
-To decide if systemd-journal is available, Netdata checks:
28
+On Linux, when systemd-journal is available, the default is `journal` for `daemon` and `collector` and `filename` for the rest. To decide if systemd-journal is available, Netdata checks:
29
30
1. `stderr` is connected to systemd-journald
31
2. `/run/systemd/journal/socket` exists
@@ -32,13 +33,16 @@ To decide if systemd-journal is available, Netdata checks:
33
34
If any of the above is detected, Netdata will select `journal` for `daemon` and `collector` sources.
35
35
-All other sources default to a file.
36
+On Windows, the default is `etw` and if that is not available it falls back to `wel`. The availability of `etw` is decided at compile time.
37
38
## Log formats
39
40
| Format | Description |
41
|---------|--------------------------------------------------------------------------------------------------------|
42
| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
43
+| etw | Event Tracing for Windows specific format. Structured logging in Event Viewer. |
44
+| wel | Windows Event Log specific format. Basic field-based logging in Event Viewer. |
45
+| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
46
| logfmt | logs data as a series of key/value pairs. The default when logging to any output other than `journal`. |
47
| json | logs data in JSON format. |
48
@@ -57,6 +61,9 @@ Each time Netdata logs, it assigns a priority to the log. It can be one of this
61
| info | the default log level about information the user should know. |
62
| debug | these are more verbose logs that can be ignored. |
63
64
+For `etw` these are mapped to `Verbose`, `Informational`, `Warning`, `Error` and `Critical`.
65
+For `wel` these are mapped to `Informational`, `Warning`, `Error`.
66
+
67
## Logs Configuration
68
69
In `netdata.conf`, there are the following settings:
@@ -108,66 +115,69 @@ Sending a `SIGHUP` to Netdata, will instruct it to re-open all its log files.
115
<details>
116
<summary>All fields exposed by Netdata</summary>
117
111
-| journal | logfmt | json | Description |
112
-|:--------------------------------------:|:------------------------------:|:------------------------------:|:---------------------------------------------------------------------------------------------------------:|
113
-| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `time` | the timestamp of the event |
114
-| `SYSLOG_IDENTIFIER` | `comm` | `comm` | the program logging the event |
115
-| `ND_LOG_SOURCE` | `source` | `source` | one of the [log sources](#log-sources) |
116
-| `PRIORITY`<br/>numeric | `level`<br/>text | `level`<br/>numeric | one of the [log levels](#log-levels) |
117
-| `ERRNO` | `errno` | `errno` | the numeric value of `errno` |
118
-| `INVOCATION_ID` | - | - | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
119
-| `CODE_LINE` | - | - | the line number of of the source code logging this event |
120
-| `CODE_FILE` | - | - | the filename of the source code logging this event |
121
-| `CODE_FUNCTION` | - | - | the function name of the source code logging this event |
122
-| `TID` | `tid` | `tid` | the thread id of the thread logging this event |
123
-| `THREAD_TAG` | `thread` | `thread` | the name of the thread logging this event |
124
-| `MESSAGE_ID` | `msg_id` | `msg_id` | see [message IDs](#message-ids) |
125
-| `ND_MODULE` | `module` | `module` | the Netdata module logging this event |
126
-| `ND_NIDL_NODE` | `node` | `node` | the hostname of the node the event is related to |
127
-| `ND_NIDL_INSTANCE` | `instance` | `instance` | the instance of the node the event is related to |
128
-| `ND_NIDL_CONTEXT` | `context` | `context` | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
129
-| `ND_NIDL_DIMENSION` | `dimension` | `dimension` | the dimension the event is related to |
130
-| `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport |
131
-| `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from |
132
-| `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from |
133
-| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `src_forwarded_host` | the contents of the HTTP header `X-Forwarded-Host` |
134
-| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `src_forwarded_for` | the contents of the HTTP header `X-Forwarded-For` |
135
-| `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child |
136
-| `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport |
137
-| `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination |
138
-| `ND_DST_PORT` | `dst_port` | `dst_port` | when the event happened during an outbound request, this is the port the request destination |
139
-| `ND_DST_CAPABILITIES` | `dst_capabilities` | `dst_capabilities` | when the request goes to a parent, this is the communication capabilities of the parent |
140
-| `ND_REQUEST_METHOD` | `req_method` | `req_method` | when the event happened during an inbound request, this is the method the request was received |
141
-| `ND_RESPONSE_CODE` | `code` | `code` | when responding to a request, this this the response code |
142
-| `ND_CONNECTION_ID` | `conn` | `conn` | when there is a connection id for an inbound connection, this is the connection id |
143
-| `ND_TRANSACTION_ID` | `transaction` | `transaction` | the transaction id (UUID) of all API requests |
144
-| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `sent_bytes` | the bytes we sent to API responses |
145
-| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `size_bytes` | the uncompressed bytes of the API responses |
146
-| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `prep_ut` | the time needed to prepare a response |
147
-| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `sent_ut` | the time needed to send a response |
148
-| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `total_ut` | the total time needed to complete a response |
149
-| `ND_ALERT_ID` | `alert_id` | `alert_id` | the alert id this event is related to |
150
-| `ND_ALERT_EVENT_ID` | `alert_event_id` | `alert_event_id` | a sequential number of the alert transition (per host) |
151
-| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `alert_unique_id` | a sequential number of the alert transition (per alert) |
152
-| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `alert_transition_id` | the unique UUID of this alert transition |
153
-| `ND_ALERT_CONFIG` | `alert_config` | `alert_config` | the alert configuration hash (UUID) |
154
-| `ND_ALERT_NAME` | `alert` | `alert` | the alert name |
155
-| `ND_ALERT_CLASS` | `alert_class` | `alert_class` | the alert classification |
156
-| `ND_ALERT_COMPONENT` | `alert_component` | `alert_component` | the alert component |
157
-| `ND_ALERT_TYPE` | `alert_type` | `alert_type` | the alert type |
158
-| `ND_ALERT_EXEC` | `alert_exec` | `alert_exec` | the alert notification program |
159
-| `ND_ALERT_RECIPIENT` | `alert_recipient` | `alert_recipient` | the alert recipient(s) |
160
-| `ND_ALERT_VALUE` | `alert_value` | `alert_value` | the current alert value |
161
-| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
162
-| `ND_ALERT_STATUS` | `alert_status` | `alert_status` | the current alert status |
163
-| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
164
-| `ND_ALERT_UNITS` | `alert_units` | `alert_units` | the units of the alert |
165
-| `ND_ALERT_SUMMARY` | `alert_summary` | `alert_summary` | the summary text of the alert |
166
-| `ND_ALERT_INFO` | `alert_info` | `alert_info` | the info text of the alert |
167
-| `ND_ALERT_DURATION` | `alert_duration` | `alert_duration` | the duration the alert was in its previous state |
168
-| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `alert_notification_timestamp` | the timestamp the notification delivery is scheduled |
169
-| `ND_REQUEST` | `request` | `request` | the full request during which the event happened |
170
-| `MESSAGE` | `msg` | `msg` | the event message |
118
+| `journal` | `logfmt` and `json` | `etw` | `wel` | Description |
119
+|:--------------------------------------:|:------------------------------:|:-----------------------------:|:-----:|:----------------------------------------------------------------------------------------------------------|
120
+| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `Timestamp` | 1 | the timestamp of the event |
121
+| `SYSLOG_IDENTIFIER` | `comm` | `Program` | 2 | the program logging the event |
122
+| `ND_LOG_SOURCE` | `source` | `NetdataLogSource` | 3 | one of the [log sources](#log-sources) |
123
+| `PRIORITY`<br/>numeric | `level`<br/>text | `Level`<br/>text | 4 | one of the [log levels](#log-levels) |
124
+| `ERRNO` | `errno` | `UnixErrno` | 5 | the numeric value of `errno` |
125
+| `INVOCATION_ID` | - | `InvocationID` | 7 | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
126
+| `CODE_LINE` | - | `CodeLine` | 8 | the line number of of the source code logging this event |
127
+| `CODE_FILE` | - | `CodeFile` | 9 | the filename of the source code logging this event |
128
+| `CODE_FUNCTION` | - | `CodeFunction` | 10 | the function name of the source code logging this event |
129
+| `TID` | `tid` | `ThreadID` | 11 | the thread id of the thread logging this event |
130
+| `THREAD_TAG` | `thread` | `ThreadName` | 12 | the name of the thread logging this event |
131
+| `MESSAGE_ID` | `msg_id` | `MessageID` | 13 | see [message IDs](#message-ids) |
132
+| `ND_MODULE` | `module` | `Module` | 14 | the Netdata module logging this event |
133
+| `ND_NIDL_NODE` | `node` | `Node` | 15 | the hostname of the node the event is related to |
134
+| `ND_NIDL_INSTANCE` | `instance` | `Instance` | 16 | the instance of the node the event is related to |
135
+| `ND_NIDL_CONTEXT` | `context` | `Context` | 17 | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
136
+| `ND_NIDL_DIMENSION` | `dimension` | `Dimension` | 18 | the dimension the event is related to |
137
+| `ND_SRC_TRANSPORT` | `src_transport` | `SourceTransport` | 19 | when the event happened during a request, this is the request transport |
138
+| `ND_SRC_IP` | `src_ip` | `SourceIP` | 24 | when the event happened during an inbound request, this is the IP the request came from |
139
+| `ND_SRC_PORT` | `src_port` | `SourcePort` | 25 | when the event happened during an inbound request, this is the port the request came from |
140
+| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `SourceForwardedHost` | 26 | the contents of the HTTP header `X-Forwarded-Host` |
141
+| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `SourceForwardedFor` | 27 | the contents of the HTTP header `X-Forwarded-For` |
142
+| `ND_SRC_CAPABILITIES` | `src_capabilities` | `SourceCapabilities` | 28 | when the request came from a child, this is the communication capabilities of the child |
143
+| `ND_DST_TRANSPORT` | `dst_transport` | `DestinationTransport` | 29 | when the event happened during an outbound request, this is the outbound request transport |
144
+| `ND_DST_IP` | `dst_ip` | `DestinationIP` | 30 | when the event happened during an outbound request, this is the IP the request destination |
145
+| `ND_DST_PORT` | `dst_port` | `DestinationPort` | 31 | when the event happened during an outbound request, this is the port the request destination |
146
+| `ND_DST_CAPABILITIES` | `dst_capabilities` | `DestinationCapabilities` | 32 | when the request goes to a parent, this is the communication capabilities of the parent |
147
+| `ND_REQUEST_METHOD` | `req_method` | `RequestMethod` | 33 | when the event happened during an inbound request, this is the method the request was received |
148
+| `ND_RESPONSE_CODE` | `code` | `ResponseCode` | 34 | when responding to a request, this this the response code |
149
+| `ND_CONNECTION_ID` | `conn` | `ConnectionID` | 35 | when there is a connection id for an inbound connection, this is the connection id |
150
+| `ND_TRANSACTION_ID` | `transaction` | `TransactionID` | 36 | the transaction id (UUID) of all API requests |
151
+| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `ResponseSentBytes` | 37 | the bytes we sent to API responses |
152
+| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `ResponseSizeBytes` | 38 | the uncompressed bytes of the API responses |
153
+| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `ResponsePreparationTimeUsec` | 39 | the time needed to prepare a response |
154
+| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `ResponseSentTimeUsec` | 40 | the time needed to send a response |
155
+| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `ResponseTotalTimeUsec` | 41 | the total time needed to complete a response |
156
+| `ND_ALERT_ID` | `alert_id` | `AlertID` | 42 | the alert id this event is related to |
157
+| `ND_ALERT_EVENT_ID` | `alert_event_id` | `AlertEventID` | 44 | a sequential number of the alert transition (per host) |
158
+| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `AlertUniqueID` | 43 | a sequential number of the alert transition (per alert) |
159
+| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `AlertTransitionID` | 45 | the unique UUID of this alert transition |
160
+| `ND_ALERT_CONFIG` | `alert_config` | `AlertConfig` | 46 | the alert configuration hash (UUID) |
161
+| `ND_ALERT_NAME` | `alert` | `AlertName` | 47 | the alert name |
162
+| `ND_ALERT_CLASS` | `alert_class` | `AlertClass` | 48 | the alert classification |
163
+| `ND_ALERT_COMPONENT` | `alert_component` | `AlertComponent` | 49 | the alert component |
164
+| `ND_ALERT_TYPE` | `alert_type` | `AlertType` | 50 | the alert type |
165
+| `ND_ALERT_EXEC` | `alert_exec` | `AlertExec` | 51 | the alert notification program |
166
+| `ND_ALERT_RECIPIENT` | `alert_recipient` | `AlertRecipient` | 52 | the alert recipient(s) |
167
+| `ND_ALERT_VALUE` | `alert_value` | `AlertValue` | 54 | the current alert value |
168
+| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `AlertOldValue` | 55 | the previous alert value |
169
+| `ND_ALERT_STATUS` | `alert_status` | `AlertStatus` | 56 | the current alert status |
170
+| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `AlertOldStatus` | 57 | the previous alert status |
171
+| `ND_ALERT_UNITS` | `alert_units` | `AlertUnits` | 59 | the units of the alert |
172
+| `ND_ALERT_SUMMARY` | `alert_summary` | `AlertSummary` | 60 | the summary text of the alert |
173
+| `ND_ALERT_INFO` | `alert_info` | `AlertInfo` | 61 | the info text of the alert |
174
+| `ND_ALERT_DURATION` | `alert_duration` | `AlertDuration` | 53 | the duration the alert was in its previous state |
175
+| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `AlertNotificationTimeUsec` | 62 | the timestamp the notification delivery is scheduled |
176
+| `ND_REQUEST` | `request` | `Request` | 63 | the full request during which the event happened |
177
+| `MESSAGE` | `msg` | `Message` | 64 | the event message |
178
+
179
+For `wel` (Windows Event Logs), all logs have an array of 64 fields strings, and their index number provides their meaning.
180
+For `etw` (Event Tracing for Windows), Netdata logs in a structured way, and field names are available.
181
182
</details>
183
@@ -212,3 +222,117 @@ journalctl -u netdata --namespace=netdata
222
# All netdata logs, the newest entries are displayed first
223
journalctl -u netdata --namespace=netdata -r
224
```
225
+
226
+## Using Event Tracing for Windows (ETW)
227
+
228
+ETW requires the publisher `Netdata` to be registered. Our Windows installer does this automatically.
229
+
230
+Registering the publisher is done via a manifest (`%SystemRoot%\System32\wevt_netdata_manifest.xml`)
231
+and its messages resources DLL (`%SystemRoot%\System32\wevt_netdata.dll`).
232
+
233
+If needed, the publisher can be registered and unregistered manually using these commands:
234
+
235
+```bat
236
+REM register the Netdata publisher
237
+wevtutil im "%SystemRoot%\System32\wevt_netdata_manifest.xml" "/mf:%SystemRoot%\System32\wevt_netdata.dll" "/rf:%SystemRoot%\System32\wevt_netdata.dll"
238
+
239
+REM unregister the Netdata publisher
240
+wevtutil um "%SystemRoot%\System32\wevt_netdata_manifest.xml"
241
+```
242
+
243
+The structure of the logs are as follows:
244
+
245
+ - Publisher `Netdata`
246
+ - Channel `Netdata/Daemon`: general messages about the Netdata service
247
+ - Channel `Netdata/Collector`: general messages about Netdata external plugins
248
+ - Channel `Netdata/Health`: alert transitions and general messages generated by Netdata's health engine
249
+ - Channel `Netdata/Access`: all accesses to Netdata APIs
250
+ - Channel `Netdata/Aclk`: for cloud connectivity tracing (disabled by default)
251
+
252
+Retention can be configured per Channel via the Event Viewer. Netdata does not set a default, so the system default is used.
253
+
254
+> **IMPORTANT**<br/>
255
+> Event Tracing for Windows (ETW) does not allow logging the percentage character `%`.
256
+> The `%` followed by a number, is recursively used for fields expansion and ETW has not
257
+> provided any way to escape the character for preventing further expansion.<br/>
258
+> <br/>
259
+> To work around this limitation, Netdata replaces all `%` which are followed by a number, with `℅`
260
+> (the Unicode character `care of`). Visually, they look similar, but when copying IPv6 addresses
261
+> or URLs from the logs, you have to be careful to manually replace `℅` with `%` before using them.
262
+
263
+## Using Windows Event Logs (WEL)
264
+
265
+WEL has a different logs structure and unfortunately WEL and ETW need to use different names if they are to be used
266
+concurrently.
267
+
268
+For WEL, Netdata logs as follows:
269
+
270
+ - Channel `NetdataWEL` (unfortunately `Netdata` cannot be used, it conflicts with the ETW Publisher name)
271
+ - Publisher `NetdataDaemon`: general messages about the Netdata service
272
+ - Publisher `NetdataCollector`: general messages about Netdata external plugins
273
+ - Publisher `NetdataHealth`: alert transitions and general messages generated by Netdata's health engine
274
+ - Publisher `NetdataAccess`: all accesses to Netdata APIs
275
+ - Publisher `NetdataAclk`: for cloud connectivity tracing (disabled by default)
276
+
277
+Publishers must have unique names system-wide, so we had to prefix them with `Netdata`.
278
+
279
+Retention can be configured per Publisher via the Event Viewer or the Registry.
280
+Netdata sets by default 20MiB for all of them, except `NetdataAclk` (5MiB) and `NetdataAccess` (35MiB),
281
+for a total of 100MiB.
282
+
283
+For WEL some registry entries are needed. Netdata automatically takes care of them when it starts.
284
+
285
+WEL does not have the problem ETW has with the percent character `%`, so Netdata logs it as-is.
286
+
287
+## Differences between ETW and WEL
288
+
289
+There are key differences between ETW and WEL.
290
+
291
+### Publishers and Providers
292
+**Publishers** are collections of ETW Providers. A Publisher is implied by a manifest file,
293
+each of which is considered a Publisher, and each manifest file can define multiple **Providers** in it.
294
+Other than that there is no entity related to **Publishers** in the system.
295
+
296
+**Publishers** are not defined for WEL.
297
+
298
+**Providers** are the applications or modules logging. Provider names must be unique across the system,
299
+for ETW and WEL together.
300
+
301
+To define a **Provider**:
302
+
303
+- ETW requires a **Publisher** manifest coupled with resources DLLs and must be registered
304
+ via `wevtutil` (handled by the Netdata Windows installer automatically).
305
+- WEL requires some registry entries and a message resources DLL (handled by Netdata automatically on startup).
306
+
307
+The Provider appears as `Source` in the Event Viewer, for both WEL and ETW.
308
+
309
+### Channels
310
+- **Channels** for WEL are collections of WEL Providers, (each WEL Provider is a single Stream of logs).
311
+- **Channels** for ETW slice the logs of each Provider into multiple Streams.
312
+
313
+WEL Channels cannot have the same name as ETW Providers. This is why Netdata's ETW provider is
314
+called `Netdata`, and WEL channel is called `NetdataWEL`.
315
+
316
+Despite the fact that ETW **Publishers** and WEL **Channels** are both collections of Providers,
317
+they are not similar. In ETW a Publisher is a collection on the publisher's Providers, but in WEL
318
+a Channel may include independent WEL Providers (e.g. the "Applications" Channel). Additionally,
319
+WEL Channels cannot include ETW Providers.
320
+
321
+### Retention
322
+Retention is always defined per Stream.
323
+
324
+- Retention in ETW is defined per ETW Channel (ETW Provider Stream).
325
+- Retention in WEL is defined per WEL Provider (each WEL Provider is a single Stream).
326
+
327
+### Messages Formatting
328
+- ETW supports recursive fields expansion, and therefore `%N` in fields is expanded recursively
329
+ (or replaced with an error message if expansion fails). Netdata replaces `%N` with `℅N` to stop
330
+ recursive expansion (since `%N` cannot be logged otherwise).
331
+- WEL performs a single field expansion, and therefore the `%` character in fields is never expanded.
332
+
333
+### Usability
334
+
335
+- ETW names all the fields and allows multiple datatypes per field, enabling log consumers to know
336
+ what each field means and its datatype.
337
+- WEL uses a simple string table for fields, and consumers need to map these string fields based on
338
+ their index.
src/libnetdata/log/log.c
deleted
-2555
@@ -1,2555 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the
4
-// source code that makes the calls, allowing our loggers to log the lines of source code that actually log
5
-#define SD_JOURNAL_SUPPRESS_LOCATION
6
-
7
-#include "../libnetdata.h"
8
-
9
-#if defined(OS_WINDOWS)
10
-#include <windows.h>
11
-#endif
12
-
13
-#ifdef __FreeBSD__
14
-#include <sys/endian.h>
15
-#endif
16
-
17
-#ifdef __APPLE__
18
-#include <machine/endian.h>
19
-#endif
20
-
21
-#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
22
-#include <execinfo.h>
23
-#endif
24
-
25
-#ifdef HAVE_SYSTEMD
26
-#include <systemd/sd-journal.h>
27
-#endif
28
-
29
-const char *program_name = "";
30
-uint64_t debug_flags = 0;
31
-int aclklog_enabled = 0;
32
-
33
-// ----------------------------------------------------------------------------
34
-
35
-struct nd_log_source;
36
-static bool nd_log_limit_reached(struct nd_log_source *source);
37
-
38
-// ----------------------------------------------------------------------------
39
-
40
-void errno_clear(void) {
41
- errno = 0;
42
-
43
-#if defined(OS_WINDOWS)
44
- SetLastError(ERROR_SUCCESS);
45
-#endif
46
-}
47
-
48
-// ----------------------------------------------------------------------------
49
-// logging method
50
-
51
-typedef enum __attribute__((__packed__)) {
52
- NDLM_DISABLED = 0,
53
- NDLM_DEVNULL,
54
- NDLM_DEFAULT,
55
- NDLM_JOURNAL,
56
- NDLM_SYSLOG,
57
- NDLM_STDOUT,
58
- NDLM_STDERR,
59
- NDLM_FILE,
60
-} ND_LOG_METHOD;
61
-
62
-static struct {
63
- ND_LOG_METHOD method;
64
- const char *name;
65
-} nd_log_methods[] = {
66
- { .method = NDLM_DISABLED, .name = "none" },
67
- { .method = NDLM_DEVNULL, .name = "/dev/null" },
68
- { .method = NDLM_DEFAULT, .name = "default" },
69
- { .method = NDLM_JOURNAL, .name = "journal" },
70
- { .method = NDLM_SYSLOG, .name = "syslog" },
71
- { .method = NDLM_STDOUT, .name = "stdout" },
72
- { .method = NDLM_STDERR, .name = "stderr" },
73
- { .method = NDLM_FILE, .name = "file" },
74
-};
75
-
76
-static ND_LOG_METHOD nd_log_method2id(const char *method) {
77
- if(!method || !*method)
78
- return NDLM_DEFAULT;
79
-
80
- size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
81
- for(size_t i = 0; i < entries ;i++) {
82
- if(strcmp(nd_log_methods[i].name, method) == 0)
83
- return nd_log_methods[i].method;
84
- }
85
-
86
- return NDLM_FILE;
87
-}
88
-
89
-static const char *nd_log_id2method(ND_LOG_METHOD method) {
90
- size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
91
- for(size_t i = 0; i < entries ;i++) {
92
- if(method == nd_log_methods[i].method)
93
- return nd_log_methods[i].name;
94
- }
95
-
96
- return "unknown";
97
-}
98
-
99
-#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR)
100
-
101
-const char *nd_log_method_for_external_plugins(const char *s) {
102
- if(s && *s) {
103
- ND_LOG_METHOD method = nd_log_method2id(s);
104
- if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method))
105
- return nd_log_id2method(method);
106
- }
107
-
108
- return nd_log_id2method(NDLM_STDERR);
109
-}
110
-
111
-// ----------------------------------------------------------------------------
112
-// workaround strerror_r()
113
-
114
-#if defined(STRERROR_R_CHAR_P)
115
-// GLIBC version of strerror_r
116
-static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
117
-#elif defined(HAVE_STRERROR_R)
118
-// POSIX version of strerror_r
119
-static const char *strerror_result(int a, const char *b) { (void)a; return b; }
120
-#elif defined(HAVE_C__GENERIC)
121
-
122
-// what a trick!
123
-// http://stackoverflow.com/questions/479207/function-overloading-in-c
124
-static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
125
-static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
126
-
127
-#define strerror_result(a, b) _Generic((a), \
128
- int: strerror_result_int, \
129
- char *: strerror_result_string \
130
- )(a, b)
131
-
132
-#else
133
-#error "cannot detect the format of function strerror_r()"
134
-#endif
135
-
136
-static const char *errno2str(int errnum, char *buf, size_t size) {
137
- return strerror_result(strerror_r(errnum, buf, size), buf);
138
-}
139
-
140
-// ----------------------------------------------------------------------------
141
-// facilities
142
-//
143
-// sys/syslog.h (Linux)
144
-// sys/sys/syslog.h (FreeBSD)
145
-// bsd/sys/syslog.h (darwin-xnu)
146
-
147
-static struct {
148
- int facility;
149
- const char *name;
150
-} nd_log_facilities[] = {
151
- { LOG_AUTH, "auth" },
152
- { LOG_AUTHPRIV, "authpriv" },
153
- { LOG_CRON, "cron" },
154
- { LOG_DAEMON, "daemon" },
155
- { LOG_FTP, "ftp" },
156
- { LOG_KERN, "kern" },
157
- { LOG_LPR, "lpr" },
158
- { LOG_MAIL, "mail" },
159
- { LOG_NEWS, "news" },
160
- { LOG_SYSLOG, "syslog" },
161
- { LOG_USER, "user" },
162
- { LOG_UUCP, "uucp" },
163
- { LOG_LOCAL0, "local0" },
164
- { LOG_LOCAL1, "local1" },
165
- { LOG_LOCAL2, "local2" },
166
- { LOG_LOCAL3, "local3" },
167
- { LOG_LOCAL4, "local4" },
168
- { LOG_LOCAL5, "local5" },
169
- { LOG_LOCAL6, "local6" },
170
- { LOG_LOCAL7, "local7" },
171
-
172
-#ifdef __FreeBSD__
173
- { LOG_CONSOLE, "console" },
174
- { LOG_NTP, "ntp" },
175
-
176
- // FreeBSD does not consider 'security' as deprecated.
177
- { LOG_SECURITY, "security" },
178
-#else
179
- // For all other O/S 'security' is mapped to 'auth'.
180
- { LOG_AUTH, "security" },
181
-#endif
182
-
183
-#ifdef __APPLE__
184
- { LOG_INSTALL, "install" },
185
- { LOG_NETINFO, "netinfo" },
186
- { LOG_RAS, "ras" },
187
- { LOG_REMOTEAUTH, "remoteauth" },
188
- { LOG_LAUNCHD, "launchd" },
189
-
190
-#endif
191
-};
192
-
193
-static int nd_log_facility2id(const char *facility) {
194
- size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
195
- for(size_t i = 0; i < entries ;i++) {
196
- if(strcmp(nd_log_facilities[i].name, facility) == 0)
197
- return nd_log_facilities[i].facility;
198
- }
199
-
200
- return LOG_DAEMON;
201
-}
202
-
203
-static const char *nd_log_id2facility(int facility) {
204
- size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
205
- for(size_t i = 0; i < entries ;i++) {
206
- if(nd_log_facilities[i].facility == facility)
207
- return nd_log_facilities[i].name;
208
- }
209
-
210
- return "daemon";
211
-}
212
-
213
-// ----------------------------------------------------------------------------
214
-// priorities
215
-
216
-static struct {
217
- ND_LOG_FIELD_PRIORITY priority;
218
- const char *name;
219
-} nd_log_priorities[] = {
220
- { .priority = NDLP_EMERG, .name = "emergency" },
221
- { .priority = NDLP_EMERG, .name = "emerg" },
222
- { .priority = NDLP_ALERT, .name = "alert" },
223
- { .priority = NDLP_CRIT, .name = "critical" },
224
- { .priority = NDLP_CRIT, .name = "crit" },
225
- { .priority = NDLP_ERR, .name = "error" },
226
- { .priority = NDLP_ERR, .name = "err" },
227
- { .priority = NDLP_WARNING, .name = "warning" },
228
- { .priority = NDLP_WARNING, .name = "warn" },
229
- { .priority = NDLP_NOTICE, .name = "notice" },
230
- { .priority = NDLP_INFO, .name = NDLP_INFO_STR },
231
- { .priority = NDLP_DEBUG, .name = "debug" },
232
-};
233
-
234
-int nd_log_priority2id(const char *priority) {
235
- size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
236
- for(size_t i = 0; i < entries ;i++) {
237
- if(strcmp(nd_log_priorities[i].name, priority) == 0)
238
- return nd_log_priorities[i].priority;
239
- }
240
-
241
- return NDLP_INFO;
242
-}
243
-
244
-const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) {
245
- size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
246
- for(size_t i = 0; i < entries ;i++) {
247
- if(priority == nd_log_priorities[i].priority)
248
- return nd_log_priorities[i].name;
249
- }
250
-
251
- return NDLP_INFO_STR;
252
-}
253
-
254
-// ----------------------------------------------------------------------------
255
-// log sources
256
-
257
-const char *nd_log_sources[] = {
258
- [NDLS_UNSET] = "UNSET",
259
- [NDLS_ACCESS] = "access",
260
- [NDLS_ACLK] = "aclk",
261
- [NDLS_COLLECTORS] = "collector",
262
- [NDLS_DAEMON] = "daemon",
263
- [NDLS_HEALTH] = "health",
264
- [NDLS_DEBUG] = "debug",
265
-};
266
-
267
-size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) {
268
- size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
269
- for(size_t i = 0; i < entries ;i++) {
270
- if(strcmp(nd_log_sources[i], source) == 0)
271
- return i;
272
- }
273
-
274
- return def;
275
-}
276
-
277
-
278
-static const char *nd_log_id2source(ND_LOG_SOURCES source) {
279
- size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
280
- if(source < entries)
281
- return nd_log_sources[source];
282
-
283
- return nd_log_sources[NDLS_COLLECTORS];
284
-}
285
-
286
-// ----------------------------------------------------------------------------
287
-// log output formats
288
-
289
-typedef enum __attribute__((__packed__)) {
290
- NDLF_JOURNAL,
291
- NDLF_LOGFMT,
292
- NDLF_JSON,
293
-} ND_LOG_FORMAT;
294
-
295
-static struct {
296
- ND_LOG_FORMAT format;
297
- const char *name;
298
-} nd_log_formats[] = {
299
- { .format = NDLF_JOURNAL, .name = "journal" },
300
- { .format = NDLF_LOGFMT, .name = "logfmt" },
301
- { .format = NDLF_JSON, .name = "json" },
302
-};
303
-
304
-static ND_LOG_FORMAT nd_log_format2id(const char *format) {
305
- if(!format || !*format)
306
- return NDLF_LOGFMT;
307
-
308
- size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
309
- for(size_t i = 0; i < entries ;i++) {
310
- if(strcmp(nd_log_formats[i].name, format) == 0)
311
- return nd_log_formats[i].format;
312
- }
313
-
314
- return NDLF_LOGFMT;
315
-}
316
-
317
-static const char *nd_log_id2format(ND_LOG_FORMAT format) {
318
- size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
319
- for(size_t i = 0; i < entries ;i++) {
320
- if(format == nd_log_formats[i].format)
321
- return nd_log_formats[i].name;
322
- }
323
-
324
- return "logfmt";
325
-}
326
-
327
-// ----------------------------------------------------------------------------
328
-// format dates
329
-
330
-void log_date(char *buffer, size_t len, time_t now) {
331
- if(unlikely(!buffer || !len))
332
- return;
333
-
334
- time_t t = now;
335
- struct tm *tmp, tmbuf;
336
-
337
- tmp = localtime_r(&t, &tmbuf);
338
-
339
- if (unlikely(!tmp)) {
340
- buffer[0] = '\0';
341
- return;
342
- }
343
-
344
- if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
345
- buffer[0] = '\0';
346
-
347
- buffer[len - 1] = '\0';
348
-}
349
-
350
-// ----------------------------------------------------------------------------
351
-
352
-struct nd_log_limit {
353
- usec_t started_monotonic_ut;
354
- uint32_t counter;
355
- uint32_t prevented;
356
-
357
- uint32_t throttle_period;
358
- uint32_t logs_per_period;
359
- uint32_t logs_per_period_backup;
360
-};
361
-
362
-#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, }
363
-#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, }
364
-
365
-struct nd_log_source {
366
- SPINLOCK spinlock;
367
- ND_LOG_METHOD method;
368
- ND_LOG_FORMAT format;
369
- const char *filename;
370
- int fd;
371
- FILE *fp;
372
-
373
- ND_LOG_FIELD_PRIORITY min_priority;
374
- const char *pending_msg;
375
- struct nd_log_limit limits;
376
-};
377
-
378
-static struct {
379
- nd_uuid_t invocation_id;
380
-
381
- ND_LOG_SOURCES overwrite_process_source;
382
-
383
- struct nd_log_source sources[_NDLS_MAX];
384
-
385
- struct {
386
- bool initialized;
387
- } journal;
388
-
389
- struct {
390
- bool initialized;
391
- int fd;
392
- char filename[FILENAME_MAX + 1];
393
- } journal_direct;
394
-
395
- struct {
396
- bool initialized;
397
- int facility;
398
- } syslog;
399
-
400
- struct {
401
- SPINLOCK spinlock;
402
- bool initialized;
403
- } std_output;
404
-
405
- struct {
406
- SPINLOCK spinlock;
407
- bool initialized;
408
- } std_error;
409
-
410
-} nd_log = {
411
- .overwrite_process_source = 0,
412
- .journal = {
413
- .initialized = false,
414
- },
415
- .journal_direct = {
416
- .initialized = false,
417
- .fd = -1,
418
- },
419
- .syslog = {
420
- .initialized = false,
421
- .facility = LOG_DAEMON,
422
- },
423
- .std_output = {
424
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
425
- .initialized = false,
426
- },
427
- .std_error = {
428
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
429
- .initialized = false,
430
- },
431
- .sources = {
432
- [NDLS_UNSET] = {
433
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
434
- .method = NDLM_DISABLED,
435
- .format = NDLF_JOURNAL,
436
- .filename = NULL,
437
- .fd = -1,
438
- .fp = NULL,
439
- .min_priority = NDLP_EMERG,
440
- .limits = ND_LOG_LIMITS_UNLIMITED,
441
- },
442
- [NDLS_ACCESS] = {
443
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
444
- .method = NDLM_DEFAULT,
445
- .format = NDLF_LOGFMT,
446
- .filename = LOG_DIR "/access.log",
447
- .fd = -1,
448
- .fp = NULL,
449
- .min_priority = NDLP_DEBUG,
450
- .limits = ND_LOG_LIMITS_UNLIMITED,
451
- },
452
- [NDLS_ACLK] = {
453
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
454
- .method = NDLM_FILE,
455
- .format = NDLF_LOGFMT,
456
- .filename = LOG_DIR "/aclk.log",
457
- .fd = -1,
458
- .fp = NULL,
459
- .min_priority = NDLP_DEBUG,
460
- .limits = ND_LOG_LIMITS_UNLIMITED,
461
- },
462
- [NDLS_COLLECTORS] = {
463
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
464
- .method = NDLM_DEFAULT,
465
- .format = NDLF_LOGFMT,
466
- .filename = LOG_DIR "/collector.log",
467
- .fd = STDERR_FILENO,
468
- .fp = NULL,
469
- .min_priority = NDLP_INFO,
470
- .limits = ND_LOG_LIMITS_DEFAULT,
471
- },
472
- [NDLS_DEBUG] = {
473
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
474
- .method = NDLM_DISABLED,
475
- .format = NDLF_LOGFMT,
476
- .filename = LOG_DIR "/debug.log",
477
- .fd = STDOUT_FILENO,
478
- .fp = NULL,
479
- .min_priority = NDLP_DEBUG,
480
- .limits = ND_LOG_LIMITS_UNLIMITED,
481
- },
482
- [NDLS_DAEMON] = {
483
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
484
- .method = NDLM_DEFAULT,
485
- .filename = LOG_DIR "/daemon.log",
486
- .format = NDLF_LOGFMT,
487
- .fd = -1,
488
- .fp = NULL,
489
- .min_priority = NDLP_INFO,
490
- .limits = ND_LOG_LIMITS_DEFAULT,
491
- },
492
- [NDLS_HEALTH] = {
493
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
494
- .method = NDLM_DEFAULT,
495
- .format = NDLF_LOGFMT,
496
- .filename = LOG_DIR "/health.log",
497
- .fd = -1,
498
- .fp = NULL,
499
- .min_priority = NDLP_DEBUG,
500
- .limits = ND_LOG_LIMITS_UNLIMITED,
501
- },
502
- },
503
-};
504
-
505
-__attribute__((constructor)) void initialize_invocation_id(void) {
506
- // check for a NETDATA_INVOCATION_ID
507
- if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) {
508
- // not found, check for systemd set INVOCATION_ID
509
- if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) {
510
- // not found, generate a new one
511
- uuid_generate_random(nd_log.invocation_id);
512
- }
513
- }
514
-
515
- char uuid[UUID_COMPACT_STR_LEN];
516
- uuid_unparse_lower_compact(nd_log.invocation_id, uuid);
517
- nd_setenv("NETDATA_INVOCATION_ID", uuid, 1);
518
-}
519
-
520
-int nd_log_health_fd(void) {
521
- if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1)
522
- return nd_log.sources[NDLS_HEALTH].fd;
523
-
524
- return STDERR_FILENO;
525
-}
526
-
527
-int nd_log_collectors_fd(void) {
528
- if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)
529
- return nd_log.sources[NDLS_COLLECTORS].fd;
530
-
531
- return STDERR_FILENO;
532
-}
533
-
534
-void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {
535
- char buf[FILENAME_MAX + 100];
536
- if(setting && *setting)
537
- strncpyz(buf, setting, sizeof(buf) - 1);
538
- else
539
- buf[0] = '\0';
540
-
541
- struct nd_log_source *ls = &nd_log.sources[source];
542
- char *output = strrchr(buf, '@');
543
-
544
- if(!output)
545
- // all of it is the output
546
- output = buf;
547
- else {
548
- // we found an '@', the next char is the output
549
- *output = '\0';
550
- output++;
551
-
552
- // parse the other params
553
- char *remaining = buf;
554
- while(remaining) {
555
- char *value = strsep_skip_consecutive_separators(&remaining, ",");
556
- if (!value || !*value) continue;
557
-
558
- char *name = strsep_skip_consecutive_separators(&value, "=");
559
- if (!name || !*name) continue;
560
-
561
- if(strcmp(name, "logfmt") == 0)
562
- ls->format = NDLF_LOGFMT;
563
- else if(strcmp(name, "json") == 0)
564
- ls->format = NDLF_JSON;
565
- else if(strcmp(name, "journal") == 0)
566
- ls->format = NDLF_JOURNAL;
567
- else if(strcmp(name, "level") == 0 && value && *value)
568
- ls->min_priority = nd_log_priority2id(value);
569
- else if(strcmp(name, "protection") == 0 && value && *value) {
570
- if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) {
571
- ls->limits = ND_LOG_LIMITS_UNLIMITED;
572
- ls->limits.counter = 0;
573
- ls->limits.prevented = 0;
574
- }
575
- else {
576
- ls->limits = ND_LOG_LIMITS_DEFAULT;
577
-
578
- char *slash = strchr(value, '/');
579
- if(slash) {
580
- *slash = '\0';
581
- slash++;
582
- ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
583
-
584
- int period;
585
- if(!duration_parse_seconds(slash, &period)) {
586
- nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing period '%s'", slash);
587
- period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
588
- }
589
-
590
- ls->limits.throttle_period = period;
591
- }
592
- else {
593
- ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
594
- ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
595
- }
596
- }
597
- }
598
- else
599
- nd_log(NDLS_DAEMON, NDLP_ERR,
600
- "Error while parsing configuration of log source '%s'. "
601
- "In config '%s', '%s' is not understood.",
602
- nd_log_id2source(source), setting, name);
603
- }
604
- }
605
-
606
- if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) {
607
- ls->method = NDLM_DISABLED;
608
- ls->filename = "/dev/null";
609
- }
610
- else if(strcmp(output, "journal") == 0) {
611
- ls->method = NDLM_JOURNAL;
612
- ls->filename = NULL;
613
- }
614
- else if(strcmp(output, "syslog") == 0) {
615
- ls->method = NDLM_SYSLOG;
616
- ls->filename = NULL;
617
- }
618
- else if(strcmp(output, "/dev/null") == 0) {
619
- ls->method = NDLM_DEVNULL;
620
- ls->filename = "/dev/null";
621
- }
622
- else if(strcmp(output, "system") == 0) {
623
- if(ls->fd == STDERR_FILENO) {
624
- ls->method = NDLM_STDERR;
625
- ls->filename = NULL;
626
- ls->fd = STDERR_FILENO;
627
- }
628
- else {
629
- ls->method = NDLM_STDOUT;
630
- ls->filename = NULL;
631
- ls->fd = STDOUT_FILENO;
632
- }
633
- }
634
- else if(strcmp(output, "stderr") == 0) {
635
- ls->method = NDLM_STDERR;
636
- ls->filename = NULL;
637
- ls->fd = STDERR_FILENO;
638
- }
639
- else if(strcmp(output, "stdout") == 0) {
640
- ls->method = NDLM_STDOUT;
641
- ls->filename = NULL;
642
- ls->fd = STDOUT_FILENO;
643
- }
644
- else {
645
- ls->method = NDLM_FILE;
646
- ls->filename = strdupz(output);
647
- }
648
-
649
-#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
650
- ls->min_priority = NDLP_DEBUG;
651
-#endif
652
-
653
- if(source == NDLS_COLLECTORS) {
654
- // set the method for the collector processes we will spawn
655
-
656
- ND_LOG_METHOD method;
657
- ND_LOG_FORMAT format = ls->format;
658
- ND_LOG_FIELD_PRIORITY priority = ls->min_priority;
659
-
660
- if(ls->method == NDLM_SYSLOG || ls->method == NDLM_JOURNAL)
661
- method = ls->method;
662
- else
663
- method = NDLM_STDERR;
664
-
665
- nd_setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1);
666
- nd_setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1);
667
- nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
668
- }
669
-}
670
-
671
-void nd_log_set_priority_level(const char *setting) {
672
- if(!setting || !*setting)
673
- setting = "info";
674
-
675
- ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting);
676
-
677
-#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
678
- priority = NDLP_DEBUG;
679
-#endif
680
-
681
- for (size_t i = 0; i < _NDLS_MAX; i++) {
682
- if (i != NDLS_DEBUG)
683
- nd_log.sources[i].min_priority = priority;
684
- }
685
-
686
- // the right one
687
- nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
688
-}
689
-
690
-void nd_log_set_facility(const char *facility) {
691
- if(!facility || !*facility)
692
- facility = "daemon";
693
-
694
- nd_log.syslog.facility = nd_log_facility2id(facility);
695
- nd_setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1);
696
-}
697
-
698
-void nd_log_set_flood_protection(size_t logs, time_t period) {
699
- nd_log.sources[NDLS_DAEMON].limits.logs_per_period =
700
- nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup;
701
- nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period =
702
- nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs;
703
-
704
- nd_log.sources[NDLS_DAEMON].limits.throttle_period =
705
- nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period;
706
-
707
- char buf[100];
708
- snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period);
709
- nd_setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1);
710
- snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs);
711
- nd_setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1);
712
-}
713
-
714
-static bool nd_log_journal_systemd_init(void) {
715
-#ifdef HAVE_SYSTEMD
716
- nd_log.journal.initialized = true;
717
-#else
718
- nd_log.journal.initialized = false;
719
-#endif
720
-
721
- return nd_log.journal.initialized;
722
-}
723
-
724
-static void nd_log_journal_direct_set_env(void) {
725
- if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL)
726
- nd_setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1);
727
-}
728
-
729
-static bool nd_log_journal_direct_init(const char *path) {
730
- if(nd_log.journal_direct.initialized) {
731
- nd_log_journal_direct_set_env();
732
- return true;
733
- }
734
-
735
- int fd;
736
- char filename[FILENAME_MAX + 1];
737
- if(!is_path_unix_socket(path)) {
738
-
739
- journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
740
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
741
-
742
- journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
743
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
744
-
745
- journal_construct_path(filename, sizeof(filename), NULL, "netdata");
746
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
747
-
748
- journal_construct_path(filename, sizeof(filename), NULL, NULL);
749
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
750
- return false;
751
- }
752
- }
753
- }
754
- }
755
- else {
756
- snprintfz(filename, sizeof(filename), "%s", path);
757
- fd = journal_direct_fd(filename);
758
- }
759
-
760
- if(fd < 0)
761
- return false;
762
-
763
- nd_log.journal_direct.fd = fd;
764
- nd_log.journal_direct.initialized = true;
765
-
766
- strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1);
767
- nd_log_journal_direct_set_env();
768
-
769
- return true;
770
-}
771
-
772
-static void nd_log_syslog_init() {
773
- if(nd_log.syslog.initialized)
774
- return;
775
-
776
- openlog(program_name, LOG_PID, nd_log.syslog.facility);
777
- nd_log.syslog.initialized = true;
778
-}
779
-
780
-void nd_log_initialize_for_external_plugins(const char *name) {
781
- // if we don't run under Netdata, log to stderr,
782
- // otherwise, use the logging method Netdata wants us to use.
783
- nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
784
- nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
785
-
786
- nd_log.overwrite_process_source = NDLS_COLLECTORS;
787
- program_name = name;
788
-
789
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
790
- nd_log.sources[i].method = STDERR_FILENO;
791
- nd_log.sources[i].fd = -1;
792
- nd_log.sources[i].fp = NULL;
793
- }
794
-
795
- nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL"));
796
- nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY"));
797
-
798
- time_t period = 1200;
799
- size_t logs = 200;
800
- const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD");
801
- if(s && *s >= '0' && *s <= '9') {
802
- period = str2l(s);
803
- if(period < 0) period = 0;
804
- }
805
-
806
- s = getenv("NETDATA_ERRORS_PER_PERIOD");
807
- if(s && *s >= '0' && *s <= '9')
808
- logs = str2u(s);
809
-
810
- nd_log_set_flood_protection(logs, period);
811
-
812
- if(!netdata_configured_host_prefix) {
813
- s = getenv("NETDATA_HOST_PREFIX");
814
- if(s && *s)
815
- netdata_configured_host_prefix = (char *)s;
816
- }
817
-
818
- ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD"));
819
- ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT"));
820
-
821
- if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) {
822
- if(is_stderr_connected_to_journal()) {
823
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal.");
824
- method = NDLM_JOURNAL;
825
- }
826
- else {
827
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr.");
828
- method = NDLM_STDERR;
829
- }
830
- }
831
-
832
- switch(method) {
833
- case NDLM_JOURNAL:
834
- if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||
835
- !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {
836
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");
837
- method = NDLM_STDERR;
838
- }
839
- break;
840
-
841
- case NDLM_SYSLOG:
842
- nd_log_syslog_init();
843
- break;
844
-
845
- default:
846
- method = NDLM_STDERR;
847
- break;
848
- }
849
-
850
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
851
- nd_log.sources[i].method = method;
852
- nd_log.sources[i].format = format;
853
- nd_log.sources[i].fd = -1;
854
- nd_log.sources[i].fp = NULL;
855
- }
856
-
857
-// nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
858
-}
859
-
860
-static bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) {
861
- if(new_fd == -1 || e->fd == -1 ||
862
- (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) ||
863
- (e->fd == STDERR_FILENO && nd_log.std_error.initialized))
864
- return false;
865
-
866
- if(new_fd != e->fd) {
867
- int t = dup2(new_fd, e->fd);
868
-
869
- bool ret = true;
870
- if (t == -1) {
871
- netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename);
872
- ret = false;
873
- }
874
- else
875
- close(new_fd);
876
-
877
- if(e->fd == STDOUT_FILENO)
878
- nd_log.std_output.initialized = true;
879
- else if(e->fd == STDERR_FILENO)
880
- nd_log.std_error.initialized = true;
881
-
882
- return ret;
883
- }
884
-
885
- return false;
886
-}
887
-
888
-static void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) {
889
- if(e->method == NDLM_DEFAULT)
890
- nd_log_set_user_settings(source, e->filename);
891
-
892
- if((e->method == NDLM_FILE && !e->filename) ||
893
- (e->method == NDLM_DEVNULL && e->fd == -1))
894
- e->method = NDLM_DISABLED;
895
-
896
- if(e->fp)
897
- fflush(e->fp);
898
-
899
- switch(e->method) {
900
- case NDLM_SYSLOG:
901
- nd_log_syslog_init();
902
- break;
903
-
904
- case NDLM_JOURNAL:
905
- nd_log_journal_direct_init(NULL);
906
- nd_log_journal_systemd_init();
907
- break;
908
-
909
- case NDLM_STDOUT:
910
- e->fp = stdout;
911
- e->fd = STDOUT_FILENO;
912
- break;
913
-
914
- case NDLM_DISABLED:
915
- break;
916
-
917
- case NDLM_DEFAULT:
918
- case NDLM_STDERR:
919
- e->method = NDLM_STDERR;
920
- e->fp = stderr;
921
- e->fd = STDERR_FILENO;
922
- break;
923
-
924
- case NDLM_DEVNULL:
925
- case NDLM_FILE: {
926
- int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
927
- if(fd == -1) {
928
- if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) {
929
- e->fd = STDERR_FILENO;
930
- e->method = NDLM_STDERR;
931
- netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename);
932
- }
933
- else
934
- netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd);
935
- }
936
- else {
937
- if (!nd_log_replace_existing_fd(e, fd)) {
938
- if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) {
939
- if(e->fd == STDOUT_FILENO)
940
- e->method = NDLM_STDOUT;
941
- else if(e->fd == STDERR_FILENO)
942
- e->method = NDLM_STDERR;
943
-
944
- // we have dup2() fd, so we can close the one we opened
945
- if(fd != STDOUT_FILENO && fd != STDERR_FILENO)
946
- close(fd);
947
- }
948
- else
949
- e->fd = fd;
950
- }
951
- }
952
-
953
- // at this point we have e->fd set properly
954
-
955
- if(e->fd == STDOUT_FILENO)
956
- e->fp = stdout;
957
- else if(e->fd == STDERR_FILENO)
958
- e->fp = stderr;
959
-
960
- if(!e->fp) {
961
- e->fp = fdopen(e->fd, "a");
962
- if (!e->fp) {
963
- netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename);
964
-
965
- if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO)
966
- close(e->fd);
967
-
968
- e->fp = stderr;
969
- e->fd = STDERR_FILENO;
970
- }
971
- }
972
- else {
973
- if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0)
974
- netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename);
975
- }
976
- }
977
- break;
978
- }
979
-}
980
-
981
-static void nd_log_stdin_init(int fd, const char *filename) {
982
- int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
983
- if(f == -1)
984
- return;
985
-
986
- if(f != fd) {
987
- dup2(f, fd);
988
- close(f);
989
- }
990
-}
991
-
992
-void nd_log_initialize(void) {
993
- nd_log_stdin_init(STDIN_FILENO, "/dev/null");
994
-
995
- for(size_t i = 0 ; i < _NDLS_MAX ; i++)
996
- nd_log_open(&nd_log.sources[i], i);
997
-}
998
-
999
-void nd_log_reopen_log_files(bool log) {
1000
- if(log)
1001
- netdata_log_info("Reopening all log files.");
1002
-
1003
- nd_log.std_output.initialized = false;
1004
- nd_log.std_error.initialized = false;
1005
- nd_log_initialize();
1006
-
1007
- if(log)
1008
- netdata_log_info("Log files re-opened.");
1009
-}
1010
-
1011
-void nd_log_reopen_log_files_for_spawn_server(void) {
1012
- if(nd_log.syslog.initialized) {
1013
- closelog();
1014
- nd_log.syslog.initialized = false;
1015
- nd_log_syslog_init();
1016
- }
1017
-
1018
- if(nd_log.journal_direct.initialized) {
1019
- close(nd_log.journal_direct.fd);
1020
- nd_log.journal_direct.fd = -1;
1021
- nd_log.journal_direct.initialized = false;
1022
- nd_log_journal_direct_init(NULL);
1023
- }
1024
-
1025
- nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED;
1026
- nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED;
1027
- nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED;
1028
- nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED;
1029
- nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED;
1030
- nd_log_reopen_log_files(false);
1031
-}
1032
-
1033
-void chown_open_file(int fd, uid_t uid, gid_t gid) {
1034
- if(fd == -1) return;
1035
-
1036
- struct stat buf;
1037
-
1038
- if(fstat(fd, &buf) == -1) {
1039
- netdata_log_error("Cannot fstat() fd %d", fd);
1040
- return;
1041
- }
1042
-
1043
- if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) {
1044
- if(fchown(fd, uid, gid) == -1)
1045
- netdata_log_error("Cannot fchown() fd %d.", fd);
1046
- }
1047
-}
1048
-
1049
-void nd_log_chown_log_files(uid_t uid, gid_t gid) {
1050
- for(size_t i = 0 ; i < _NDLS_MAX ; i++) {
1051
- if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO)
1052
- chown_open_file(nd_log.sources[i].fd, uid, gid);
1053
- }
1054
-}
1055
-
1056
-// ----------------------------------------------------------------------------
1057
-// annotators
1058
-struct log_field;
1059
-static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf);
1060
-static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf);
1061
-static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf);
1062
-
1063
-#if defined(OS_WINDOWS)
1064
-static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf);
1065
-#endif
1066
-
1067
-// ----------------------------------------------------------------------------
1068
-
1069
-typedef void (*annotator_t)(BUFFER *wb, const char *key, struct log_field *lf);
1070
-
1071
-struct log_field {
1072
- const char *journal;
1073
- const char *logfmt;
1074
- annotator_t logfmt_annotator;
1075
- struct log_stack_entry entry;
1076
-};
1077
-
1078
-#define THREAD_LOG_STACK_MAX 50
1079
-
1080
-static __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
1081
-static __thread size_t thread_log_stack_next = 0;
1082
-
1083
-static __thread struct log_field thread_log_fields[_NDF_MAX] = {
1084
- // THE ORDER DEFINES THE ORDER FIELDS WILL APPEAR IN logfmt
1085
-
1086
- [NDF_STOP] = { // processing will not stop on this - so it is ok to be first
1087
- .journal = NULL,
1088
- .logfmt = NULL,
1089
- .logfmt_annotator = NULL,
1090
- },
1091
- [NDF_TIMESTAMP_REALTIME_USEC] = {
1092
- .journal = NULL,
1093
- .logfmt = "time",
1094
- .logfmt_annotator = timestamp_usec_annotator,
1095
- },
1096
- [NDF_SYSLOG_IDENTIFIER] = {
1097
- .journal = "SYSLOG_IDENTIFIER", // standard journald field
1098
- .logfmt = "comm",
1099
- },
1100
- [NDF_LOG_SOURCE] = {
1101
- .journal = "ND_LOG_SOURCE",
1102
- .logfmt = "source",
1103
- },
1104
- [NDF_PRIORITY] = {
1105
- .journal = "PRIORITY", // standard journald field
1106
- .logfmt = "level",
1107
- .logfmt_annotator = priority_annotator,
1108
- },
1109
- [NDF_ERRNO] = {
1110
- .journal = "ERRNO", // standard journald field
1111
- .logfmt = "errno",
1112
- .logfmt_annotator = errno_annotator,
1113
- },
1114
-#if defined(OS_WINDOWS)
1115
- [NDF_WINERROR] = {
1116
- .journal = "WINERROR",
1117
- .logfmt = "winerror",
1118
- .logfmt_annotator = winerror_annotator,
1119
- },
1120
-#endif
1121
- [NDF_INVOCATION_ID] = {
1122
- .journal = "INVOCATION_ID", // standard journald field
1123
- .logfmt = NULL,
1124
- },
1125
- [NDF_LINE] = {
1126
- .journal = "CODE_LINE", // standard journald field
1127
- .logfmt = NULL,
1128
- },
1129
- [NDF_FILE] = {
1130
- .journal = "CODE_FILE", // standard journald field
1131
- .logfmt = NULL,
1132
- },
1133
- [NDF_FUNC] = {
1134
- .journal = "CODE_FUNC", // standard journald field
1135
- .logfmt = NULL,
1136
- },
1137
- [NDF_TID] = {
1138
- .journal = "TID", // standard journald field
1139
- .logfmt = "tid",
1140
- },
1141
- [NDF_THREAD_TAG] = {
1142
- .journal = "THREAD_TAG",
1143
- .logfmt = "thread",
1144
- },
1145
- [NDF_MESSAGE_ID] = {
1146
- .journal = "MESSAGE_ID",
1147
- .logfmt = "msg_id",
1148
- },
1149
- [NDF_MODULE] = {
1150
- .journal = "ND_MODULE",
1151
- .logfmt = "module",
1152
- },
1153
- [NDF_NIDL_NODE] = {
1154
- .journal = "ND_NIDL_NODE",
1155
- .logfmt = "node",
1156
- },
1157
- [NDF_NIDL_INSTANCE] = {
1158
- .journal = "ND_NIDL_INSTANCE",
1159
- .logfmt = "instance",
1160
- },
1161
- [NDF_NIDL_CONTEXT] = {
1162
- .journal = "ND_NIDL_CONTEXT",
1163
- .logfmt = "context",
1164
- },
1165
- [NDF_NIDL_DIMENSION] = {
1166
- .journal = "ND_NIDL_DIMENSION",
1167
- .logfmt = "dimension",
1168
- },
1169
- [NDF_SRC_TRANSPORT] = {
1170
- .journal = "ND_SRC_TRANSPORT",
1171
- .logfmt = "src_transport",
1172
- },
1173
- [NDF_ACCOUNT_ID] = {
1174
- .journal = "ND_ACCOUNT_ID",
1175
- .logfmt = "account",
1176
- },
1177
- [NDF_USER_NAME] = {
1178
- .journal = "ND_USER_NAME",
1179
- .logfmt = "user",
1180
- },
1181
- [NDF_USER_ROLE] = {
1182
- .journal = "ND_USER_ROLE",
1183
- .logfmt = "role",
1184
- },
1185
- [NDF_USER_ACCESS] = {
1186
- .journal = "ND_USER_PERMISSIONS",
1187
- .logfmt = "permissions",
1188
- },
1189
- [NDF_SRC_IP] = {
1190
- .journal = "ND_SRC_IP",
1191
- .logfmt = "src_ip",
1192
- },
1193
- [NDF_SRC_FORWARDED_HOST] = {
1194
- .journal = "ND_SRC_FORWARDED_HOST",
1195
- .logfmt = "src_forwarded_host",
1196
- },
1197
- [NDF_SRC_FORWARDED_FOR] = {
1198
- .journal = "ND_SRC_FORWARDED_FOR",
1199
- .logfmt = "src_forwarded_for",
1200
- },
1201
- [NDF_SRC_PORT] = {
1202
- .journal = "ND_SRC_PORT",
1203
- .logfmt = "src_port",
1204
- },
1205
- [NDF_SRC_CAPABILITIES] = {
1206
- .journal = "ND_SRC_CAPABILITIES",
1207
- .logfmt = "src_capabilities",
1208
- },
1209
- [NDF_DST_TRANSPORT] = {
1210
- .journal = "ND_DST_TRANSPORT",
1211
- .logfmt = "dst_transport",
1212
- },
1213
- [NDF_DST_IP] = {
1214
- .journal = "ND_DST_IP",
1215
- .logfmt = "dst_ip",
1216
- },
1217
- [NDF_DST_PORT] = {
1218
- .journal = "ND_DST_PORT",
1219
- .logfmt = "dst_port",
1220
- },
1221
- [NDF_DST_CAPABILITIES] = {
1222
- .journal = "ND_DST_CAPABILITIES",
1223
- .logfmt = "dst_capabilities",
1224
- },
1225
- [NDF_REQUEST_METHOD] = {
1226
- .journal = "ND_REQUEST_METHOD",
1227
- .logfmt = "req_method",
1228
- },
1229
- [NDF_RESPONSE_CODE] = {
1230
- .journal = "ND_RESPONSE_CODE",
1231
- .logfmt = "code",
1232
- },
1233
- [NDF_CONNECTION_ID] = {
1234
- .journal = "ND_CONNECTION_ID",
1235
- .logfmt = "conn",
1236
- },
1237
- [NDF_TRANSACTION_ID] = {
1238
- .journal = "ND_TRANSACTION_ID",
1239
- .logfmt = "transaction",
1240
- },
1241
- [NDF_RESPONSE_SENT_BYTES] = {
1242
- .journal = "ND_RESPONSE_SENT_BYTES",
1243
- .logfmt = "sent_bytes",
1244
- },
1245
- [NDF_RESPONSE_SIZE_BYTES] = {
1246
- .journal = "ND_RESPONSE_SIZE_BYTES",
1247
- .logfmt = "size_bytes",
1248
- },
1249
- [NDF_RESPONSE_PREPARATION_TIME_USEC] = {
1250
- .journal = "ND_RESPONSE_PREP_TIME_USEC",
1251
- .logfmt = "prep_ut",
1252
- },
1253
- [NDF_RESPONSE_SENT_TIME_USEC] = {
1254
- .journal = "ND_RESPONSE_SENT_TIME_USEC",
1255
- .logfmt = "sent_ut",
1256
- },
1257
- [NDF_RESPONSE_TOTAL_TIME_USEC] = {
1258
- .journal = "ND_RESPONSE_TOTAL_TIME_USEC",
1259
- .logfmt = "total_ut",
1260
- },
1261
- [NDF_ALERT_ID] = {
1262
- .journal = "ND_ALERT_ID",
1263
- .logfmt = "alert_id",
1264
- },
1265
- [NDF_ALERT_UNIQUE_ID] = {
1266
- .journal = "ND_ALERT_UNIQUE_ID",
1267
- .logfmt = "alert_unique_id",
1268
- },
1269
- [NDF_ALERT_TRANSITION_ID] = {
1270
- .journal = "ND_ALERT_TRANSITION_ID",
1271
- .logfmt = "alert_transition_id",
1272
- },
1273
- [NDF_ALERT_EVENT_ID] = {
1274
- .journal = "ND_ALERT_EVENT_ID",
1275
- .logfmt = "alert_event_id",
1276
- },
1277
- [NDF_ALERT_CONFIG_HASH] = {
1278
- .journal = "ND_ALERT_CONFIG",
1279
- .logfmt = "alert_config",
1280
- },
1281
- [NDF_ALERT_NAME] = {
1282
- .journal = "ND_ALERT_NAME",
1283
- .logfmt = "alert",
1284
- },
1285
- [NDF_ALERT_CLASS] = {
1286
- .journal = "ND_ALERT_CLASS",
1287
- .logfmt = "alert_class",
1288
- },
1289
- [NDF_ALERT_COMPONENT] = {
1290
- .journal = "ND_ALERT_COMPONENT",
1291
- .logfmt = "alert_component",
1292
- },
1293
- [NDF_ALERT_TYPE] = {
1294
- .journal = "ND_ALERT_TYPE",
1295
- .logfmt = "alert_type",
1296
- },
1297
- [NDF_ALERT_EXEC] = {
1298
- .journal = "ND_ALERT_EXEC",
1299
- .logfmt = "alert_exec",
1300
- },
1301
- [NDF_ALERT_RECIPIENT] = {
1302
- .journal = "ND_ALERT_RECIPIENT",
1303
- .logfmt = "alert_recipient",
1304
- },
1305
- [NDF_ALERT_VALUE] = {
1306
- .journal = "ND_ALERT_VALUE",
1307
- .logfmt = "alert_value",
1308
- },
1309
- [NDF_ALERT_VALUE_OLD] = {
1310
- .journal = "ND_ALERT_VALUE_OLD",
1311
- .logfmt = "alert_value_old",
1312
- },
1313
- [NDF_ALERT_STATUS] = {
1314
- .journal = "ND_ALERT_STATUS",
1315
- .logfmt = "alert_status",
1316
- },
1317
- [NDF_ALERT_STATUS_OLD] = {
1318
- .journal = "ND_ALERT_STATUS_OLD",
1319
- .logfmt = "alert_value_old",
1320
- },
1321
- [NDF_ALERT_UNITS] = {
1322
- .journal = "ND_ALERT_UNITS",
1323
- .logfmt = "alert_units",
1324
- },
1325
- [NDF_ALERT_SUMMARY] = {
1326
- .journal = "ND_ALERT_SUMMARY",
1327
- .logfmt = "alert_summary",
1328
- },
1329
- [NDF_ALERT_INFO] = {
1330
- .journal = "ND_ALERT_INFO",
1331
- .logfmt = "alert_info",
1332
- },
1333
- [NDF_ALERT_DURATION] = {
1334
- .journal = "ND_ALERT_DURATION",
1335
- .logfmt = "alert_duration",
1336
- },
1337
- [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = {
1338
- .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC",
1339
- .logfmt = "alert_notification_timestamp",
1340
- .logfmt_annotator = timestamp_usec_annotator,
1341
- },
1342
-
1343
- // put new items here
1344
- // leave the request URL and the message last
1345
-
1346
- [NDF_REQUEST] = {
1347
- .journal = "ND_REQUEST",
1348
- .logfmt = "request",
1349
- },
1350
- [NDF_MESSAGE] = {
1351
- .journal = "MESSAGE",
1352
- .logfmt = "msg",
1353
- },
1354
-};
1355
-
1356
-#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0]))
1357
-
1358
-ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len) {
1359
- for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) {
1360
- if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0)
1361
- return i;
1362
- }
1363
-
1364
- return NDF_STOP;
1365
-}
1366
-
1367
-void log_stack_pop(void *ptr) {
1368
- if(!ptr) return;
1369
-
1370
- struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr;
1371
-
1372
- if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) {
1373
- fatal("You cannot pop in the middle of the stack, or an item not in the stack");
1374
- return;
1375
- }
1376
-
1377
- thread_log_stack_next--;
1378
-}
1379
-
1380
-void log_stack_push(struct log_stack_entry *lgs) {
1381
- if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return;
1382
- thread_log_stack_base[thread_log_stack_next++] = lgs;
1383
-}
1384
-
1385
-// ----------------------------------------------------------------------------
1386
-// json formatter
1387
-
1388
-static void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) {
1389
-
1390
- // --- FIELD_PARSER_VERSIONS ---
1391
- //
1392
- // IMPORTANT:
1393
- // THERE ARE 6 VERSIONS OF THIS CODE
1394
- //
1395
- // 1. journal (direct socket API),
1396
- // 2. journal (libsystemd API),
1397
- // 3. logfmt,
1398
- // 4. json,
1399
- // 5. convert to uint64
1400
- // 6. convert to int64
1401
- //
1402
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
1403
-
1404
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
1405
- CLEAN_BUFFER *tmp = NULL;
1406
-
1407
- for (size_t i = 0; i < fields_max; i++) {
1408
- if (!fields[i].entry.set || !fields[i].logfmt)
1409
- continue;
1410
-
1411
- const char *key = fields[i].logfmt;
1412
-
1413
- const char *s = NULL;
1414
- switch(fields[i].entry.type) {
1415
- case NDFT_TXT:
1416
- s = fields[i].entry.txt;
1417
- break;
1418
- case NDFT_STR:
1419
- s = string2str(fields[i].entry.str);
1420
- break;
1421
- case NDFT_BFR:
1422
- s = buffer_tostring(fields[i].entry.bfr);
1423
- break;
1424
- case NDFT_U64:
1425
- buffer_json_member_add_uint64(wb, key, fields[i].entry.u64);
1426
- break;
1427
- case NDFT_I64:
1428
- buffer_json_member_add_int64(wb, key, fields[i].entry.i64);
1429
- break;
1430
- case NDFT_DBL:
1431
- buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
1432
- break;
1433
- case NDFT_UUID:
1434
- if(!uuid_is_null(*fields[i].entry.uuid)) {
1435
- char u[UUID_COMPACT_STR_LEN];
1436
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1437
- buffer_json_member_add_string(wb, key, u);
1438
- }
1439
- break;
1440
- case NDFT_CALLBACK: {
1441
- if(!tmp)
1442
- tmp = buffer_create(1024, NULL);
1443
- else
1444
- buffer_flush(tmp);
1445
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
1446
- s = buffer_tostring(tmp);
1447
- else
1448
- s = NULL;
1449
- }
1450
- break;
1451
- default:
1452
- s = "UNHANDLED";
1453
- break;
1454
- }
1455
-
1456
- if(s && *s)
1457
- buffer_json_member_add_string(wb, key, s);
1458
- }
1459
-
1460
- buffer_json_finalize(wb);
1461
-}
1462
-
1463
-// ----------------------------------------------------------------------------
1464
-// logfmt formatter
1465
-
1466
-
1467
-static int64_t log_field_to_int64(struct log_field *lf) {
1468
-
1469
- // --- FIELD_PARSER_VERSIONS ---
1470
- //
1471
- // IMPORTANT:
1472
- // THERE ARE 6 VERSIONS OF THIS CODE
1473
- //
1474
- // 1. journal (direct socket API),
1475
- // 2. journal (libsystemd API),
1476
- // 3. logfmt,
1477
- // 4. json,
1478
- // 5. convert to uint64
1479
- // 6. convert to int64
1480
- //
1481
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
1482
-
1483
- CLEAN_BUFFER *tmp = NULL;
1484
- const char *s = NULL;
1485
-
1486
- switch(lf->entry.type) {
1487
- case NDFT_UUID:
1488
- case NDFT_UNSET:
1489
- return 0;
1490
-
1491
- case NDFT_TXT:
1492
- s = lf->entry.txt;
1493
- break;
1494
-
1495
- case NDFT_STR:
1496
- s = string2str(lf->entry.str);
1497
- break;
1498
-
1499
- case NDFT_BFR:
1500
- s = buffer_tostring(lf->entry.bfr);
1501
- break;
1502
-
1503
- case NDFT_CALLBACK:
1504
- tmp = buffer_create(0, NULL);
1505
-
1506
- if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
1507
- s = buffer_tostring(tmp);
1508
- else
1509
- s = NULL;
1510
- break;
1511
-
1512
- case NDFT_U64:
1513
- return (int64_t)lf->entry.u64;
1514
-
1515
- case NDFT_I64:
1516
- return (int64_t)lf->entry.i64;
1517
-
1518
- case NDFT_DBL:
1519
- return (int64_t)lf->entry.dbl;
1520
- }
1521
-
1522
- if(s && *s)
1523
- return str2ll(s, NULL);
1524
-
1525
- return 0;
1526
-}
1527
-
1528
-static uint64_t log_field_to_uint64(struct log_field *lf) {
1529
-
1530
- // --- FIELD_PARSER_VERSIONS ---
1531
- //
1532
- // IMPORTANT:
1533
- // THERE ARE 6 VERSIONS OF THIS CODE
1534
- //
1535
- // 1. journal (direct socket API),
1536
- // 2. journal (libsystemd API),
1537
- // 3. logfmt,
1538
- // 4. json,
1539
- // 5. convert to uint64
1540
- // 6. convert to int64
1541
- //
1542
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
1543
-
1544
- CLEAN_BUFFER *tmp = NULL;
1545
- const char *s = NULL;
1546
-
1547
- switch(lf->entry.type) {
1548
- case NDFT_UUID:
1549
- case NDFT_UNSET:
1550
- return 0;
1551
-
1552
- case NDFT_TXT:
1553
- s = lf->entry.txt;
1554
- break;
1555
-
1556
- case NDFT_STR:
1557
- s = string2str(lf->entry.str);
1558
- break;
1559
-
1560
- case NDFT_BFR:
1561
- s = buffer_tostring(lf->entry.bfr);
1562
- break;
1563
-
1564
- case NDFT_CALLBACK:
1565
- tmp = buffer_create(0, NULL);
1566
-
1567
- if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
1568
- s = buffer_tostring(tmp);
1569
- else
1570
- s = NULL;
1571
- break;
1572
-
1573
- case NDFT_U64:
1574
- return lf->entry.u64;
1575
-
1576
- case NDFT_I64:
1577
- return lf->entry.i64;
1578
-
1579
- case NDFT_DBL:
1580
- return (uint64_t) lf->entry.dbl;
1581
- }
1582
-
1583
- if(s && *s)
1584
- return str2uint64_t(s, NULL);
1585
-
1586
- return 0;
1587
-}
1588
-
1589
-static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
1590
- usec_t ut = log_field_to_uint64(lf);
1591
-
1592
- if(!ut)
1593
- return;
1594
-
1595
- char datetime[RFC3339_MAX_LENGTH];
1596
- rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
1597
-
1598
- if(buffer_strlen(wb))
1599
- buffer_fast_strcat(wb, " ", 1);
1600
-
1601
- buffer_strcat(wb, key);
1602
- buffer_fast_strcat(wb, "=", 1);
1603
- buffer_json_strcat(wb, datetime);
1604
-}
1605
-
1606
-static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
1607
- int64_t errnum = log_field_to_int64(lf);
1608
-
1609
- if(errnum == 0)
1610
- return;
1611
-
1612
- char buf[1024];
1613
- const char *s = errno2str((int)errnum, buf, sizeof(buf));
1614
-
1615
- if(buffer_strlen(wb))
1616
- buffer_fast_strcat(wb, " ", 1);
1617
-
1618
- buffer_strcat(wb, key);
1619
- buffer_fast_strcat(wb, "=\"", 2);
1620
- buffer_print_int64(wb, errnum);
1621
- buffer_fast_strcat(wb, ", ", 2);
1622
- buffer_json_strcat(wb, s);
1623
- buffer_fast_strcat(wb, "\"", 1);
1624
-}
1625
-
1626
-#if defined(OS_WINDOWS)
1627
-static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
1628
- DWORD errnum = log_field_to_uint64(lf);
1629
-
1630
- if (errnum == 0)
1631
- return;
1632
-
1633
- char buf[1024];
1634
- wchar_t wbuf[1024];
1635
- DWORD size = FormatMessageW(
1636
- FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
1637
- NULL,
1638
- errnum,
1639
- MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
1640
- wbuf,
1641
- (DWORD)(sizeof(wbuf) / sizeof(wchar_t) - 1),
1642
- NULL
1643
- );
1644
-
1645
- if (size > 0) {
1646
- // Remove \r\n at the end
1647
- while (size > 0 && (wbuf[size - 1] == L'\r' || wbuf[size - 1] == L'\n'))
1648
- wbuf[--size] = L'\0';
1649
-
1650
- // Convert wide string to UTF-8
1651
- int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, buf, sizeof(buf), NULL, NULL);
1652
- if (utf8_size == 0)
1653
- snprintf(buf, sizeof(buf) - 1, "unknown error code");
1654
- buf[sizeof(buf) - 1] = '\0';
1655
- }
1656
- else
1657
- snprintf(buf, sizeof(buf) - 1, "unknown error code");
1658
-
1659
- if (buffer_strlen(wb))
1660
- buffer_fast_strcat(wb, " ", 1);
1661
-
1662
- buffer_strcat(wb, key);
1663
- buffer_fast_strcat(wb, "=\"", 2);
1664
- buffer_print_int64(wb, errnum);
1665
- buffer_fast_strcat(wb, ", ", 2);
1666
- buffer_json_strcat(wb, buf);
1667
- buffer_fast_strcat(wb, "\"", 1);
1668
-}
1669
-#endif
1670
-
1671
-static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
1672
- uint64_t pri = log_field_to_uint64(lf);
1673
-
1674
- if(buffer_strlen(wb))
1675
- buffer_fast_strcat(wb, " ", 1);
1676
-
1677
- buffer_strcat(wb, key);
1678
- buffer_fast_strcat(wb, "=", 1);
1679
- buffer_strcat(wb, nd_log_id2priority(pri));
1680
-}
1681
-
1682
-static bool needs_quotes_for_logfmt(const char *s)
1683
-{
1684
- static bool safe_for_logfmt[256] = {
1685
- [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true,
1686
- ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true,
1687
- ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true,
1688
- ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true,
1689
- ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true,
1690
- ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true,
1691
- ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true,
1692
- ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true,
1693
- ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true,
1694
- ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true,
1695
- ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true,
1696
- ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true,
1697
- ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true,
1698
- ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true,
1699
- };
1700
-
1701
- if(!*s)
1702
- return true;
1703
-
1704
- while(*s) {
1705
- if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s])
1706
- return true;
1707
-
1708
- s++;
1709
- }
1710
-
1711
- return false;
1712
-}
1713
-
1714
-static void string_to_logfmt(BUFFER *wb, const char *s)
1715
-{
1716
- bool spaces = needs_quotes_for_logfmt(s);
1717
-
1718
- if(spaces)
1719
- buffer_fast_strcat(wb, "\"", 1);
1720
-
1721
- buffer_json_strcat(wb, s);
1722
-
1723
- if(spaces)
1724
- buffer_fast_strcat(wb, "\"", 1);
1725
-}
1726
-
1727
-static void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max)
1728
-{
1729
-
1730
- // --- FIELD_PARSER_VERSIONS ---
1731
- //
1732
- // IMPORTANT:
1733
- // THERE ARE 6 VERSIONS OF THIS CODE
1734
- //
1735
- // 1. journal (direct socket API),
1736
- // 2. journal (libsystemd API),
1737
- // 3. logfmt,
1738
- // 4. json,
1739
- // 5. convert to uint64
1740
- // 6. convert to int64
1741
- //
1742
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
1743
-
1744
- CLEAN_BUFFER *tmp = NULL;
1745
-
1746
- for (size_t i = 0; i < fields_max; i++) {
1747
- if (!fields[i].entry.set || !fields[i].logfmt)
1748
- continue;
1749
-
1750
- const char *key = fields[i].logfmt;
1751
-
1752
- if(fields[i].logfmt_annotator)
1753
- fields[i].logfmt_annotator(wb, key, &fields[i]);
1754
- else {
1755
- if(buffer_strlen(wb))
1756
- buffer_fast_strcat(wb, " ", 1);
1757
-
1758
- switch(fields[i].entry.type) {
1759
- case NDFT_TXT:
1760
- if(*fields[i].entry.txt) {
1761
- buffer_strcat(wb, key);
1762
- buffer_fast_strcat(wb, "=", 1);
1763
- string_to_logfmt(wb, fields[i].entry.txt);
1764
- }
1765
- break;
1766
- case NDFT_STR:
1767
- buffer_strcat(wb, key);
1768
- buffer_fast_strcat(wb, "=", 1);
1769
- string_to_logfmt(wb, string2str(fields[i].entry.str));
1770
- break;
1771
- case NDFT_BFR:
1772
- if(buffer_strlen(fields[i].entry.bfr)) {
1773
- buffer_strcat(wb, key);
1774
- buffer_fast_strcat(wb, "=", 1);
1775
- string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr));
1776
- }
1777
- break;
1778
- case NDFT_U64:
1779
- buffer_strcat(wb, key);
1780
- buffer_fast_strcat(wb, "=", 1);
1781
- buffer_print_uint64(wb, fields[i].entry.u64);
1782
- break;
1783
- case NDFT_I64:
1784
- buffer_strcat(wb, key);
1785
- buffer_fast_strcat(wb, "=", 1);
1786
- buffer_print_int64(wb, fields[i].entry.i64);
1787
- break;
1788
- case NDFT_DBL:
1789
- buffer_strcat(wb, key);
1790
- buffer_fast_strcat(wb, "=", 1);
1791
- buffer_print_netdata_double(wb, fields[i].entry.dbl);
1792
- break;
1793
- case NDFT_UUID:
1794
- if(!uuid_is_null(*fields[i].entry.uuid)) {
1795
- char u[UUID_COMPACT_STR_LEN];
1796
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1797
- buffer_strcat(wb, key);
1798
- buffer_fast_strcat(wb, "=", 1);
1799
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
1800
- }
1801
- break;
1802
- case NDFT_CALLBACK: {
1803
- if(!tmp)
1804
- tmp = buffer_create(1024, NULL);
1805
- else
1806
- buffer_flush(tmp);
1807
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) {
1808
- buffer_strcat(wb, key);
1809
- buffer_fast_strcat(wb, "=", 1);
1810
- string_to_logfmt(wb, buffer_tostring(tmp));
1811
- }
1812
- }
1813
- break;
1814
- default:
1815
- buffer_strcat(wb, "UNHANDLED");
1816
- break;
1817
- }
1818
- }
1819
- }
1820
-}
1821
-
1822
-// ----------------------------------------------------------------------------
1823
-// journal logger
1824
-
1825
-bool nd_log_journal_socket_available(void) {
1826
- if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
1827
- char filename[FILENAME_MAX + 1];
1828
-
1829
- snprintfz(filename, sizeof(filename), "%s%s",
1830
- netdata_configured_host_prefix, "/run/systemd/journal/socket");
1831
-
1832
- if(is_path_unix_socket(filename))
1833
- return true;
1834
- }
1835
-
1836
- return is_path_unix_socket("/run/systemd/journal/socket");
1837
-}
1838
-
1839
-static bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) {
1840
-#ifdef HAVE_SYSTEMD
1841
-
1842
- // --- FIELD_PARSER_VERSIONS ---
1843
- //
1844
- // IMPORTANT:
1845
- // THERE ARE 6 VERSIONS OF THIS CODE
1846
- //
1847
- // 1. journal (direct socket API),
1848
- // 2. journal (libsystemd API),
1849
- // 3. logfmt,
1850
- // 4. json,
1851
- // 5. convert to uint64
1852
- // 6. convert to int64
1853
- //
1854
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
1855
-
1856
- struct iovec iov[fields_max];
1857
- int iov_count = 0;
1858
-
1859
- memset(iov, 0, sizeof(iov));
1860
-
1861
- CLEAN_BUFFER *tmp = NULL;
1862
-
1863
- for (size_t i = 0; i < fields_max; i++) {
1864
- if (!fields[i].entry.set || !fields[i].journal)
1865
- continue;
1866
-
1867
- const char *key = fields[i].journal;
1868
- char *value = NULL;
1869
- int rc = 0;
1870
- switch (fields[i].entry.type) {
1871
- case NDFT_TXT:
1872
- if(*fields[i].entry.txt)
1873
- rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt);
1874
- break;
1875
- case NDFT_STR:
1876
- rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str));
1877
- break;
1878
- case NDFT_BFR:
1879
- if(buffer_strlen(fields[i].entry.bfr))
1880
- rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr));
1881
- break;
1882
- case NDFT_U64:
1883
- rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64);
1884
- break;
1885
- case NDFT_I64:
1886
- rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64);
1887
- break;
1888
- case NDFT_DBL:
1889
- rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
1890
- break;
1891
- case NDFT_UUID:
1892
- if(!uuid_is_null(*fields[i].entry.uuid)) {
1893
- char u[UUID_COMPACT_STR_LEN];
1894
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1895
- rc = asprintf(&value, "%s=%s", key, u);
1896
- }
1897
- break;
1898
- case NDFT_CALLBACK: {
1899
- if(!tmp)
1900
- tmp = buffer_create(1024, NULL);
1901
- else
1902
- buffer_flush(tmp);
1903
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
1904
- rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp));
1905
- }
1906
- break;
1907
- default:
1908
- rc = asprintf(&value, "%s=%s", key, "UNHANDLED");
1909
- break;
1910
- }
1911
-
1912
- if (rc != -1 && value) {
1913
- iov[iov_count].iov_base = value;
1914
- iov[iov_count].iov_len = strlen(value);
1915
- iov_count++;
1916
- }
1917
- }
1918
-
1919
- int r = sd_journal_sendv(iov, iov_count);
1920
-
1921
- // Clean up allocated memory
1922
- for (int i = 0; i < iov_count; i++) {
1923
- if (iov[i].iov_base != NULL) {
1924
- free(iov[i].iov_base);
1925
- }
1926
- }
1927
-
1928
- return r == 0;
1929
-#else
1930
- return false;
1931
-#endif
1932
-}
1933
-
1934
-static bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) {
1935
- if(!nd_log.journal_direct.initialized)
1936
- return false;
1937
-
1938
- // --- FIELD_PARSER_VERSIONS ---
1939
- //
1940
- // IMPORTANT:
1941
- // THERE ARE 6 VERSIONS OF THIS CODE
1942
- //
1943
- // 1. journal (direct socket API),
1944
- // 2. journal (libsystemd API),
1945
- // 3. logfmt,
1946
- // 4. json,
1947
- // 5. convert to uint64
1948
- // 6. convert to int64
1949
- //
1950
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
1951
-
1952
- CLEAN_BUFFER *wb = buffer_create(4096, NULL);
1953
- CLEAN_BUFFER *tmp = NULL;
1954
-
1955
- for (size_t i = 0; i < fields_max; i++) {
1956
- if (!fields[i].entry.set || !fields[i].journal)
1957
- continue;
1958
-
1959
- const char *key = fields[i].journal;
1960
-
1961
- const char *s = NULL;
1962
- switch(fields[i].entry.type) {
1963
- case NDFT_TXT:
1964
- s = fields[i].entry.txt;
1965
- break;
1966
- case NDFT_STR:
1967
- s = string2str(fields[i].entry.str);
1968
- break;
1969
- case NDFT_BFR:
1970
- s = buffer_tostring(fields[i].entry.bfr);
1971
- break;
1972
- case NDFT_U64:
1973
- buffer_strcat(wb, key);
1974
- buffer_putc(wb, '=');
1975
- buffer_print_uint64(wb, fields[i].entry.u64);
1976
- buffer_putc(wb, '\n');
1977
- break;
1978
- case NDFT_I64:
1979
- buffer_strcat(wb, key);
1980
- buffer_putc(wb, '=');
1981
- buffer_print_int64(wb, fields[i].entry.i64);
1982
- buffer_putc(wb, '\n');
1983
- break;
1984
- case NDFT_DBL:
1985
- buffer_strcat(wb, key);
1986
- buffer_putc(wb, '=');
1987
- buffer_print_netdata_double(wb, fields[i].entry.dbl);
1988
- buffer_putc(wb, '\n');
1989
- break;
1990
- case NDFT_UUID:
1991
- if(!uuid_is_null(*fields[i].entry.uuid)) {
1992
- char u[UUID_COMPACT_STR_LEN];
1993
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1994
- buffer_strcat(wb, key);
1995
- buffer_putc(wb, '=');
1996
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
1997
- buffer_putc(wb, '\n');
1998
- }
1999
- break;
2000
- case NDFT_CALLBACK: {
2001
- if(!tmp)
2002
- tmp = buffer_create(1024, NULL);
2003
- else
2004
- buffer_flush(tmp);
2005
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
2006
- s = buffer_tostring(tmp);
2007
- else
2008
- s = NULL;
2009
- }
2010
- break;
2011
- default:
2012
- s = "UNHANDLED";
2013
- break;
2014
- }
2015
-
2016
- if(s && *s) {
2017
- buffer_strcat(wb, key);
2018
- if(!strchr(s, '\n')) {
2019
- buffer_putc(wb, '=');
2020
- buffer_strcat(wb, s);
2021
- buffer_putc(wb, '\n');
2022
- }
2023
- else {
2024
- buffer_putc(wb, '\n');
2025
- size_t size = strlen(s);
2026
- uint64_t le_size = htole64(size);
2027
- buffer_memcat(wb, &le_size, sizeof(le_size));
2028
- buffer_memcat(wb, s, size);
2029
- buffer_putc(wb, '\n');
2030
- }
2031
- }
2032
- }
2033
-
2034
- return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb));
2035
-}
2036
-
2037
-// ----------------------------------------------------------------------------
2038
-// syslog logger - uses logfmt
2039
-
2040
-static bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) {
2041
- CLEAN_BUFFER *wb = buffer_create(1024, NULL);
2042
-
2043
- nd_logger_logfmt(wb, fields, fields_max);
2044
- syslog(priority, "%s", buffer_tostring(wb));
2045
-
2046
- return true;
2047
-}
2048
-
2049
-// ----------------------------------------------------------------------------
2050
-// file logger - uses logfmt
2051
-
2052
-static bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
2053
- BUFFER *wb = buffer_create(1024, NULL);
2054
-
2055
- if(format == NDLF_JSON)
2056
- nd_logger_json(wb, fields, fields_max);
2057
- else
2058
- nd_logger_logfmt(wb, fields, fields_max);
2059
-
2060
- int r = fprintf(fp, "%s\n", buffer_tostring(wb));
2061
- fflush(fp);
2062
-
2063
- buffer_free(wb);
2064
- return r > 0;
2065
-}
2066
-
2067
-// ----------------------------------------------------------------------------
2068
-// logger router
2069
-
2070
-static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) {
2071
- *spinlock = NULL;
2072
- ND_LOG_METHOD output = nd_log.sources[source].method;
2073
-
2074
- switch(output) {
2075
- case NDLM_JOURNAL:
2076
- if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) {
2077
- output = NDLM_FILE;
2078
- *fpp = stderr;
2079
- *spinlock = &nd_log.std_error.spinlock;
2080
- }
2081
- else {
2082
- *fpp = NULL;
2083
- *spinlock = NULL;
2084
- }
2085
- break;
2086
-
2087
- case NDLM_SYSLOG:
2088
- if(unlikely(!nd_log.syslog.initialized)) {
2089
- output = NDLM_FILE;
2090
- *spinlock = &nd_log.std_error.spinlock;
2091
- *fpp = stderr;
2092
- }
2093
- else {
2094
- *spinlock = NULL;
2095
- *fpp = NULL;
2096
- }
2097
- break;
2098
-
2099
- case NDLM_FILE:
2100
- if(!nd_log.sources[source].fp) {
2101
- *fpp = stderr;
2102
- *spinlock = &nd_log.std_error.spinlock;
2103
- }
2104
- else {
2105
- *fpp = nd_log.sources[source].fp;
2106
- *spinlock = &nd_log.sources[source].spinlock;
2107
- }
2108
- break;
2109
-
2110
- case NDLM_STDOUT:
2111
- output = NDLM_FILE;
2112
- *fpp = stdout;
2113
- *spinlock = &nd_log.std_output.spinlock;
2114
- break;
2115
-
2116
- default:
2117
- case NDLM_DEFAULT:
2118
- case NDLM_STDERR:
2119
- output = NDLM_FILE;
2120
- *fpp = stderr;
2121
- *spinlock = &nd_log.std_error.spinlock;
2122
- break;
2123
-
2124
- case NDLM_DISABLED:
2125
- case NDLM_DEVNULL:
2126
- output = NDLM_DISABLED;
2127
- *fpp = NULL;
2128
- *spinlock = NULL;
2129
- break;
2130
- }
2131
-
2132
- return output;
2133
-}
2134
-
2135
-// ----------------------------------------------------------------------------
2136
-// high level logger
2137
-
2138
-static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,
2139
- ND_LOG_METHOD output, struct nd_log_source *source,
2140
- struct log_field *fields, size_t fields_max) {
2141
- if(spinlock)
2142
- spinlock_lock(spinlock);
2143
-
2144
- // check the limits
2145
- if(limit && nd_log_limit_reached(source))
2146
- goto cleanup;
2147
-
2148
- if(output == NDLM_JOURNAL) {
2149
- if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) {
2150
- // we can't log to journal, let's log to stderr
2151
- if(spinlock)
2152
- spinlock_unlock(spinlock);
2153
-
2154
- output = NDLM_FILE;
2155
- spinlock = &nd_log.std_error.spinlock;
2156
- fp = stderr;
2157
-
2158
- if(spinlock)
2159
- spinlock_lock(spinlock);
2160
- }
2161
- }
2162
-
2163
- if(output == NDLM_SYSLOG)
2164
- nd_logger_syslog(priority, source->format, fields, fields_max);
2165
-
2166
- if(output == NDLM_FILE)
2167
- nd_logger_file(fp, source->format, fields, fields_max);
2168
-
2169
-
2170
-cleanup:
2171
- if(spinlock)
2172
- spinlock_unlock(spinlock);
2173
-}
2174
-
2175
-static void nd_logger_unset_all_thread_fields(void) {
2176
- size_t fields_max = THREAD_FIELDS_MAX;
2177
- for(size_t i = 0; i < fields_max ; i++)
2178
- thread_log_fields[i].entry.set = false;
2179
-}
2180
-
2181
-static void nd_logger_merge_log_stack_to_thread_fields(void) {
2182
- for(size_t c = 0; c < thread_log_stack_next ;c++) {
2183
- struct log_stack_entry *lgs = thread_log_stack_base[c];
2184
-
2185
- for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) {
2186
- if(lgs[i].id >= _NDF_MAX || !lgs[i].set)
2187
- continue;
2188
-
2189
- struct log_stack_entry *e = &lgs[i];
2190
- ND_LOG_STACK_FIELD_TYPE type = lgs[i].type;
2191
-
2192
- // do not add empty / unset fields
2193
- if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
2194
- (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
2195
- (type == NDFT_STR && !e->str) ||
2196
- (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) ||
2197
- (type == NDFT_CALLBACK && !e->cb.formatter) ||
2198
- type == NDFT_UNSET)
2199
- continue;
2200
-
2201
- thread_log_fields[lgs[i].id].entry = *e;
2202
- }
2203
- }
2204
-}
2205
-
2206
-static void nd_logger(const char *file, const char *function, const unsigned long line,
2207
- ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit,
2208
- int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) {
2209
-
2210
- SPINLOCK *spinlock;
2211
- FILE *fp;
2212
- ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock);
2213
- if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
2214
- return;
2215
-
2216
- // mark all fields as unset
2217
- nd_logger_unset_all_thread_fields();
2218
-
2219
- // flatten the log stack into the fields
2220
- nd_logger_merge_log_stack_to_thread_fields();
2221
-
2222
- // set the common fields that are automatically set by the logging subsystem
2223
-
2224
- if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
2225
- thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
2226
-
2227
- if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set))
2228
- thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source));
2229
- else {
2230
- ND_LOG_SOURCES src = source;
2231
-
2232
- if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT)
2233
- src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source);
2234
- else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64)
2235
- src = thread_log_fields[NDF_LOG_SOURCE].entry.u64;
2236
-
2237
- if(src != source && src < _NDLS_MAX) {
2238
- source = src;
2239
- output = nd_logger_select_output(source, &fp, &spinlock);
2240
- if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
2241
- return;
2242
- }
2243
- }
2244
-
2245
- if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set))
2246
- thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name);
2247
-
2248
- if(likely(!thread_log_fields[NDF_LINE].entry.set)) {
2249
- thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line);
2250
- thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file);
2251
- thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function);
2252
- }
2253
-
2254
- if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) {
2255
- thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority);
2256
- }
2257
-
2258
- if(likely(!thread_log_fields[NDF_TID].entry.set))
2259
- thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached());
2260
-
2261
- if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) {
2262
- const char *thread_tag = nd_thread_tag();
2263
- thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag);
2264
-
2265
- // TODO: fix the ND_MODULE in logging by setting proper module name in threads
2266
-// if(!thread_log_fields[NDF_MODULE].entry.set)
2267
-// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag);
2268
- }
2269
-
2270
- if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set))
2271
- thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec());
2272
-
2273
- if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)
2274
- thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);
2275
-
2276
-#if defined(OS_WINDOWS)
2277
- if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set)
2278
- thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror);
2279
-#endif
2280
-
2281
- CLEAN_BUFFER *wb = NULL;
2282
- if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {
2283
- wb = buffer_create(1024, NULL);
2284
- buffer_vsprintf(wb, fmt, ap);
2285
- thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb));
2286
- }
2287
-
2288
- nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source],
2289
- thread_log_fields, THREAD_FIELDS_MAX);
2290
-
2291
- if(nd_log.sources[source].pending_msg) {
2292
- // log a pending message
2293
-
2294
- nd_logger_unset_all_thread_fields();
2295
-
2296
- thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){
2297
- .set = true,
2298
- .type = NDFT_U64,
2299
- .u64 = now_realtime_usec(),
2300
- };
2301
-
2302
- thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){
2303
- .set = true,
2304
- .type = NDFT_TXT,
2305
- .txt = nd_log_id2source(source),
2306
- };
2307
-
2308
- thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){
2309
- .set = true,
2310
- .type = NDFT_TXT,
2311
- .txt = program_name,
2312
- };
2313
-
2314
- thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){
2315
- .set = true,
2316
- .type = NDFT_TXT,
2317
- .txt = nd_log.sources[source].pending_msg,
2318
- };
2319
-
2320
- nd_logger_log_fields(spinlock, fp, false, priority, output,
2321
- &nd_log.sources[source],
2322
- thread_log_fields, THREAD_FIELDS_MAX);
2323
-
2324
- freez((void *)nd_log.sources[source].pending_msg);
2325
- nd_log.sources[source].pending_msg = NULL;
2326
- }
2327
-
2328
- errno_clear();
2329
-}
2330
-
2331
-static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {
2332
- if(source >= _NDLS_MAX)
2333
- source = NDLS_DAEMON;
2334
-
2335
- if(nd_log.overwrite_process_source)
2336
- source = nd_log.overwrite_process_source;
2337
-
2338
- return source;
2339
-}
2340
-
2341
-// ----------------------------------------------------------------------------
2342
-// public API for loggers
2343
-
2344
-void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )
2345
-{
2346
- int saved_errno = errno;
2347
-
2348
- size_t saved_winerror = 0;
2349
-#if defined(OS_WINDOWS)
2350
- saved_winerror = GetLastError();
2351
-#endif
2352
-
2353
- source = nd_log_validate_source(source);
2354
-
2355
- if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
2356
- return;
2357
-
2358
- va_list args;
2359
- va_start(args, fmt);
2360
- nd_logger(file, function, line, source, priority,
2361
- source == NDLS_DAEMON || source == NDLS_COLLECTORS,
2362
- saved_errno, saved_winerror, fmt, args);
2363
- va_end(args);
2364
-}
2365
-
2366
-void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
2367
- int saved_errno = errno;
2368
-
2369
- size_t saved_winerror = 0;
2370
-#if defined(OS_WINDOWS)
2371
- saved_winerror = GetLastError();
2372
-#endif
2373
-
2374
- source = nd_log_validate_source(source);
2375
-
2376
- if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
2377
- return;
2378
-
2379
- if(erl->sleep_ut)
2380
- sleep_usec(erl->sleep_ut);
2381
-
2382
- spinlock_lock(&erl->spinlock);
2383
-
2384
- erl->count++;
2385
- time_t now = now_boottime_sec();
2386
- if(now - erl->last_logged < erl->log_every) {
2387
- spinlock_unlock(&erl->spinlock);
2388
- return;
2389
- }
2390
-
2391
- spinlock_unlock(&erl->spinlock);
2392
-
2393
- va_list args;
2394
- va_start(args, fmt);
2395
- nd_logger(file, function, line, source, priority,
2396
- source == NDLS_DAEMON || source == NDLS_COLLECTORS,
2397
- saved_errno, saved_winerror, fmt, args);
2398
- va_end(args);
2399
- erl->last_logged = now;
2400
- erl->count = 0;
2401
-}
2402
-
2403
-void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
2404
- int saved_errno = errno;
2405
-
2406
- size_t saved_winerror = 0;
2407
-#if defined(OS_WINDOWS)
2408
- saved_winerror = GetLastError();
2409
-#endif
2410
-
2411
- ND_LOG_SOURCES source = NDLS_DAEMON;
2412
- source = nd_log_validate_source(source);
2413
-
2414
- va_list args;
2415
- va_start(args, fmt);
2416
- nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args);
2417
- va_end(args);
2418
-
2419
- char date[LOG_DATE_LENGTH];
2420
- log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
2421
-
2422
- char action_data[70+1];
2423
- snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);
2424
-
2425
- const char *thread_tag = nd_thread_tag();
2426
- const char *tag_to_send = thread_tag;
2427
-
2428
- // anonymize thread names
2429
- if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)
2430
- tag_to_send = THREAD_TAG_STREAM_RECEIVER;
2431
- if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)
2432
- tag_to_send = THREAD_TAG_STREAM_SENDER;
2433
-
2434
- char action_result[60+1];
2435
- snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);
2436
-
2437
-#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
2438
- int fd = nd_log.sources[NDLS_DAEMON].fd;
2439
- if(fd == -1)
2440
- fd = STDERR_FILENO;
2441
-
2442
- int nptrs;
2443
- void *buffer[10000];
2444
-
2445
- nptrs = backtrace(buffer, sizeof(buffer));
2446
- if(nptrs)
2447
- backtrace_symbols_fd(buffer, nptrs, fd);
2448
-#endif
2449
-
2450
-#ifdef NETDATA_INTERNAL_CHECKS
2451
- abort();
2452
-#endif
2453
-
2454
- netdata_cleanup_and_exit(1, "FATAL", action_result, action_data);
2455
-}
2456
-
2457
-// ----------------------------------------------------------------------------
2458
-// log limits
2459
-
2460
-void nd_log_limits_reset(void) {
2461
- usec_t now_ut = now_monotonic_usec();
2462
-
2463
- spinlock_lock(&nd_log.std_output.spinlock);
2464
- spinlock_lock(&nd_log.std_error.spinlock);
2465
-
2466
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
2467
- spinlock_lock(&nd_log.sources[i].spinlock);
2468
- nd_log.sources[i].limits.prevented = 0;
2469
- nd_log.sources[i].limits.counter = 0;
2470
- nd_log.sources[i].limits.started_monotonic_ut = now_ut;
2471
- nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup;
2472
- spinlock_unlock(&nd_log.sources[i].spinlock);
2473
- }
2474
-
2475
- spinlock_unlock(&nd_log.std_output.spinlock);
2476
- spinlock_unlock(&nd_log.std_error.spinlock);
2477
-}
2478
-
2479
-void nd_log_limits_unlimited(void) {
2480
- nd_log_limits_reset();
2481
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
2482
- nd_log.sources[i].limits.logs_per_period = 0;
2483
- }
2484
-}
2485
-
2486
-static bool nd_log_limit_reached(struct nd_log_source *source) {
2487
- if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0)
2488
- return false;
2489
-
2490
- usec_t now_ut = now_monotonic_usec();
2491
- if(!source->limits.started_monotonic_ut)
2492
- source->limits.started_monotonic_ut = now_ut;
2493
-
2494
- source->limits.counter++;
2495
-
2496
- if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) {
2497
- if(source->limits.prevented) {
2498
- BUFFER *wb = buffer_create(1024, NULL);
2499
- buffer_sprintf(wb,
2500
- "LOG FLOOD PROTECTION: resuming logging "
2501
- "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).",
2502
- source->limits.prevented,
2503
- source->limits.throttle_period);
2504
-
2505
- if(source->pending_msg)
2506
- freez((void *)source->pending_msg);
2507
-
2508
- source->pending_msg = strdupz(buffer_tostring(wb));
2509
-
2510
- buffer_free(wb);
2511
- }
2512
-
2513
- // restart the period accounting
2514
- source->limits.started_monotonic_ut = now_ut;
2515
- source->limits.counter = 1;
2516
- source->limits.prevented = 0;
2517
-
2518
- // log this error
2519
- return false;
2520
- }
2521
-
2522
- if(source->limits.counter > source->limits.logs_per_period) {
2523
- if(!source->limits.prevented) {
2524
- BUFFER *wb = buffer_create(1024, NULL);
2525
- buffer_sprintf(wb,
2526
- "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs "
2527
- "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.",
2528
- source->limits.counter,
2529
- (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC),
2530
- source->limits.logs_per_period,
2531
- source->limits.throttle_period,
2532
- program_name,
2533
- (int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC)
2534
- );
2535
-
2536
- if(source->pending_msg)
2537
- freez((void *)source->pending_msg);
2538
-
2539
- source->pending_msg = strdupz(buffer_tostring(wb));
2540
-
2541
- buffer_free(wb);
2542
- }
2543
-
2544
- source->limits.prevented++;
2545
-
2546
- // prevent logging this error
2547
-#ifdef NETDATA_INTERNAL_CHECKS
2548
- return false;
2549
-#else
2550
- return true;
2551
-#endif
2552
- }
2553
-
2554
- return false;
2555
-}
src/libnetdata/log/nd_log-annotators.c
new
+84
@@ -0,0 +1,84 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+const char *timestamp_usec_annotator(struct log_field *lf) {
6
+ usec_t ut = log_field_to_uint64(lf);
7
+
8
+ if(!ut)
9
+ return NULL;
10
+
11
+ static __thread char datetime[RFC3339_MAX_LENGTH];
12
+ rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
13
+ return datetime;
14
+}
15
+
16
+const char *errno_annotator(struct log_field *lf) {
17
+ int64_t errnum = log_field_to_int64(lf);
18
+
19
+ if(errnum == 0)
20
+ return NULL;
21
+
22
+ static __thread char buf[256];
23
+ size_t len = print_uint64(buf, errnum);
24
+ buf[len++] = ',';
25
+ buf[len++] = ' ';
26
+
27
+ char *msg_to = &buf[len];
28
+ size_t msg_size = sizeof(buf) - len;
29
+
30
+ const char *s = errno2str((int)errnum, msg_to, msg_size);
31
+ if(s != msg_to)
32
+ strncpyz(msg_to, s, msg_size - 1);
33
+
34
+ return buf;
35
+}
36
+
37
+#if defined(OS_WINDOWS)
38
+const char *winerror_annotator(struct log_field *lf) {
39
+ DWORD errnum = log_field_to_uint64(lf);
40
+
41
+ if (errnum == 0)
42
+ return NULL;
43
+
44
+ static __thread char buf[256];
45
+ size_t len = print_uint64(buf, errnum);
46
+ buf[len++] = ',';
47
+ buf[len++] = ' ';
48
+
49
+ char *msg_to = &buf[len];
50
+ size_t msg_size = sizeof(buf) - len;
51
+
52
+ wchar_t wbuf[1024];
53
+ DWORD size = FormatMessageW(
54
+ FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
55
+ NULL,
56
+ errnum,
57
+ MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
58
+ wbuf,
59
+ (DWORD)(sizeof(wbuf) / sizeof(wchar_t) - 1),
60
+ NULL
61
+ );
62
+
63
+ if (size > 0) {
64
+ // Remove \r\n at the end
65
+ while (size > 0 && (wbuf[size - 1] == L'\r' || wbuf[size - 1] == L'\n'))
66
+ wbuf[--size] = L'\0';
67
+
68
+ // Convert wide string to UTF-8
69
+ int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, msg_to, (int)msg_size, NULL, NULL);
70
+ if (utf8_size == 0)
71
+ snprintf(msg_to, msg_size - 1, "unknown error code");
72
+ msg_to[msg_size - 1] = '\0';
73
+ }
74
+ else
75
+ snprintf(msg_to, msg_size - 1, "unknown error code");
76
+
77
+ return buf;
78
+}
79
+#endif
80
+
81
+const char *priority_annotator(struct log_field *lf) {
82
+ uint64_t pri = log_field_to_uint64(lf);
83
+ return nd_log_id2priority(pri);
84
+}
src/libnetdata/log/nd_log-common.h
new
+147
@@ -0,0 +1,147 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ND_LOG_COMMON_H
4
+#define NETDATA_ND_LOG_COMMON_H
5
+
6
+#include <syslog.h>
7
+
8
+typedef enum __attribute__((__packed__)) {
9
+ NDLS_UNSET = 0, // internal use only
10
+ NDLS_ACCESS, // access.log
11
+ NDLS_ACLK, // aclk.log
12
+ NDLS_COLLECTORS, // collector.log
13
+ NDLS_DAEMON, // error.log
14
+ NDLS_HEALTH, // health.log
15
+ NDLS_DEBUG, // debug.log
16
+
17
+ // terminator
18
+ _NDLS_MAX,
19
+} ND_LOG_SOURCES;
20
+
21
+typedef enum __attribute__((__packed__)) {
22
+ NDLP_EMERG = LOG_EMERG, // from syslog.h
23
+ NDLP_ALERT = LOG_ALERT, // from syslog.h
24
+ NDLP_CRIT = LOG_CRIT, // from syslog.h
25
+ NDLP_ERR = LOG_ERR, // from syslog.h
26
+ NDLP_WARNING = LOG_WARNING, // from syslog.h
27
+ NDLP_NOTICE = LOG_NOTICE, // from syslog.h
28
+ NDLP_INFO = LOG_INFO, // from syslog.h
29
+ NDLP_DEBUG = LOG_DEBUG, // from syslog.h
30
+
31
+ // terminator
32
+ _NDLP_MAX,
33
+} ND_LOG_FIELD_PRIORITY;
34
+
35
+typedef enum __attribute__((__packed__)) {
36
+ // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
37
+ // so that it easy to audit for missing fields
38
+
39
+ // NEVER RENUMBER THIS LIST
40
+ // The Windows Events Log has them at fixed positions
41
+
42
+ NDF_STOP = 0,
43
+ NDF_TIMESTAMP_REALTIME_USEC = 1, // the timestamp of the log message - added automatically
44
+ NDF_SYSLOG_IDENTIFIER = 2, // the syslog identifier of the application - added automatically
45
+ NDF_LOG_SOURCE = 3, // DAEMON, COLLECTORS, HEALTH, MSGID_ACCESS, ACLK - set at the log call
46
+ NDF_PRIORITY = 4, // the syslog priority (severity) - set at the log call
47
+ NDF_ERRNO = 5, // the ERRNO at the time of the log call - added automatically
48
+ NDF_WINERROR = 6, // Windows GetLastError()
49
+ NDF_INVOCATION_ID = 7, // the INVOCATION_ID of Netdata - added automatically
50
+ NDF_LINE = 8, // the source code file line number - added automatically
51
+ NDF_FILE = 9, // the source code filename - added automatically
52
+ NDF_FUNC = 10, // the source code function - added automatically
53
+ NDF_TID = 11, // the thread ID of the thread logging - added automatically
54
+ NDF_THREAD_TAG = 12, // the thread tag of the thread logging - added automatically
55
+ NDF_MESSAGE_ID = 13, // for specific events
56
+ NDF_MODULE = 14, // for internal plugin module, all other get the NDF_THREAD_TAG
57
+
58
+ NDF_NIDL_NODE = 15, // the node / rrdhost currently being worked
59
+ NDF_NIDL_INSTANCE = 16, // the instance / rrdset currently being worked
60
+ NDF_NIDL_CONTEXT = 17, // the context of the instance currently being worked
61
+ NDF_NIDL_DIMENSION = 18, // the dimension / rrddim currently being worked
62
+
63
+ // web server, aclk and stream receiver
64
+ NDF_SRC_TRANSPORT = 19, // the transport we received the request, one of: http, https, pluginsd
65
+
66
+ // Netdata Cloud Related
67
+ NDF_ACCOUNT_ID = 20,
68
+ NDF_USER_NAME = 21,
69
+ NDF_USER_ROLE = 22,
70
+ NDF_USER_ACCESS = 23,
71
+
72
+ // web server and stream receiver
73
+ NDF_SRC_IP = 24, // the streaming / web server source IP
74
+ NDF_SRC_PORT = 25, // the streaming / web server source Port
75
+ NDF_SRC_FORWARDED_HOST = 26,
76
+ NDF_SRC_FORWARDED_FOR = 27,
77
+ NDF_SRC_CAPABILITIES = 28, // the stream receiver capabilities
78
+
79
+ // stream sender (established links)
80
+ NDF_DST_TRANSPORT = 29, // the transport we send the request, one of: http, https
81
+ NDF_DST_IP = 30, // the destination streaming IP
82
+ NDF_DST_PORT = 31, // the destination streaming Port
83
+ NDF_DST_CAPABILITIES = 32, // the destination streaming capabilities
84
+
85
+ // web server, aclk and stream receiver
86
+ NDF_REQUEST_METHOD = 33, // for http like requests, the http request method
87
+ NDF_RESPONSE_CODE = 34, // for http like requests, the http response code, otherwise a status string
88
+
89
+ // web server (all), aclk (queries)
90
+ NDF_CONNECTION_ID = 35, // the web server connection ID
91
+ NDF_TRANSACTION_ID = 36, // the web server and API transaction ID
92
+ NDF_RESPONSE_SENT_BYTES = 37, // for http like requests, the response bytes
93
+ NDF_RESPONSE_SIZE_BYTES = 38, // for http like requests, the uncompressed response size
94
+ NDF_RESPONSE_PREPARATION_TIME_USEC = 39, // for http like requests, the preparation time
95
+ NDF_RESPONSE_SENT_TIME_USEC = 40, // for http like requests, the time to send the response back
96
+ NDF_RESPONSE_TOTAL_TIME_USEC = 41, // for http like requests, the total time to complete the response
97
+
98
+ // health alerts
99
+ NDF_ALERT_ID = 42,
100
+ NDF_ALERT_UNIQUE_ID = 43,
101
+ NDF_ALERT_EVENT_ID = 44,
102
+ NDF_ALERT_TRANSITION_ID = 45,
103
+ NDF_ALERT_CONFIG_HASH = 46,
104
+ NDF_ALERT_NAME = 47,
105
+ NDF_ALERT_CLASS = 48,
106
+ NDF_ALERT_COMPONENT = 49,
107
+ NDF_ALERT_TYPE = 50,
108
+ NDF_ALERT_EXEC = 51,
109
+ NDF_ALERT_RECIPIENT = 52,
110
+ NDF_ALERT_DURATION = 53,
111
+ NDF_ALERT_VALUE = 54,
112
+ NDF_ALERT_VALUE_OLD = 55,
113
+ NDF_ALERT_STATUS = 56,
114
+ NDF_ALERT_STATUS_OLD = 57,
115
+ NDF_ALERT_SOURCE = 58,
116
+ NDF_ALERT_UNITS = 59,
117
+ NDF_ALERT_SUMMARY = 60,
118
+ NDF_ALERT_INFO = 61,
119
+ NDF_ALERT_NOTIFICATION_REALTIME_USEC = 62,
120
+ // NDF_ALERT_FLAGS,
121
+
122
+ // put new items here
123
+ // leave the request URL and the message last
124
+
125
+ NDF_REQUEST = 63, // the request we are currently working on
126
+ NDF_MESSAGE = 64, // the log message, if any
127
+
128
+ // terminator
129
+ _NDF_MAX,
130
+} ND_LOG_FIELD_ID;
131
+
132
+typedef enum __attribute__((__packed__)) {
133
+ NDFT_UNSET = 0,
134
+ NDFT_TXT,
135
+ NDFT_STR,
136
+ NDFT_BFR,
137
+ NDFT_U64,
138
+ NDFT_I64,
139
+ NDFT_DBL,
140
+ NDFT_UUID,
141
+ NDFT_CALLBACK,
142
+
143
+ // terminator
144
+ _NDFT_MAX,
145
+} ND_LOG_STACK_FIELD_TYPE;
146
+
147
+#endif //NETDATA_ND_LOG_COMMON_H
src/libnetdata/log/nd_log-config.c
new
+207
@@ -0,0 +1,207 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {
6
+ char buf[FILENAME_MAX + 100];
7
+ if(setting && *setting)
8
+ strncpyz(buf, setting, sizeof(buf) - 1);
9
+ else
10
+ buf[0] = '\0';
11
+
12
+ struct nd_log_source *ls = &nd_log.sources[source];
13
+ char *output = strrchr(buf, '@');
14
+
15
+ if(!output)
16
+ // all of it is the output
17
+ output = buf;
18
+ else {
19
+ // we found an '@', the next char is the output
20
+ *output = '\0';
21
+ output++;
22
+
23
+ // parse the other params
24
+ char *remaining = buf;
25
+ while(remaining) {
26
+ char *value = strsep_skip_consecutive_separators(&remaining, ",");
27
+ if (!value || !*value) continue;
28
+
29
+ char *name = strsep_skip_consecutive_separators(&value, "=");
30
+ if (!name || !*name) continue;
31
+
32
+ if(strcmp(name, "logfmt") == 0)
33
+ ls->format = NDLF_LOGFMT;
34
+ else if(strcmp(name, "json") == 0)
35
+ ls->format = NDLF_JSON;
36
+ else if(strcmp(name, "journal") == 0)
37
+ ls->format = NDLF_JOURNAL;
38
+#if defined(OS_WINDOWS)
39
+#if defined(HAVE_ETW)
40
+ else if(strcmp(name, ETW_NAME) == 0)
41
+ ls->format = NDLF_ETW;
42
+#endif
43
+#if defined(HAVE_WEL)
44
+ else if(strcmp(name, WEL_NAME) == 0)
45
+ ls->format = NDLF_WEL;
46
+#endif
47
+#endif
48
+ else if(strcmp(name, "level") == 0 && value && *value)
49
+ ls->min_priority = nd_log_priority2id(value);
50
+ else if(strcmp(name, "protection") == 0 && value && *value) {
51
+ if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) {
52
+ ls->limits = ND_LOG_LIMITS_UNLIMITED;
53
+ ls->limits.counter = 0;
54
+ ls->limits.prevented = 0;
55
+ }
56
+ else {
57
+ ls->limits = ND_LOG_LIMITS_DEFAULT;
58
+
59
+ char *slash = strchr(value, '/');
60
+ if(slash) {
61
+ *slash = '\0';
62
+ slash++;
63
+ ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
64
+
65
+ int period;
66
+ if(!duration_parse_seconds(slash, &period)) {
67
+ nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing period '%s'", slash);
68
+ period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
69
+ }
70
+
71
+ ls->limits.throttle_period = period;
72
+ }
73
+ else {
74
+ ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
75
+ ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
76
+ }
77
+ }
78
+ }
79
+ else
80
+ nd_log(NDLS_DAEMON, NDLP_ERR,
81
+ "Error while parsing configuration of log source '%s'. "
82
+ "In config '%s', '%s' is not understood.",
83
+ nd_log_id2source(source), setting, name);
84
+ }
85
+ }
86
+
87
+ if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) {
88
+ ls->method = NDLM_DISABLED;
89
+ ls->filename = "/dev/null";
90
+ }
91
+ else if(strcmp(output, "journal") == 0) {
92
+ ls->method = NDLM_JOURNAL;
93
+ ls->filename = NULL;
94
+ }
95
+#if defined(OS_WINDOWS)
96
+#if defined(HAVE_ETW)
97
+ else if(strcmp(output, ETW_NAME) == 0) {
98
+ ls->method = NDLM_ETW;
99
+ ls->filename = NULL;
100
+ }
101
+#endif
102
+#if defined(HAVE_WEL)
103
+ else if(strcmp(output, WEL_NAME) == 0) {
104
+ ls->method = NDLM_WEL;
105
+ ls->filename = NULL;
106
+ }
107
+#endif
108
+#endif
109
+ else if(strcmp(output, "syslog") == 0) {
110
+ ls->method = NDLM_SYSLOG;
111
+ ls->filename = NULL;
112
+ }
113
+ else if(strcmp(output, "/dev/null") == 0) {
114
+ ls->method = NDLM_DEVNULL;
115
+ ls->filename = "/dev/null";
116
+ }
117
+ else if(strcmp(output, "system") == 0) {
118
+ if(ls->fd == STDERR_FILENO) {
119
+ ls->method = NDLM_STDERR;
120
+ ls->filename = NULL;
121
+ ls->fd = STDERR_FILENO;
122
+ }
123
+ else {
124
+ ls->method = NDLM_STDOUT;
125
+ ls->filename = NULL;
126
+ ls->fd = STDOUT_FILENO;
127
+ }
128
+ }
129
+ else if(strcmp(output, "stderr") == 0) {
130
+ ls->method = NDLM_STDERR;
131
+ ls->filename = NULL;
132
+ ls->fd = STDERR_FILENO;
133
+ }
134
+ else if(strcmp(output, "stdout") == 0) {
135
+ ls->method = NDLM_STDOUT;
136
+ ls->filename = NULL;
137
+ ls->fd = STDOUT_FILENO;
138
+ }
139
+ else {
140
+ ls->method = NDLM_FILE;
141
+ ls->filename = strdupz(output);
142
+ }
143
+
144
+#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
145
+ ls->min_priority = NDLP_DEBUG;
146
+#endif
147
+
148
+ if(source == NDLS_COLLECTORS) {
149
+ // set the method for the collector processes we will spawn
150
+
151
+ ND_LOG_METHOD method = NDLM_STDERR;
152
+ ND_LOG_FORMAT format = NDLF_LOGFMT;
153
+ ND_LOG_FIELD_PRIORITY priority = ls->min_priority;
154
+
155
+ if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ls->method)) {
156
+ method = ls->method;
157
+ format = ls->format;
158
+ }
159
+
160
+ nd_setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1);
161
+ nd_setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1);
162
+ nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
163
+ }
164
+}
165
+
166
+void nd_log_set_priority_level(const char *setting) {
167
+ if(!setting || !*setting)
168
+ setting = "info";
169
+
170
+ ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting);
171
+
172
+#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
173
+ priority = NDLP_DEBUG;
174
+#endif
175
+
176
+ for (size_t i = 0; i < _NDLS_MAX; i++) {
177
+ if (i != NDLS_DEBUG)
178
+ nd_log.sources[i].min_priority = priority;
179
+ }
180
+
181
+ // the right one
182
+ nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
183
+}
184
+
185
+void nd_log_set_facility(const char *facility) {
186
+ if(!facility || !*facility)
187
+ facility = "daemon";
188
+
189
+ nd_log.syslog.facility = nd_log_facility2id(facility);
190
+ nd_setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1);
191
+}
192
+
193
+void nd_log_set_flood_protection(size_t logs, time_t period) {
194
+ nd_log.sources[NDLS_DAEMON].limits.logs_per_period =
195
+ nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup;
196
+ nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period =
197
+ nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs;
198
+
199
+ nd_log.sources[NDLS_DAEMON].limits.throttle_period =
200
+ nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period;
201
+
202
+ char buf[100];
203
+ snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period);
204
+ nd_setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1);
205
+ snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs);
206
+ nd_setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1);
207
+}
src/libnetdata/log/nd_log-field-formatters.c
new
+127
@@ -0,0 +1,127 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+int64_t log_field_to_int64(struct log_field *lf) {
6
+
7
+ // --- FIELD_PARSER_VERSIONS ---
8
+ //
9
+ // IMPORTANT:
10
+ // THERE ARE 6 VERSIONS OF THIS CODE
11
+ //
12
+ // 1. journal (direct socket API),
13
+ // 2. journal (libsystemd API),
14
+ // 3. logfmt,
15
+ // 4. json,
16
+ // 5. convert to uint64
17
+ // 6. convert to int64
18
+ //
19
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
20
+
21
+ CLEAN_BUFFER *tmp = NULL;
22
+ const char *s = NULL;
23
+
24
+ switch(lf->entry.type) {
25
+ default:
26
+ case NDFT_UUID:
27
+ case NDFT_UNSET:
28
+ return 0;
29
+
30
+ case NDFT_TXT:
31
+ s = lf->entry.txt;
32
+ break;
33
+
34
+ case NDFT_STR:
35
+ s = string2str(lf->entry.str);
36
+ break;
37
+
38
+ case NDFT_BFR:
39
+ s = buffer_tostring(lf->entry.bfr);
40
+ break;
41
+
42
+ case NDFT_CALLBACK:
43
+ tmp = buffer_create(0, NULL);
44
+
45
+ if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
46
+ s = buffer_tostring(tmp);
47
+ else
48
+ s = NULL;
49
+ break;
50
+
51
+ case NDFT_U64:
52
+ return (int64_t)lf->entry.u64;
53
+
54
+ case NDFT_I64:
55
+ return (int64_t)lf->entry.i64;
56
+
57
+ case NDFT_DBL:
58
+ return (int64_t)lf->entry.dbl;
59
+ }
60
+
61
+ if(s && *s)
62
+ return str2ll(s, NULL);
63
+
64
+ return 0;
65
+}
66
+
67
+uint64_t log_field_to_uint64(struct log_field *lf) {
68
+
69
+ // --- FIELD_PARSER_VERSIONS ---
70
+ //
71
+ // IMPORTANT:
72
+ // THERE ARE 6 VERSIONS OF THIS CODE
73
+ //
74
+ // 1. journal (direct socket API),
75
+ // 2. journal (libsystemd API),
76
+ // 3. logfmt,
77
+ // 4. json,
78
+ // 5. convert to uint64
79
+ // 6. convert to int64
80
+ //
81
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
82
+
83
+ CLEAN_BUFFER *tmp = NULL;
84
+ const char *s = NULL;
85
+
86
+ switch(lf->entry.type) {
87
+ default:
88
+ case NDFT_UUID:
89
+ case NDFT_UNSET:
90
+ return 0;
91
+
92
+ case NDFT_TXT:
93
+ s = lf->entry.txt;
94
+ break;
95
+
96
+ case NDFT_STR:
97
+ s = string2str(lf->entry.str);
98
+ break;
99
+
100
+ case NDFT_BFR:
101
+ s = buffer_tostring(lf->entry.bfr);
102
+ break;
103
+
104
+ case NDFT_CALLBACK:
105
+ tmp = buffer_create(0, NULL);
106
+
107
+ if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
108
+ s = buffer_tostring(tmp);
109
+ else
110
+ s = NULL;
111
+ break;
112
+
113
+ case NDFT_U64:
114
+ return lf->entry.u64;
115
+
116
+ case NDFT_I64:
117
+ return lf->entry.i64;
118
+
119
+ case NDFT_DBL:
120
+ return (uint64_t) lf->entry.dbl;
121
+ }
122
+
123
+ if(s && *s)
124
+ return str2uint64_t(s, NULL);
125
+
126
+ return 0;
127
+}
src/libnetdata/log/nd_log-format-json.c
new
+78
@@ -0,0 +1,78 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) {
6
+
7
+ // --- FIELD_PARSER_VERSIONS ---
8
+ //
9
+ // IMPORTANT:
10
+ // THERE ARE 6 VERSIONS OF THIS CODE
11
+ //
12
+ // 1. journal (direct socket API),
13
+ // 2. journal (libsystemd API),
14
+ // 3. logfmt,
15
+ // 4. json,
16
+ // 5. convert to uint64
17
+ // 6. convert to int64
18
+ //
19
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
20
+
21
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
22
+ CLEAN_BUFFER *tmp = NULL;
23
+
24
+ for (size_t i = 0; i < fields_max; i++) {
25
+ if (!fields[i].entry.set || !fields[i].logfmt)
26
+ continue;
27
+
28
+ const char *key = fields[i].logfmt;
29
+
30
+ const char *s = NULL;
31
+ switch(fields[i].entry.type) {
32
+ case NDFT_TXT:
33
+ s = fields[i].entry.txt;
34
+ break;
35
+ case NDFT_STR:
36
+ s = string2str(fields[i].entry.str);
37
+ break;
38
+ case NDFT_BFR:
39
+ s = buffer_tostring(fields[i].entry.bfr);
40
+ break;
41
+ case NDFT_U64:
42
+ buffer_json_member_add_uint64(wb, key, fields[i].entry.u64);
43
+ break;
44
+ case NDFT_I64:
45
+ buffer_json_member_add_int64(wb, key, fields[i].entry.i64);
46
+ break;
47
+ case NDFT_DBL:
48
+ buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
49
+ break;
50
+ case NDFT_UUID:
51
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
52
+ char u[UUID_COMPACT_STR_LEN];
53
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
54
+ buffer_json_member_add_string(wb, key, u);
55
+ }
56
+ break;
57
+ case NDFT_CALLBACK: {
58
+ if(!tmp)
59
+ tmp = buffer_create(1024, NULL);
60
+ else
61
+ buffer_flush(tmp);
62
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
63
+ s = buffer_tostring(tmp);
64
+ else
65
+ s = NULL;
66
+ }
67
+ break;
68
+ default:
69
+ s = "UNHANDLED";
70
+ break;
71
+ }
72
+
73
+ if(s && *s)
74
+ buffer_json_member_add_string(wb, key, s);
75
+ }
76
+
77
+ buffer_json_finalize(wb);
78
+}
src/libnetdata/log/nd_log-format-logfmt.c
new
+151
@@ -0,0 +1,151 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+static bool needs_quotes_for_logfmt(const char *s)
6
+{
7
+ static bool safe_for_logfmt[256] = {
8
+ [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true,
9
+ ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true,
10
+ ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true,
11
+ ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true,
12
+ ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true,
13
+ ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true,
14
+ ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true,
15
+ ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true,
16
+ ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true,
17
+ ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true,
18
+ ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true,
19
+ ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true,
20
+ ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true,
21
+ ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true,
22
+ };
23
+
24
+ if(!*s)
25
+ return true;
26
+
27
+ while(*s) {
28
+ if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s])
29
+ return true;
30
+
31
+ s++;
32
+ }
33
+
34
+ return false;
35
+}
36
+
37
+static void string_to_logfmt(BUFFER *wb, const char *s)
38
+{
39
+ bool spaces = needs_quotes_for_logfmt(s);
40
+
41
+ if(spaces)
42
+ buffer_fast_strcat(wb, "\"", 1);
43
+
44
+ buffer_json_strcat(wb, s);
45
+
46
+ if(spaces)
47
+ buffer_fast_strcat(wb, "\"", 1);
48
+}
49
+
50
+void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max) {
51
+
52
+ // --- FIELD_PARSER_VERSIONS ---
53
+ //
54
+ // IMPORTANT:
55
+ // THERE ARE 6 VERSIONS OF THIS CODE
56
+ //
57
+ // 1. journal (direct socket API),
58
+ // 2. journal (libsystemd API),
59
+ // 3. logfmt,
60
+ // 4. json,
61
+ // 5. convert to uint64
62
+ // 6. convert to int64
63
+ //
64
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
65
+
66
+ CLEAN_BUFFER *tmp = NULL;
67
+
68
+ for (size_t i = 0; i < fields_max; i++) {
69
+ if (!fields[i].entry.set || !fields[i].logfmt)
70
+ continue;
71
+
72
+ const char *key = fields[i].logfmt;
73
+
74
+ if(fields[i].annotator) {
75
+ const char *s = fields[i].annotator(&fields[i]);
76
+ if(!s) continue;
77
+
78
+ if(buffer_strlen(wb))
79
+ buffer_fast_strcat(wb, " ", 1);
80
+
81
+ buffer_strcat(wb, key);
82
+ buffer_fast_strcat(wb, "=", 1);
83
+ string_to_logfmt(wb, s);
84
+ }
85
+ else {
86
+ if(buffer_strlen(wb))
87
+ buffer_fast_strcat(wb, " ", 1);
88
+
89
+ switch(fields[i].entry.type) {
90
+ case NDFT_TXT:
91
+ if(*fields[i].entry.txt) {
92
+ buffer_strcat(wb, key);
93
+ buffer_fast_strcat(wb, "=", 1);
94
+ string_to_logfmt(wb, fields[i].entry.txt);
95
+ }
96
+ break;
97
+ case NDFT_STR:
98
+ buffer_strcat(wb, key);
99
+ buffer_fast_strcat(wb, "=", 1);
100
+ string_to_logfmt(wb, string2str(fields[i].entry.str));
101
+ break;
102
+ case NDFT_BFR:
103
+ if(buffer_strlen(fields[i].entry.bfr)) {
104
+ buffer_strcat(wb, key);
105
+ buffer_fast_strcat(wb, "=", 1);
106
+ string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr));
107
+ }
108
+ break;
109
+ case NDFT_U64:
110
+ buffer_strcat(wb, key);
111
+ buffer_fast_strcat(wb, "=", 1);
112
+ buffer_print_uint64(wb, fields[i].entry.u64);
113
+ break;
114
+ case NDFT_I64:
115
+ buffer_strcat(wb, key);
116
+ buffer_fast_strcat(wb, "=", 1);
117
+ buffer_print_int64(wb, fields[i].entry.i64);
118
+ break;
119
+ case NDFT_DBL:
120
+ buffer_strcat(wb, key);
121
+ buffer_fast_strcat(wb, "=", 1);
122
+ buffer_print_netdata_double(wb, fields[i].entry.dbl);
123
+ break;
124
+ case NDFT_UUID:
125
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
126
+ char u[UUID_COMPACT_STR_LEN];
127
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
128
+ buffer_strcat(wb, key);
129
+ buffer_fast_strcat(wb, "=", 1);
130
+ buffer_fast_strcat(wb, u, sizeof(u) - 1);
131
+ }
132
+ break;
133
+ case NDFT_CALLBACK: {
134
+ if(!tmp)
135
+ tmp = buffer_create(1024, NULL);
136
+ else
137
+ buffer_flush(tmp);
138
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) {
139
+ buffer_strcat(wb, key);
140
+ buffer_fast_strcat(wb, "=", 1);
141
+ string_to_logfmt(wb, buffer_tostring(tmp));
142
+ }
143
+ }
144
+ break;
145
+ default:
146
+ buffer_strcat(wb, "UNHANDLED");
147
+ break;
148
+ }
149
+ }
150
+ }
151
+}
src/libnetdata/log/nd_log-init.c
new
+301
@@ -0,0 +1,301 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+// --------------------------------------------------------------------------------------------------------------------
6
+
7
+__attribute__((constructor)) void initialize_invocation_id(void) {
8
+ // check for a NETDATA_INVOCATION_ID
9
+ if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) {
10
+ // not found, check for systemd set INVOCATION_ID
11
+ if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) {
12
+ // not found, generate a new one
13
+ uuid_generate_random(nd_log.invocation_id);
14
+ }
15
+ }
16
+
17
+ char uuid[UUID_COMPACT_STR_LEN];
18
+ uuid_unparse_lower_compact(nd_log.invocation_id, uuid);
19
+ nd_setenv("NETDATA_INVOCATION_ID", uuid, 1);
20
+}
21
+
22
+// --------------------------------------------------------------------------------------------------------------------
23
+
24
+void nd_log_initialize_for_external_plugins(const char *name) {
25
+ // if we don't run under Netdata, log to stderr,
26
+ // otherwise, use the logging method Netdata wants us to use.
27
+#if defined(OS_WINDOWS)
28
+#if defined(HAVE_ETW)
29
+ nd_setenv("NETDATA_LOG_METHOD", ETW_NAME, 0);
30
+ nd_setenv("NETDATA_LOG_FORMAT", ETW_NAME, 0);
31
+#elif defined(HAVE_WEL)
32
+ nd_setenv("NETDATA_LOG_METHOD", WEL_NAME, 0);
33
+ nd_setenv("NETDATA_LOG_FORMAT", WEL_NAME, 0);
34
+#else
35
+ nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
36
+ nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
37
+#endif
38
+#else
39
+ nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
40
+ nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
41
+#endif
42
+
43
+ nd_log.overwrite_process_source = NDLS_COLLECTORS;
44
+ program_name = name;
45
+
46
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
47
+ nd_log.sources[i].method = STDERR_FILENO;
48
+ nd_log.sources[i].fd = -1;
49
+ nd_log.sources[i].fp = NULL;
50
+ }
51
+
52
+ nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL"));
53
+ nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY"));
54
+
55
+ time_t period = 1200;
56
+ size_t logs = 200;
57
+ const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD");
58
+ if(s && *s >= '0' && *s <= '9') {
59
+ period = str2l(s);
60
+ if(period < 0) period = 0;
61
+ }
62
+
63
+ s = getenv("NETDATA_ERRORS_PER_PERIOD");
64
+ if(s && *s >= '0' && *s <= '9')
65
+ logs = str2u(s);
66
+
67
+ nd_log_set_flood_protection(logs, period);
68
+
69
+ if(!netdata_configured_host_prefix) {
70
+ s = getenv("NETDATA_HOST_PREFIX");
71
+ if(s && *s)
72
+ netdata_configured_host_prefix = (char *)s;
73
+ }
74
+
75
+ ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD"));
76
+ ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT"));
77
+
78
+ if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) {
79
+ if(is_stderr_connected_to_journal()) {
80
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal.");
81
+ method = NDLM_JOURNAL;
82
+ }
83
+ else {
84
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr.");
85
+ method = NDLM_STDERR;
86
+ }
87
+ }
88
+
89
+ switch(method) {
90
+ case NDLM_JOURNAL:
91
+ if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||
92
+ !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {
93
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");
94
+ method = NDLM_STDERR;
95
+ }
96
+ break;
97
+
98
+#if defined(OS_WINDOWS)
99
+#if defined(HAVE_ETW)
100
+ case NDLM_ETW:
101
+ if(!nd_log_init_etw()) {
102
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Events Tracing for Windows (ETW). Using stderr.");
103
+ method = NDLM_STDERR;
104
+ }
105
+ break;
106
+#endif
107
+#if defined(HAVE_WEL)
108
+ case NDLM_WEL:
109
+ if(!nd_log_init_wel()) {
110
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Windows Event Log (WEL). Using stderr.");
111
+ method = NDLM_STDERR;
112
+ }
113
+ break;
114
+#endif
115
+#endif
116
+
117
+ case NDLM_SYSLOG:
118
+ nd_log_init_syslog();
119
+ break;
120
+
121
+ default:
122
+ method = NDLM_STDERR;
123
+ break;
124
+ }
125
+
126
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
127
+ nd_log.sources[i].method = method;
128
+ nd_log.sources[i].format = format;
129
+ nd_log.sources[i].fd = -1;
130
+ nd_log.sources[i].fp = NULL;
131
+ }
132
+
133
+ // nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
134
+}
135
+
136
+// --------------------------------------------------------------------------------------------------------------------
137
+
138
+void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) {
139
+ if(e->method == NDLM_DEFAULT)
140
+ nd_log_set_user_settings(source, e->filename);
141
+
142
+ if((e->method == NDLM_FILE && !e->filename) ||
143
+ (e->method == NDLM_DEVNULL && e->fd == -1))
144
+ e->method = NDLM_DISABLED;
145
+
146
+ if(e->fp)
147
+ fflush(e->fp);
148
+
149
+ switch(e->method) {
150
+ case NDLM_SYSLOG:
151
+ nd_log_init_syslog();
152
+ break;
153
+
154
+ case NDLM_JOURNAL:
155
+ nd_log_journal_direct_init(NULL);
156
+ nd_log_journal_systemd_init();
157
+ break;
158
+
159
+#if defined(OS_WINDOWS)
160
+#if defined(HAVE_ETW)
161
+ case NDLM_ETW:
162
+ nd_log_init_etw();
163
+ break;
164
+#endif
165
+#if defined(HAVE_WEL)
166
+ case NDLM_WEL:
167
+ nd_log_init_wel();
168
+ break;
169
+#endif
170
+#endif
171
+
172
+ case NDLM_STDOUT:
173
+ e->fp = stdout;
174
+ e->fd = STDOUT_FILENO;
175
+ break;
176
+
177
+ case NDLM_DISABLED:
178
+ break;
179
+
180
+ case NDLM_DEFAULT:
181
+ case NDLM_STDERR:
182
+ e->method = NDLM_STDERR;
183
+ e->fp = stderr;
184
+ e->fd = STDERR_FILENO;
185
+ break;
186
+
187
+ case NDLM_DEVNULL:
188
+ case NDLM_FILE: {
189
+ int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
190
+ if(fd == -1) {
191
+ if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) {
192
+ e->fd = STDERR_FILENO;
193
+ e->method = NDLM_STDERR;
194
+ netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename);
195
+ }
196
+ else
197
+ netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd);
198
+ }
199
+ else {
200
+ if (!nd_log_replace_existing_fd(e, fd)) {
201
+ if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) {
202
+ if(e->fd == STDOUT_FILENO)
203
+ e->method = NDLM_STDOUT;
204
+ else if(e->fd == STDERR_FILENO)
205
+ e->method = NDLM_STDERR;
206
+
207
+ // we have dup2() fd, so we can close the one we opened
208
+ if(fd != STDOUT_FILENO && fd != STDERR_FILENO)
209
+ close(fd);
210
+ }
211
+ else
212
+ e->fd = fd;
213
+ }
214
+ }
215
+
216
+ // at this point we have e->fd set properly
217
+
218
+ if(e->fd == STDOUT_FILENO)
219
+ e->fp = stdout;
220
+ else if(e->fd == STDERR_FILENO)
221
+ e->fp = stderr;
222
+
223
+ if(!e->fp) {
224
+ e->fp = fdopen(e->fd, "a");
225
+ if (!e->fp) {
226
+ netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename);
227
+
228
+ if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO)
229
+ close(e->fd);
230
+
231
+ e->fp = stderr;
232
+ e->fd = STDERR_FILENO;
233
+ }
234
+ }
235
+ else {
236
+ if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0)
237
+ netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename);
238
+ }
239
+ }
240
+ break;
241
+ }
242
+}
243
+
244
+// --------------------------------------------------------------------------------------------------------------------
245
+
246
+void nd_log_stdin_init(int fd, const char *filename) {
247
+ int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
248
+ if(f == -1)
249
+ return;
250
+
251
+ if(f != fd) {
252
+ dup2(f, fd);
253
+ close(f);
254
+ }
255
+}
256
+
257
+void nd_log_initialize(void) {
258
+ nd_log_stdin_init(STDIN_FILENO, "/dev/null");
259
+
260
+ for(size_t i = 0 ; i < _NDLS_MAX ; i++)
261
+ nd_log_open(&nd_log.sources[i], i);
262
+}
263
+
264
+void nd_log_reopen_log_files(bool log) {
265
+ if(log)
266
+ netdata_log_info("Reopening all log files.");
267
+
268
+ nd_log.std_output.initialized = false;
269
+ nd_log.std_error.initialized = false;
270
+ nd_log.journal_direct.initialized = false;
271
+ nd_log.journal.initialized = false;
272
+ nd_log_initialize();
273
+
274
+ if(log)
275
+ netdata_log_info("Log files re-opened.");
276
+}
277
+
278
+void nd_log_reopen_log_files_for_spawn_server(void) {
279
+ gettid_uncached();
280
+
281
+ if(nd_log.syslog.initialized) {
282
+ closelog();
283
+ nd_log.syslog.initialized = false;
284
+ nd_log_init_syslog();
285
+ }
286
+
287
+ if(nd_log.journal_direct.initialized) {
288
+ close(nd_log.journal_direct.fd);
289
+ nd_log.journal_direct.fd = -1;
290
+ nd_log.journal_direct.initialized = false;
291
+ nd_log_journal_direct_init(NULL);
292
+ }
293
+
294
+ nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED;
295
+ nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED;
296
+ nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED;
297
+ nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED;
298
+ nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED;
299
+ nd_log_reopen_log_files(false);
300
+}
301
+
src/libnetdata/log/nd_log-internals.c
new
+821
@@ -0,0 +1,821 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+// --------------------------------------------------------------------------------------------------------------------
6
+// workaround strerror_r()
7
+
8
+#if defined(STRERROR_R_CHAR_P)
9
+// GLIBC version of strerror_r
10
+static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
11
+#elif defined(HAVE_STRERROR_R)
12
+// POSIX version of strerror_r
13
+static const char *strerror_result(int a, const char *b) { (void)a; return b; }
14
+#elif defined(HAVE_C__GENERIC)
15
+
16
+// what a trick!
17
+// http://stackoverflow.com/questions/479207/function-overloading-in-c
18
+static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
19
+static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
20
+
21
+#define strerror_result(a, b) _Generic((a), \
22
+ int: strerror_result_int, \
23
+ char *: strerror_result_string \
24
+ )(a, b)
25
+
26
+#else
27
+#error "cannot detect the format of function strerror_r()"
28
+#endif
29
+
30
+const char *errno2str(int errnum, char *buf, size_t size) {
31
+ return strerror_result(strerror_r(errnum, buf, size), buf);
32
+}
33
+
34
+// --------------------------------------------------------------------------------------------------------------------
35
+// logging method
36
+
37
+static struct {
38
+ ND_LOG_METHOD method;
39
+ const char *name;
40
+} nd_log_methods[] = {
41
+ { .method = NDLM_DISABLED, .name = "none" },
42
+ { .method = NDLM_DEVNULL, .name = "/dev/null" },
43
+ { .method = NDLM_DEFAULT, .name = "default" },
44
+ { .method = NDLM_JOURNAL, .name = "journal" },
45
+ { .method = NDLM_SYSLOG, .name = "syslog" },
46
+ { .method = NDLM_STDOUT, .name = "stdout" },
47
+ { .method = NDLM_STDERR, .name = "stderr" },
48
+ { .method = NDLM_FILE, .name = "file" },
49
+#if defined(OS_WINDOWS)
50
+#if defined(HAVE_ETW)
51
+ { .method = NDLM_ETW, .name = ETW_NAME },
52
+#endif
53
+#if defined(HAVE_WEL)
54
+ { .method = NDLM_WEL, .name = WEL_NAME },
55
+#endif
56
+#endif
57
+};
58
+
59
+ND_LOG_METHOD nd_log_method2id(const char *method) {
60
+ if(!method || !*method)
61
+ return NDLM_DEFAULT;
62
+
63
+ size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
64
+ for(size_t i = 0; i < entries ;i++) {
65
+ if(strcmp(nd_log_methods[i].name, method) == 0)
66
+ return nd_log_methods[i].method;
67
+ }
68
+
69
+ return NDLM_FILE;
70
+}
71
+
72
+const char *nd_log_id2method(ND_LOG_METHOD method) {
73
+ size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
74
+ for(size_t i = 0; i < entries ;i++) {
75
+ if(method == nd_log_methods[i].method)
76
+ return nd_log_methods[i].name;
77
+ }
78
+
79
+ return "unknown";
80
+}
81
+
82
+const char *nd_log_method_for_external_plugins(const char *s) {
83
+ if(s && *s) {
84
+ ND_LOG_METHOD method = nd_log_method2id(s);
85
+ if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method))
86
+ return nd_log_id2method(method);
87
+ }
88
+
89
+ return nd_log_id2method(NDLM_STDERR);
90
+}
91
+
92
+// --------------------------------------------------------------------------------------------------------------------
93
+// facilities
94
+//
95
+// sys/syslog.h (Linux)
96
+// sys/sys/syslog.h (FreeBSD)
97
+// bsd/sys/syslog.h (darwin-xnu)
98
+
99
+static struct {
100
+ int facility;
101
+ const char *name;
102
+} nd_log_facilities[] = {
103
+ { LOG_AUTH, "auth" },
104
+ { LOG_AUTHPRIV, "authpriv" },
105
+ { LOG_CRON, "cron" },
106
+ { LOG_DAEMON, "daemon" },
107
+ { LOG_FTP, "ftp" },
108
+ { LOG_KERN, "kern" },
109
+ { LOG_LPR, "lpr" },
110
+ { LOG_MAIL, "mail" },
111
+ { LOG_NEWS, "news" },
112
+ { LOG_SYSLOG, "syslog" },
113
+ { LOG_USER, "user" },
114
+ { LOG_UUCP, "uucp" },
115
+ { LOG_LOCAL0, "local0" },
116
+ { LOG_LOCAL1, "local1" },
117
+ { LOG_LOCAL2, "local2" },
118
+ { LOG_LOCAL3, "local3" },
119
+ { LOG_LOCAL4, "local4" },
120
+ { LOG_LOCAL5, "local5" },
121
+ { LOG_LOCAL6, "local6" },
122
+ { LOG_LOCAL7, "local7" },
123
+
124
+#ifdef __FreeBSD__
125
+ { LOG_CONSOLE, "console" },
126
+ { LOG_NTP, "ntp" },
127
+
128
+ // FreeBSD does not consider 'security' as deprecated.
129
+ { LOG_SECURITY, "security" },
130
+#else
131
+ // For all other O/S 'security' is mapped to 'auth'.
132
+ { LOG_AUTH, "security" },
133
+#endif
134
+
135
+#ifdef __APPLE__
136
+ { LOG_INSTALL, "install" },
137
+ { LOG_NETINFO, "netinfo" },
138
+ { LOG_RAS, "ras" },
139
+ { LOG_REMOTEAUTH, "remoteauth" },
140
+ { LOG_LAUNCHD, "launchd" },
141
+
142
+#endif
143
+};
144
+
145
+int nd_log_facility2id(const char *facility) {
146
+ size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
147
+ for(size_t i = 0; i < entries ;i++) {
148
+ if(strcmp(nd_log_facilities[i].name, facility) == 0)
149
+ return nd_log_facilities[i].facility;
150
+ }
151
+
152
+ return LOG_DAEMON;
153
+}
154
+
155
+const char *nd_log_id2facility(int facility) {
156
+ size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
157
+ for(size_t i = 0; i < entries ;i++) {
158
+ if(nd_log_facilities[i].facility == facility)
159
+ return nd_log_facilities[i].name;
160
+ }
161
+
162
+ return "daemon";
163
+}
164
+
165
+// --------------------------------------------------------------------------------------------------------------------
166
+// priorities
167
+
168
+static struct {
169
+ ND_LOG_FIELD_PRIORITY priority;
170
+ const char *name;
171
+} nd_log_priorities[] = {
172
+ { .priority = NDLP_EMERG, .name = "emergency" },
173
+ { .priority = NDLP_EMERG, .name = "emerg" },
174
+ { .priority = NDLP_ALERT, .name = "alert" },
175
+ { .priority = NDLP_CRIT, .name = "critical" },
176
+ { .priority = NDLP_CRIT, .name = "crit" },
177
+ { .priority = NDLP_ERR, .name = "error" },
178
+ { .priority = NDLP_ERR, .name = "err" },
179
+ { .priority = NDLP_WARNING, .name = "warning" },
180
+ { .priority = NDLP_WARNING, .name = "warn" },
181
+ { .priority = NDLP_NOTICE, .name = "notice" },
182
+ { .priority = NDLP_INFO, .name = NDLP_INFO_STR },
183
+ { .priority = NDLP_DEBUG, .name = "debug" },
184
+};
185
+
186
+int nd_log_priority2id(const char *priority) {
187
+ size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
188
+ for(size_t i = 0; i < entries ;i++) {
189
+ if(strcmp(nd_log_priorities[i].name, priority) == 0)
190
+ return nd_log_priorities[i].priority;
191
+ }
192
+
193
+ return NDLP_INFO;
194
+}
195
+
196
+const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) {
197
+ size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
198
+ for(size_t i = 0; i < entries ;i++) {
199
+ if(priority == nd_log_priorities[i].priority)
200
+ return nd_log_priorities[i].name;
201
+ }
202
+
203
+ return NDLP_INFO_STR;
204
+}
205
+
206
+// --------------------------------------------------------------------------------------------------------------------
207
+// log sources
208
+
209
+const char *nd_log_sources[] = {
210
+ [NDLS_UNSET] = "UNSET",
211
+ [NDLS_ACCESS] = "access",
212
+ [NDLS_ACLK] = "aclk",
213
+ [NDLS_COLLECTORS] = "collector",
214
+ [NDLS_DAEMON] = "daemon",
215
+ [NDLS_HEALTH] = "health",
216
+ [NDLS_DEBUG] = "debug",
217
+};
218
+
219
+size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) {
220
+ size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
221
+ for(size_t i = 0; i < entries ;i++) {
222
+ if(strcmp(nd_log_sources[i], source) == 0)
223
+ return i;
224
+ }
225
+
226
+ return def;
227
+}
228
+
229
+
230
+const char *nd_log_id2source(ND_LOG_SOURCES source) {
231
+ size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
232
+ if(source < entries)
233
+ return nd_log_sources[source];
234
+
235
+ return nd_log_sources[NDLS_COLLECTORS];
236
+}
237
+
238
+// --------------------------------------------------------------------------------------------------------------------
239
+// log output formats
240
+
241
+static struct {
242
+ ND_LOG_FORMAT format;
243
+ const char *name;
244
+} nd_log_formats[] = {
245
+ { .format = NDLF_JOURNAL, .name = "journal" },
246
+ { .format = NDLF_LOGFMT, .name = "logfmt" },
247
+ { .format = NDLF_JSON, .name = "json" },
248
+#if defined(OS_WINDOWS)
249
+#if defined(HAVE_ETW)
250
+ { .format = NDLF_ETW, .name = ETW_NAME },
251
+#endif
252
+#if defined(HAVE_WEL)
253
+ { .format = NDLF_WEL, .name = WEL_NAME },
254
+#endif
255
+#endif
256
+};
257
+
258
+ND_LOG_FORMAT nd_log_format2id(const char *format) {
259
+ if(!format || !*format)
260
+ return NDLF_LOGFMT;
261
+
262
+ size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
263
+ for(size_t i = 0; i < entries ;i++) {
264
+ if(strcmp(nd_log_formats[i].name, format) == 0)
265
+ return nd_log_formats[i].format;
266
+ }
267
+
268
+ return NDLF_LOGFMT;
269
+}
270
+
271
+const char *nd_log_id2format(ND_LOG_FORMAT format) {
272
+ size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
273
+ for(size_t i = 0; i < entries ;i++) {
274
+ if(format == nd_log_formats[i].format)
275
+ return nd_log_formats[i].name;
276
+ }
277
+
278
+ return "logfmt";
279
+}
280
+
281
+// --------------------------------------------------------------------------------------------------------------------
282
+
283
+struct nd_log nd_log = {
284
+ .overwrite_process_source = 0,
285
+ .journal = {
286
+ .initialized = false,
287
+ },
288
+ .journal_direct = {
289
+ .initialized = false,
290
+ .fd = -1,
291
+ },
292
+ .syslog = {
293
+ .initialized = false,
294
+ .facility = LOG_DAEMON,
295
+ },
296
+#if defined(OS_WINDOWS)
297
+ .eventlog = {
298
+ .initialized = false,
299
+ },
300
+#endif
301
+ .std_output = {
302
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
303
+ .initialized = false,
304
+ },
305
+ .std_error = {
306
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
307
+ .initialized = false,
308
+ },
309
+ .sources = {
310
+ [NDLS_UNSET] = {
311
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
312
+ .method = NDLM_DISABLED,
313
+ .format = NDLF_JOURNAL,
314
+ .filename = NULL,
315
+ .fd = -1,
316
+ .fp = NULL,
317
+ .min_priority = NDLP_EMERG,
318
+ .limits = ND_LOG_LIMITS_UNLIMITED,
319
+ },
320
+ [NDLS_ACCESS] = {
321
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
322
+ .method = NDLM_DEFAULT,
323
+ .format = NDLF_LOGFMT,
324
+ .filename = LOG_DIR "/access.log",
325
+ .fd = -1,
326
+ .fp = NULL,
327
+ .min_priority = NDLP_DEBUG,
328
+ .limits = ND_LOG_LIMITS_UNLIMITED,
329
+ },
330
+ [NDLS_ACLK] = {
331
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
332
+ .method = NDLM_FILE,
333
+ .format = NDLF_LOGFMT,
334
+ .filename = LOG_DIR "/aclk.log",
335
+ .fd = -1,
336
+ .fp = NULL,
337
+ .min_priority = NDLP_DEBUG,
338
+ .limits = ND_LOG_LIMITS_UNLIMITED,
339
+ },
340
+ [NDLS_COLLECTORS] = {
341
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
342
+ .method = NDLM_DEFAULT,
343
+ .format = NDLF_LOGFMT,
344
+ .filename = LOG_DIR "/collector.log",
345
+ .fd = STDERR_FILENO,
346
+ .fp = NULL,
347
+ .min_priority = NDLP_INFO,
348
+ .limits = ND_LOG_LIMITS_DEFAULT,
349
+ },
350
+ [NDLS_DEBUG] = {
351
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
352
+ .method = NDLM_DISABLED,
353
+ .format = NDLF_LOGFMT,
354
+ .filename = LOG_DIR "/debug.log",
355
+ .fd = STDOUT_FILENO,
356
+ .fp = NULL,
357
+ .min_priority = NDLP_DEBUG,
358
+ .limits = ND_LOG_LIMITS_UNLIMITED,
359
+ },
360
+ [NDLS_DAEMON] = {
361
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
362
+ .method = NDLM_DEFAULT,
363
+ .filename = LOG_DIR "/daemon.log",
364
+ .format = NDLF_LOGFMT,
365
+ .fd = -1,
366
+ .fp = NULL,
367
+ .min_priority = NDLP_INFO,
368
+ .limits = ND_LOG_LIMITS_DEFAULT,
369
+ },
370
+ [NDLS_HEALTH] = {
371
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
372
+ .method = NDLM_DEFAULT,
373
+ .format = NDLF_LOGFMT,
374
+ .filename = LOG_DIR "/health.log",
375
+ .fd = -1,
376
+ .fp = NULL,
377
+ .min_priority = NDLP_DEBUG,
378
+ .limits = ND_LOG_LIMITS_UNLIMITED,
379
+ },
380
+ },
381
+};
382
+
383
+// --------------------------------------------------------------------------------------------------------------------
384
+
385
+__thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
386
+__thread size_t thread_log_stack_next = 0;
387
+__thread struct log_field thread_log_fields[_NDF_MAX] = {
388
+ // THE ORDER HERE IS IRRELEVANT (but keep them sorted by their number)
389
+
390
+ [NDF_STOP] = { // processing will not stop on this - so it is ok to be first
391
+ .journal = NULL,
392
+ .logfmt = NULL,
393
+ .eventlog = NULL,
394
+ .annotator = NULL,
395
+ },
396
+ [NDF_TIMESTAMP_REALTIME_USEC] = {
397
+ .journal = NULL,
398
+ .eventlog = "Timestamp",
399
+ .logfmt = "time",
400
+ .annotator = timestamp_usec_annotator,
401
+ },
402
+ [NDF_SYSLOG_IDENTIFIER] = {
403
+ .journal = "SYSLOG_IDENTIFIER", // standard journald field
404
+ .eventlog = "Program",
405
+ .logfmt = "comm",
406
+ },
407
+ [NDF_LOG_SOURCE] = {
408
+ .journal = "ND_LOG_SOURCE",
409
+ .eventlog = "NetdataLogSource",
410
+ .logfmt = "source",
411
+ },
412
+ [NDF_PRIORITY] = {
413
+ .journal = "PRIORITY", // standard journald field
414
+ .eventlog = "Level",
415
+ .logfmt = "level",
416
+ .annotator = priority_annotator,
417
+ },
418
+ [NDF_ERRNO] = {
419
+ .journal = "ERRNO", // standard journald field
420
+ .eventlog = "UnixErrno",
421
+ .logfmt = "errno",
422
+ .annotator = errno_annotator,
423
+ },
424
+ [NDF_WINERROR] = {
425
+#if defined(OS_WINDOWS)
426
+ .journal = "WINERROR",
427
+ .eventlog = "WindowsLastError",
428
+ .logfmt = "winerror",
429
+ .annotator = winerror_annotator,
430
+#endif
431
+ },
432
+ [NDF_INVOCATION_ID] = {
433
+ .journal = "INVOCATION_ID", // standard journald field
434
+ .eventlog = "InvocationID",
435
+ .logfmt = NULL,
436
+ },
437
+ [NDF_LINE] = {
438
+ .journal = "CODE_LINE", // standard journald field
439
+ .eventlog = "CodeLine",
440
+ .logfmt = NULL,
441
+ },
442
+ [NDF_FILE] = {
443
+ .journal = "CODE_FILE", // standard journald field
444
+ .eventlog = "CodeFile",
445
+ .logfmt = NULL,
446
+ },
447
+ [NDF_FUNC] = {
448
+ .journal = "CODE_FUNC", // standard journald field
449
+ .eventlog = "CodeFunction",
450
+ .logfmt = NULL,
451
+ },
452
+ [NDF_TID] = {
453
+ .journal = "TID", // standard journald field
454
+ .eventlog = "ThreadID",
455
+ .logfmt = "tid",
456
+ },
457
+ [NDF_THREAD_TAG] = {
458
+ .journal = "THREAD_TAG",
459
+ .eventlog = "ThreadName",
460
+ .logfmt = "thread",
461
+ },
462
+ [NDF_MESSAGE_ID] = {
463
+ .journal = "MESSAGE_ID",
464
+ .eventlog = "MessageID",
465
+ .logfmt = "msg_id",
466
+ },
467
+ [NDF_MODULE] = {
468
+ .journal = "ND_MODULE",
469
+ .eventlog = "Module",
470
+ .logfmt = "module",
471
+ },
472
+ [NDF_NIDL_NODE] = {
473
+ .journal = "ND_NIDL_NODE",
474
+ .eventlog = "Node",
475
+ .logfmt = "node",
476
+ },
477
+ [NDF_NIDL_INSTANCE] = {
478
+ .journal = "ND_NIDL_INSTANCE",
479
+ .eventlog = "Instance",
480
+ .logfmt = "instance",
481
+ },
482
+ [NDF_NIDL_CONTEXT] = {
483
+ .journal = "ND_NIDL_CONTEXT",
484
+ .eventlog = "Context",
485
+ .logfmt = "context",
486
+ },
487
+ [NDF_NIDL_DIMENSION] = {
488
+ .journal = "ND_NIDL_DIMENSION",
489
+ .eventlog = "Dimension",
490
+ .logfmt = "dimension",
491
+ },
492
+ [NDF_SRC_TRANSPORT] = {
493
+ .journal = "ND_SRC_TRANSPORT",
494
+ .eventlog = "SourceTransport",
495
+ .logfmt = "src_transport",
496
+ },
497
+ [NDF_ACCOUNT_ID] = {
498
+ .journal = "ND_ACCOUNT_ID",
499
+ .eventlog = "AccountID",
500
+ .logfmt = "account",
501
+ },
502
+ [NDF_USER_NAME] = {
503
+ .journal = "ND_USER_NAME",
504
+ .eventlog = "UserName",
505
+ .logfmt = "user",
506
+ },
507
+ [NDF_USER_ROLE] = {
508
+ .journal = "ND_USER_ROLE",
509
+ .eventlog = "UserRole",
510
+ .logfmt = "role",
511
+ },
512
+ [NDF_USER_ACCESS] = {
513
+ .journal = "ND_USER_PERMISSIONS",
514
+ .eventlog = "UserPermissions",
515
+ .logfmt = "permissions",
516
+ },
517
+ [NDF_SRC_IP] = {
518
+ .journal = "ND_SRC_IP",
519
+ .eventlog = "SourceIP",
520
+ .logfmt = "src_ip",
521
+ },
522
+ [NDF_SRC_FORWARDED_HOST] = {
523
+ .journal = "ND_SRC_FORWARDED_HOST",
524
+ .eventlog = "SourceForwardedHost",
525
+ .logfmt = "src_forwarded_host",
526
+ },
527
+ [NDF_SRC_FORWARDED_FOR] = {
528
+ .journal = "ND_SRC_FORWARDED_FOR",
529
+ .eventlog = "SourceForwardedFor",
530
+ .logfmt = "src_forwarded_for",
531
+ },
532
+ [NDF_SRC_PORT] = {
533
+ .journal = "ND_SRC_PORT",
534
+ .eventlog = "SourcePort",
535
+ .logfmt = "src_port",
536
+ },
537
+ [NDF_SRC_CAPABILITIES] = {
538
+ .journal = "ND_SRC_CAPABILITIES",
539
+ .eventlog = "SourceCapabilities",
540
+ .logfmt = "src_capabilities",
541
+ },
542
+ [NDF_DST_TRANSPORT] = {
543
+ .journal = "ND_DST_TRANSPORT",
544
+ .eventlog = "DestinationTransport",
545
+ .logfmt = "dst_transport",
546
+ },
547
+ [NDF_DST_IP] = {
548
+ .journal = "ND_DST_IP",
549
+ .eventlog = "DestinationIP",
550
+ .logfmt = "dst_ip",
551
+ },
552
+ [NDF_DST_PORT] = {
553
+ .journal = "ND_DST_PORT",
554
+ .eventlog = "DestinationPort",
555
+ .logfmt = "dst_port",
556
+ },
557
+ [NDF_DST_CAPABILITIES] = {
558
+ .journal = "ND_DST_CAPABILITIES",
559
+ .eventlog = "DestinationCapabilities",
560
+ .logfmt = "dst_capabilities",
561
+ },
562
+ [NDF_REQUEST_METHOD] = {
563
+ .journal = "ND_REQUEST_METHOD",
564
+ .eventlog = "RequestMethod",
565
+ .logfmt = "req_method",
566
+ },
567
+ [NDF_RESPONSE_CODE] = {
568
+ .journal = "ND_RESPONSE_CODE",
569
+ .eventlog = "ResponseCode",
570
+ .logfmt = "code",
571
+ },
572
+ [NDF_CONNECTION_ID] = {
573
+ .journal = "ND_CONNECTION_ID",
574
+ .eventlog = "ConnectionID",
575
+ .logfmt = "conn",
576
+ },
577
+ [NDF_TRANSACTION_ID] = {
578
+ .journal = "ND_TRANSACTION_ID",
579
+ .eventlog = "TransactionID",
580
+ .logfmt = "transaction",
581
+ },
582
+ [NDF_RESPONSE_SENT_BYTES] = {
583
+ .journal = "ND_RESPONSE_SENT_BYTES",
584
+ .eventlog = "ResponseSentBytes",
585
+ .logfmt = "sent_bytes",
586
+ },
587
+ [NDF_RESPONSE_SIZE_BYTES] = {
588
+ .journal = "ND_RESPONSE_SIZE_BYTES",
589
+ .eventlog = "ResponseSizeBytes",
590
+ .logfmt = "size_bytes",
591
+ },
592
+ [NDF_RESPONSE_PREPARATION_TIME_USEC] = {
593
+ .journal = "ND_RESPONSE_PREP_TIME_USEC",
594
+ .eventlog = "ResponsePreparationTimeUsec",
595
+ .logfmt = "prep_ut",
596
+ },
597
+ [NDF_RESPONSE_SENT_TIME_USEC] = {
598
+ .journal = "ND_RESPONSE_SENT_TIME_USEC",
599
+ .eventlog = "ResponseSentTimeUsec",
600
+ .logfmt = "sent_ut",
601
+ },
602
+ [NDF_RESPONSE_TOTAL_TIME_USEC] = {
603
+ .journal = "ND_RESPONSE_TOTAL_TIME_USEC",
604
+ .eventlog = "ResponseTotalTimeUsec",
605
+ .logfmt = "total_ut",
606
+ },
607
+ [NDF_ALERT_ID] = {
608
+ .journal = "ND_ALERT_ID",
609
+ .eventlog = "AlertID",
610
+ .logfmt = "alert_id",
611
+ },
612
+ [NDF_ALERT_UNIQUE_ID] = {
613
+ .journal = "ND_ALERT_UNIQUE_ID",
614
+ .eventlog = "AlertUniqueID",
615
+ .logfmt = "alert_unique_id",
616
+ },
617
+ [NDF_ALERT_TRANSITION_ID] = {
618
+ .journal = "ND_ALERT_TRANSITION_ID",
619
+ .eventlog = "AlertTransitionID",
620
+ .logfmt = "alert_transition_id",
621
+ },
622
+ [NDF_ALERT_EVENT_ID] = {
623
+ .journal = "ND_ALERT_EVENT_ID",
624
+ .eventlog = "AlertEventID",
625
+ .logfmt = "alert_event_id",
626
+ },
627
+ [NDF_ALERT_CONFIG_HASH] = {
628
+ .journal = "ND_ALERT_CONFIG",
629
+ .eventlog = "AlertConfig",
630
+ .logfmt = "alert_config",
631
+ },
632
+ [NDF_ALERT_NAME] = {
633
+ .journal = "ND_ALERT_NAME",
634
+ .eventlog = "AlertName",
635
+ .logfmt = "alert",
636
+ },
637
+ [NDF_ALERT_CLASS] = {
638
+ .journal = "ND_ALERT_CLASS",
639
+ .eventlog = "AlertClass",
640
+ .logfmt = "alert_class",
641
+ },
642
+ [NDF_ALERT_COMPONENT] = {
643
+ .journal = "ND_ALERT_COMPONENT",
644
+ .eventlog = "AlertComponent",
645
+ .logfmt = "alert_component",
646
+ },
647
+ [NDF_ALERT_TYPE] = {
648
+ .journal = "ND_ALERT_TYPE",
649
+ .eventlog = "AlertType",
650
+ .logfmt = "alert_type",
651
+ },
652
+ [NDF_ALERT_EXEC] = {
653
+ .journal = "ND_ALERT_EXEC",
654
+ .eventlog = "AlertExec",
655
+ .logfmt = "alert_exec",
656
+ },
657
+ [NDF_ALERT_RECIPIENT] = {
658
+ .journal = "ND_ALERT_RECIPIENT",
659
+ .eventlog = "AlertRecipient",
660
+ .logfmt = "alert_recipient",
661
+ },
662
+ [NDF_ALERT_VALUE] = {
663
+ .journal = "ND_ALERT_VALUE",
664
+ .eventlog = "AlertValue",
665
+ .logfmt = "alert_value",
666
+ },
667
+ [NDF_ALERT_VALUE_OLD] = {
668
+ .journal = "ND_ALERT_VALUE_OLD",
669
+ .eventlog = "AlertOldValue",
670
+ .logfmt = "alert_value_old",
671
+ },
672
+ [NDF_ALERT_STATUS] = {
673
+ .journal = "ND_ALERT_STATUS",
674
+ .eventlog = "AlertStatus",
675
+ .logfmt = "alert_status",
676
+ },
677
+ [NDF_ALERT_STATUS_OLD] = {
678
+ .journal = "ND_ALERT_STATUS_OLD",
679
+ .eventlog = "AlertOldStatus",
680
+ .logfmt = "alert_value_old",
681
+ },
682
+ [NDF_ALERT_UNITS] = {
683
+ .journal = "ND_ALERT_UNITS",
684
+ .eventlog = "AlertUnits",
685
+ .logfmt = "alert_units",
686
+ },
687
+ [NDF_ALERT_SUMMARY] = {
688
+ .journal = "ND_ALERT_SUMMARY",
689
+ .eventlog = "AlertSummary",
690
+ .logfmt = "alert_summary",
691
+ },
692
+ [NDF_ALERT_INFO] = {
693
+ .journal = "ND_ALERT_INFO",
694
+ .eventlog = "AlertInfo",
695
+ .logfmt = "alert_info",
696
+ },
697
+ [NDF_ALERT_DURATION] = {
698
+ .journal = "ND_ALERT_DURATION",
699
+ .eventlog = "AlertDuration",
700
+ .logfmt = "alert_duration",
701
+ },
702
+ [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = {
703
+ .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC",
704
+ .eventlog = "AlertNotificationTime",
705
+ .logfmt = "alert_notification_timestamp",
706
+ .annotator = timestamp_usec_annotator,
707
+ },
708
+
709
+ // put new items here
710
+ // leave the request URL and the message last
711
+
712
+ [NDF_REQUEST] = {
713
+ .journal = "ND_REQUEST",
714
+ .eventlog = "Request",
715
+ .logfmt = "request",
716
+ },
717
+ [NDF_MESSAGE] = {
718
+ .journal = "MESSAGE",
719
+ .eventlog = "Message",
720
+ .logfmt = "msg",
721
+ },
722
+};
723
+
724
+// --------------------------------------------------------------------------------------------------------------------
725
+
726
+void log_stack_pop(void *ptr) {
727
+ if(!ptr) return;
728
+
729
+ struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr;
730
+
731
+ if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) {
732
+ fatal("You cannot pop in the middle of the stack, or an item not in the stack");
733
+ return;
734
+ }
735
+
736
+ thread_log_stack_next--;
737
+}
738
+
739
+void log_stack_push(struct log_stack_entry *lgs) {
740
+ if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return;
741
+ thread_log_stack_base[thread_log_stack_next++] = lgs;
742
+}
743
+
744
+// --------------------------------------------------------------------------------------------------------------------
745
+
746
+ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len) {
747
+ for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) {
748
+ if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0)
749
+ return i;
750
+ }
751
+
752
+ return NDF_STOP;
753
+}
754
+
755
+// --------------------------------------------------------------------------------------------------------------------
756
+
757
+int nd_log_health_fd(void) {
758
+ if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1)
759
+ return nd_log.sources[NDLS_HEALTH].fd;
760
+
761
+ return STDERR_FILENO;
762
+}
763
+
764
+int nd_log_collectors_fd(void) {
765
+ if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)
766
+ return nd_log.sources[NDLS_COLLECTORS].fd;
767
+
768
+ return STDERR_FILENO;
769
+}
770
+
771
+// --------------------------------------------------------------------------------------------------------------------
772
+
773
+void log_date(char *buffer, size_t len, time_t now) {
774
+ if(unlikely(!buffer || !len))
775
+ return;
776
+
777
+ time_t t = now;
778
+ struct tm *tmp, tmbuf;
779
+
780
+ tmp = localtime_r(&t, &tmbuf);
781
+
782
+ if (unlikely(!tmp)) {
783
+ buffer[0] = '\0';
784
+ return;
785
+ }
786
+
787
+ if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
788
+ buffer[0] = '\0';
789
+
790
+ buffer[len - 1] = '\0';
791
+}
792
+
793
+// --------------------------------------------------------------------------------------------------------------------
794
+
795
+bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) {
796
+ if(new_fd == -1 || e->fd == -1 ||
797
+ (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) ||
798
+ (e->fd == STDERR_FILENO && nd_log.std_error.initialized))
799
+ return false;
800
+
801
+ if(new_fd != e->fd) {
802
+ int t = dup2(new_fd, e->fd);
803
+
804
+ bool ret = true;
805
+ if (t == -1) {
806
+ netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename);
807
+ ret = false;
808
+ }
809
+ else
810
+ close(new_fd);
811
+
812
+ if(e->fd == STDOUT_FILENO)
813
+ nd_log.std_output.initialized = true;
814
+ else if(e->fd == STDERR_FILENO)
815
+ nd_log.std_error.initialized = true;
816
+
817
+ return ret;
818
+ }
819
+
820
+ return false;
821
+}
src/libnetdata/log/nd_log-internals.h
new
+251
@@ -0,0 +1,251 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ND_LOG_INTERNALS_H
4
+#define NETDATA_ND_LOG_INTERNALS_H
5
+
6
+#include "../libnetdata.h"
7
+
8
+#if defined(OS_WINDOWS)
9
+#include <windows.h>
10
+#endif
11
+
12
+#ifdef __FreeBSD__
13
+#include <sys/endian.h>
14
+#endif
15
+
16
+#ifdef __APPLE__
17
+#include <machine/endian.h>
18
+#endif
19
+
20
+#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
21
+#include <execinfo.h>
22
+#endif
23
+
24
+#ifdef HAVE_SYSTEMD
25
+#include <systemd/sd-journal.h>
26
+#endif
27
+
28
+const char *errno2str(int errnum, char *buf, size_t size);
29
+
30
+// --------------------------------------------------------------------------------------------------------------------
31
+// ND_LOG_METHOD
32
+
33
+typedef enum __attribute__((__packed__)) {
34
+ NDLM_DISABLED = 0,
35
+ NDLM_DEVNULL,
36
+ NDLM_DEFAULT,
37
+ NDLM_JOURNAL,
38
+ NDLM_SYSLOG,
39
+ NDLM_STDOUT,
40
+ NDLM_STDERR,
41
+ NDLM_FILE,
42
+#if defined(OS_WINDOWS)
43
+#if defined(HAVE_ETW)
44
+ NDLM_ETW,
45
+#endif
46
+#if defined(HAVE_WEL)
47
+ NDLM_WEL,
48
+#endif
49
+#endif
50
+} ND_LOG_METHOD;
51
+
52
+// all the log methods are finally mapped to these
53
+#if defined(HAVE_ETW)
54
+#define ETW_CONDITION(ndlo) ((ndlo) == NDLM_ETW)
55
+#else
56
+#define ETW_CONDITION(ndlo) (false)
57
+#endif
58
+
59
+#if defined(HAVE_WEL)
60
+#define WEL_CONDITION(ndlo) ((ndlo) == NDLM_WEL)
61
+#else
62
+#define WEL_CONDITION(ndlo) (false)
63
+#endif
64
+
65
+#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo))
66
+#define IS_FINAL_LOG_METHOD(ndlo) ((ndlo) == NDLM_FILE || (ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo))
67
+
68
+ND_LOG_METHOD nd_log_method2id(const char *method);
69
+const char *nd_log_id2method(ND_LOG_METHOD method);
70
+
71
+// --------------------------------------------------------------------------------------------------------------------
72
+// ND_LOG_FORMAT
73
+
74
+typedef enum __attribute__((__packed__)) {
75
+ NDLF_JOURNAL,
76
+ NDLF_LOGFMT,
77
+ NDLF_JSON,
78
+#if defined(OS_WINDOWS)
79
+#if defined(HAVE_ETW)
80
+ NDLF_ETW, // Event Tracing for Windows
81
+#endif
82
+#if defined(HAVE_WEL)
83
+ NDLF_WEL, // Windows Event Log
84
+#endif
85
+#endif
86
+} ND_LOG_FORMAT;
87
+
88
+#define ETW_NAME "etw"
89
+#define WEL_NAME "wel"
90
+
91
+const char *nd_log_id2format(ND_LOG_FORMAT format);
92
+ND_LOG_FORMAT nd_log_format2id(const char *format);
93
+
94
+size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def);
95
+const char *nd_log_id2source(ND_LOG_SOURCES source);
96
+
97
+const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
98
+int nd_log_priority2id(const char *priority);
99
+
100
+const char *nd_log_id2facility(int facility);
101
+int nd_log_facility2id(const char *facility);
102
+
103
+#include "nd_log_limit.h"
104
+
105
+struct nd_log_source {
106
+ SPINLOCK spinlock;
107
+ ND_LOG_METHOD method;
108
+ ND_LOG_FORMAT format;
109
+ const char *filename;
110
+ int fd;
111
+ FILE *fp;
112
+
113
+ ND_LOG_FIELD_PRIORITY min_priority;
114
+ const char *pending_msg;
115
+ struct nd_log_limit limits;
116
+
117
+#if defined(OS_WINDOWS)
118
+ ND_LOG_SOURCES source;
119
+ HANDLE hEventLog;
120
+ USHORT channelID;
121
+ UCHAR Opcode;
122
+ USHORT Task;
123
+ ULONGLONG Keyword;
124
+#endif
125
+};
126
+
127
+struct nd_log {
128
+ nd_uuid_t invocation_id;
129
+
130
+ ND_LOG_SOURCES overwrite_process_source;
131
+
132
+ struct nd_log_source sources[_NDLS_MAX];
133
+
134
+ struct {
135
+ bool initialized;
136
+ } journal;
137
+
138
+ struct {
139
+ bool initialized;
140
+ int fd;
141
+ char filename[FILENAME_MAX + 1];
142
+ } journal_direct;
143
+
144
+ struct {
145
+ bool initialized;
146
+ int facility;
147
+ } syslog;
148
+
149
+ struct {
150
+ bool etw; // when set use etw, otherwise wel
151
+ bool initialized;
152
+ } eventlog;
153
+
154
+ struct {
155
+ SPINLOCK spinlock;
156
+ bool initialized;
157
+ } std_output;
158
+
159
+ struct {
160
+ SPINLOCK spinlock;
161
+ bool initialized;
162
+ } std_error;
163
+
164
+};
165
+
166
+// --------------------------------------------------------------------------------------------------------------------
167
+
168
+struct log_field;
169
+typedef const char *(*annotator_t)(struct log_field *lf);
170
+
171
+struct log_field {
172
+ const char *journal;
173
+ const char *logfmt;
174
+ const char *eventlog;
175
+ annotator_t annotator;
176
+ struct log_stack_entry entry;
177
+};
178
+
179
+#define THREAD_LOG_STACK_MAX 50
180
+#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0]))
181
+
182
+extern __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
183
+extern __thread size_t thread_log_stack_next;
184
+extern __thread struct log_field thread_log_fields[_NDF_MAX];
185
+
186
+// --------------------------------------------------------------------------------------------------------------------
187
+
188
+extern struct nd_log nd_log;
189
+bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd);
190
+void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source);
191
+void nd_log_stdin_init(int fd, const char *filename);
192
+
193
+// --------------------------------------------------------------------------------------------------------------------
194
+// annotators
195
+
196
+struct log_field;
197
+const char *errno_annotator(struct log_field *lf);
198
+const char *priority_annotator(struct log_field *lf);
199
+const char *timestamp_usec_annotator(struct log_field *lf);
200
+
201
+#if defined(OS_WINDOWS)
202
+const char *winerror_annotator(struct log_field *lf);
203
+#endif
204
+
205
+// --------------------------------------------------------------------------------------------------------------------
206
+// field formatters
207
+
208
+uint64_t log_field_to_uint64(struct log_field *lf);
209
+int64_t log_field_to_int64(struct log_field *lf);
210
+
211
+// --------------------------------------------------------------------------------------------------------------------
212
+// common text formatters
213
+
214
+void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max);
215
+void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max);
216
+
217
+// --------------------------------------------------------------------------------------------------------------------
218
+// output to syslog
219
+
220
+void nd_log_init_syslog(void);
221
+void nd_log_reset_syslog(void);
222
+bool nd_logger_syslog(int priority, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max);
223
+
224
+// --------------------------------------------------------------------------------------------------------------------
225
+// output to systemd-journal
226
+
227
+bool nd_log_journal_systemd_init(void);
228
+bool nd_log_journal_direct_init(const char *path);
229
+bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max);
230
+bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields_max);
231
+
232
+// --------------------------------------------------------------------------------------------------------------------
233
+// output to file
234
+
235
+bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max);
236
+
237
+// --------------------------------------------------------------------------------------------------------------------
238
+// output to windows events log
239
+
240
+#if defined(OS_WINDOWS)
241
+#if defined(HAVE_ETW)
242
+bool nd_log_init_etw(void);
243
+bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max);
244
+#endif
245
+#if defined(HAVE_WEL)
246
+bool nd_log_init_wel(void);
247
+bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max);
248
+#endif
249
+#endif
250
+
251
+#endif //NETDATA_ND_LOG_INTERNALS_H
src/libnetdata/log/nd_log-to-file.c
new
+41
@@ -0,0 +1,41 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+void chown_open_file(int fd, uid_t uid, gid_t gid) {
6
+ if(fd == -1) return;
7
+
8
+ struct stat buf;
9
+
10
+ if(fstat(fd, &buf) == -1) {
11
+ netdata_log_error("Cannot fstat() fd %d", fd);
12
+ return;
13
+ }
14
+
15
+ if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) {
16
+ if(fchown(fd, uid, gid) == -1)
17
+ netdata_log_error("Cannot fchown() fd %d.", fd);
18
+ }
19
+}
20
+
21
+void nd_log_chown_log_files(uid_t uid, gid_t gid) {
22
+ for(size_t i = 0 ; i < _NDLS_MAX ; i++) {
23
+ if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO)
24
+ chown_open_file(nd_log.sources[i].fd, uid, gid);
25
+ }
26
+}
27
+
28
+bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
29
+ BUFFER *wb = buffer_create(1024, NULL);
30
+
31
+ if(format == NDLF_JSON)
32
+ nd_logger_json(wb, fields, fields_max);
33
+ else
34
+ nd_logger_logfmt(wb, fields, fields_max);
35
+
36
+ int r = fprintf(fp, "%s\n", buffer_tostring(wb));
37
+ fflush(fp);
38
+
39
+ buffer_free(wb);
40
+ return r > 0;
41
+}
src/libnetdata/log/nd_log-to-syslog.c
new
+20
@@ -0,0 +1,20 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+void nd_log_init_syslog(void) {
6
+ if(nd_log.syslog.initialized)
7
+ return;
8
+
9
+ openlog(program_name, LOG_PID, nd_log.syslog.facility);
10
+ nd_log.syslog.initialized = true;
11
+}
12
+
13
+bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) {
14
+ CLEAN_BUFFER *wb = buffer_create(1024, NULL);
15
+
16
+ nd_logger_logfmt(wb, fields, fields_max);
17
+ syslog(priority, "%s", buffer_tostring(wb));
18
+
19
+ return true;
20
+}
src/libnetdata/log/nd_log-to-systemd-journal.c
new
+273
@@ -0,0 +1,273 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+bool nd_log_journal_systemd_init(void) {
6
+#ifdef HAVE_SYSTEMD
7
+ nd_log.journal.initialized = true;
8
+#else
9
+ nd_log.journal.initialized = false;
10
+#endif
11
+
12
+ return nd_log.journal.initialized;
13
+}
14
+
15
+bool nd_log_journal_socket_available(void) {
16
+ if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
17
+ char filename[FILENAME_MAX + 1];
18
+
19
+ snprintfz(filename, sizeof(filename), "%s%s",
20
+ netdata_configured_host_prefix, "/run/systemd/journal/socket");
21
+
22
+ if(is_path_unix_socket(filename))
23
+ return true;
24
+ }
25
+
26
+ return is_path_unix_socket("/run/systemd/journal/socket");
27
+}
28
+
29
+static void nd_log_journal_direct_set_env(void) {
30
+ if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL)
31
+ nd_setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1);
32
+}
33
+
34
+bool nd_log_journal_direct_init(const char *path) {
35
+ if(nd_log.journal_direct.initialized) {
36
+ nd_log_journal_direct_set_env();
37
+ return true;
38
+ }
39
+
40
+ int fd;
41
+ char filename[FILENAME_MAX + 1];
42
+ if(!is_path_unix_socket(path)) {
43
+
44
+ journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
45
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
46
+
47
+ journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
48
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
49
+
50
+ journal_construct_path(filename, sizeof(filename), NULL, "netdata");
51
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
52
+
53
+ journal_construct_path(filename, sizeof(filename), NULL, NULL);
54
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
55
+ return false;
56
+ }
57
+ }
58
+ }
59
+ }
60
+ else {
61
+ snprintfz(filename, sizeof(filename), "%s", path);
62
+ fd = journal_direct_fd(filename);
63
+ }
64
+
65
+ if(fd < 0)
66
+ return false;
67
+
68
+ nd_log.journal_direct.fd = fd;
69
+ nd_log.journal_direct.initialized = true;
70
+
71
+ strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1);
72
+ nd_log_journal_direct_set_env();
73
+
74
+ return true;
75
+}
76
+
77
+bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) {
78
+#ifdef HAVE_SYSTEMD
79
+
80
+ // --- FIELD_PARSER_VERSIONS ---
81
+ //
82
+ // IMPORTANT:
83
+ // THERE ARE 6 VERSIONS OF THIS CODE
84
+ //
85
+ // 1. journal (direct socket API),
86
+ // 2. journal (libsystemd API),
87
+ // 3. logfmt,
88
+ // 4. json,
89
+ // 5. convert to uint64
90
+ // 6. convert to int64
91
+ //
92
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
93
+
94
+ struct iovec iov[fields_max];
95
+ int iov_count = 0;
96
+
97
+ memset(iov, 0, sizeof(iov));
98
+
99
+ CLEAN_BUFFER *tmp = NULL;
100
+
101
+ for (size_t i = 0; i < fields_max; i++) {
102
+ if (!fields[i].entry.set || !fields[i].journal)
103
+ continue;
104
+
105
+ const char *key = fields[i].journal;
106
+ char *value = NULL;
107
+ int rc = 0;
108
+ switch (fields[i].entry.type) {
109
+ case NDFT_TXT:
110
+ if(*fields[i].entry.txt)
111
+ rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt);
112
+ break;
113
+ case NDFT_STR:
114
+ rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str));
115
+ break;
116
+ case NDFT_BFR:
117
+ if(buffer_strlen(fields[i].entry.bfr))
118
+ rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr));
119
+ break;
120
+ case NDFT_U64:
121
+ rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64);
122
+ break;
123
+ case NDFT_I64:
124
+ rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64);
125
+ break;
126
+ case NDFT_DBL:
127
+ rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
128
+ break;
129
+ case NDFT_UUID:
130
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
131
+ char u[UUID_COMPACT_STR_LEN];
132
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
133
+ rc = asprintf(&value, "%s=%s", key, u);
134
+ }
135
+ break;
136
+ case NDFT_CALLBACK: {
137
+ if(!tmp)
138
+ tmp = buffer_create(1024, NULL);
139
+ else
140
+ buffer_flush(tmp);
141
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
142
+ rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp));
143
+ }
144
+ break;
145
+ default:
146
+ rc = asprintf(&value, "%s=%s", key, "UNHANDLED");
147
+ break;
148
+ }
149
+
150
+ if (rc != -1 && value) {
151
+ iov[iov_count].iov_base = value;
152
+ iov[iov_count].iov_len = strlen(value);
153
+ iov_count++;
154
+ }
155
+ }
156
+
157
+ int r = sd_journal_sendv(iov, iov_count);
158
+
159
+ // Clean up allocated memory
160
+ for (int i = 0; i < iov_count; i++) {
161
+ if (iov[i].iov_base != NULL) {
162
+ free(iov[i].iov_base);
163
+ }
164
+ }
165
+
166
+ return r == 0;
167
+#else
168
+ return false;
169
+#endif
170
+}
171
+
172
+bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) {
173
+ if(!nd_log.journal_direct.initialized)
174
+ return false;
175
+
176
+ // --- FIELD_PARSER_VERSIONS ---
177
+ //
178
+ // IMPORTANT:
179
+ // THERE ARE 6 VERSIONS OF THIS CODE
180
+ //
181
+ // 1. journal (direct socket API),
182
+ // 2. journal (libsystemd API),
183
+ // 3. logfmt,
184
+ // 4. json,
185
+ // 5. convert to uint64
186
+ // 6. convert to int64
187
+ //
188
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
189
+
190
+ CLEAN_BUFFER *wb = buffer_create(4096, NULL);
191
+ CLEAN_BUFFER *tmp = NULL;
192
+
193
+ for (size_t i = 0; i < fields_max; i++) {
194
+ if (!fields[i].entry.set || !fields[i].journal)
195
+ continue;
196
+
197
+ const char *key = fields[i].journal;
198
+
199
+ const char *s = NULL;
200
+ switch(fields[i].entry.type) {
201
+ case NDFT_TXT:
202
+ s = fields[i].entry.txt;
203
+ break;
204
+ case NDFT_STR:
205
+ s = string2str(fields[i].entry.str);
206
+ break;
207
+ case NDFT_BFR:
208
+ s = buffer_tostring(fields[i].entry.bfr);
209
+ break;
210
+ case NDFT_U64:
211
+ buffer_strcat(wb, key);
212
+ buffer_putc(wb, '=');
213
+ buffer_print_uint64(wb, fields[i].entry.u64);
214
+ buffer_putc(wb, '\n');
215
+ break;
216
+ case NDFT_I64:
217
+ buffer_strcat(wb, key);
218
+ buffer_putc(wb, '=');
219
+ buffer_print_int64(wb, fields[i].entry.i64);
220
+ buffer_putc(wb, '\n');
221
+ break;
222
+ case NDFT_DBL:
223
+ buffer_strcat(wb, key);
224
+ buffer_putc(wb, '=');
225
+ buffer_print_netdata_double(wb, fields[i].entry.dbl);
226
+ buffer_putc(wb, '\n');
227
+ break;
228
+ case NDFT_UUID:
229
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
230
+ char u[UUID_COMPACT_STR_LEN];
231
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
232
+ buffer_strcat(wb, key);
233
+ buffer_putc(wb, '=');
234
+ buffer_fast_strcat(wb, u, sizeof(u) - 1);
235
+ buffer_putc(wb, '\n');
236
+ }
237
+ break;
238
+ case NDFT_CALLBACK: {
239
+ if(!tmp)
240
+ tmp = buffer_create(1024, NULL);
241
+ else
242
+ buffer_flush(tmp);
243
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
244
+ s = buffer_tostring(tmp);
245
+ else
246
+ s = NULL;
247
+ }
248
+ break;
249
+ default:
250
+ s = "UNHANDLED";
251
+ break;
252
+ }
253
+
254
+ if(s && *s) {
255
+ buffer_strcat(wb, key);
256
+ if(!strchr(s, '\n')) {
257
+ buffer_putc(wb, '=');
258
+ buffer_strcat(wb, s);
259
+ buffer_putc(wb, '\n');
260
+ }
261
+ else {
262
+ buffer_putc(wb, '\n');
263
+ size_t size = strlen(s);
264
+ uint64_t le_size = htole64(size);
265
+ buffer_memcat(wb, &le_size, sizeof(le_size));
266
+ buffer_memcat(wb, s, size);
267
+ buffer_putc(wb, '\n');
268
+ }
269
+ }
270
+ }
271
+
272
+ return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb));
273
+}
src/libnetdata/log/nd_log-to-windows-common.h
new
+188
@@ -0,0 +1,188 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
4
+#define NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
5
+
6
+// Helper macro to create wide string literals
7
+#define WIDEN2(x) L ## x
8
+#define WIDEN(x) WIDEN2(x)
9
+
10
+#define NETDATA_ETW_PROVIDER_GUID_STR "{96c5ca72-9bd8-4634-81e5-000014e7da7a}"
11
+#define NETDATA_ETW_PROVIDER_GUID_STR_W WIDEN(NETDATA_ETW_PROVIDER_GUID)
12
+
13
+#define NETDATA_CHANNEL_NAME "Netdata"
14
+#define NETDATA_CHANNEL_NAME_W WIDEN(NETDATA_CHANNEL_NAME)
15
+
16
+#define NETDATA_WEL_CHANNEL_NAME "NetdataWEL"
17
+#define NETDATA_WEL_CHANNEL_NAME_W WIDEN(NETDATA_WEL_CHANNEL_NAME)
18
+
19
+#define NETDATA_ETW_CHANNEL_NAME "Netdata"
20
+#define NETDATA_ETW_CHANNEL_NAME_W WIDEN(NETDATA_ETW_CHANNEL_NAME)
21
+
22
+#define NETDATA_ETW_PROVIDER_NAME "Netdata"
23
+#define NETDATA_ETW_PROVIDER_NAME_W WIDEN(NETDATA_ETW_PROVIDER_NAME)
24
+
25
+#define NETDATA_WEL_PROVIDER_PREFIX "Netdata"
26
+#define NETDATA_WEL_PROVIDER_PREFIX_W WIDEN(NETDATA_WEL_PROVIDER_PREFIX)
27
+
28
+#define NETDATA_WEL_PROVIDER_ACCESS NETDATA_WEL_PROVIDER_PREFIX "Access"
29
+#define NETDATA_WEL_PROVIDER_ACCESS_W WIDEN(NETDATA_WEL_PROVIDER_ACCESS)
30
+
31
+#define NETDATA_WEL_PROVIDER_ACLK NETDATA_WEL_PROVIDER_PREFIX "Aclk"
32
+#define NETDATA_WEL_PROVIDER_ACLK_W WIDEN(NETDATA_WEL_PROVIDER_ACLK)
33
+
34
+#define NETDATA_WEL_PROVIDER_COLLECTORS NETDATA_WEL_PROVIDER_PREFIX "Collectors"
35
+#define NETDATA_WEL_PROVIDER_COLLECTORS_W WIDEN(NETDATA_WEL_PROVIDER_COLLECTORS)
36
+
37
+#define NETDATA_WEL_PROVIDER_DAEMON NETDATA_WEL_PROVIDER_PREFIX "Daemon"
38
+#define NETDATA_WEL_PROVIDER_DAEMON_W WIDEN(NETDATA_WEL_PROVIDER_DAEMON)
39
+
40
+#define NETDATA_WEL_PROVIDER_HEALTH NETDATA_WEL_PROVIDER_PREFIX "Health"
41
+#define NETDATA_WEL_PROVIDER_HEALTH_W WIDEN(NETDATA_WEL_PROVIDER_HEALTH)
42
+
43
+
44
+#define NETDATA_ETW_SUBCHANNEL_ACCESS "Access"
45
+#define NETDATA_ETW_SUBCHANNEL_ACCESS_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACCESS)
46
+
47
+#define NETDATA_ETW_SUBCHANNEL_ACLK "Aclk"
48
+#define NETDATA_ETW_SUBCHANNEL_ACLK_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACLK)
49
+
50
+#define NETDATA_ETW_SUBCHANNEL_COLLECTORS "Collectors"
51
+#define NETDATA_ETW_SUBCHANNEL_COLLECTORS_W WIDEN(NETDATA_ETW_SUBCHANNEL_COLLECTORS)
52
+
53
+#define NETDATA_ETW_SUBCHANNEL_DAEMON "Daemon"
54
+#define NETDATA_ETW_SUBCHANNEL_DAEMON_W WIDEN(NETDATA_ETW_SUBCHANNEL_DAEMON)
55
+
56
+#define NETDATA_ETW_SUBCHANNEL_HEALTH "Health"
57
+#define NETDATA_ETW_SUBCHANNEL_HEALTH_W WIDEN(NETDATA_ETW_SUBCHANNEL_HEALTH)
58
+
59
+// Define shift values
60
+#define EVENT_ID_SEV_SHIFT 30
61
+#define EVENT_ID_C_SHIFT 29
62
+#define EVENT_ID_R_SHIFT 28
63
+#define EVENT_ID_FACILITY_SHIFT 16
64
+#define EVENT_ID_CODE_SHIFT 0
65
+
66
+#define EVENT_ID_PRIORITY_SHIFT 0 // Shift 0 bits
67
+#define EVENT_ID_SOURCE_SHIFT 4 // Shift 4 bits
68
+
69
+// Define masks
70
+#define EVENT_ID_SEV_MASK 0xC0000000 // Bits 31-30
71
+#define EVENT_ID_C_MASK 0x20000000 // Bit 29
72
+#define EVENT_ID_R_MASK 0x10000000 // Bit 28
73
+#define EVENT_ID_FACILITY_MASK 0x0FFF0000 // Bits 27-16
74
+#define EVENT_ID_CODE_MASK 0x0000FFFF // Bits 15-0
75
+
76
+#define EVENT_ID_PRIORITY_MASK 0x000F // Bits 0-3
77
+#define EVENT_ID_SOURCE_MASK 0x00F0 // Bits 4-7
78
+
79
+typedef enum __attribute__((packed)) {
80
+ MSGID_MESSAGE_ONLY = 1,
81
+ MSGID_MESSAGE_ERRNO,
82
+ MSGID_REQUEST_ONLY,
83
+ MSGID_ALERT_TRANSITION,
84
+ MSGID_ACCESS,
85
+ MSGID_ACCESS_FORWARDER,
86
+ MSGID_ACCESS_USER,
87
+ MSGID_ACCESS_FORWARDER_USER,
88
+ MSGID_ACCESS_MESSAGE,
89
+ MSGID_ACCESS_MESSAGE_REQUEST,
90
+ MSGID_ACCESS_MESSAGE_USER,
91
+
92
+ // terminator
93
+ _MSGID_MAX,
94
+} MESSAGE_ID;
95
+
96
+static inline uint32_t get_event_type_from_priority(ND_LOG_FIELD_PRIORITY priority) {
97
+ switch (priority) {
98
+ case NDLP_EMERG:
99
+ case NDLP_ALERT:
100
+ case NDLP_CRIT:
101
+ case NDLP_ERR:
102
+ return EVENTLOG_ERROR_TYPE;
103
+
104
+ case NDLP_WARNING:
105
+ return EVENTLOG_WARNING_TYPE;
106
+
107
+ case NDLP_NOTICE:
108
+ case NDLP_INFO:
109
+ case NDLP_DEBUG:
110
+ default:
111
+ return EVENTLOG_INFORMATION_TYPE;
112
+ }
113
+}
114
+
115
+static inline uint8_t get_severity_from_priority(ND_LOG_FIELD_PRIORITY priority) {
116
+ switch (priority) {
117
+ case NDLP_EMERG:
118
+ case NDLP_ALERT:
119
+ case NDLP_CRIT:
120
+ case NDLP_ERR:
121
+ return STATUS_SEVERITY_ERROR;
122
+
123
+ case NDLP_WARNING:
124
+ return STATUS_SEVERITY_WARNING;
125
+
126
+ case NDLP_NOTICE:
127
+ case NDLP_INFO:
128
+ case NDLP_DEBUG:
129
+ default:
130
+ return STATUS_SEVERITY_INFORMATIONAL;
131
+ }
132
+}
133
+
134
+static inline uint8_t get_level_from_priority(ND_LOG_FIELD_PRIORITY priority) {
135
+ switch (priority) {
136
+ // return 0 = log an event regardless of any filtering applied
137
+
138
+ case NDLP_EMERG:
139
+ case NDLP_ALERT:
140
+ case NDLP_CRIT:
141
+ return 1;
142
+
143
+ case NDLP_ERR:
144
+ return 2;
145
+
146
+ case NDLP_WARNING:
147
+ return 3;
148
+
149
+ case NDLP_NOTICE:
150
+ case NDLP_INFO:
151
+ return 4;
152
+
153
+ case NDLP_DEBUG:
154
+ default:
155
+ return 5;
156
+ }
157
+}
158
+
159
+static inline const char *get_level_from_priority_str(ND_LOG_FIELD_PRIORITY priority) {
160
+ switch (priority) {
161
+ // return "win:LogAlways" to log an event regardless of any filtering applied
162
+
163
+ case NDLP_EMERG:
164
+ case NDLP_ALERT:
165
+ case NDLP_CRIT:
166
+ return "win:Critical";
167
+
168
+ case NDLP_ERR:
169
+ return "win:Error";
170
+
171
+ case NDLP_WARNING:
172
+ return "win:Warning";
173
+
174
+ case NDLP_NOTICE:
175
+ case NDLP_INFO:
176
+ return "win:Informational";
177
+
178
+ case NDLP_DEBUG:
179
+ default:
180
+ return "win:Verbose";
181
+ }
182
+}
183
+
184
+static inline uint16_t construct_event_code(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) {
185
+ return (source << 12 | priority << 8 | messageID << 0);
186
+}
187
+
188
+#endif //NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
src/libnetdata/log/nd_log-to-windows-events.c
new
+560
@@ -0,0 +1,560 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL))
6
+#include <windows.h>
7
+#include <winevt.h>
8
+#include <evntprov.h>
9
+#include <wchar.h>
10
+#include <guiddef.h>
11
+#include <wctype.h>
12
+
13
+// --------------------------------------------------------------------------------------------------------------------
14
+// construct an event id
15
+
16
+// load message resources generated header
17
+#include "wevt_netdata.h"
18
+
19
+// include the common definitions with the message resources and manifest generator
20
+#include "nd_log-to-windows-common.h"
21
+
22
+#if defined(HAVE_ETW)
23
+// we need the manifest, only in ETW mode
24
+
25
+// eliminate compiler warnings and load manifest generated header
26
+#undef EXTERN_C
27
+#define EXTERN_C
28
+#undef __declspec
29
+#define __declspec(x)
30
+#include "wevt_netdata_manifest.h"
31
+
32
+static REGHANDLE regHandle;
33
+#endif
34
+
35
+// Function to construct EventID
36
+static DWORD complete_event_id(DWORD facility, DWORD severity, DWORD event_code) {
37
+ DWORD event_id = 0;
38
+
39
+ // Set Severity
40
+ event_id |= ((DWORD)(severity) << EVENT_ID_SEV_SHIFT) & EVENT_ID_SEV_MASK;
41
+
42
+ // Set Customer Code Flag (C)
43
+ event_id |= (0x0 << EVENT_ID_C_SHIFT) & EVENT_ID_C_MASK;
44
+
45
+ // Set Reserved Bit (R) - typically 0
46
+ event_id |= (0x0 << EVENT_ID_R_SHIFT) & EVENT_ID_R_MASK;
47
+
48
+ // Set Facility
49
+ event_id |= ((DWORD)(facility) << EVENT_ID_FACILITY_SHIFT) & EVENT_ID_FACILITY_MASK;
50
+
51
+ // Set Code
52
+ event_id |= ((DWORD)(event_code) << EVENT_ID_CODE_SHIFT) & EVENT_ID_CODE_MASK;
53
+
54
+ return event_id;
55
+}
56
+
57
+DWORD construct_event_id(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) {
58
+ DWORD event_code = construct_event_code(source, priority, messageID);
59
+ return complete_event_id(FACILITY_NETDATA, get_severity_from_priority(priority), event_code);
60
+}
61
+
62
+static bool check_event_id(ND_LOG_SOURCES source __maybe_unused, ND_LOG_FIELD_PRIORITY priority __maybe_unused, MESSAGE_ID messageID __maybe_unused, DWORD event_code __maybe_unused) {
63
+#ifdef NETDATA_INTERNAL_CHECKS
64
+ DWORD generated = construct_event_id(source, priority, messageID);
65
+ if(generated != event_code) {
66
+
67
+ // this is just used for a break point, to see the values in hex
68
+ char current[UINT64_HEX_MAX_LENGTH];
69
+ print_uint64_hex(current, generated);
70
+
71
+ char wanted[UINT64_HEX_MAX_LENGTH];
72
+ print_uint64_hex(wanted, event_code);
73
+
74
+ const char *got = current;
75
+ const char *good = wanted;
76
+ internal_fatal(true, "EventIDs mismatch, expected %s, got %s", good, got);
77
+ }
78
+#endif
79
+
80
+ return true;
81
+}
82
+
83
+// --------------------------------------------------------------------------------------------------------------------
84
+// initialization
85
+
86
+// Define provider names per source (only when not using ETW)
87
+static const wchar_t *wel_provider_per_source[_NDLS_MAX] = {
88
+ [NDLS_UNSET] = NULL, // not used, linked to NDLS_DAEMON
89
+ [NDLS_ACCESS] = NETDATA_WEL_PROVIDER_ACCESS_W, //
90
+ [NDLS_ACLK] = NETDATA_WEL_PROVIDER_ACLK_W, //
91
+ [NDLS_COLLECTORS] = NETDATA_WEL_PROVIDER_COLLECTORS_W,//
92
+ [NDLS_DAEMON] = NETDATA_WEL_PROVIDER_DAEMON_W, //
93
+ [NDLS_HEALTH] = NETDATA_WEL_PROVIDER_HEALTH_W, //
94
+ [NDLS_DEBUG] = NULL, // used, linked to NDLS_DAEMON
95
+};
96
+
97
+bool wel_replace_program_with_wevt_netdata_dll(wchar_t *str, size_t size) {
98
+ const wchar_t *replacement = L"\\wevt_netdata.dll";
99
+
100
+ // Find the last occurrence of '\\' to isolate the filename
101
+ wchar_t *lastBackslash = wcsrchr(str, L'\\');
102
+
103
+ if (lastBackslash != NULL) {
104
+ // Calculate new length after replacement
105
+ size_t newLen = (lastBackslash - str) + wcslen(replacement);
106
+
107
+ // Ensure new length does not exceed buffer size
108
+ if (newLen >= size)
109
+ return false; // Not enough space in the buffer
110
+
111
+ // Terminate the string at the last backslash
112
+ *lastBackslash = L'\0';
113
+
114
+ // Append the replacement filename
115
+ wcsncat(str, replacement, size - wcslen(str) - 1);
116
+
117
+ // Check if the new file exists
118
+ if (GetFileAttributesW(str) != INVALID_FILE_ATTRIBUTES)
119
+ return true; // The file exists
120
+ else
121
+ return false; // The file does not exist
122
+ }
123
+
124
+ return false; // No backslash found (likely invalid input)
125
+}
126
+
127
+static bool wel_add_to_registry(const wchar_t *channel, const wchar_t *provider, DWORD defaultMaxSize) {
128
+ // Build the registry path: SYSTEM\CurrentControlSet\Services\EventLog\<LogName>\<SourceName>
129
+ wchar_t key[MAX_PATH];
130
+ if(!provider)
131
+ swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls", channel);
132
+ else
133
+ swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls\\%ls", channel, provider);
134
+
135
+ HKEY hRegKey;
136
+ DWORD disposition;
137
+ LONG result = RegCreateKeyExW(HKEY_LOCAL_MACHINE, key,
138
+ 0, NULL, REG_OPTION_NON_VOLATILE, KEY_SET_VALUE, NULL, &hRegKey, &disposition);
139
+
140
+ if (result != ERROR_SUCCESS)
141
+ return false; // Could not create the registry key
142
+
143
+ // Check if MaxSize is already set
144
+ DWORD maxSize = 0;
145
+ DWORD size = sizeof(maxSize);
146
+ if (RegQueryValueExW(hRegKey, L"MaxSize", NULL, NULL, (LPBYTE)&maxSize, &size) != ERROR_SUCCESS) {
147
+ // MaxSize is not set, set it to the default value
148
+ RegSetValueExW(hRegKey, L"MaxSize", 0, REG_DWORD, (const BYTE*)&defaultMaxSize, sizeof(defaultMaxSize));
149
+ }
150
+
151
+ wchar_t modulePath[MAX_PATH];
152
+ if (GetModuleFileNameW(NULL, modulePath, MAX_PATH) == 0) {
153
+ RegCloseKey(hRegKey);
154
+ return false;
155
+ }
156
+
157
+ if(wel_replace_program_with_wevt_netdata_dll(modulePath, _countof(modulePath))) {
158
+ RegSetValueExW(hRegKey, L"EventMessageFile", 0, REG_EXPAND_SZ,
159
+ (LPBYTE)modulePath, (wcslen(modulePath) + 1) * sizeof(wchar_t));
160
+
161
+ DWORD types_supported = EVENTLOG_SUCCESS | EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE;
162
+ RegSetValueExW(hRegKey, L"TypesSupported", 0, REG_DWORD, (LPBYTE)&types_supported, sizeof(DWORD));
163
+ }
164
+
165
+ RegCloseKey(hRegKey);
166
+ return true;
167
+}
168
+
169
+#if defined(HAVE_ETW)
170
+static void etw_set_source_meta(struct nd_log_source *source, USHORT channelID, const EVENT_DESCRIPTOR *ed) {
171
+ // It turns out that the keyword varies per only per channel!
172
+ // so, to log with the right keyword, Task, Opcode we copy the ids from the header
173
+ // the messages compiler (mc.exe) generated from the manifest.
174
+
175
+ source->channelID = channelID;
176
+ source->Opcode = ed->Opcode;
177
+ source->Task = ed->Task;
178
+ source->Keyword = ed->Keyword;
179
+}
180
+
181
+static bool etw_register_provider(void) {
182
+ // Register the ETW provider
183
+ if (EventRegister(&NETDATA_ETW_PROVIDER_GUID, NULL, NULL, ®Handle) != ERROR_SUCCESS)
184
+ return false;
185
+
186
+ etw_set_source_meta(&nd_log.sources[NDLS_DAEMON], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
187
+ etw_set_source_meta(&nd_log.sources[NDLS_COLLECTORS], CHANNEL_COLLECTORS, &ED_COLLECTORS_INFO_MESSAGE_ONLY);
188
+ etw_set_source_meta(&nd_log.sources[NDLS_ACCESS], CHANNEL_ACCESS, &ED_ACCESS_INFO_MESSAGE_ONLY);
189
+ etw_set_source_meta(&nd_log.sources[NDLS_HEALTH], CHANNEL_HEALTH, &ED_HEALTH_INFO_MESSAGE_ONLY);
190
+ etw_set_source_meta(&nd_log.sources[NDLS_ACLK], CHANNEL_ACLK, &ED_ACLK_INFO_MESSAGE_ONLY);
191
+ etw_set_source_meta(&nd_log.sources[NDLS_UNSET], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
192
+ etw_set_source_meta(&nd_log.sources[NDLS_DEBUG], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
193
+
194
+ return true;
195
+}
196
+#endif
197
+
198
+bool nd_log_init_windows(void) {
199
+ if(nd_log.eventlog.initialized)
200
+ return true;
201
+
202
+ // validate we have the right keys
203
+ if(
204
+ !check_event_id(NDLS_COLLECTORS, NDLP_INFO, MSGID_MESSAGE_ONLY, MC_COLLECTORS_INFO_MESSAGE_ONLY) ||
205
+ !check_event_id(NDLS_DAEMON, NDLP_ERR, MSGID_MESSAGE_ONLY, MC_DAEMON_ERR_MESSAGE_ONLY) ||
206
+ !check_event_id(NDLS_ACCESS, NDLP_WARNING, MSGID_ACCESS_USER, MC_ACCESS_WARN_ACCESS_USER) ||
207
+ !check_event_id(NDLS_HEALTH, NDLP_CRIT, MSGID_ALERT_TRANSITION, MC_HEALTH_CRIT_ALERT_TRANSITION) ||
208
+ !check_event_id(NDLS_DEBUG, NDLP_ALERT, MSGID_ACCESS_FORWARDER_USER, MC_DEBUG_ALERT_ACCESS_FORWARDER_USER))
209
+ return false;
210
+
211
+#if defined(HAVE_ETW)
212
+ if(nd_log.eventlog.etw && !etw_register_provider())
213
+ return false;
214
+#endif
215
+
216
+// if(!nd_log.eventlog.etw && !wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, NULL, 50 * 1024 * 1024))
217
+// return false;
218
+
219
+ // Loop through each source and add it to the registry
220
+ for(size_t i = 0; i < _NDLS_MAX; i++) {
221
+ nd_log.sources[i].source = i;
222
+
223
+ const wchar_t *sub_channel = wel_provider_per_source[i];
224
+
225
+ if(!sub_channel)
226
+ // we will map these to NDLS_DAEMON
227
+ continue;
228
+
229
+ DWORD defaultMaxSize = 0;
230
+ switch (i) {
231
+ case NDLS_ACLK:
232
+ defaultMaxSize = 5 * 1024 * 1024;
233
+ break;
234
+
235
+ case NDLS_HEALTH:
236
+ defaultMaxSize = 35 * 1024 * 1024;
237
+ break;
238
+
239
+ default:
240
+ case NDLS_ACCESS:
241
+ case NDLS_COLLECTORS:
242
+ case NDLS_DAEMON:
243
+ defaultMaxSize = 20 * 1024 * 1024;
244
+ break;
245
+ }
246
+
247
+ if(!nd_log.eventlog.etw) {
248
+ if(!wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, sub_channel, defaultMaxSize))
249
+ return false;
250
+
251
+ // when not using a manifest, each source is a provider
252
+ nd_log.sources[i].hEventLog = RegisterEventSourceW(NULL, sub_channel);
253
+ if (!nd_log.sources[i].hEventLog)
254
+ return false;
255
+ }
256
+ }
257
+
258
+ if(!nd_log.eventlog.etw) {
259
+ // Map the unset ones to NDLS_DAEMON
260
+ for (size_t i = 0; i < _NDLS_MAX; i++) {
261
+ if (!nd_log.sources[i].hEventLog)
262
+ nd_log.sources[i].hEventLog = nd_log.sources[NDLS_DAEMON].hEventLog;
263
+ }
264
+ }
265
+
266
+ nd_log.eventlog.initialized = true;
267
+ return true;
268
+}
269
+
270
+bool nd_log_init_etw(void) {
271
+ nd_log.eventlog.etw = true;
272
+ return nd_log_init_windows();
273
+}
274
+
275
+bool nd_log_init_wel(void) {
276
+ nd_log.eventlog.etw = false;
277
+ return nd_log_init_windows();
278
+}
279
+
280
+// --------------------------------------------------------------------------------------------------------------------
281
+// we pass all our fields to the windows events logs
282
+// numbered the same way we have them in memory.
283
+//
284
+// to avoid runtime memory allocations, we use a static allocations with ready to use buffers
285
+// which are immediately available for logging.
286
+
287
+#define SMALL_WIDE_BUFFERS_SIZE 256
288
+#define MEDIUM_WIDE_BUFFERS_SIZE 2048
289
+#define BIG_WIDE_BUFFERS_SIZE 16384
290
+static wchar_t small_wide_buffers[_NDF_MAX][SMALL_WIDE_BUFFERS_SIZE];
291
+static wchar_t medium_wide_buffers[2][MEDIUM_WIDE_BUFFERS_SIZE];
292
+static wchar_t big_wide_buffers[2][BIG_WIDE_BUFFERS_SIZE];
293
+
294
+static struct {
295
+ size_t size;
296
+ wchar_t *buf;
297
+} fields_buffers[_NDF_MAX] = { 0 };
298
+
299
+#if defined(HAVE_ETW)
300
+static EVENT_DATA_DESCRIPTOR etw_eventData[_NDF_MAX - 1];
301
+#endif
302
+
303
+static LPCWSTR wel_messages[_NDF_MAX - 1];
304
+
305
+__attribute__((constructor)) void wevents_initialize_buffers(void) {
306
+ for(size_t i = 0; i < _NDF_MAX ;i++) {
307
+ fields_buffers[i].buf = small_wide_buffers[i];
308
+ fields_buffers[i].size = SMALL_WIDE_BUFFERS_SIZE;
309
+ }
310
+
311
+ fields_buffers[NDF_NIDL_INSTANCE].buf = medium_wide_buffers[0];
312
+ fields_buffers[NDF_NIDL_INSTANCE].size = MEDIUM_WIDE_BUFFERS_SIZE;
313
+
314
+ fields_buffers[NDF_REQUEST].buf = big_wide_buffers[0];
315
+ fields_buffers[NDF_REQUEST].size = BIG_WIDE_BUFFERS_SIZE;
316
+ fields_buffers[NDF_MESSAGE].buf = big_wide_buffers[1];
317
+ fields_buffers[NDF_MESSAGE].size = BIG_WIDE_BUFFERS_SIZE;
318
+
319
+ for(size_t i = 1; i < _NDF_MAX ;i++)
320
+ wel_messages[i - 1] = fields_buffers[i].buf;
321
+}
322
+
323
+// --------------------------------------------------------------------------------------------------------------------
324
+
325
+#define is_field_set(fields, fields_max, field) ((field) < (fields_max) && (fields)[field].entry.set)
326
+
327
+static const char *get_field_value_unsafe(struct log_field *fields, ND_LOG_FIELD_ID i, size_t fields_max, BUFFER **tmp) {
328
+ if(!is_field_set(fields, fields_max, i) || !fields[i].eventlog)
329
+ return "";
330
+
331
+ static char number_str[MAX(MAX(UINT64_MAX_LENGTH, DOUBLE_MAX_LENGTH), UUID_STR_LEN)];
332
+
333
+ const char *s = NULL;
334
+ if (fields[i].annotator)
335
+ s = fields[i].annotator(&fields[i]);
336
+
337
+ else
338
+ switch (fields[i].entry.type) {
339
+ case NDFT_TXT:
340
+ s = fields[i].entry.txt;
341
+ break;
342
+ case NDFT_STR:
343
+ s = string2str(fields[i].entry.str);
344
+ break;
345
+ case NDFT_BFR:
346
+ s = buffer_tostring(fields[i].entry.bfr);
347
+ break;
348
+ case NDFT_U64:
349
+ print_uint64(number_str, fields[i].entry.u64);
350
+ s = number_str;
351
+ break;
352
+ case NDFT_I64:
353
+ print_int64(number_str, fields[i].entry.i64);
354
+ s = number_str;
355
+ break;
356
+ case NDFT_DBL:
357
+ print_netdata_double(number_str, fields[i].entry.dbl);
358
+ s = number_str;
359
+ break;
360
+ case NDFT_UUID:
361
+ if (!uuid_is_null(*fields[i].entry.uuid)) {
362
+ uuid_unparse_lower(*fields[i].entry.uuid, number_str);
363
+ s = number_str;
364
+ }
365
+ break;
366
+ case NDFT_CALLBACK:
367
+ if (!*tmp)
368
+ *tmp = buffer_create(1024, NULL);
369
+ else
370
+ buffer_flush(*tmp);
371
+
372
+ if (fields[i].entry.cb.formatter(*tmp, fields[i].entry.cb.formatter_data))
373
+ s = buffer_tostring(*tmp);
374
+ else
375
+ s = NULL;
376
+ break;
377
+
378
+ default:
379
+ s = "UNHANDLED";
380
+ break;
381
+ }
382
+
383
+ if(!s || !*s) return "";
384
+ return s;
385
+}
386
+static void etw_replace_percent_with_unicode(wchar_t *s, size_t size) {
387
+ size_t original_len = wcslen(s);
388
+
389
+ // Traverse the string, replacing '%' with the Unicode fullwidth percent sign
390
+ for (size_t i = 0; i < original_len && i < size - 1; i++) {
391
+ if (s[i] == L'%' && iswdigit(s[i + 1])) {
392
+ // s[i] = 0xFF05; // Replace '%' with fullwidth percent sign '％'
393
+ // s[i] = 0x29BC; // ⦼
394
+ s[i] = 0x2105; // ℅
395
+ }
396
+ }
397
+
398
+ // Ensure null termination if needed
399
+ s[size - 1] = L'\0';
400
+}
401
+
402
+static void wevt_generate_all_fields_unsafe(struct log_field *fields, size_t fields_max, BUFFER **tmp) {
403
+ for (size_t i = 0; i < fields_max; i++) {
404
+ fields_buffers[i].buf[0] = L'\0';
405
+
406
+ if (!fields[i].entry.set || !fields[i].eventlog)
407
+ continue;
408
+
409
+ const char *s = get_field_value_unsafe(fields, i, fields_max, tmp);
410
+ if (s && *s) {
411
+ utf8_to_utf16(fields_buffers[i].buf, (int) fields_buffers[i].size, s, -1);
412
+
413
+ if(nd_log.eventlog.etw)
414
+ // UNBELIEVABLE! they do recursive parameter expansion in ETW...
415
+ etw_replace_percent_with_unicode(fields_buffers[i].buf, fields_buffers[i].size);
416
+ }
417
+ }
418
+}
419
+
420
+static bool has_user_role_permissions(struct log_field *fields, size_t fields_max, BUFFER **tmp) {
421
+ const char *t;
422
+
423
+ t = get_field_value_unsafe(fields, NDF_USER_NAME, fields_max, tmp);
424
+ if (*t) return true;
425
+
426
+ t = get_field_value_unsafe(fields, NDF_USER_ROLE, fields_max, tmp);
427
+ if (*t && strcmp(t, "none") != 0) return true;
428
+
429
+ t = get_field_value_unsafe(fields, NDF_USER_ACCESS, fields_max, tmp);
430
+ if (*t && strcmp(t, "0x0") != 0) return true;
431
+
432
+ return false;
433
+}
434
+
435
+static bool nd_logger_windows(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
436
+ if (!nd_log.eventlog.initialized)
437
+ return false;
438
+
439
+ ND_LOG_FIELD_PRIORITY priority = NDLP_INFO;
440
+ if (fields[NDF_PRIORITY].entry.set)
441
+ priority = (ND_LOG_FIELD_PRIORITY) fields[NDF_PRIORITY].entry.u64;
442
+
443
+ DWORD wType = get_event_type_from_priority(priority);
444
+ (void) wType;
445
+
446
+ CLEAN_BUFFER *tmp = NULL;
447
+
448
+ static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
449
+ spinlock_lock(&spinlock);
450
+ wevt_generate_all_fields_unsafe(fields, fields_max, &tmp);
451
+
452
+ MESSAGE_ID messageID;
453
+ switch (source->source) {
454
+ default:
455
+ case NDLS_DEBUG:
456
+ case NDLS_DAEMON:
457
+ case NDLS_COLLECTORS:
458
+ messageID = MSGID_MESSAGE_ONLY;
459
+ break;
460
+
461
+ case NDLS_HEALTH:
462
+ messageID = MSGID_ALERT_TRANSITION;
463
+ break;
464
+
465
+ case NDLS_ACCESS:
466
+ if (is_field_set(fields, fields_max, NDF_MESSAGE)) {
467
+ messageID = MSGID_ACCESS_MESSAGE;
468
+
469
+ if (has_user_role_permissions(fields, fields_max, &tmp))
470
+ messageID = MSGID_ACCESS_MESSAGE_USER;
471
+ else if (*get_field_value_unsafe(fields, NDF_REQUEST, fields_max, &tmp))
472
+ messageID = MSGID_ACCESS_MESSAGE_REQUEST;
473
+ } else if (is_field_set(fields, fields_max, NDF_RESPONSE_CODE)) {
474
+ messageID = MSGID_ACCESS;
475
+
476
+ if (*get_field_value_unsafe(fields, NDF_SRC_FORWARDED_FOR, fields_max, &tmp))
477
+ messageID = MSGID_ACCESS_FORWARDER;
478
+
479
+ if (has_user_role_permissions(fields, fields_max, &tmp)) {
480
+ if (messageID == MSGID_ACCESS)
481
+ messageID = MSGID_ACCESS_USER;
482
+ else
483
+ messageID = MSGID_ACCESS_FORWARDER_USER;
484
+ }
485
+ } else
486
+ messageID = MSGID_REQUEST_ONLY;
487
+ break;
488
+
489
+ case NDLS_ACLK:
490
+ messageID = MSGID_MESSAGE_ONLY;
491
+ break;
492
+ }
493
+
494
+ if (messageID == MSGID_MESSAGE_ONLY && (
495
+ *get_field_value_unsafe(fields, NDF_ERRNO, fields_max, &tmp) ||
496
+ *get_field_value_unsafe(fields, NDF_WINERROR, fields_max, &tmp))) {
497
+ messageID = MSGID_MESSAGE_ERRNO;
498
+ }
499
+
500
+ DWORD eventID = construct_event_id(source->source, priority, messageID);
501
+
502
+ // wType
503
+ //
504
+ // without a manifest => this determines the Level of the event
505
+ // with a manifest => Level from the manifest is used (wType ignored)
506
+ // [however it is good to have, in case the manifest is not accessible somehow]
507
+ //
508
+
509
+ // wCategory
510
+ //
511
+ // without a manifest => numeric Task values appear
512
+ // with a manifest => Task from the manifest is used (wCategory ignored)
513
+
514
+ BOOL rc;
515
+#if defined(HAVE_ETW)
516
+ if (nd_log.eventlog.etw) {
517
+ // metadata based logging - ETW
518
+
519
+ for (size_t i = 1; i < _NDF_MAX; i++)
520
+ EventDataDescCreate(&etw_eventData[i - 1], fields_buffers[i].buf,
521
+ (wcslen(fields_buffers[i].buf) + 1) * sizeof(WCHAR));
522
+
523
+ EVENT_DESCRIPTOR EventDesc = {
524
+ .Id = eventID & EVENT_ID_CODE_MASK, // ETW needs the raw event id
525
+ .Version = 0,
526
+ .Channel = source->channelID,
527
+ .Level = get_level_from_priority(priority),
528
+ .Opcode = source->Opcode,
529
+ .Task = source->Task,
530
+ .Keyword = source->Keyword,
531
+ };
532
+
533
+ rc = ERROR_SUCCESS == EventWrite(regHandle, &EventDesc, _NDF_MAX - 1, etw_eventData);
534
+
535
+ }
536
+ else
537
+#endif
538
+ {
539
+ // eventID based logging - WEL
540
+ rc = ReportEventW(source->hEventLog, wType, 0, eventID, NULL, _NDF_MAX - 1, 0, wel_messages, NULL);
541
+ }
542
+
543
+ spinlock_unlock(&spinlock);
544
+
545
+ return rc == TRUE;
546
+}
547
+
548
+#if defined(HAVE_ETW)
549
+bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
550
+ return nd_logger_windows(source, fields, fields_max);
551
+}
552
+#endif
553
+
554
+#if defined(HAVE_WEL)
555
+bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
556
+ return nd_logger_windows(source, fields, fields_max);
557
+}
558
+#endif
559
+
560
+#endif
src/libnetdata/log/nd_log.c
new
+465
@@ -0,0 +1,465 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the
4
+// source code that makes the calls, allowing our loggers to log the lines of source code that actually log
5
+#define SD_JOURNAL_SUPPRESS_LOCATION
6
+
7
+#include "../libnetdata.h"
8
+#include "nd_log-internals.h"
9
+
10
+const char *program_name = "";
11
+uint64_t debug_flags = 0;
12
+int aclklog_enabled = 0;
13
+
14
+// --------------------------------------------------------------------------------------------------------------------
15
+
16
+void errno_clear(void) {
17
+ errno = 0;
18
+
19
+#if defined(OS_WINDOWS)
20
+ SetLastError(ERROR_SUCCESS);
21
+#endif
22
+}
23
+
24
+// --------------------------------------------------------------------------------------------------------------------
25
+// logger router
26
+
27
+static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) {
28
+ *spinlock = NULL;
29
+ ND_LOG_METHOD output = nd_log.sources[source].method;
30
+
31
+ switch(output) {
32
+ case NDLM_JOURNAL:
33
+ if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) {
34
+ output = NDLM_FILE;
35
+ *fpp = stderr;
36
+ *spinlock = &nd_log.std_error.spinlock;
37
+ }
38
+ else {
39
+ *fpp = NULL;
40
+ *spinlock = NULL;
41
+ }
42
+ break;
43
+
44
+#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL))
45
+#if defined(HAVE_ETW)
46
+ case NDLM_ETW:
47
+#endif
48
+#if defined(HAVE_WEL)
49
+ case NDLM_WEL:
50
+#endif
51
+ if(unlikely(!nd_log.eventlog.initialized)) {
52
+ output = NDLM_FILE;
53
+ *fpp = stderr;
54
+ *spinlock = &nd_log.std_error.spinlock;
55
+ }
56
+ else {
57
+ *fpp = NULL;
58
+ *spinlock = NULL;
59
+ }
60
+ break;
61
+#endif
62
+
63
+ case NDLM_SYSLOG:
64
+ if(unlikely(!nd_log.syslog.initialized)) {
65
+ output = NDLM_FILE;
66
+ *spinlock = &nd_log.std_error.spinlock;
67
+ *fpp = stderr;
68
+ }
69
+ else {
70
+ *spinlock = NULL;
71
+ *fpp = NULL;
72
+ }
73
+ break;
74
+
75
+ case NDLM_FILE:
76
+ if(!nd_log.sources[source].fp) {
77
+ *fpp = stderr;
78
+ *spinlock = &nd_log.std_error.spinlock;
79
+ }
80
+ else {
81
+ *fpp = nd_log.sources[source].fp;
82
+ *spinlock = &nd_log.sources[source].spinlock;
83
+ }
84
+ break;
85
+
86
+ case NDLM_STDOUT:
87
+ output = NDLM_FILE;
88
+ *fpp = stdout;
89
+ *spinlock = &nd_log.std_output.spinlock;
90
+ break;
91
+
92
+ default:
93
+ case NDLM_DEFAULT:
94
+ case NDLM_STDERR:
95
+ output = NDLM_FILE;
96
+ *fpp = stderr;
97
+ *spinlock = &nd_log.std_error.spinlock;
98
+ break;
99
+
100
+ case NDLM_DISABLED:
101
+ case NDLM_DEVNULL:
102
+ output = NDLM_DISABLED;
103
+ *fpp = NULL;
104
+ *spinlock = NULL;
105
+ break;
106
+ }
107
+
108
+ return output;
109
+}
110
+
111
+// --------------------------------------------------------------------------------------------------------------------
112
+// high level logger
113
+
114
+static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,
115
+ ND_LOG_METHOD output, struct nd_log_source *source,
116
+ struct log_field *fields, size_t fields_max) {
117
+ if(spinlock)
118
+ spinlock_lock(spinlock);
119
+
120
+ // check the limits
121
+ if(limit && nd_log_limit_reached(source))
122
+ goto cleanup;
123
+
124
+ if(output == NDLM_JOURNAL) {
125
+ if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) {
126
+ // we can't log to journal, let's log to stderr
127
+ if(spinlock)
128
+ spinlock_unlock(spinlock);
129
+
130
+ output = NDLM_FILE;
131
+ spinlock = &nd_log.std_error.spinlock;
132
+ fp = stderr;
133
+
134
+ if(spinlock)
135
+ spinlock_lock(spinlock);
136
+ }
137
+ }
138
+
139
+#if defined(OS_WINDOWS)
140
+#if defined(HAVE_ETW)
141
+ if(output == NDLM_ETW) {
142
+ if(!nd_logger_etw(source, fields, fields_max)) {
143
+ // we can't log to windows events, let's log to stderr
144
+ if(spinlock)
145
+ spinlock_unlock(spinlock);
146
+
147
+ output = NDLM_FILE;
148
+ spinlock = &nd_log.std_error.spinlock;
149
+ fp = stderr;
150
+
151
+ if(spinlock)
152
+ spinlock_lock(spinlock);
153
+ }
154
+ }
155
+#endif
156
+#if defined(HAVE_WEL)
157
+ if(output == NDLM_WEL) {
158
+ if(!nd_logger_wel(source, fields, fields_max)) {
159
+ // we can't log to windows events, let's log to stderr
160
+ if(spinlock)
161
+ spinlock_unlock(spinlock);
162
+
163
+ output = NDLM_FILE;
164
+ spinlock = &nd_log.std_error.spinlock;
165
+ fp = stderr;
166
+
167
+ if(spinlock)
168
+ spinlock_lock(spinlock);
169
+ }
170
+ }
171
+#endif
172
+#endif
173
+
174
+ if(output == NDLM_SYSLOG)
175
+ nd_logger_syslog(priority, source->format, fields, fields_max);
176
+
177
+ if(output == NDLM_FILE)
178
+ nd_logger_file(fp, source->format, fields, fields_max);
179
+
180
+
181
+cleanup:
182
+ if(spinlock)
183
+ spinlock_unlock(spinlock);
184
+}
185
+
186
+static void nd_logger_unset_all_thread_fields(void) {
187
+ size_t fields_max = THREAD_FIELDS_MAX;
188
+ for(size_t i = 0; i < fields_max ; i++)
189
+ thread_log_fields[i].entry.set = false;
190
+}
191
+
192
+static void nd_logger_merge_log_stack_to_thread_fields(void) {
193
+ for(size_t c = 0; c < thread_log_stack_next ;c++) {
194
+ struct log_stack_entry *lgs = thread_log_stack_base[c];
195
+
196
+ for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) {
197
+ if(lgs[i].id >= _NDF_MAX || !lgs[i].set)
198
+ continue;
199
+
200
+ struct log_stack_entry *e = &lgs[i];
201
+ ND_LOG_STACK_FIELD_TYPE type = lgs[i].type;
202
+
203
+ // do not add empty / unset fields
204
+ if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
205
+ (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
206
+ (type == NDFT_STR && !e->str) ||
207
+ (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) ||
208
+ (type == NDFT_CALLBACK && !e->cb.formatter) ||
209
+ type == NDFT_UNSET)
210
+ continue;
211
+
212
+ thread_log_fields[lgs[i].id].entry = *e;
213
+ }
214
+ }
215
+}
216
+
217
+static void nd_logger(const char *file, const char *function, const unsigned long line,
218
+ ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit,
219
+ int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) {
220
+
221
+ SPINLOCK *spinlock;
222
+ FILE *fp;
223
+ ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock);
224
+ if(!IS_FINAL_LOG_METHOD(output))
225
+ return;
226
+
227
+ // mark all fields as unset
228
+ nd_logger_unset_all_thread_fields();
229
+
230
+ // flatten the log stack into the fields
231
+ nd_logger_merge_log_stack_to_thread_fields();
232
+
233
+ // set the common fields that are automatically set by the logging subsystem
234
+
235
+ if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
236
+ thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
237
+
238
+ if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set))
239
+ thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source));
240
+ else {
241
+ ND_LOG_SOURCES src = source;
242
+
243
+ if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT)
244
+ src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source);
245
+ else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64)
246
+ src = thread_log_fields[NDF_LOG_SOURCE].entry.u64;
247
+
248
+ if(src != source && src < _NDLS_MAX) {
249
+ source = src;
250
+ output = nd_logger_select_output(source, &fp, &spinlock);
251
+ if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
252
+ return;
253
+ }
254
+ }
255
+
256
+ if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set))
257
+ thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name);
258
+
259
+ if(likely(!thread_log_fields[NDF_LINE].entry.set)) {
260
+ thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line);
261
+ thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file);
262
+ thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function);
263
+ }
264
+
265
+ if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) {
266
+ thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority);
267
+ }
268
+
269
+ if(likely(!thread_log_fields[NDF_TID].entry.set))
270
+ thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached());
271
+
272
+ if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) {
273
+ const char *thread_tag = nd_thread_tag();
274
+ thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag);
275
+
276
+ // TODO: fix the ND_MODULE in logging by setting proper module name in threads
277
+// if(!thread_log_fields[NDF_MODULE].entry.set)
278
+// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag);
279
+ }
280
+
281
+ if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set))
282
+ thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec());
283
+
284
+ if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)
285
+ thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);
286
+
287
+ if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set)
288
+ thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror);
289
+
290
+ CLEAN_BUFFER *wb = NULL;
291
+ if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {
292
+ wb = buffer_create(1024, NULL);
293
+ buffer_vsprintf(wb, fmt, ap);
294
+ thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb));
295
+ }
296
+
297
+ nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source],
298
+ thread_log_fields, THREAD_FIELDS_MAX);
299
+
300
+ if(nd_log.sources[source].pending_msg) {
301
+ // log a pending message
302
+
303
+ nd_logger_unset_all_thread_fields();
304
+
305
+ thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){
306
+ .set = true,
307
+ .type = NDFT_U64,
308
+ .u64 = now_realtime_usec(),
309
+ };
310
+
311
+ thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){
312
+ .set = true,
313
+ .type = NDFT_TXT,
314
+ .txt = nd_log_id2source(source),
315
+ };
316
+
317
+ thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){
318
+ .set = true,
319
+ .type = NDFT_TXT,
320
+ .txt = program_name,
321
+ };
322
+
323
+ thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){
324
+ .set = true,
325
+ .type = NDFT_TXT,
326
+ .txt = nd_log.sources[source].pending_msg,
327
+ };
328
+
329
+ nd_logger_log_fields(spinlock, fp, false, priority, output,
330
+ &nd_log.sources[source],
331
+ thread_log_fields, THREAD_FIELDS_MAX);
332
+
333
+ freez((void *)nd_log.sources[source].pending_msg);
334
+ nd_log.sources[source].pending_msg = NULL;
335
+ }
336
+
337
+ errno_clear();
338
+}
339
+
340
+static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {
341
+ if(source >= _NDLS_MAX)
342
+ source = NDLS_DAEMON;
343
+
344
+ if(nd_log.overwrite_process_source)
345
+ source = nd_log.overwrite_process_source;
346
+
347
+ return source;
348
+}
349
+
350
+// --------------------------------------------------------------------------------------------------------------------
351
+// public API for loggers
352
+
353
+void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )
354
+{
355
+ int saved_errno = errno;
356
+
357
+ size_t saved_winerror = 0;
358
+#if defined(OS_WINDOWS)
359
+ saved_winerror = GetLastError();
360
+#endif
361
+
362
+ source = nd_log_validate_source(source);
363
+
364
+ if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
365
+ return;
366
+
367
+ va_list args;
368
+ va_start(args, fmt);
369
+ nd_logger(file, function, line, source, priority,
370
+ source == NDLS_DAEMON || source == NDLS_COLLECTORS,
371
+ saved_errno, saved_winerror, fmt, args);
372
+ va_end(args);
373
+}
374
+
375
+void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
376
+ int saved_errno = errno;
377
+
378
+ size_t saved_winerror = 0;
379
+#if defined(OS_WINDOWS)
380
+ saved_winerror = GetLastError();
381
+#endif
382
+
383
+ source = nd_log_validate_source(source);
384
+
385
+ if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
386
+ return;
387
+
388
+ if(erl->sleep_ut)
389
+ sleep_usec(erl->sleep_ut);
390
+
391
+ spinlock_lock(&erl->spinlock);
392
+
393
+ erl->count++;
394
+ time_t now = now_boottime_sec();
395
+ if(now - erl->last_logged < erl->log_every) {
396
+ spinlock_unlock(&erl->spinlock);
397
+ return;
398
+ }
399
+
400
+ spinlock_unlock(&erl->spinlock);
401
+
402
+ va_list args;
403
+ va_start(args, fmt);
404
+ nd_logger(file, function, line, source, priority,
405
+ source == NDLS_DAEMON || source == NDLS_COLLECTORS,
406
+ saved_errno, saved_winerror, fmt, args);
407
+ va_end(args);
408
+ erl->last_logged = now;
409
+ erl->count = 0;
410
+}
411
+
412
+void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
413
+ int saved_errno = errno;
414
+
415
+ size_t saved_winerror = 0;
416
+#if defined(OS_WINDOWS)
417
+ saved_winerror = GetLastError();
418
+#endif
419
+
420
+ ND_LOG_SOURCES source = NDLS_DAEMON;
421
+ source = nd_log_validate_source(source);
422
+
423
+ va_list args;
424
+ va_start(args, fmt);
425
+ nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args);
426
+ va_end(args);
427
+
428
+ char date[LOG_DATE_LENGTH];
429
+ log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
430
+
431
+ char action_data[70+1];
432
+ snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);
433
+
434
+ const char *thread_tag = nd_thread_tag();
435
+ const char *tag_to_send = thread_tag;
436
+
437
+ // anonymize thread names
438
+ if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)
439
+ tag_to_send = THREAD_TAG_STREAM_RECEIVER;
440
+ if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)
441
+ tag_to_send = THREAD_TAG_STREAM_SENDER;
442
+
443
+ char action_result[60+1];
444
+ snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);
445
+
446
+#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
447
+ int fd = nd_log.sources[NDLS_DAEMON].fd;
448
+ if(fd == -1)
449
+ fd = STDERR_FILENO;
450
+
451
+ int nptrs;
452
+ void *buffer[10000];
453
+
454
+ nptrs = backtrace(buffer, sizeof(buffer));
455
+ if(nptrs)
456
+ backtrace_symbols_fd(buffer, nptrs, fd);
457
+#endif
458
+
459
+#ifdef NETDATA_INTERNAL_CHECKS
460
+ abort();
461
+#endif
462
+
463
+ netdata_cleanup_and_exit(1, "FATAL", action_result, action_data);
464
+}
465
+
src/libnetdata/log/nd_log.h
renamed
+5
-136
@@ -1,149 +1,18 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
-#ifndef NETDATA_LOG_H
4
-#define NETDATA_LOG_H 1
3
+#ifndef NETDATA_ND_LOG_H
4
+#define NETDATA_ND_LOG_H 1
5
6
# ifdef __cplusplus
7
extern "C" {
8
# endif
9
10
#include "../libnetdata.h"
11
+#include "nd_log-common.h"
12
13
#define ND_LOG_DEFAULT_THROTTLE_LOGS 1000
14
#define ND_LOG_DEFAULT_THROTTLE_PERIOD 60
15
15
-typedef enum __attribute__((__packed__)) {
16
- NDLS_UNSET = 0, // internal use only
17
- NDLS_ACCESS, // access.log
18
- NDLS_ACLK, // aclk.log
19
- NDLS_COLLECTORS, // collector.log
20
- NDLS_DAEMON, // error.log
21
- NDLS_HEALTH, // health.log
22
- NDLS_DEBUG, // debug.log
23
-
24
- // terminator
25
- _NDLS_MAX,
26
-} ND_LOG_SOURCES;
27
-
28
-typedef enum __attribute__((__packed__)) {
29
- NDLP_EMERG = LOG_EMERG,
30
- NDLP_ALERT = LOG_ALERT,
31
- NDLP_CRIT = LOG_CRIT,
32
- NDLP_ERR = LOG_ERR,
33
- NDLP_WARNING = LOG_WARNING,
34
- NDLP_NOTICE = LOG_NOTICE,
35
- NDLP_INFO = LOG_INFO,
36
- NDLP_DEBUG = LOG_DEBUG,
37
-} ND_LOG_FIELD_PRIORITY;
38
-
39
-typedef enum __attribute__((__packed__)) {
40
- // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
41
- // so that it easy to audit for missing fields
42
-
43
- NDF_STOP = 0,
44
- NDF_TIMESTAMP_REALTIME_USEC, // the timestamp of the log message - added automatically
45
- NDF_SYSLOG_IDENTIFIER, // the syslog identifier of the application - added automatically
46
- NDF_LOG_SOURCE, // DAEMON, COLLECTORS, HEALTH, ACCESS, ACLK - set at the log call
47
- NDF_PRIORITY, // the syslog priority (severity) - set at the log call
48
- NDF_ERRNO, // the ERRNO at the time of the log call - added automatically
49
-#if defined(OS_WINDOWS)
50
- NDF_WINERROR, // Windows GetLastError()
51
-#endif
52
- NDF_INVOCATION_ID, // the INVOCATION_ID of Netdata - added automatically
53
- NDF_LINE, // the source code file line number - added automatically
54
- NDF_FILE, // the source code filename - added automatically
55
- NDF_FUNC, // the source code function - added automatically
56
- NDF_TID, // the thread ID of the thread logging - added automatically
57
- NDF_THREAD_TAG, // the thread tag of the thread logging - added automatically
58
- NDF_MESSAGE_ID, // for specific events
59
- NDF_MODULE, // for internal plugin module, all other get the NDF_THREAD_TAG
60
-
61
- NDF_NIDL_NODE, // the node / rrdhost currently being worked
62
- NDF_NIDL_INSTANCE, // the instance / rrdset currently being worked
63
- NDF_NIDL_CONTEXT, // the context of the instance currently being worked
64
- NDF_NIDL_DIMENSION, // the dimension / rrddim currently being worked
65
-
66
- // web server, aclk and stream receiver
67
- NDF_SRC_TRANSPORT, // the transport we received the request, one of: http, https, pluginsd
68
-
69
- // Netdata Cloud Related
70
- NDF_ACCOUNT_ID,
71
- NDF_USER_NAME,
72
- NDF_USER_ROLE,
73
- NDF_USER_ACCESS,
74
-
75
- // web server and stream receiver
76
- NDF_SRC_IP, // the streaming / web server source IP
77
- NDF_SRC_PORT, // the streaming / web server source Port
78
- NDF_SRC_FORWARDED_HOST,
79
- NDF_SRC_FORWARDED_FOR,
80
- NDF_SRC_CAPABILITIES, // the stream receiver capabilities
81
-
82
- // stream sender (established links)
83
- NDF_DST_TRANSPORT, // the transport we send the request, one of: http, https
84
- NDF_DST_IP, // the destination streaming IP
85
- NDF_DST_PORT, // the destination streaming Port
86
- NDF_DST_CAPABILITIES, // the destination streaming capabilities
87
-
88
- // web server, aclk and stream receiver
89
- NDF_REQUEST_METHOD, // for http like requests, the http request method
90
- NDF_RESPONSE_CODE, // for http like requests, the http response code, otherwise a status string
91
-
92
- // web server (all), aclk (queries)
93
- NDF_CONNECTION_ID, // the web server connection ID
94
- NDF_TRANSACTION_ID, // the web server and API transaction ID
95
- NDF_RESPONSE_SENT_BYTES, // for http like requests, the response bytes
96
- NDF_RESPONSE_SIZE_BYTES, // for http like requests, the uncompressed response size
97
- NDF_RESPONSE_PREPARATION_TIME_USEC, // for http like requests, the preparation time
98
- NDF_RESPONSE_SENT_TIME_USEC, // for http like requests, the time to send the response back
99
- NDF_RESPONSE_TOTAL_TIME_USEC, // for http like requests, the total time to complete the response
100
-
101
- // health alerts
102
- NDF_ALERT_ID,
103
- NDF_ALERT_UNIQUE_ID,
104
- NDF_ALERT_EVENT_ID,
105
- NDF_ALERT_TRANSITION_ID,
106
- NDF_ALERT_CONFIG_HASH,
107
- NDF_ALERT_NAME,
108
- NDF_ALERT_CLASS,
109
- NDF_ALERT_COMPONENT,
110
- NDF_ALERT_TYPE,
111
- NDF_ALERT_EXEC,
112
- NDF_ALERT_RECIPIENT,
113
- NDF_ALERT_DURATION,
114
- NDF_ALERT_VALUE,
115
- NDF_ALERT_VALUE_OLD,
116
- NDF_ALERT_STATUS,
117
- NDF_ALERT_STATUS_OLD,
118
- NDF_ALERT_SOURCE,
119
- NDF_ALERT_UNITS,
120
- NDF_ALERT_SUMMARY,
121
- NDF_ALERT_INFO,
122
- NDF_ALERT_NOTIFICATION_REALTIME_USEC,
123
- // NDF_ALERT_FLAGS,
124
-
125
- // put new items here
126
- // leave the request URL and the message last
127
-
128
- NDF_REQUEST, // the request we are currently working on
129
- NDF_MESSAGE, // the log message, if any
130
-
131
- // terminator
132
- _NDF_MAX,
133
-} ND_LOG_FIELD_ID;
134
-
135
-typedef enum __attribute__((__packed__)) {
136
- NDFT_UNSET = 0,
137
- NDFT_TXT,
138
- NDFT_STR,
139
- NDFT_BFR,
140
- NDFT_U64,
141
- NDFT_I64,
142
- NDFT_DBL,
143
- NDFT_UUID,
144
- NDFT_CALLBACK,
145
-} ND_LOG_STACK_FIELD_TYPE;
146
-
16
void errno_clear(void);
17
void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting);
18
void nd_log_set_facility(const char *facility);
@@ -156,7 +25,7 @@ void nd_log_set_flood_protection(size_t logs, time_t period);
25
void nd_log_initialize_for_external_plugins(const char *name);
26
void nd_log_reopen_log_files_for_spawn_server(void);
27
bool nd_log_journal_socket_available(void);
159
-ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len);
28
+ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len);
29
int nd_log_priority2id(const char *priority);
30
const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
31
const char *nd_log_method_for_external_plugins(const char *s);
@@ -306,4 +175,4 @@ void netdata_logger_fatal( const char *file, const char *function, unsigned long
175
}
176
# endif
177
309
-#endif /* NETDATA_LOG_H */
178
+#endif /* NETDATA_ND_LOG_H */
src/libnetdata/log/nd_log_limit.c
new
+100
@@ -0,0 +1,100 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log_limit.h"
4
+
5
+void nd_log_limits_reset(void) {
6
+ usec_t now_ut = now_monotonic_usec();
7
+
8
+ spinlock_lock(&nd_log.std_output.spinlock);
9
+ spinlock_lock(&nd_log.std_error.spinlock);
10
+
11
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
12
+ spinlock_lock(&nd_log.sources[i].spinlock);
13
+ nd_log.sources[i].limits.prevented = 0;
14
+ nd_log.sources[i].limits.counter = 0;
15
+ nd_log.sources[i].limits.started_monotonic_ut = now_ut;
16
+ nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup;
17
+ spinlock_unlock(&nd_log.sources[i].spinlock);
18
+ }
19
+
20
+ spinlock_unlock(&nd_log.std_output.spinlock);
21
+ spinlock_unlock(&nd_log.std_error.spinlock);
22
+}
23
+
24
+void nd_log_limits_unlimited(void) {
25
+ nd_log_limits_reset();
26
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
27
+ nd_log.sources[i].limits.logs_per_period = 0;
28
+ }
29
+}
30
+
31
+bool nd_log_limit_reached(struct nd_log_source *source) {
32
+ if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0)
33
+ return false;
34
+
35
+ usec_t now_ut = now_monotonic_usec();
36
+ if(!source->limits.started_monotonic_ut)
37
+ source->limits.started_monotonic_ut = now_ut;
38
+
39
+ source->limits.counter++;
40
+
41
+ if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) {
42
+ if(source->limits.prevented) {
43
+ BUFFER *wb = buffer_create(1024, NULL);
44
+ buffer_sprintf(wb,
45
+ "LOG FLOOD PROTECTION: resuming logging "
46
+ "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).",
47
+ source->limits.prevented,
48
+ source->limits.throttle_period);
49
+
50
+ if(source->pending_msg)
51
+ freez((void *)source->pending_msg);
52
+
53
+ source->pending_msg = strdupz(buffer_tostring(wb));
54
+
55
+ buffer_free(wb);
56
+ }
57
+
58
+ // restart the period accounting
59
+ source->limits.started_monotonic_ut = now_ut;
60
+ source->limits.counter = 1;
61
+ source->limits.prevented = 0;
62
+
63
+ // log this error
64
+ return false;
65
+ }
66
+
67
+ if(source->limits.counter > source->limits.logs_per_period) {
68
+ if(!source->limits.prevented) {
69
+ BUFFER *wb = buffer_create(1024, NULL);
70
+ buffer_sprintf(wb,
71
+ "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs "
72
+ "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.",
73
+ source->limits.counter,
74
+ (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC),
75
+ source->limits.logs_per_period,
76
+ source->limits.throttle_period,
77
+ program_name,
78
+ (int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC)
79
+ );
80
+
81
+ if(source->pending_msg)
82
+ freez((void *)source->pending_msg);
83
+
84
+ source->pending_msg = strdupz(buffer_tostring(wb));
85
+
86
+ buffer_free(wb);
87
+ }
88
+
89
+ source->limits.prevented++;
90
+
91
+ // prevent logging this error
92
+#ifdef NETDATA_INTERNAL_CHECKS
93
+ return false;
94
+#else
95
+ return true;
96
+#endif
97
+ }
98
+
99
+ return false;
100
+}
src/libnetdata/log/nd_log_limit.h
new
+26
@@ -0,0 +1,26 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ND_LOG_LIMIT_H
4
+#define NETDATA_ND_LOG_LIMIT_H
5
+
6
+#include "../libnetdata.h"
7
+
8
+struct nd_log_source;
9
+bool nd_log_limit_reached(struct nd_log_source *source);
10
+
11
+struct nd_log_limit {
12
+ usec_t started_monotonic_ut;
13
+ uint32_t counter;
14
+ uint32_t prevented;
15
+
16
+ uint32_t throttle_period;
17
+ uint32_t logs_per_period;
18
+ uint32_t logs_per_period_backup;
19
+};
20
+
21
+#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, }
22
+#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, }
23
+
24
+#include "nd_log-internals.h"
25
+
26
+#endif //NETDATA_ND_LOG_LIMIT_H
src/libnetdata/log/nd_wevents_manifest.xml
new
+295
@@ -0,0 +1,295 @@
1
+<?xml version="1.0" encoding="UTF-8"?>
2
+<instrumentationManifest
3
+ xmlns="http://schemas.microsoft.com/win/2004/08/events"
4
+ xmlns:win="http://manifests.microsoft.com/win/2004/08/windows/events"
5
+ xmlns:xs="http://www.w3.org/2001/XMLSchema">
6
+ <instrumentation>
7
+ <events>
8
+
9
+ <provider name="Netdata"
10
+ guid="{96c5ca72-9bd8-4634-81e5-000014e7da7a}"
11
+ symbol="ND_PROVIDER_NAME"
12
+ messageFileName="%SystemRoot%\System32\nd_wevents.dll"
13
+ resourceFileName="%SystemRoot%\System32\nd_wevents.dll"
14
+ parameterFileName="%SystemRoot%\System32\nd_wevents.dll"
15
+ message="$(string.ND_PROVIDER_NAME)">
16
+
17
+ <!-- Define the channels -->
18
+ <channels>
19
+ <channel name="Netdata/Daemon"
20
+ symbol="ND_CHANNEL_DAEMON"
21
+ type="Operational"/>
22
+
23
+ <channel name="Netdata/Collectors"
24
+ symbol="ND_CHANNEL_COLLECTORS"
25
+ type="Operational"/>
26
+
27
+ <channel name="Netdata/Access"
28
+ symbol="ND_CHANNEL_ACCESS"
29
+ type="Operational"/>
30
+
31
+ <channel symbol="ND_CHANNEL_HEALTH"
32
+ name="Netdata/Alerts"
33
+ type="Operational"/>
34
+
35
+ <channel name="Netdata/ACLK"
36
+ symbol="ND_CHANNEL_ACLK"
37
+ type="Operational"/>
38
+ </channels>
39
+
40
+ <levels>
41
+ </levels>
42
+
43
+ <opcodes>
44
+ </opcodes>
45
+
46
+ <tasks>
47
+ <task name="Daemon" value="1" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Daemon)"/>
48
+ <task name="Collector" value="2" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Collector)"/>
49
+ <task name="Access" value="3" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Access)"/>
50
+ <task name="Health" value="4" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Health)"/>
51
+ <task name="Aclk" value="5" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Aclk)"/>
52
+ </tasks>
53
+
54
+ <templates>
55
+ <template tid="NetdataLogTemplate">
56
+ <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->
57
+ <data name="Timestamp" inType="win:UnicodeString"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->
58
+ <data name="Program" inType="win:UnicodeString"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->
59
+ <data name="NetdataLogSource" inType="win:UnicodeString"/> <!-- 3 (NDF_LOG_SOURCE) -->
60
+ <data name="Level" inType="win:UnicodeString"/> <!-- 4 (NDF_PRIORITY) -->
61
+ <data name="UnixErrno" inType="win:UnicodeString"/> <!-- 5 (NDF_ERRNO) -->
62
+ <data name="WindowsLastError" inType="win:UnicodeString"/> <!-- 6 (NDF_WINERROR) -->
63
+ <data name="InvocationID" inType="win:UnicodeString"/> <!-- 7 (NDF_INVOCATION_ID) -->
64
+ <data name="CodeLine" inType="win:UInt32"/> <!-- 8 (NDF_LINE) -->
65
+ <data name="CodeFile" inType="win:UnicodeString"/> <!-- 9 (NDF_FILE) -->
66
+ <data name="CodeFunction" inType="win:UnicodeString"/> <!-- 10 (NDF_FUNC) -->
67
+ <data name="ThreadID" inType="win:UInt32"/> <!-- 11 (NDF_TID) -->
68
+ <data name="ThreadName" inType="win:UnicodeString"/> <!-- 12 (NDF_THREAD_TAG) -->
69
+ <data name="MessageID" inType="win:UnicodeString"/> <!-- 13 (NDF_MESSAGE_ID) -->
70
+ <data name="Module" inType="win:UnicodeString"/> <!-- 14 (NDF_MODULE) -->
71
+ <data name="Node" inType="win:UnicodeString"/> <!-- 15 (NDF_NIDL_NODE) -->
72
+ <data name="Instance" inType="win:UnicodeString"/> <!-- 16 (NDF_NIDL_INSTANCE) -->
73
+ <data name="Context" inType="win:UnicodeString"/> <!-- 17 (NDF_NIDL_CONTEXT) -->
74
+ <data name="Dimension" inType="win:UnicodeString"/> <!-- 18 (NDF_NIDL_DIMENSION) -->
75
+ <data name="SourceTransport" inType="win:UnicodeString"/> <!-- 19 (NDF_SRC_TRANSPORT) -->
76
+ <data name="AccountID" inType="win:UnicodeString"/> <!-- 20 (NDF_ACCOUNT_ID) -->
77
+ <data name="UserName" inType="win:UnicodeString"/> <!-- 21 (NDF_USER_NAME) -->
78
+ <data name="UserRole" inType="win:UnicodeString"/> <!-- 22 (NDF_USER_ROLE) -->
79
+ <data name="UserPermissions" inType="win:UnicodeString"/> <!-- 23 (NDF_USER_ACCESS) -->
80
+ <data name="SourceIP" inType="win:UnicodeString"/> <!-- 24 (NDF_SRC_IP) -->
81
+ <data name="SourceForwardedHost" inType="win:UnicodeString"/> <!-- 25 (NDF_SRC_PORT) -->
82
+ <data name="SourceForwardedFor" inType="win:UnicodeString"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->
83
+ <data name="SourcePort" inType="win:UInt32"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->
84
+ <data name="SourceCapabilities" inType="win:UnicodeString"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->
85
+ <data name="DestinationTransport" inType="win:UnicodeString"/> <!-- 29 (NDF_DST_TRANSPORT) -->
86
+ <data name="DestinationIP" inType="win:UnicodeString"/> <!-- 30 (NDF_DST_IP) -->
87
+ <data name="DestinationPort" inType="win:UInt32"/> <!-- 31 (NDF_DST_PORT) -->
88
+ <data name="DestinationCapabilities" inType="win:UnicodeString"/> <!-- 32 (NDF_DST_CAPABILITIES) -->
89
+ <data name="RequestMethod" inType="win:UnicodeString"/> <!-- 33 (NDF_REQUEST_METHOD) -->
90
+ <data name="ResponseCode" inType="win:UInt32"/> <!-- 34 (NDF_RESPONSE_CODE) -->
91
+ <data name="ConnectionID" inType="win:UnicodeString"/> <!-- 35 (NDF_CONNECTION_ID) -->
92
+ <data name="TransactionID" inType="win:UnicodeString"/> <!-- 36 (NDF_TRANSACTION_ID) -->
93
+ <data name="ResponseSentBytes" inType="win:UInt64"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->
94
+ <data name="ResponseSizeBytes" inType="win:UInt64"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->
95
+ <data name="ResponsePreparationTimeUsec" inType="win:UInt64"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->
96
+ <data name="ResponseSentTimeUsec" inType="win:UInt64"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->
97
+ <data name="ResponseTotalTimeUsec" inType="win:UInt64"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->
98
+ <data name="AlertID" inType="win:UnicodeString"/> <!-- 42 (NDF_ALERT_ID) -->
99
+ <data name="AlertUniqueID" inType="win:UnicodeString"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->
100
+ <data name="AlertTransitionID" inType="win:UnicodeString"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->
101
+ <data name="AlertEventID" inType="win:UnicodeString"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->
102
+ <data name="AlertConfig" inType="win:UnicodeString"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->
103
+ <data name="AlertName" inType="win:UnicodeString"/> <!-- 47 (NDF_ALERT_NAME) -->
104
+ <data name="AlertClass" inType="win:UnicodeString"/> <!-- 48 (NDF_ALERT_CLASS) -->
105
+ <data name="AlertComponent" inType="win:UnicodeString"/> <!-- 49 (NDF_ALERT_COMPONENT) -->
106
+ <data name="AlertType" inType="win:UnicodeString"/> <!-- 50 (NDF_ALERT_TYPE) -->
107
+ <data name="AlertExec" inType="win:UnicodeString"/> <!-- 51 (NDF_ALERT_EXEC) -->
108
+ <data name="AlertRecipient" inType="win:UnicodeString"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->
109
+ <data name="AlertDuration" inType="win:UInt64"/> <!-- 53 (NDF_ALERT_DURATION) -->
110
+ <data name="AlertValue" inType="win:Double"/> <!-- 54 (NDF_ALERT_VALUE) -->
111
+ <data name="AlertOldValue" inType="win:Double"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->
112
+ <data name="AlertStatus" inType="win:UnicodeString"/> <!-- 56 (NDF_ALERT_STATUS) -->
113
+ <data name="AlertOldStatus" inType="win:UnicodeString"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->
114
+ <data name="Source" inType="win:UnicodeString"/> <!-- 58 (NDF_ALERT_SOURCE) -->
115
+ <data name="AlertUnits" inType="win:UnicodeString"/> <!-- 59 (NDF_ALERT_UNITS) -->
116
+ <data name="AlertSummary" inType="win:UnicodeString"/> <!-- 60 (NDF_ALERT_SUMMARY) -->
117
+ <data name="AlertInfo" inType="win:UnicodeString"/> <!-- 61 (NDF_ALERT_INFO) -->
118
+ <data name="AlertNotificationTime" inType="win:UInt64"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->
119
+ <data name="Request" inType="win:UnicodeString"/> <!-- 63 (NDF_REQUEST) -->
120
+ <data name="Message" inType="win:UnicodeString"/> <!-- 64 (NDF_MESSAGE) -->
121
+ </template>
122
+ </templates>
123
+
124
+ <events>
125
+ <!-- Daemon Events -->
126
+ <event symbol="ND_EVENT_DAEMON_INFO"
127
+ value="0x1000"
128
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
129
+ channel="Netdata/Daemon"
130
+ level="win:Informational"
131
+ task="Daemon"
132
+ opcode="win:Info"
133
+ template="NetdataLogTemplate"/>
134
+
135
+ <event symbol="ND_EVENT_DAEMON_WARNING"
136
+ value="0x1001"
137
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
138
+ channel="Netdata/Daemon"
139
+ level="win:Warning"
140
+ task="Daemon"
141
+ opcode="win:Info"
142
+ template="NetdataLogTemplate"/>
143
+
144
+ <event symbol="ND_EVENT_DAEMON_ERROR"
145
+ value="0x1002"
146
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
147
+ channel="Netdata/Daemon"
148
+ level="win:Error"
149
+ task="Daemon"
150
+ opcode="win:Info"
151
+ template="NetdataLogTemplate"/>
152
+
153
+ <!-- Collector Events -->
154
+ <event symbol="ND_EVENT_COLLECTOR_INFO"
155
+ value="0x2000"
156
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
157
+ channel="Netdata/Collectors"
158
+ level="win:Informational"
159
+ task="Collector"
160
+ opcode="win:Info"
161
+ template="NetdataLogTemplate"/>
162
+
163
+ <event symbol="ND_EVENT_COLLECTOR_WARNING"
164
+ value="0x2001"
165
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
166
+ channel="Netdata/Collectors"
167
+ level="win:Warning"
168
+ task="Collector"
169
+ opcode="win:Info"
170
+ template="NetdataLogTemplate"/>
171
+
172
+ <event symbol="ND_EVENT_COLLECTOR_ERROR"
173
+ value="0x2002"
174
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
175
+ channel="Netdata/Collectors"
176
+ level="win:Error"
177
+ task="Collector"
178
+ opcode="win:Info"
179
+ template="NetdataLogTemplate"/>
180
+
181
+ <!-- Access Events -->
182
+ <event symbol="ND_EVENT_ACCESS_INFO"
183
+ value="0x3000"
184
+ message="$(string.ND_ACCESS_EVENT_MESSAGE)"
185
+ channel="Netdata/Access"
186
+ level="win:Informational"
187
+ task="Access"
188
+ opcode="win:Info"
189
+ template="NetdataLogTemplate"/>
190
+
191
+ <event symbol="ND_EVENT_ACCESS_WARNING"
192
+ value="0x3001"
193
+ message="$(string.ND_ACCESS_EVENT_MESSAGE)"
194
+ channel="Netdata/Access"
195
+ level="win:Warning"
196
+ task="Access"
197
+ opcode="win:Info"
198
+ template="NetdataLogTemplate"/>
199
+
200
+ <event symbol="ND_EVENT_ACCESS_ERROR"
201
+ value="0x3002"
202
+ message="$(string.ND_ACCESS_EVENT_MESSAGE)"
203
+ channel="Netdata/Access"
204
+ level="win:Error"
205
+ task="Access"
206
+ opcode="win:Info"
207
+ template="NetdataLogTemplate"/>
208
+
209
+ <!-- Health Events -->
210
+ <event symbol="ND_EVENT_HEALTH_INFO"
211
+ value="0x4000"
212
+ message="$(string.ND_HEALTH_EVENT_MESSAGE)"
213
+ channel="Netdata/Alerts"
214
+ level="win:Informational"
215
+ task="Health"
216
+ opcode="win:Info"
217
+ template="NetdataLogTemplate"/>
218
+
219
+ <event symbol="ND_EVENT_HEALTH_WARNING"
220
+ value="0x4001"
221
+ message="$(string.ND_HEALTH_EVENT_MESSAGE)"
222
+ channel="Netdata/Alerts"
223
+ level="win:Warning"
224
+ task="Health"
225
+ opcode="win:Info"
226
+ template="NetdataLogTemplate"/>
227
+
228
+ <event symbol="ND_EVENT_HEALTH_ERROR"
229
+ value="0x4002"
230
+ message="$(string.ND_HEALTH_EVENT_MESSAGE)"
231
+ channel="Netdata/Alerts"
232
+ level="win:Error"
233
+ task="Health"
234
+ opcode="win:Info"
235
+ template="NetdataLogTemplate"/>
236
+
237
+ <!-- ACLK Events -->
238
+ <event symbol="ND_EVENT_ACLK_INFO"
239
+ value="0x5000"
240
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
241
+ channel="Netdata/ACLK"
242
+ level="win:Informational"
243
+ task="Aclk"
244
+ opcode="win:Info"
245
+ template="NetdataLogTemplate"/>
246
+
247
+ <event symbol="ND_EVENT_ACLK_WARNING"
248
+ value="0x5001"
249
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
250
+ channel="Netdata/ACLK"
251
+ level="win:Warning"
252
+ task="Aclk"
253
+ opcode="win:Info"
254
+ template="NetdataLogTemplate"/>
255
+
256
+ <event symbol="ND_EVENT_ACLK_ERROR"
257
+ value="0x5002"
258
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
259
+ channel="Netdata/ACLK"
260
+ level="win:Error"
261
+ task="Aclk"
262
+ opcode="win:Info"
263
+ template="NetdataLogTemplate"/>
264
+
265
+ </events>
266
+ </provider>
267
+ </events>
268
+ </instrumentation>
269
+
270
+ <localization>
271
+ <resources culture="en-US">
272
+ <stringTable>
273
+ <string id="Task.Daemon" value="ND Daemon Log"/>
274
+ <string id="Task.Collector" value="ND Collector Log"/>
275
+ <string id="Task.Access" value="ND Access Log"/>
276
+ <string id="Task.Health" value="ND Health Log"/>
277
+ <string id="Task.Aclk" value="ND ACLK Log"/>
278
+
279
+ <string id="ND_PROVIDER_NAME" value="Netdata"/>
280
+ <string id="ND_GENERIC_LOG_MESSAGE" value="%64"/>
281
+ <string id="ND_ACCESS_EVENT_MESSAGE"
282
+ value="Transaction %36, method: %33, path: %63
283
+
284
+ Source IP : %24, Forwarded-For: %27
285
+ User : %21, role: %22, permissions: %23
286
+ Timings (usec): prep %39, sent %40, total %41
287
+ Response Size : sent %37, uncompressed %38
288
+ Response Code : %34
289
+"/>
290
+ <string id="ND_HEALTH_EVENT_MESSAGE"
291
+ value="Alert '%47' of instance '%16' on node '%15', transitioned from %57 to %56"/>
292
+ </stringTable>
293
+ </resources>
294
+ </localization>
295
+</instrumentationManifest>
src/libnetdata/log/systemd-cat-native.c
+1
-1
@@ -611,7 +611,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) {
611
if(equal) {
612
const char *field = line->buffer;
613
size_t field_len = equal - line->buffer;
614
- ND_LOG_FIELD_ID id = nd_log_field_id_by_name(field, field_len);
614
+ ND_LOG_FIELD_ID id = nd_log_field_id_by_journal_name(field, field_len);
615
if(id != NDF_STOP) {
616
const char *value = ++equal;
617
src/libnetdata/log/systemd-journal-helpers.c
renamed
+1
-1
@@ -1,6 +1,6 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
-#include "journal.h"
3
+#include "systemd-journal-helpers.h"
4
5
bool is_path_unix_socket(const char *path) {
6
// Check if the path is valid
src/libnetdata/log/systemd-journal-helpers.h
renamed
+3
-3
@@ -2,8 +2,8 @@
2
3
#include "../libnetdata.h"
4
5
-#ifndef NETDATA_LOG_JOURNAL_H
6
-#define NETDATA_LOG_JOURNAL_H
5
+#ifndef NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
6
+#define NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
7
8
#define JOURNAL_DIRECT_SOCKET "/run/systemd/journal/socket"
9
@@ -15,4 +15,4 @@ bool journal_direct_send(int fd, const char *msg, size_t msg_len);
15
bool is_path_unix_socket(const char *path);
16
bool is_stderr_connected_to_journal(void);
17
18
-#endif //NETDATA_LOG_JOURNAL_H
18
+#endif // NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
src/libnetdata/log/wevt_netdata_compile.bat
new
+121
@@ -0,0 +1,121 @@
1
+@echo off
2
+setlocal enabledelayedexpansion
3
+
4
+echo PATH=%PATH%
5
+
6
+if "%~1"=="" (
7
+ echo Error: Missing .mc file path.
8
+ goto :usage
9
+)
10
+if "%~2"=="" (
11
+ echo Error: Missing destination directory.
12
+ goto :usage
13
+)
14
+
15
+REM Set variables
16
+set "SRC_DIR=%~1"
17
+set "BIN_DIR=%~2"
18
+set "MC_FILE=%BIN_DIR%\wevt_netdata.mc"
19
+set "MAN_FILE=%BIN_DIR%\wevt_netdata_manifest.xml"
20
+set "BASE_NAME=wevt_netdata"
21
+set "SDK_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
22
+set "VS_PATH=C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
23
+
24
+if not exist "%SRC_DIR%" (
25
+ echo Error: Source directory does not exist.
26
+ exit /b 1
27
+)
28
+
29
+if not exist "%BIN_DIR%" (
30
+ echo Error: Destination directory does not exist.
31
+ exit /b 1
32
+)
33
+
34
+if not exist "%MC_FILE%" (
35
+ echo Error: %MC_FILE% not found.
36
+ exit /b 1
37
+)
38
+
39
+if not exist "%MAN_FILE%" (
40
+ echo Error: %MAN_FILE% not found.
41
+ exit /b 1
42
+)
43
+
44
+REM Add SDK paths to PATH
45
+set "PATH=C:\Windows\System32;%SDK_PATH%;%VS_PATH%;%PATH%"
46
+
47
+REM Check if commands are available
48
+where mc >nul 2>nul
49
+if %errorlevel% neq 0 (
50
+ echo Error: mc.exe not found in PATH.
51
+ exit /b 1
52
+)
53
+where rc >nul 2>nul
54
+if %errorlevel% neq 0 (
55
+ echo Error: rc.exe not found in PATH.
56
+ exit /b 1
57
+)
58
+where link >nul 2>nul
59
+if %errorlevel% neq 0 (
60
+ echo Error: link.exe not found in PATH.
61
+ exit /b 1
62
+)
63
+where wevtutil >nul 2>nul
64
+if %errorlevel% neq 0 (
65
+ echo Error: wevtutil.exe not found in PATH.
66
+ exit /b 1
67
+)
68
+
69
+REM Change to the destination directory
70
+cd /d "%BIN_DIR%"
71
+
72
+echo.
73
+echo Running mc.exe...
74
+mc -v -b -U "%MC_FILE%" "%MAN_FILE%"
75
+if %errorlevel% neq 0 (
76
+ echo Error: mc.exe failed on messages.
77
+ exit /b 1
78
+)
79
+
80
+if not exist "%BASE_NAME%.rc" (
81
+ echo Error: %BASE_NAME%.rc not found.
82
+ exit /b 1
83
+)
84
+
85
+echo.
86
+echo Modifying %BASE_NAME%.rc to include the manifest...
87
+copy "%MAN_FILE%" %BASE_NAME%_manifest.man
88
+echo 1 2004 "%BASE_NAME%_manifest.man" >> %BASE_NAME%.rc
89
+
90
+echo.
91
+echo %BASE_NAME%.rc contents:
92
+type %BASE_NAME%.rc
93
+
94
+echo.
95
+echo Running rc.exe...
96
+rc /v /fo %BASE_NAME%.res %BASE_NAME%.rc
97
+if %errorlevel% neq 0 (
98
+ echo Error: rc.exe failed.
99
+ exit /b 1
100
+)
101
+
102
+if not exist "%BASE_NAME%.res" (
103
+ echo Error: %BASE_NAME%.res not found.
104
+ exit /b 1
105
+)
106
+
107
+echo.
108
+echo Running link.exe...
109
+link /dll /noentry /machine:x64 /out:%BASE_NAME%.dll %BASE_NAME%.res
110
+if %errorlevel% neq 0 (
111
+ echo Error: link.exe failed.
112
+ exit /b 1
113
+)
114
+
115
+echo.
116
+echo Process completed successfully.
117
+exit /b 0
118
+
119
+:usage
120
+echo Usage: %~nx0 [path_to_mc_file] [destination_directory]
121
+exit /b 1
src/libnetdata/log/wevt_netdata_compile.sh
new
+48
@@ -0,0 +1,48 @@
1
+#!/bin/bash
2
+
3
+mylocation=$(dirname "${0}")
4
+
5
+# Check if both parameters are provided
6
+if [ $# -ne 2 ]; then
7
+ echo "Error: Incorrect number of parameters."
8
+ echo "Usage: $0 <source_directory> <destination_directory>"
9
+ exit 1
10
+fi
11
+
12
+# Get the parameters
13
+src_dir="$1"
14
+dest_dir="$2"
15
+
16
+# Get the directory of this script
17
+SCRIPT_DIR="$(dirname "$0")"
18
+
19
+# Create a temporary batch file
20
+temp_bat=$(mktemp --suffix=.bat)
21
+
22
+# Write the contents to the temporary batch file
23
+# Use cygpath directly within the heredoc
24
+cat << EOF > "$temp_bat"
25
+@echo off
26
+set "PATH=%SYSTEMROOT%;$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --sdk -w);$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --visualstudio -w)"
27
+call "$(cygpath -w -a "$SCRIPT_DIR/wevt_netdata_compile.bat")" "$(cygpath -w -a "$src_dir")" "$(cygpath -w -a "$dest_dir")"
28
+EOF
29
+
30
+# Execute the temporary batch file
31
+echo
32
+echo "Executing Windows Batch File..."
33
+echo
34
+cat "$temp_bat"
35
+cmd.exe //c "$(cygpath -w -a "$temp_bat")"
36
+exit_status=$?
37
+
38
+# Remove the temporary batch file
39
+rm "$temp_bat"
40
+
41
+# Check the exit status
42
+if [ $exit_status -eq 0 ]; then
43
+ echo "nd_wevents_compile.bat executed successfully."
44
+else
45
+ echo "nd_wevents_compile.bat failed with exit status $exit_status."
46
+fi
47
+
48
+exit $exit_status
src/libnetdata/log/wevt_netdata_install.bat
new
+52
@@ -0,0 +1,52 @@
1
+@echo off
2
+setlocal enabledelayedexpansion
3
+
4
+set "MAN_SRC=%~dp0wevt_netdata_manifest.xml"
5
+set "DLL_SRC=%~dp0wevt_netdata.dll"
6
+set "DLL_DST=%SystemRoot%\System32\wevt_netdata.dll"
7
+
8
+where wevtutil >nul 2>nul
9
+if %errorlevel% neq 0 (
10
+ echo Error: wevtutil.exe not found in PATH.
11
+ exit /b 1
12
+)
13
+
14
+echo.
15
+echo Uninstalling previous manifest (if any)...
16
+wevtutil um "%MAN_SRC%"
17
+
18
+echo.
19
+echo Copying %DLL_SRC% to %DLL_DST%
20
+copy /y "%DLL_SRC%" "%DLL_DST%"
21
+if %errorlevel% neq 0 (
22
+ echo Error: Failed to copy %DLL_SRC% to %DLL_DST%
23
+ exit /b 1
24
+)
25
+
26
+echo.
27
+echo Granting access to %DLL_DST% for Windows Event Logging...
28
+icacls "%DLL_DST%" /grant "NT SERVICE\EventLog":R
29
+if %errorlevel% neq 0 (
30
+ echo Error: Failed to grant access to %DLL_DST%.
31
+ exit /b 1
32
+)
33
+
34
+echo.
35
+echo Importing the manifest...
36
+wevtutil im "%MAN_SRC%" /rf:"%DLL_DST%" /mf:"%DLL_DST%"
37
+if %errorlevel% neq 0 (
38
+ echo Error: Failed to import the manifest.
39
+ exit /b 1
40
+)
41
+
42
+echo.
43
+echo Verifying Netdata Publisher for Event Tracing for Windows (ETW)...
44
+wevtutil gp "Netdata"
45
+if %errorlevel% neq 0 (
46
+ echo Error: Failed to get publisher Netdata.
47
+ exit /b 1
48
+)
49
+
50
+echo.
51
+echo Netdata Event Tracing for Windows manifest installed successfully.
52
+exit /b 0
src/libnetdata/log/wevt_netdata_mc_generate.c
new
+518
@@ -0,0 +1,518 @@
1
+#include <stdio.h>
2
+#include <stdlib.h>
3
+#include <inttypes.h>
4
+#include <ctype.h>
5
+#include <stdbool.h>
6
+#include <string.h>
7
+
8
+// from winnt.h
9
+#define EVENTLOG_SUCCESS 0x0000
10
+#define EVENTLOG_ERROR_TYPE 0x0001
11
+#define EVENTLOG_WARNING_TYPE 0x0002
12
+#define EVENTLOG_INFORMATION_TYPE 0x0004
13
+#define EVENTLOG_AUDIT_SUCCESS 0x0008
14
+#define EVENTLOG_AUDIT_FAILURE 0x0010
15
+
16
+// the severities we define in .mc file
17
+#define STATUS_SEVERITY_INFORMATIONAL 0x1
18
+#define STATUS_SEVERITY_WARNING 0x2
19
+#define STATUS_SEVERITY_ERROR 0x3
20
+
21
+#define FACILITY_APPLICATION 0x0fff
22
+
23
+#include "nd_log-common.h"
24
+#include "nd_log-to-windows-common.h"
25
+
26
+const char *get_msg_symbol(MESSAGE_ID msg) {
27
+ switch(msg) {
28
+ case MSGID_MESSAGE_ONLY:
29
+ return "MESSAGE_ONLY";
30
+
31
+ case MSGID_MESSAGE_ERRNO:
32
+ return "MESSAGE_ERRNO";
33
+
34
+ case MSGID_REQUEST_ONLY:
35
+ return "REQUEST_ONLY";
36
+
37
+ case MSGID_ACCESS_MESSAGE:
38
+ return "ACCESS_MESSAGE";
39
+
40
+ case MSGID_ACCESS_MESSAGE_REQUEST:
41
+ return "ACCESS_MESSAGE_REQUEST";
42
+
43
+ case MSGID_ACCESS_MESSAGE_USER:
44
+ return "ACCESS_MESSAGE_USER";
45
+
46
+ case MSGID_ACCESS:
47
+ return "ACCESS";
48
+
49
+ case MSGID_ACCESS_USER:
50
+ return "ACCESS_USER";
51
+
52
+ case MSGID_ACCESS_FORWARDER:
53
+ return "ACCESS_FORWARDER";
54
+
55
+ case MSGID_ACCESS_FORWARDER_USER:
56
+ return "ACCESS_FORWARDER_USER";
57
+
58
+ case MSGID_ALERT_TRANSITION:
59
+ return "ALERT_TRANSITION";
60
+
61
+ default:
62
+ fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg);
63
+ exit(1);
64
+ }
65
+}
66
+
67
+const char *get_msg_format(MESSAGE_ID msg) {
68
+ switch(msg) {
69
+ case MSGID_MESSAGE_ONLY:
70
+ return "%2(%12): %64\r\n";
71
+
72
+ case MSGID_MESSAGE_ERRNO:
73
+ return "%2(%12): %64%n\r\n"
74
+ "%n\r\n"
75
+ " Unix Errno : %5%n\r\n"
76
+ " Windows Error: %6%n\r\n"
77
+ ;
78
+
79
+ case MSGID_REQUEST_ONLY:
80
+ return "%2(%12): %63\r\n";
81
+
82
+ case MSGID_ACCESS_MESSAGE:
83
+ return "%64\r\n";
84
+
85
+ case MSGID_ACCESS_MESSAGE_REQUEST:
86
+ return "%64%n\r\n"
87
+ "%n\r\n"
88
+ " Request: %63%n\r\n"
89
+ ;
90
+
91
+ case MSGID_ACCESS_MESSAGE_USER:
92
+ return "%64%n\r\n"
93
+ "%n\r\n"
94
+ " User: %21, role: %22, permissions: %23%n\r\n"
95
+ ;
96
+
97
+ case MSGID_ACCESS:
98
+ return "%33 %63%n\r\n"
99
+ "%n\r\n"
100
+ " Response Code : %34%n\r\n"
101
+ " Transaction ID: %36%n\r\n"
102
+ " Source IP : %24%n\r\n"
103
+ ;
104
+
105
+ case MSGID_ACCESS_USER:
106
+ return "%33 %63%n\r\n"
107
+ "%n\r\n"
108
+ " Response Code : %34%n\r\n"
109
+ " Transaction ID: %36%n\r\n"
110
+ " Source IP : %24%n\r\n"
111
+ " User : %21, role: %22, permissions: %23%n\r\n"
112
+ ;
113
+
114
+ case MSGID_ACCESS_FORWARDER:
115
+ return "%33 %63%n\r\n"
116
+ "%n\r\n"
117
+ " Response Code : %34%n\r\n"
118
+ " Transaction ID: %36%n\r\n"
119
+ " Source IP : %24, For %27%n\r\n"
120
+ ;
121
+
122
+ case MSGID_ACCESS_FORWARDER_USER:
123
+ return "%33 %63%n\r\n"
124
+ "%n\r\n"
125
+ " Response Code : %34%n\r\n"
126
+ " Transaction ID: %36%n\r\n"
127
+ " Source IP : %24, For %27%n\r\n"
128
+ " User : %21, role: %22, permissions: %23%n\r\n"
129
+ ;
130
+
131
+ case MSGID_ALERT_TRANSITION:
132
+ return "Alert '%47' of instance '%16' on node '%15' transitioned from %57 to %56\r\n";
133
+
134
+ default:
135
+ fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg);
136
+ exit(1);
137
+ }
138
+}
139
+
140
+int main(int argc, const char **argv) {
141
+ (void)argc; (void)argv;
142
+
143
+ const char *header = NULL, *footer = NULL, *s_header = NULL, *s_footer = NULL;
144
+
145
+ bool manifest = false;
146
+ if(argc == 2 && strcmp(argv[1], "--manifest") == 0) {
147
+ manifest = true;
148
+
149
+ header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n"
150
+ "<!--\r\n"
151
+ "\r\n"
152
+ " THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n"
153
+ "\r\n"
154
+ " This XML file can be verified by running mc.exe (the MS tool) with this manifest as param.\r\n"
155
+ "\r\n"
156
+ " \"c:\\Program Files (x86)\\Windows Kits\\10\\bin\\10.0.26100.0\\x64\\mc.exe\" wevt_netdata_manifest.xml wevt_netdata.mc\r\n"
157
+ "\r\n"
158
+ " -->\r\n"
159
+ "<instrumentationManifest\r\n"
160
+ " xmlns=\"http://schemas.microsoft.com/win/2004/08/events\"\r\n"
161
+ " xmlns:win=\"http://manifests.microsoft.com/win/2004/08/windows/events\"\r\n"
162
+ " xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\r\n"
163
+ " <instrumentation>\r\n"
164
+ " <events>\r\n"
165
+ "\r\n"
166
+ " <provider name=\"" NETDATA_ETW_PROVIDER_NAME "\"\r\n"
167
+ " guid=\"" NETDATA_ETW_PROVIDER_GUID_STR "\"\r\n"
168
+ " symbol=\"NETDATA_ETW_PROVIDER_GUID\"\r\n"
169
+ " messageFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n"
170
+ " resourceFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n"
171
+ " message=\"$(string.ND_PROVIDER_NAME)\">\r\n"
172
+ "\r\n"
173
+ " <!-- Define the provider sub-channels -->\r\n"
174
+ " <channels>\r\n"
175
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON "\"\r\n"
176
+ " symbol=\"CHANNEL_DAEMON\"\r\n"
177
+ " type=\"Operational\"\r\n"
178
+ " message=\"$(string.Channel.Daemon)\"\r\n"
179
+ " enabled=\"true\"\r\n"
180
+ " />\r\n"
181
+ "\r\n"
182
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS "\"\r\n"
183
+ " symbol=\"CHANNEL_COLLECTORS\"\r\n"
184
+ " type=\"Operational\"\r\n"
185
+ " message=\"$(string.Channel.Collectors)\"\r\n"
186
+ " enabled=\"true\"\r\n"
187
+ " />\r\n"
188
+ "\r\n"
189
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS "\"\r\n"
190
+ " symbol=\"CHANNEL_ACCESS\"\r\n"
191
+ " type=\"Operational\"\r\n"
192
+ " message=\"$(string.Channel.Access)\"\r\n"
193
+ " enabled=\"true\"\r\n"
194
+ " />\r\n"
195
+ "\r\n"
196
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH "\"\r\n"
197
+ " symbol=\"CHANNEL_HEALTH\"\r\n"
198
+ " type=\"Operational\"\r\n"
199
+ " message=\"$(string.Channel.Health)\"\r\n"
200
+ " enabled=\"true\"\r\n"
201
+ " />\r\n"
202
+ "\r\n"
203
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK "\"\r\n"
204
+ " symbol=\"CHANNEL_ACLK\"\r\n"
205
+ " type=\"Operational\"\r\n"
206
+ " message=\"$(string.Channel.Aclk)\"\r\n"
207
+ " enabled=\"true\"\r\n"
208
+ " />\r\n"
209
+ " </channels>\r\n"
210
+ "\r\n"
211
+ " <levels>\r\n"
212
+ " </levels>\r\n"
213
+ "\r\n"
214
+ " <opcodes>\r\n"
215
+ " </opcodes>\r\n"
216
+ "\r\n"
217
+ " <tasks>\r\n"
218
+ " </tasks>\r\n"
219
+ "\r\n"
220
+ " <templates>\r\n"
221
+ " <template tid=\"AllFieldsTemplate\">\r\n"
222
+ " <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->\r\n"
223
+ " <data name=\"Timestamp\" inType=\"win:UnicodeString\"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->\r\n"
224
+ " <data name=\"Program\" inType=\"win:UnicodeString\"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->\r\n"
225
+ " <data name=\"NetdataLogSource\" inType=\"win:UnicodeString\"/> <!-- 3 (NDF_LOG_SOURCE) -->\r\n"
226
+ " <data name=\"Level\" inType=\"win:UnicodeString\"/> <!-- 4 (NDF_PRIORITY) -->\r\n"
227
+ " <data name=\"UnixErrno\" inType=\"win:UnicodeString\"/> <!-- 5 (NDF_ERRNO) -->\r\n"
228
+ " <data name=\"WindowsLastError\" inType=\"win:UnicodeString\"/> <!-- 6 (NDF_WINERROR) -->\r\n"
229
+ " <data name=\"InvocationID\" inType=\"win:UnicodeString\"/> <!-- 7 (NDF_INVOCATION_ID) -->\r\n"
230
+ " <data name=\"CodeLine\" inType=\"win:UnicodeString\"/> <!-- 8 (NDF_LINE) -->\r\n"
231
+ " <data name=\"CodeFile\" inType=\"win:UnicodeString\"/> <!-- 9 (NDF_FILE) -->\r\n"
232
+ " <data name=\"CodeFunction\" inType=\"win:UnicodeString\"/> <!-- 10 (NDF_FUNC) -->\r\n"
233
+ " <data name=\"ThreadID\" inType=\"win:UnicodeString\"/> <!-- 11 (NDF_TID) -->\r\n"
234
+ " <data name=\"ThreadName\" inType=\"win:UnicodeString\"/> <!-- 12 (NDF_THREAD_TAG) -->\r\n"
235
+ " <data name=\"MessageID\" inType=\"win:UnicodeString\"/> <!-- 13 (NDF_MESSAGE_ID) -->\r\n"
236
+ " <data name=\"Module\" inType=\"win:UnicodeString\"/> <!-- 14 (NDF_MODULE) -->\r\n"
237
+ " <data name=\"Node\" inType=\"win:UnicodeString\"/> <!-- 15 (NDF_NIDL_NODE) -->\r\n"
238
+ " <data name=\"Instance\" inType=\"win:UnicodeString\"/> <!-- 16 (NDF_NIDL_INSTANCE) -->\r\n"
239
+ " <data name=\"Context\" inType=\"win:UnicodeString\"/> <!-- 17 (NDF_NIDL_CONTEXT) -->\r\n"
240
+ " <data name=\"Dimension\" inType=\"win:UnicodeString\"/> <!-- 18 (NDF_NIDL_DIMENSION) -->\r\n"
241
+ " <data name=\"SourceTransport\" inType=\"win:UnicodeString\"/> <!-- 19 (NDF_SRC_TRANSPORT) -->\r\n"
242
+ " <data name=\"AccountID\" inType=\"win:UnicodeString\"/> <!-- 20 (NDF_ACCOUNT_ID) -->\r\n"
243
+ " <data name=\"UserName\" inType=\"win:UnicodeString\"/> <!-- 21 (NDF_USER_NAME) -->\r\n"
244
+ " <data name=\"UserRole\" inType=\"win:UnicodeString\"/> <!-- 22 (NDF_USER_ROLE) -->\r\n"
245
+ " <data name=\"UserPermissions\" inType=\"win:UnicodeString\"/> <!-- 23 (NDF_USER_ACCESS) -->\r\n"
246
+ " <data name=\"SourceIP\" inType=\"win:UnicodeString\"/> <!-- 24 (NDF_SRC_IP) -->\r\n"
247
+ " <data name=\"SourceForwardedHost\" inType=\"win:UnicodeString\"/> <!-- 25 (NDF_SRC_PORT) -->\r\n"
248
+ " <data name=\"SourceForwardedFor\" inType=\"win:UnicodeString\"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->\r\n"
249
+ " <data name=\"SourcePort\" inType=\"win:UnicodeString\"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->\r\n"
250
+ " <data name=\"SourceCapabilities\" inType=\"win:UnicodeString\"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->\r\n"
251
+ " <data name=\"DestinationTransport\" inType=\"win:UnicodeString\"/> <!-- 29 (NDF_DST_TRANSPORT) -->\r\n"
252
+ " <data name=\"DestinationIP\" inType=\"win:UnicodeString\"/> <!-- 30 (NDF_DST_IP) -->\r\n"
253
+ " <data name=\"DestinationPort\" inType=\"win:UnicodeString\"/> <!-- 31 (NDF_DST_PORT) -->\r\n"
254
+ " <data name=\"DestinationCapabilities\" inType=\"win:UnicodeString\"/> <!-- 32 (NDF_DST_CAPABILITIES) -->\r\n"
255
+ " <data name=\"RequestMethod\" inType=\"win:UnicodeString\"/> <!-- 33 (NDF_REQUEST_METHOD) -->\r\n"
256
+ " <data name=\"ResponseCode\" inType=\"win:UnicodeString\"/> <!-- 34 (NDF_RESPONSE_CODE) -->\r\n"
257
+ " <data name=\"ConnectionID\" inType=\"win:UnicodeString\"/> <!-- 35 (NDF_CONNECTION_ID) -->\r\n"
258
+ " <data name=\"TransactionID\" inType=\"win:UnicodeString\"/> <!-- 36 (NDF_TRANSACTION_ID) -->\r\n"
259
+ " <data name=\"ResponseSentBytes\" inType=\"win:UnicodeString\"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->\r\n"
260
+ " <data name=\"ResponseSizeBytes\" inType=\"win:UnicodeString\"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->\r\n"
261
+ " <data name=\"ResponsePreparationTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->\r\n"
262
+ " <data name=\"ResponseSentTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->\r\n"
263
+ " <data name=\"ResponseTotalTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->\r\n"
264
+ " <data name=\"AlertID\" inType=\"win:UnicodeString\"/> <!-- 42 (NDF_ALERT_ID) -->\r\n"
265
+ " <data name=\"AlertUniqueID\" inType=\"win:UnicodeString\"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->\r\n"
266
+ " <data name=\"AlertTransitionID\" inType=\"win:UnicodeString\"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->\r\n"
267
+ " <data name=\"AlertEventID\" inType=\"win:UnicodeString\"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->\r\n"
268
+ " <data name=\"AlertConfig\" inType=\"win:UnicodeString\"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->\r\n"
269
+ " <data name=\"AlertName\" inType=\"win:UnicodeString\"/> <!-- 47 (NDF_ALERT_NAME) -->\r\n"
270
+ " <data name=\"AlertClass\" inType=\"win:UnicodeString\"/> <!-- 48 (NDF_ALERT_CLASS) -->\r\n"
271
+ " <data name=\"AlertComponent\" inType=\"win:UnicodeString\"/> <!-- 49 (NDF_ALERT_COMPONENT) -->\r\n"
272
+ " <data name=\"AlertType\" inType=\"win:UnicodeString\"/> <!-- 50 (NDF_ALERT_TYPE) -->\r\n"
273
+ " <data name=\"AlertExec\" inType=\"win:UnicodeString\"/> <!-- 51 (NDF_ALERT_EXEC) -->\r\n"
274
+ " <data name=\"AlertRecipient\" inType=\"win:UnicodeString\"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->\r\n"
275
+ " <data name=\"AlertDuration\" inType=\"win:UnicodeString\"/> <!-- 53 (NDF_ALERT_DURATION) -->\r\n"
276
+ " <data name=\"AlertValue\" inType=\"win:UnicodeString\"/> <!-- 54 (NDF_ALERT_VALUE) -->\r\n"
277
+ " <data name=\"AlertOldValue\" inType=\"win:UnicodeString\"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->\r\n"
278
+ " <data name=\"AlertStatus\" inType=\"win:UnicodeString\"/> <!-- 56 (NDF_ALERT_STATUS) -->\r\n"
279
+ " <data name=\"AlertOldStatus\" inType=\"win:UnicodeString\"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->\r\n"
280
+ " <data name=\"Source\" inType=\"win:UnicodeString\"/> <!-- 58 (NDF_ALERT_SOURCE) -->\r\n"
281
+ " <data name=\"AlertUnits\" inType=\"win:UnicodeString\"/> <!-- 59 (NDF_ALERT_UNITS) -->\r\n"
282
+ " <data name=\"AlertSummary\" inType=\"win:UnicodeString\"/> <!-- 60 (NDF_ALERT_SUMMARY) -->\r\n"
283
+ " <data name=\"AlertInfo\" inType=\"win:UnicodeString\"/> <!-- 61 (NDF_ALERT_INFO) -->\r\n"
284
+ " <data name=\"AlertNotificationTime\" inType=\"win:UnicodeString\"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->\r\n"
285
+ " <data name=\"Request\" inType=\"win:UnicodeString\"/> <!-- 63 (NDF_REQUEST) -->\r\n"
286
+ " <data name=\"Message\" inType=\"win:UnicodeString\"/> <!-- 64 (NDF_MESSAGE) -->\r\n"
287
+ " </template>\r\n"
288
+ " </templates>\r\n"
289
+ "\r\n"
290
+ " <events>\r\n"
291
+ ;
292
+
293
+ footer = " </events>\r\n"
294
+ " </provider>\r\n"
295
+ " </events>\r\n"
296
+ " </instrumentation>\r\n"
297
+ ;
298
+
299
+ s_header = " <localization>\r\n"
300
+ " <resources culture=\"en-US\">\r\n"
301
+ " <stringTable>\r\n"
302
+ " <string id=\"ND_PROVIDER_NAME\" value=\"" NETDATA_ETW_PROVIDER_NAME "\"/>\r\n"
303
+ "\r\n"
304
+ " <string id=\"Channel.Daemon\" value=\"Daemon\"/>\r\n"
305
+ " <string id=\"Channel.Collectors\" value=\"Collectors\"/>\r\n"
306
+ " <string id=\"Channel.Access\" value=\"Access\"/>\r\n"
307
+ " <string id=\"Channel.Health\" value=\"Health\"/>\r\n"
308
+ " <string id=\"Channel.Aclk\" value=\"Aclk\"/>\r\n"
309
+ "\r\n"
310
+ ;
311
+
312
+ s_footer = " </stringTable>\r\n"
313
+ " </resources>\r\n"
314
+ " </localization>\r\n"
315
+ "</instrumentationManifest>\r\n"
316
+ ;
317
+ }
318
+ else {
319
+ header = ";// THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n"
320
+ "\r\n"
321
+ "MessageIdTypedef=DWORD\r\n"
322
+ "\r\n"
323
+ "SeverityNames=(\r\n"
324
+ " Informational=0x1:STATUS_SEVERITY_INFORMATIONAL\r\n"
325
+ " Warning=0x2:STATUS_SEVERITY_WARNING\r\n"
326
+ " Error=0x3:STATUS_SEVERITY_ERROR\r\n"
327
+ " )\r\n"
328
+ "\r\n"
329
+ "FacilityNames=(\r\n"
330
+ " " NETDATA_CHANNEL_NAME "=0x0FFF:FACILITY_NETDATA\r\n"
331
+ " )\r\n"
332
+ "\r\n"
333
+ "LanguageNames=(\r\n"
334
+ " English=0x409:MSG00409\r\n"
335
+ " )\r\n"
336
+ "\r\n"
337
+ ;
338
+
339
+ footer = "";
340
+ }
341
+
342
+ bool done[UINT16_MAX] = { 0 };
343
+ char symbol[1024];
344
+
345
+ printf("%s", header);
346
+ for(size_t src = 1; src < _NDLS_MAX ;src++) {
347
+ for(size_t pri = 0; pri < _NDLP_MAX ;pri++) {
348
+ uint8_t severity = get_severity_from_priority(pri);
349
+
350
+ for(size_t msg = 1; msg < _MSGID_MAX ;msg++) {
351
+
352
+ if(src >= 16) {
353
+ fprintf(stderr, "\n\nSource %zu is bigger than 4 bits!\n\n", src);
354
+ return 1;
355
+ }
356
+
357
+ if(pri >= 16) {
358
+ fprintf(stderr, "\n\nPriority %zu is bigger than 4 bits!\n\n", pri);
359
+ return 1;
360
+ }
361
+
362
+ if(msg >= 256) {
363
+ fprintf(stderr, "\n\nMessageID %zu is bigger than 8 bits!\n\n", msg);
364
+ return 1;
365
+ }
366
+
367
+ uint16_t eventID = construct_event_code(src, pri, msg);
368
+ if((eventID & 0xFFFF) != eventID) {
369
+ fprintf(stderr, "\n\nEventID 0x%x is bigger than 16 bits!\n\n", eventID);
370
+ return 1;
371
+ }
372
+
373
+ if(done[eventID]) continue;
374
+ done[eventID] = true;
375
+
376
+ const char *level = get_level_from_priority_str(pri);
377
+ const char *pri_txt;
378
+ switch(pri) {
379
+ case NDLP_EMERG:
380
+ pri_txt = "EMERG";
381
+ break;
382
+
383
+ case NDLP_CRIT:
384
+ pri_txt = "CRIT";
385
+ break;
386
+
387
+ case NDLP_ALERT:
388
+ pri_txt = "ALERT";
389
+ break;
390
+
391
+ case NDLP_ERR:
392
+ pri_txt = "ERR";
393
+ break;
394
+
395
+ case NDLP_WARNING:
396
+ pri_txt = "WARN";
397
+ break;
398
+
399
+ case NDLP_INFO:
400
+ pri_txt = "INFO";
401
+ break;
402
+
403
+ case NDLP_NOTICE:
404
+ pri_txt = "NOTICE";
405
+ break;
406
+
407
+ case NDLP_DEBUG:
408
+ pri_txt = "DEBUG";
409
+ break;
410
+
411
+ default:
412
+ fprintf(stderr, "\n\nInvalid priority %zu!\n\n\n", pri);
413
+ return 1;
414
+ }
415
+
416
+ const char *channel;
417
+ const char *src_txt;
418
+ switch(src) {
419
+ case NDLS_COLLECTORS:
420
+ src_txt = "COLLECTORS";
421
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS;
422
+ break;
423
+
424
+ case NDLS_ACCESS:
425
+ src_txt = "ACCESS";
426
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS;
427
+ break;
428
+
429
+ case NDLS_HEALTH:
430
+ src_txt = "HEALTH";
431
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH;
432
+ break;
433
+
434
+ case NDLS_DEBUG:
435
+ src_txt = "DEBUG";
436
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON;
437
+ break;
438
+
439
+ case NDLS_DAEMON:
440
+ src_txt = "DAEMON";
441
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON;
442
+ break;
443
+
444
+ case NDLS_ACLK:
445
+ src_txt = "ACLK";
446
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK;
447
+ break;
448
+
449
+ default:
450
+ fprintf(stderr, "\n\nInvalid source %zu!\n\n\n", src);
451
+ return 1;
452
+ }
453
+
454
+ const char *msg_txt = get_msg_symbol(msg);
455
+ const char *format = get_msg_format(msg);
456
+
457
+ const char *severity_txt;
458
+ switch (severity) {
459
+ case STATUS_SEVERITY_INFORMATIONAL:
460
+ severity_txt = "Informational";
461
+ break;
462
+
463
+ case STATUS_SEVERITY_ERROR:
464
+ severity_txt = "Error";
465
+ break;
466
+
467
+ case STATUS_SEVERITY_WARNING:
468
+ severity_txt = "Warning";
469
+ break;
470
+
471
+ default:
472
+ fprintf(stderr, "\n\nInvalid severity id %u!\n\n\n", severity);
473
+ return 1;
474
+ }
475
+
476
+ if(manifest)
477
+ snprintf(symbol, sizeof(symbol), "ED_%s_%s_%s", src_txt, pri_txt, msg_txt);
478
+ else
479
+ snprintf(symbol, sizeof(symbol), "MC_%s_%s_%s", src_txt, pri_txt, msg_txt);
480
+
481
+ if(manifest)
482
+ printf(" <event symbol=\"%s\"\r\n"
483
+ " value=\"0x%x\"\r\n"
484
+ " message=\"$(string.msg.MAN_%s)\"\r\n"
485
+ " channel=\"%s\"\r\n"
486
+ " level=\"%s\"\r\n"
487
+ " task=\"win:None\"\r\n"
488
+ " opcode=\"win:Info\"\r\n"
489
+ " template=\"AllFieldsTemplate\"/>\r\n\r\n",
490
+ symbol, eventID, msg_txt, channel, level);
491
+ else
492
+ printf("MessageId=0x%x\r\n"
493
+ "Severity=%s\r\n"
494
+ "Facility=" NETDATA_CHANNEL_NAME "\r\n"
495
+ "SymbolicName=%s\r\n"
496
+ "Language=English\r\n"
497
+ "%s"
498
+ ".\r\n"
499
+ "\r\n",
500
+ eventID, severity_txt, symbol, format);
501
+ }
502
+ }
503
+ }
504
+ printf("%s", footer);
505
+
506
+ if(s_header) {
507
+ printf("%s", s_header);
508
+
509
+ for(size_t msg = 1; msg < _MSGID_MAX ;msg++) {
510
+ const char *msg_txt = get_msg_symbol(msg);
511
+ const char *format = get_msg_format(msg);
512
+ printf(" <string id=\"msg.MAN_%s\" value=\"%s\"/>\r\n", msg_txt, format);
513
+ }
514
+
515
+ printf("%s", s_footer);
516
+ }
517
+}
518
+
src/libnetdata/os/windows-perflib/perflib.c
+6
-20
@@ -386,44 +386,30 @@ static inline PERF_COUNTER_DEFINITION *getCounterDefinition(PERF_DATA_BLOCK *pDa
386
// --------------------------------------------------------------------------------------------------------------------
387
388
static inline BOOL getEncodedStringToUTF8(char *dst, size_t dst_len, DWORD CodePage, char *start, DWORD length) {
389
+ static __thread wchar_t unicode[PERFLIB_MAX_NAME_LENGTH];
390
+
391
WCHAR *tempBuffer; // Temporary buffer for Unicode data
392
DWORD charsCopied = 0;
391
- BOOL free_tempBuffer;
393
394
if (CodePage == 0) {
395
// Input is already Unicode (UTF-16)
396
tempBuffer = (WCHAR *)start;
397
charsCopied = length / sizeof(WCHAR); // Convert byte length to number of WCHARs
397
- free_tempBuffer = FALSE;
398
}
399
else {
400
- // Convert the multi-byte instance name to Unicode (UTF-16)
401
- // Calculate maximum possible characters in UTF-16
402
-
403
- int charCount = MultiByteToWideChar(CodePage, 0, start, (int)length, NULL, 0);
404
- tempBuffer = (WCHAR *)malloc(charCount * sizeof(WCHAR));
405
- if (!tempBuffer) return FALSE;
406
-
407
- charsCopied = MultiByteToWideChar(CodePage, 0, start, (int)length, tempBuffer, charCount);
408
- if (charsCopied == 0) {
409
- free(tempBuffer);
410
- dst[0] = '\0';
411
- return FALSE;
412
- }
413
-
414
- free_tempBuffer = TRUE;
400
+ tempBuffer = unicode;
401
+ charsCopied = any_to_utf16(CodePage, unicode, _countof(unicode), start, (int)length);
402
+ if(!charsCopied) return FALSE;
403
}
404
405
// Now convert from Unicode (UTF-16) to UTF-8
406
int bytesCopied = WideCharToMultiByte(CP_UTF8, 0, tempBuffer, (int)charsCopied, dst, (int)dst_len, NULL, NULL);
407
if (bytesCopied == 0) {
420
- if (free_tempBuffer) free(tempBuffer);
408
dst[0] = '\0'; // Ensure the buffer is null-terminated even on failure
409
return FALSE;
410
}
411
425
- dst[bytesCopied] = '\0'; // Ensure buffer is null-terminated
426
- if (free_tempBuffer) free(tempBuffer); // Free temporary buffer if used
412
+ dst[bytesCopied - 1] = '\0'; // Ensure buffer is null-terminated
413
return TRUE;
414
}
415
src/libnetdata/os/windows-perflib/perflib.h
+1
@@ -25,6 +25,7 @@ const char *RegistryFindNameByID(DWORD id);
25
const char *RegistryFindHelpByID(DWORD id);
26
DWORD RegistryFindIDByName(const char *name);
27
#define PERFLIB_REGISTRY_NAME_NOT_FOUND (DWORD)-1
28
+#define PERFLIB_MAX_NAME_LENGTH 1024
29
30
PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id);
31
void perflibFreePerformanceData(void);
src/libnetdata/spawn_server/log-forwarder.c
new
+322
@@ -0,0 +1,322 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "../libnetdata.h"
4
+#include "log-forwarder.h"
5
+
6
+typedef struct LOG_FORWARDER_ENTRY {
7
+ int fd;
8
+ char *cmd;
9
+ pid_t pid;
10
+ BUFFER *wb;
11
+ size_t pfds_idx;
12
+ bool delete;
13
+
14
+ struct LOG_FORWARDER_ENTRY *prev;
15
+ struct LOG_FORWARDER_ENTRY *next;
16
+} LOG_FORWARDER_ENTRY;
17
+
18
+typedef struct LOG_FORWARDER {
19
+ LOG_FORWARDER_ENTRY *entries;
20
+ ND_THREAD *thread;
21
+ SPINLOCK spinlock;
22
+ int pipe_fds[2]; // Pipe for notifications
23
+ bool running;
24
+} LOG_FORWARDER;
25
+
26
+static void *log_forwarder_thread_func(void *arg);
27
+
28
+// --------------------------------------------------------------------------------------------------------------------
29
+// helper functions
30
+
31
+static inline LOG_FORWARDER_ENTRY *log_forwarder_find_entry_unsafe(LOG_FORWARDER *lf, int fd) {
32
+ for (LOG_FORWARDER_ENTRY *entry = lf->entries; entry; entry = entry->next) {
33
+ if (entry->fd == fd)
34
+ return entry;
35
+ }
36
+
37
+ return NULL;
38
+}
39
+
40
+static inline void log_forwarder_del_entry_unsafe(LOG_FORWARDER *lf, LOG_FORWARDER_ENTRY *entry) {
41
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(lf->entries, entry, prev, next);
42
+ buffer_free(entry->wb);
43
+ freez(entry->cmd);
44
+ close(entry->fd);
45
+ freez(entry);
46
+}
47
+
48
+static inline void log_forwarder_wake_up_worker(LOG_FORWARDER *lf) {
49
+ char ch = 0;
50
+ ssize_t bytes_written = write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
51
+ if (bytes_written != 1)
52
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to write to notification pipe");
53
+}
54
+
55
+// --------------------------------------------------------------------------------------------------------------------
56
+// starting / stopping
57
+
58
+LOG_FORWARDER *log_forwarder_start(void) {
59
+ LOG_FORWARDER *lf = callocz(1, sizeof(LOG_FORWARDER));
60
+
61
+ spinlock_init(&lf->spinlock);
62
+ if (pipe(lf->pipe_fds) != 0) {
63
+ freez(lf);
64
+ return NULL;
65
+ }
66
+
67
+ // make sure read() will not block on this pipe
68
+ sock_setnonblock(lf->pipe_fds[PIPE_READ]);
69
+
70
+ lf->running = true;
71
+ lf->thread = nd_thread_create("log-fw", NETDATA_THREAD_OPTION_JOINABLE, log_forwarder_thread_func, lf);
72
+
73
+ return lf;
74
+}
75
+
76
+static inline void mark_all_entries_for_deletion_unsafe(LOG_FORWARDER *lf) {
77
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ;entry = entry->next)
78
+ entry->delete = true;
79
+}
80
+
81
+void log_forwarder_stop(LOG_FORWARDER *lf) {
82
+ if(!lf || !lf->running) return;
83
+
84
+ // Signal the thread to stop
85
+ spinlock_lock(&lf->spinlock);
86
+ lf->running = false;
87
+
88
+ // mark them all for deletion
89
+ mark_all_entries_for_deletion_unsafe(lf);
90
+
91
+ // Send a byte to the pipe to wake up the thread
92
+ char ch = 0;
93
+ write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
94
+ spinlock_unlock(&lf->spinlock);
95
+
96
+ // Wait for the thread to finish
97
+ close(lf->pipe_fds[PIPE_WRITE]); // force it to quit
98
+ nd_thread_join(lf->thread);
99
+ close(lf->pipe_fds[PIPE_READ]);
100
+
101
+ freez(lf);
102
+}
103
+
104
+// --------------------------------------------------------------------------------------------------------------------
105
+// managing entries
106
+
107
+void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd) {
108
+ if(!lf || !lf->running || fd < 0) return;
109
+
110
+ LOG_FORWARDER_ENTRY *entry = callocz(1, sizeof(LOG_FORWARDER_ENTRY));
111
+ entry->fd = fd;
112
+ entry->cmd = NULL;
113
+ entry->pid = 0;
114
+ entry->pfds_idx = 0;
115
+ entry->delete = false;
116
+ entry->wb = buffer_create(0, NULL);
117
+
118
+ spinlock_lock(&lf->spinlock);
119
+
120
+ // Append to the entries list
121
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(lf->entries, entry, prev, next);
122
+
123
+ // Send a byte to the pipe to wake up the thread
124
+ log_forwarder_wake_up_worker(lf);
125
+
126
+ spinlock_unlock(&lf->spinlock);
127
+}
128
+
129
+bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd) {
130
+ if(!lf || !lf->running || fd < 0) return false;
131
+
132
+ bool ret = false;
133
+
134
+ spinlock_lock(&lf->spinlock);
135
+
136
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
137
+ if(entry) {
138
+ entry->delete = true;
139
+
140
+ // Send a byte to the pipe to wake up the thread
141
+ log_forwarder_wake_up_worker(lf);
142
+
143
+ ret = true;
144
+ }
145
+
146
+ spinlock_unlock(&lf->spinlock);
147
+
148
+ return ret;
149
+}
150
+
151
+void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd) {
152
+ if(!lf || !lf->running || fd < 0 || !cmd || !*cmd) return;
153
+
154
+ spinlock_lock(&lf->spinlock);
155
+
156
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
157
+ if (entry) {
158
+ freez(entry->cmd);
159
+ entry->cmd = strdupz(cmd);
160
+ }
161
+
162
+ spinlock_unlock(&lf->spinlock);
163
+}
164
+
165
+void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid) {
166
+ if(!lf || !lf->running || fd < 0) return;
167
+
168
+ spinlock_lock(&lf->spinlock);
169
+
170
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
171
+ if (entry)
172
+ entry->pid = pid;
173
+
174
+ spinlock_unlock(&lf->spinlock);
175
+}
176
+
177
+// --------------------------------------------------------------------------------------------------------------------
178
+// log forwarder thread
179
+
180
+static inline void log_forwarder_log(LOG_FORWARDER *lf __maybe_unused, LOG_FORWARDER_ENTRY *entry, const char *msg) {
181
+ const char *s = msg;
182
+ while(*s && isspace((uint8_t)*s)) s++;
183
+ if(*s == '\0') return; // do not log empty lines
184
+
185
+ ND_LOG_STACK lgs[] = {
186
+ ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, entry->cmd ? entry->cmd : "unknown"),
187
+ ND_LOG_FIELD_I64(NDF_TID, entry->pid),
188
+ ND_LOG_FIELD_END(),
189
+ };
190
+ ND_LOG_STACK_PUSH(lgs);
191
+
192
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "STDERR: %s", msg);
193
+}
194
+
195
+// returns the number of entries active
196
+static inline size_t log_forwarder_remove_deleted_unsafe(LOG_FORWARDER *lf) {
197
+ size_t entries = 0;
198
+
199
+ LOG_FORWARDER_ENTRY *entry = lf->entries;
200
+ while(entry) {
201
+ LOG_FORWARDER_ENTRY *next = entry->next;
202
+
203
+ if(entry->delete) {
204
+ if (buffer_strlen(entry->wb))
205
+ // there is something not logged in it - log it
206
+ log_forwarder_log(lf, entry, buffer_tostring(entry->wb));
207
+
208
+ log_forwarder_del_entry_unsafe(lf, entry);
209
+ }
210
+ else
211
+ entries++;
212
+
213
+ entry = next;
214
+ }
215
+
216
+ return entries;
217
+}
218
+
219
+static void *log_forwarder_thread_func(void *arg) {
220
+ LOG_FORWARDER *lf = (LOG_FORWARDER *)arg;
221
+
222
+ while (1) {
223
+ spinlock_lock(&lf->spinlock);
224
+ if (!lf->running) {
225
+ mark_all_entries_for_deletion_unsafe(lf);
226
+ log_forwarder_remove_deleted_unsafe(lf);
227
+ spinlock_unlock(&lf->spinlock);
228
+ break;
229
+ }
230
+
231
+ // Count the number of fds
232
+ size_t nfds = 1 + log_forwarder_remove_deleted_unsafe(lf);
233
+
234
+ struct pollfd pfds[nfds];
235
+
236
+ // First, the notification pipe
237
+ pfds[0].fd = lf->pipe_fds[PIPE_READ];
238
+ pfds[0].events = POLLIN;
239
+
240
+ int idx = 1;
241
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next, idx++) {
242
+ pfds[idx].fd = entry->fd;
243
+ pfds[idx].events = POLLIN;
244
+ entry->pfds_idx = idx;
245
+ }
246
+
247
+ spinlock_unlock(&lf->spinlock);
248
+
249
+ int timeout = 200; // 200ms
250
+ int ret = poll(pfds, nfds, timeout);
251
+
252
+ if (ret > 0) {
253
+ // Check the notification pipe
254
+ if (pfds[0].revents & POLLIN) {
255
+ // Read and discard the data
256
+ char buf[256];
257
+ ssize_t bytes_read = read(lf->pipe_fds[PIPE_READ], buf, sizeof(buf));
258
+ // Ignore the data; proceed regardless of the result
259
+ if (bytes_read == -1) {
260
+ if (errno != EAGAIN && errno != EWOULDBLOCK) {
261
+ // Handle read error if necessary
262
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to read from notification pipe");
263
+ return NULL;
264
+ }
265
+ }
266
+ }
267
+
268
+ // Now check the other fds
269
+ spinlock_lock(&lf->spinlock);
270
+
271
+ size_t to_remove = 0;
272
+
273
+ // read or mark them for deletion
274
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next) {
275
+ if (entry->pfds_idx < 1 || entry->pfds_idx >= nfds || !(pfds[entry->pfds_idx].revents & POLLIN))
276
+ continue;
277
+
278
+ BUFFER *wb = entry->wb;
279
+ buffer_need_bytes(wb, 1024);
280
+
281
+ ssize_t bytes_read = read(entry->fd, &wb->buffer[wb->len], wb->size - wb->len - 1);
282
+ if(bytes_read > 0)
283
+ wb->len += bytes_read;
284
+ else if(bytes_read == 0 || (bytes_read == -1 && errno != EINTR && errno != EAGAIN)) {
285
+ // EOF or error
286
+ entry->delete = true;
287
+ to_remove++;
288
+ }
289
+
290
+ // log as many lines are they have been received
291
+ char *start = (char *)buffer_tostring(wb);
292
+ char *newline = strchr(start, '\n');
293
+ while(newline) {
294
+ *newline = '\0';
295
+ log_forwarder_log(lf, entry, start);
296
+
297
+ start = ++newline;
298
+ newline = strchr(newline, '\n');
299
+ }
300
+
301
+ if(start != wb->buffer) {
302
+ wb->len = strlen(start);
303
+ if (wb->len)
304
+ memmove(wb->buffer, start, wb->len);
305
+ }
306
+
307
+ entry->pfds_idx = 0;
308
+ }
309
+
310
+ spinlock_unlock(&lf->spinlock);
311
+ }
312
+ else if (ret == 0) {
313
+ // Timeout, nothing to do
314
+ continue;
315
+
316
+ }
317
+ else
318
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Log forwarder: poll() error");
319
+ }
320
+
321
+ return NULL;
322
+}
src/libnetdata/spawn_server/log-forwarder.h
new
+17
@@ -0,0 +1,17 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_LOG_FORWARDER_H
4
+#define NETDATA_LOG_FORWARDER_H
5
+
6
+#include "../libnetdata.h"
7
+
8
+typedef struct LOG_FORWARDER LOG_FORWARDER;
9
+
10
+LOG_FORWARDER *log_forwarder_start(void); // done once, at spawn_server_create()
11
+void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd); // to add a new fd
12
+void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd); // set the syslog identifier
13
+void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid); // set the pid of the child process
14
+bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd); // to remove an fd
15
+void log_forwarder_stop(LOG_FORWARDER *lf); // done once, at spawn_server_destroy()
16
+
17
+#endif //NETDATA_LOG_FORWARDER_H
src/libnetdata/spawn_server/spawn_server_internals.h
+3
@@ -6,6 +6,7 @@
6
#include "../libnetdata.h"
7
#include "spawn_server.h"
8
#include "spawn_library.h"
9
+#include "log-forwarder.h"
10
11
#if defined(OS_WINDOWS)
12
#define SPAWN_SERVER_VERSION_WINDOWS 1
@@ -62,6 +63,7 @@ struct spawn_server {
63
#endif
64
65
#if defined(SPAWN_SERVER_VERSION_WINDOWS)
66
+ LOG_FORWARDER *log_forwarder;
67
#endif
68
};
69
@@ -70,6 +72,7 @@ struct spawn_instance {
72
int sock;
73
int write_fd;
74
int read_fd;
75
+ int stderr_fd;
76
pid_t child_pid;
77
78
#if defined(SPAWN_SERVER_VERSION_UV)
src/libnetdata/spawn_server/spawn_server_nofork.c
+6
-5
@@ -59,6 +59,7 @@ static void spawn_server_run_child(SPAWN_SERVER *server, SPAWN_REQUEST *rq) {
59
60
// close all open file descriptors of the parent, but keep ours
61
os_close_all_non_std_open_fds_except(rq->fds, 4, 0);
62
+ nd_log_reopen_log_files_for_spawn_server();
63
64
// set the process name
65
os_setproctitle("spawn-child", server->argc, server->argv);
@@ -363,7 +364,6 @@ static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN
364
else if (pid == 0) {
365
// the child
366
366
- gettid_uncached(); // make sure the logger logs valid pids
367
spawn_server_run_child(server, rq);
368
exit(63);
369
}
@@ -984,22 +984,24 @@ static bool spawn_server_create_listening_socket(SPAWN_SERVER *server) {
984
}
985
986
static void replace_stdio_with_dev_null() {
987
+ // we cannot log in this function - the logger is not yet initialized after fork()
988
+
989
int dev_null_fd = open("/dev/null", O_RDWR);
990
if (dev_null_fd == -1) {
989
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));
991
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));
992
return;
993
}
994
995
// Redirect stdin (fd 0)
996
if (dup2(dev_null_fd, STDIN_FILENO) == -1) {
995
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));
997
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));
998
close(dev_null_fd);
999
return;
1000
}
1001
1002
// Redirect stdout (fd 1)
1003
if (dup2(dev_null_fd, STDOUT_FILENO) == -1) {
1002
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));
1004
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));
1005
close(dev_null_fd);
1006
return;
1007
}
@@ -1070,7 +1072,6 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name
1072
pid_t pid = fork();
1073
if (pid == 0) {
1074
// the child - the spawn server
1073
- gettid_uncached(); // make sure the logger logs valid pids
1075
1076
{
1077
char buf[15];
src/libnetdata/spawn_server/spawn_server_windows.c
+50
-13
@@ -40,11 +40,18 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, c
40
server->name = strdupz(name);
41
else
42
server->name = strdupz("unnamed");
43
+
44
+ server->log_forwarder = log_forwarder_start();
45
+
46
return server;
47
}
48
49
void spawn_server_destroy(SPAWN_SERVER *server) {
50
if (server) {
51
+ if (server->log_forwarder) {
52
+ log_forwarder_stop(server->log_forwarder);
53
+ server->log_forwarder = NULL;
54
+ }
55
freez((void *)server->name);
56
freez(server);
57
}
@@ -136,13 +143,13 @@ int set_fd_blocking(int fd) {
143
// }
144
//}
145
139
-SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {
146
+SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd __maybe_unused, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {
147
static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
148
149
if (type != SPAWN_INSTANCE_TYPE_EXEC)
150
return NULL;
151
145
- int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 };
152
+ int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 }, pipe_stderr[2] = { -1, -1 };
153
154
errno_clear();
155
@@ -166,12 +173,21 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
173
goto cleanup;
174
}
175
176
+ if (pipe(pipe_stderr) == -1) {
177
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
178
+ "SPAWN PARENT: Cannot create stderr pipe() for request No %zu, command: %s",
179
+ instance->request_id, command);
180
+ goto cleanup;
181
+ }
182
+
183
// Ensure pipes are in blocking mode
184
if (set_fd_blocking(pipe_stdin[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdin[PIPE_WRITE]) == -1 ||
171
- set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1) {
185
+ set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1 ||
186
+ set_fd_blocking(pipe_stderr[PIPE_READ]) == -1 || set_fd_blocking(pipe_stderr[PIPE_WRITE]) == -1) {
187
nd_log(NDLS_COLLECTORS, NDLP_ERR,
188
"SPAWN PARENT: Failed to set blocking I/O on pipes for request No %zu, command: %s",
189
instance->request_id, command);
190
+ goto cleanup;
191
}
192
193
// do not run multiple times this section
@@ -181,9 +197,9 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
197
// Convert POSIX file descriptors to Windows handles
198
HANDLE stdin_read_handle = (HANDLE)_get_osfhandle(pipe_stdin[PIPE_READ]);
199
HANDLE stdout_write_handle = (HANDLE)_get_osfhandle(pipe_stdout[PIPE_WRITE]);
184
- HANDLE stderr_handle = (HANDLE)_get_osfhandle(stderr_fd);
200
+ HANDLE stderr_write_handle = (HANDLE)_get_osfhandle(pipe_stderr[PIPE_WRITE]);
201
186
- if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_handle == INVALID_HANDLE_VALUE) {
202
+ if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_write_handle == INVALID_HANDLE_VALUE) {
203
spinlock_unlock(&spinlock);
204
nd_log(NDLS_COLLECTORS, NDLP_ERR,
205
"SPAWN PARENT: Invalid handle value(s) for request No %zu, command: %s",
@@ -194,7 +210,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
210
// Set handle inheritance
211
if (!SetHandleInformation(stdin_read_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||
212
!SetHandleInformation(stdout_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||
197
- !SetHandleInformation(stderr_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
213
+ !SetHandleInformation(stderr_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
214
spinlock_unlock(&spinlock);
215
nd_log(NDLS_COLLECTORS, NDLP_ERR,
216
"SPAWN PARENT: Cannot set handle(s) inheritance for request No %zu, command: %s",
@@ -210,18 +226,18 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
226
si.dwFlags = STARTF_USESTDHANDLES;
227
si.hStdInput = stdin_read_handle;
228
si.hStdOutput = stdout_write_handle;
213
- si.hStdError = stderr_handle;
229
+ si.hStdError = stderr_write_handle;
230
231
// Retrieve the current environment block
232
char* env_block = GetEnvironmentStrings();
233
// print_environment_block(env_block);
234
219
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
235
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
236
"SPAWN PARENT: Running request No %zu, command: '%s'",
237
instance->request_id, command);
238
223
- int fds[3] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], stderr_fd };
224
- os_close_all_non_std_open_fds_except(fds, 3, CLOSE_RANGE_CLOEXEC);
239
+ int fds_to_keep_open[] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], pipe_stderr[PIPE_WRITE] };
240
+ os_close_all_non_std_open_fds_except(fds_to_keep_open, 3, CLOSE_RANGE_CLOEXEC);
241
242
// Spawn the process
243
errno_clear();
@@ -247,6 +263,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
263
// Close unused pipe ends
264
close(pipe_stdin[PIPE_READ]); pipe_stdin[PIPE_READ] = -1;
265
close(pipe_stdout[PIPE_WRITE]); pipe_stdout[PIPE_WRITE] = -1;
266
+ close(pipe_stderr[PIPE_WRITE]); pipe_stderr[PIPE_WRITE] = -1;
267
268
// Store process information in instance
269
instance->dwProcessId = pi.dwProcessId;
@@ -256,9 +273,15 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
273
// Convert handles to POSIX file descriptors
274
instance->write_fd = pipe_stdin[PIPE_WRITE];
275
instance->read_fd = pipe_stdout[PIPE_READ];
276
+ instance->stderr_fd = pipe_stderr[PIPE_READ];
277
+
278
+ // Add stderr_fd to the log forwarder
279
+ log_forwarder_add_fd(server->log_forwarder, instance->stderr_fd);
280
+ log_forwarder_annotate_fd_name(server->log_forwarder, instance->stderr_fd, command);
281
+ log_forwarder_annotate_fd_pid(server->log_forwarder, instance->stderr_fd, spawn_server_instance_pid(instance));
282
283
errno_clear();
261
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
284
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
285
"SPAWN PARENT: created process for request No %zu, pid %d (winpid %d), command: %s",
286
instance->request_id, (int)instance->child_pid, (int)pi.dwProcessId, command);
287
@@ -269,6 +292,8 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
292
if (pipe_stdin[PIPE_WRITE] >= 0) close(pipe_stdin[PIPE_WRITE]);
293
if (pipe_stdout[PIPE_READ] >= 0) close(pipe_stdout[PIPE_READ]);
294
if (pipe_stdout[PIPE_WRITE] >= 0) close(pipe_stdout[PIPE_WRITE]);
295
+ if (pipe_stderr[PIPE_READ] >= 0) close(pipe_stderr[PIPE_READ]);
296
+ if (pipe_stderr[PIPE_WRITE] >= 0) close(pipe_stderr[PIPE_WRITE]);
297
freez(instance);
298
return NULL;
299
}
@@ -314,7 +339,7 @@ static void TerminateChildProcesses(SPAWN_INSTANCE *si) {
339
if (pe.th32ParentProcessID == si->dwProcessId) {
340
HANDLE hChildProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pe.th32ProcessID);
341
if (hChildProcess) {
317
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
342
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING,
343
"SPAWN PARENT: killing subprocess %u of request No %zu, pid %d (winpid %u)",
344
pe.th32ProcessID, si->request_id, (int)si->child_pid, si->dwProcessId);
345
@@ -378,6 +403,12 @@ int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
403
// to have them, to avoid abnormal shutdown of the plugins
404
if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; }
405
if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; }
406
+ if(si->stderr_fd != -1) {
407
+ if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd))
408
+ close(si->stderr_fd);
409
+
410
+ si->stderr_fd = -1;
411
+ }
412
413
errno_clear();
414
if(TerminateProcess(si->process_handle, STATUS_CONTROL_C_EXIT) == 0)
@@ -394,6 +425,12 @@ int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
425
int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) {
426
if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; }
427
if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; }
428
+ if(si->stderr_fd != -1) {
429
+ if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd))
430
+ close(si->stderr_fd);
431
+
432
+ si->stderr_fd = -1;
433
+ }
434
435
// wait for the process to end
436
WaitForSingleObject(si->process_handle, INFINITE);
@@ -404,7 +441,7 @@ int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
441
442
char *err = GetErrorString(exit_code);
443
407
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
444
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
445
"SPAWN PARENT: child of request No %zu, pid %d (winpid %u), exited with code %u (0x%x): %s",
446
si->request_id, (int)si->child_pid, si->dwProcessId,
447
(unsigned)exit_code, (unsigned)exit_code, err ? err : "(no reason text)");
src/libnetdata/string/utf8.c
new
+192
@@ -0,0 +1,192 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "../libnetdata.h"
4
+
5
+#if defined(OS_WINDOWS)
6
+#include <windows.h>
7
+
8
+/*
9
+ * Convert any CodePage to UTF16
10
+ * Goals:
11
+ * 1. Destination is always NULL terminated
12
+ * 2. If the destination buffer is not enough, return as much as possible data (truncate)
13
+ * 3. Always return the number of wide characters written, including the null terminator
14
+ */
15
+
16
+size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len) {
17
+ if(!src || src_len == 0) {
18
+ // invalid input
19
+ if(dst && dst_size)
20
+ *dst = L'\0';
21
+ return 0;
22
+ }
23
+
24
+ if(!dst || !dst_size) {
25
+ // the caller wants to know the buffer to allocate for the conversion
26
+ int required = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0);
27
+ if(required <= 0) return 0; // error in the conversion
28
+
29
+ // Add 1 for null terminator only if src_len is not -1
30
+ // so that the caller can call us again to get the entire string (not truncated)
31
+ return (size_t)required + ((src_len != -1) ? 1 : 0);
32
+ }
33
+
34
+ // do the conversion directly to the destination buffer
35
+ int rc = MultiByteToWideChar(CodePage, 0, src, src_len, dst, (int)dst_size);
36
+ if(rc <= 0) {
37
+ // conversion failed, let's see why...
38
+ DWORD status = GetLastError();
39
+ if(status == ERROR_INSUFFICIENT_BUFFER) {
40
+ // it cannot fit entirely, let's allocate a new buffer to convert it
41
+ // and then truncate it to the destination buffer
42
+
43
+ // clear errno and LastError to clear the error of the
44
+ // MultiByteToWideChar() that failed
45
+ errno_clear();
46
+
47
+ // get the required size
48
+ int required_size = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0);
49
+
50
+ // mallocz() never fails (exits the program on NULL)
51
+ wchar_t *tmp = mallocz(required_size * sizeof(wchar_t));
52
+
53
+ // convert it, now it should fit
54
+ rc = MultiByteToWideChar(CodePage, 0, src, src_len, tmp, required_size);
55
+ if (rc <= 0) {
56
+ // it failed!
57
+ *dst = L'\0';
58
+ freez(tmp);
59
+ return 0;
60
+ }
61
+
62
+ size_t len = rc;
63
+
64
+ // copy as much as we can
65
+ memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(wchar_t));
66
+
67
+ // null terminate it
68
+ dst[MIN(len, (dst_size - 1))] = L'\0';
69
+
70
+ // free the temporary buffer
71
+ freez(tmp);
72
+
73
+ // return the actual bytes written
74
+ return MIN(len, dst_size);
75
+ }
76
+
77
+ // empty the destination
78
+ *dst = L'\0';
79
+ return 0;
80
+ }
81
+
82
+ size_t len = rc;
83
+
84
+ if(len >= dst_size) {
85
+ // truncate it to fit the null
86
+ dst[dst_size - 1] = L'\0';
87
+ return dst_size;
88
+ }
89
+
90
+ if(dst[len - 1] != L'\0') {
91
+ // the result is not null terminated
92
+ // append the null
93
+ dst[len] = L'\0';
94
+ return len + 1;
95
+ }
96
+
97
+ // the result is already null terminated
98
+ return len;
99
+}
100
+
101
+/*
102
+ * Convert UTF16 (wide-character string) to UTF8
103
+ * Goals:
104
+ * 1. Destination is always NULL terminated
105
+ * 2. If the destination buffer is not enough, return as much as possible data (truncate)
106
+ * 3. Always return the number of bytes written, including the null terminator
107
+ */
108
+
109
+size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len) {
110
+ if (!src || src_len == 0) {
111
+ // invalid input
112
+ if(dst && dst_size)
113
+ *dst = L'\0';
114
+ return 0;
115
+ }
116
+
117
+ if (!dst || dst_size == 0) {
118
+ // The caller wants to know the buffer size required for the conversion
119
+ int required = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL);
120
+ if (required <= 0) return 0; // error in the conversion
121
+
122
+ // Add 1 for null terminator only if src_len is not -1
123
+ return (size_t)required + ((src_len != -1) ? 1 : 0);
124
+ }
125
+
126
+ // Perform the conversion directly into the destination buffer
127
+ int rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, dst, (int)dst_size, NULL, NULL);
128
+ if (rc <= 0) {
129
+ // Conversion failed, let's see why...
130
+ DWORD status = GetLastError();
131
+ if (status == ERROR_INSUFFICIENT_BUFFER) {
132
+ // It cannot fit entirely, let's allocate a new buffer to convert it
133
+ // and then truncate it to the destination buffer
134
+
135
+ // Clear errno and LastError to clear the error of the
136
+ // WideCharToMultiByte() that failed
137
+ errno_clear();
138
+
139
+ // Get the required size
140
+ int required_size = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL);
141
+
142
+ // mallocz() never fails (exits the program on NULL)
143
+ char *tmp = mallocz(required_size * sizeof(char));
144
+
145
+ // Convert it, now it should fit
146
+ rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, tmp, required_size, NULL, NULL);
147
+ if (rc <= 0) {
148
+ // Conversion failed
149
+ *dst = '\0';
150
+ freez(tmp);
151
+ return 0;
152
+ }
153
+
154
+ size_t len = rc;
155
+
156
+ // Copy as much as we can
157
+ memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(char));
158
+
159
+ // Null-terminate it
160
+ dst[MIN(len, (dst_size - 1))] = '\0';
161
+
162
+ // Free the temporary buffer
163
+ freez(tmp);
164
+
165
+ // Return the actual bytes written
166
+ return MIN(len, dst_size);
167
+ }
168
+
169
+ // Empty the destination
170
+ *dst = '\0';
171
+ return 0;
172
+ }
173
+
174
+ size_t len = rc;
175
+
176
+ if (len >= dst_size) {
177
+ // Truncate it to fit the null terminator
178
+ dst[dst_size - 1] = '\0';
179
+ return dst_size;
180
+ }
181
+
182
+ if (dst[len - 1] != '\0') {
183
+ // The result is not null-terminated
184
+ // Append the null terminator
185
+ dst[len] = '\0';
186
+ return len + 1;
187
+ }
188
+
189
+ // The result is already null-terminated
190
+ return len;
191
+}
192
+#endif
src/libnetdata/string/utf8.h
+21
@@ -6,4 +6,25 @@
6
#define IS_UTF8_BYTE(x) ((uint8_t)(x) & (uint8_t)0x80)
7
#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x) && ((uint8_t)(x) & (uint8_t)0x40))
8
9
+#ifndef _countof
10
+#define _countof(x) (sizeof(x) / sizeof(*(x)))
11
+#endif
12
+
13
+#if defined(OS_WINDOWS)
14
+
15
+// return an always null terminated wide string, truncate to given size if destination is not big enough,
16
+// src_len can be -1 use all of it.
17
+// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
18
+size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len);
19
+
20
+// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it.
21
+// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
22
+#define utf8_to_utf16(utf16, utf16_count, src, src_len) any_to_utf16(CP_UTF8, utf16, utf16_count, src, src_len)
23
+
24
+// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it.
25
+// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
26
+size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len);
27
+
28
+#endif
29
+
30
#endif /* NETDATA_STRING_UTF8_H */
src/libnetdata/template-enum.h
+11
@@ -41,6 +41,7 @@
41
42
#define BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(type) \
43
type type ## _2id_one(const char *str); \
44
+ const char *type##_2str_one(type id); \
45
const char *type##_2json(BUFFER *wb, const char *key, type id);
46
47
#define BITMAP_STR_DEFINE_FUNCTIONS(type, def, def_str) \
@@ -57,6 +58,16 @@
58
return def; \
59
} \
60
\
61
+ const char *type##_2str_one(type id) \
62
+ { \
63
+ for (size_t i = 0; type ## _names[i].name; i++) { \
64
+ if (id == type ## _names[i].id) \
65
+ return type ## _names[i].name; \
66
+ } \
67
+ \
68
+ return def_str; \
69
+ } \
70
+ \
71
const char *type##_2json(BUFFER *wb, const char *key, type id) \
72
{ \
73
buffer_json_member_add_array(wb, key); \