Properly integrate dlib into our build system. (#20086)
* Properly integrate dlib into our build system. - Use the current latest version of dlib (v19.24.8). This fixes a number of warnings in the dlib code that are spilling over into our build output. This also is required for the other changes, as the version of dlib we were using was still expecting CMake versions earlier than 3.0. - Use dlib's CMake build infrastructure instead of their crazy shared source thing. This gives us much better integration into our build system, enables us to suppress warnings on a per-source-file level, and also means that warnings will be much better localized. In theory it should also marginally improve build times, though this likely won’t be visible for any native builds. - Use CMake's FetchContent for dlib instead of a git submodule. This gives us smaller source tarballs and better decouples our sources from our dependencies, as well as making it marginally easier to work wtih our git repository due to having one less submodule. * Disable all of dlib’s external dependencies. We weren’t using any of them anyway, and this both speeds up the build and resolves errors on some systems. * Allow usage of local dlib sources for builds.
Austin S. Hemmelgarn committed
Apr 9, 2025 at 12:50 UTC
e1ab60c2cbc81219da780bc0401c29fca37cc2d1
6 files changed
+73
-13
.gitmodules
-5
@@ -1,11 +1,6 @@
1
[submodule "aclk/aclk-schemas"]
2
path = src/aclk/aclk-schemas
3
url = https://github.com/netdata/aclk-schemas.git
4
-[submodule "ml/kmeans/dlib"]
5
- path = src/ml/dlib
6
- url = https://github.com/davisking/dlib.git
7
- shallow = true
8
- ignore = dirty
4
[submodule "src/web/server/h2o/libh2o"]
5
path = src/web/server/h2o/libh2o
6
url = https://github.com/h2o/h2o.git
CMakeLists.txt
+14
-5
@@ -164,6 +164,11 @@ mark_as_advanced(DEFAULT_FEATURE_STATE)
164
165
# High-level features
166
option(ENABLE_ML "Enable machine learning features" ${DEFAULT_FEATURE_STATE})
167
+
168
+if(ENABLE_ML)
169
+ set(NETDATA_DLIB_SOURCE_DIR "" CACHE PATH "Path to local dlib sources for building ML code")
170
+endif()
171
+
172
option(ENABLE_DBENGINE "Enable dbengine metrics storage" True)
173
option(ENABLE_DASHBOARD "Enable local dashboard" True)
174
mark_as_advanced(ENABLE_DASHBOARD)
@@ -394,6 +399,7 @@ endif()
399
include(NetdataJSONC)
400
include(NetdataYAML)
401
include(NetdataBacktrace)
402
+include(NetdataDlib)
403
404
if(ENABLE_LEGACY_EBPF_PROGRAMS)
405
include(NetdataEBPFLegacy)
@@ -418,6 +424,10 @@ if(ENABLE_SENTRY)
424
netdata_bundle_sentry()
425
endif()
426
427
+if(ENABLE_ML)
428
+ netdata_bundle_dlib()
429
+endif()
430
+
431
#
432
# check include files
433
#
@@ -1489,7 +1499,6 @@ if(ENABLE_ML)
1499
set(ML_FILES
1500
src/ml/ad_charts.h
1501
src/ml/ad_charts.cc
1492
- src/ml/dlib/dlib/all/source.cpp
1502
src/ml/ml.cc
1503
src/ml/ml_calculated_number.h
1504
src/ml/ml_host.h
@@ -2977,16 +2986,16 @@ add_executable(netdata
2986
"$<$<BOOL:${OS_WINDOWS}>:${NETDATA_RES_FILES}>"
2987
)
2988
2989
+if(ENABLE_ML)
2990
+ netdata_add_dlib_to_target(netdata)
2991
+endif()
2992
+
2993
if(OS_WINDOWS)
2994
add_executable(NetdataClaim ${CLAIM_WINDOWS_FILES} ${NETDATA_CLAIM_RES_FILES})
2995
target_link_libraries(NetdataClaim shell32 gdi32 msftedit)
2996
target_compile_options(NetdataClaim PUBLIC -mwindows)
2997
endif()
2998
2986
-target_compile_definitions(netdata PRIVATE
2987
- "$<$<BOOL:${ENABLE_ML}>:DLIB_NO_GUI_SUPPORT>"
2988
-)
2989
-
2999
target_compile_options(netdata PRIVATE
3000
"$<$<BOOL:${ENABLE_EXPORTER_MONGODB}>:${MONGOC_CFLAGS_OTHER}>"
3001
"$<$<BOOL:${ENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE}>:${SNAPPY_CFLAGS_OTHER}>"
packaging/cmake/Modules/NetdataDlib.cmake
new
+57
@@ -0,0 +1,57 @@
1
+# SPDX-License-Identifier: GPL-3.0-or-later
2
+# Functions and macros for handling of dlib
3
+
4
+function(netdata_bundle_dlib)
5
+ include(FetchContent)
6
+ include(NetdataFetchContentExtra)
7
+
8
+ message(STATUS "Preparing vendored copy of dlib")
9
+
10
+ if(NETDATA_DLIB_SOURCE_PATH)
11
+ set(FETCHCONTENT_SOURCE_DIR_DLIB "${NETDATA_DLIB_SOURCE_PATH}")
12
+ message(STATUS "Using local dlib source: ${NETDATA_DLIB_SOURCE_DIR}")
13
+ endif()
14
+
15
+ set(FETCHCONTENT_FULLY_DISCONNECTED Off)
16
+ set(repo https://github.com/davisking/dlib.git)
17
+ set(tag 636c0bcd1e4f428d167699891bc12b404d2d1b41) # v19.24.8
18
+
19
+ set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
20
+ set(DLIB_NO_GUI_SUPPORT ON)
21
+ set(DLIB_JPEG_SUPPORT OFF)
22
+ set(DLIB_LINK_WITH_SQLITE3 OFF)
23
+ set(DLIB_USE_BLAS OFF)
24
+ set(DLIB_USE_LAPACK OFF)
25
+ set(DLIB_USE_CUDA OFF)
26
+ set(DLIB_PNG_SUPPORT OFF)
27
+ set(DLIB_GIF_SUPPORT OFF)
28
+ set(DLIB_WEBP_SUPPORT OFF)
29
+ set(DLIB_JXL_SUPPORT OFF)
30
+ set(DLIB_USE_MKL_FFT OFF)
31
+ set(DLIB_USE_FFMPEG OFF)
32
+
33
+ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28)
34
+ FetchContent_Declare(dlib
35
+ GIT_REPOSITORY ${repo}
36
+ GIT_TAG ${tag}
37
+ CMAKE_ARGS ${NETDATA_CMAKE_PROPAGATE_TOOLCHAIN_ARGS}
38
+ EXCLUDE_FROM_ALL
39
+ )
40
+ else()
41
+ FetchContent_Declare(dlib
42
+ GIT_REPOSITORY ${repo}
43
+ GIT_TAG ${tag}
44
+ CMAKE_ARGS ${NETDATA_CMAKE_PROPAGATE_TOOLCHAIN_ARGS}
45
+ )
46
+ endif()
47
+
48
+ FetchContent_MakeAvailable_NoInstall(dlib)
49
+
50
+ message(STATUS "Finished preparing vendored copy of JSON-C")
51
+endfunction()
52
+
53
+function(netdata_add_dlib_to_target _target)
54
+ get_target_property(NETDATA_DLIB_INCLUDE_DIRS dlib INTERFACE_INCLUDE_DIRECTORIES)
55
+ target_include_directories(${_target} PRIVATE ${NETDATA_DLIB_INCLUDE_DIRS})
56
+ target_link_libraries(${_target} PRIVATE dlib)
57
+endfunction()
src/ml/dlib
deleted
-1
@@ -1 +0,0 @@
1
-Subproject commit 021cbbb1c2ddec39d8dd4cb6abfbbafdf1cf4482
src/ml/ml_calculated_number.h
+1
-1
@@ -3,7 +3,7 @@
3
#ifndef NETDATA_ML_CALCULATED_NUMBER_H
4
#define NETDATA_ML_CALCULATED_NUMBER_H
5
6
-#include "dlib/dlib/matrix.h"
6
+#include <dlib/matrix.h>
7
8
// CentOS 7 shenanigans
9
#include <cmath>
src/ml/ml_kmeans.cc
+1
-1
@@ -2,7 +2,7 @@
2
3
#include "ml_kmeans.h"
4
#include "libnetdata/libnetdata.h"
5
-#include "dlib/dlib/clustering.h"
5
+#include <dlib/clustering.h>
6
7
void
8
ml_kmeans_init(ml_kmeans_t *kmeans)