@cryptotaxi247 / netdata-1 / commits / 7e3899076

Add profile.plugin (#13962)

* Add profile plugin * Rebase master * Fix typo. * Track number of backfilled points per second. * Rebase master

vkalintiris committed Jul 3, 2023 at 12:25 UTC 7e38990764b998a689e4454c891ca01a279a194a
7 files changed +291
CMakeLists.txt
+5
@@ -555,6 +555,10 @@ set(TIMEX_PLUGIN_FILES
555 collectors/timex.plugin/plugin_timex.c
556 )
557
558 +set(PROFILE_PLUGIN_FILES
559 + collectors/profile.plugin/plugin_profile.cc
560 + )
561 +
562 set(FREEIPMI_PLUGIN_FILES
563 collectors/freeipmi.plugin/freeipmi_plugin.c
564 )
@@ -1278,6 +1282,7 @@ ENDIF()
1282
1283 list(APPEND NETDATA_FILES ${ACLK_ALWAYS_BUILD})
1284 list(APPEND NETDATA_FILES ${TIMEX_PLUGIN_FILES})
1285 +list(APPEND NETDATA_FILES ${PROFILE_PLUGIN_FILES})
1286
1287 # -----------------------------------------------------------------------------
1288 # netdata
Makefile.am
+5
@@ -406,6 +406,10 @@ PROC_PLUGIN_FILES = \
406 collectors/proc.plugin/sys_class_infiniband.c \
407 $(NULL)
408
409 +PROFILE_PLUGIN_FILES = \
410 + collectors/profile.plugin/plugin_profile.cc \
411 + $(NULL)
412 +
413 TC_PLUGIN_FILES = \
414 collectors/tc.plugin/plugin_tc.c \
415 $(NULL)
@@ -1076,6 +1080,7 @@ NETDATA_FILES = \
1080 $(ACLK_FILES) \
1081 $(SPAWN_PLUGIN_FILES) \
1082 $(TIMEX_PLUGIN_FILES) \
1083 + $(PROFILE_PLUGIN_FILES) \
1084 $(NULL)
1085
1086 if FREEBSD
collectors/profile.plugin/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
collectors/profile.plugin/README.md new
+34
@@ -0,0 +1,34 @@
1 +# profile.plugin
2 +
3 +This plugin allows someone to backfill an agent with random data.
4 +
5 +A user can specify:
6 +
7 + - The number charts they want,
8 + - the number of dimensions per chart,
9 + - the desire update every collection frequency,
10 + - the number of seconds to backfill.
11 + - the number of collection threads.
12 +
13 +## Configuration
14 +
15 +Edit the `netdata.conf` configuration file using [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) from the [Netdata config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#the-netdata-config-directory), which is typically at `/etc/netdata`.
16 +
17 +Scroll down to the `[plugin:profile]` section to find the available options:
18 +
19 +```
20 +[plugin:profile]
21 + update every = 5
22 + number of charts = 200
23 + number of dimensions per chart = 5
24 + seconds to backfill = 86400
25 + number of threads = 16
26 +```
27 +
28 +The `number of threads` option will create the specified number of collection
29 +threads. The rest of the options apply to each thread individually, eg. the
30 +above configuration will create 3200 charts, 16000 dimensions in total, which will be
31 +backfilled for the duration of 1 day.
32 +
33 +Note that all but the 1st chart created in each thread will be marked as hidden
34 +in order to ease the load on the dashboard's UI.
collectors/profile.plugin/plugin_profile.cc new
+228
@@ -0,0 +1,228 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifdef __cplusplus
4 +extern "C" {
5 +#endif
6 +
7 +#include "daemon/common.h"
8 +
9 +#ifdef __cplusplus
10 +}
11 +#endif
12 +
13 +#include <random>
14 +#include <thread>
15 +#include <vector>
16 +
17 +#define PLUGIN_PROFILE_NAME "profile.plugin"
18 +
19 +#define CONFIG_SECTION_PROFILE "plugin:profile"
20 +
21 +class Generator {
22 +public:
23 + Generator(size_t N) : Offset(0) {
24 + std::random_device RandDev;
25 + std::mt19937 Gen(RandDev());
26 + std::uniform_int_distribution<int> D(-16, 16);
27 +
28 + V.reserve(N);
29 + for (size_t Idx = 0; Idx != N; Idx++)
30 + V.push_back(D(Gen));
31 + }
32 +
33 + double getRandValue() {
34 + return V[Offset++ % V.size()];
35 + }
36 +
37 +private:
38 + size_t Offset;
39 + std::vector<double> V;
40 +};
41 +
42 +class Profiler {
43 +public:
44 + Profiler(size_t ID, size_t NumCharts, size_t NumDimsPerChart, time_t SecondsToBackfill, int UpdateEvery) :
45 + ID(ID),
46 + NumCharts(NumCharts),
47 + NumDimsPerChart(NumDimsPerChart),
48 + SecondsToBackfill(SecondsToBackfill),
49 + UpdateEvery(UpdateEvery),
50 + Gen(1024 * 1024)
51 + {}
52 +
53 + void create() {
54 + char ChartId[1024];
55 + char DimId[1024];
56 +
57 + Charts.reserve(NumCharts);
58 + for (size_t I = 0; I != NumCharts; I++) {
59 + size_t CID = ID + Charts.size() + 1;
60 +
61 + snprintfz(ChartId, 1024 - 1, "chart_%zu", CID);
62 +
63 + RRDSET *RS = rrdset_create_localhost(
64 + "profile", // type
65 + ChartId, // id
66 + nullptr, // name,
67 + "profile_family", // family
68 + "profile_context", // context
69 + "profile_title", // title
70 + "profile_units", // units
71 + "profile_plugin", // plugin
72 + "profile_module", // module
73 + 12345678 + CID, // priority
74 + UpdateEvery, // update_every
75 + RRDSET_TYPE_LINE // chart_type
76 + );
77 + if (I != 0)
78 + rrdset_flag_set(RS, RRDSET_FLAG_HIDDEN);
79 + Charts.push_back(RS);
80 +
81 + Dimensions.reserve(NumDimsPerChart);
82 + for (size_t J = 0; J != NumDimsPerChart; J++) {
83 + snprintfz(DimId, 1024 - 1, "dim_%zu", J);
84 +
85 + RRDDIM *RD = rrddim_add(
86 + RS, // st
87 + DimId, // id
88 + nullptr, // name
89 + 1, // multiplier
90 + 1, // divisor
91 + RRD_ALGORITHM_ABSOLUTE // algorithm
92 + );
93 +
94 + Dimensions.push_back(RD);
95 + }
96 + }
97 + }
98 +
99 + void update(const struct timeval &Now) {
100 + for (RRDSET *RS: Charts) {
101 + for (RRDDIM *RD : Dimensions) {
102 + rrddim_timed_set_by_pointer(RS, RD, Now, Gen.getRandValue());
103 + }
104 +
105 + rrdset_timed_done(RS, Now, RS->counter_done != 0);
106 + }
107 + }
108 +
109 + void run() {
110 + #define WORKER_JOB_CREATE_CHARTS 0
111 + #define WORKER_JOB_UPDATE_CHARTS 1
112 + #define WORKER_JOB_METRIC_DURATION_TO_BACKFILL 2
113 + #define WORKER_JOB_METRIC_POINTS_BACKFILLED 3
114 +
115 + worker_register("PROFILER");
116 + worker_register_job_name(WORKER_JOB_CREATE_CHARTS, "create charts");
117 + worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "update charts");
118 + worker_register_job_custom_metric(WORKER_JOB_METRIC_DURATION_TO_BACKFILL, "duration to backfill", "seconds", WORKER_METRIC_ABSOLUTE);
119 + worker_register_job_custom_metric(WORKER_JOB_METRIC_POINTS_BACKFILLED, "points backfilled", "points", WORKER_METRIC_ABSOLUTE);
120 +
121 + heartbeat_t HB;
122 + heartbeat_init(&HB);
123 +
124 + worker_is_busy(WORKER_JOB_CREATE_CHARTS);
125 + create();
126 +
127 + struct timeval CollectionTV;
128 + now_realtime_timeval(&CollectionTV);
129 +
130 + if (SecondsToBackfill) {
131 + CollectionTV.tv_sec -= SecondsToBackfill;
132 + CollectionTV.tv_sec -= (CollectionTV.tv_sec % UpdateEvery);
133 +
134 + CollectionTV.tv_usec = 0;
135 + }
136 +
137 + size_t BackfilledPoints = 0;
138 + struct timeval NowTV, PrevTV;
139 + now_realtime_timeval(&NowTV);
140 + PrevTV = NowTV;
141 +
142 + while (service_running(SERVICE_COLLECTORS)) {
143 + worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
144 +
145 + update(CollectionTV);
146 + CollectionTV.tv_sec += UpdateEvery;
147 +
148 + now_realtime_timeval(&NowTV);
149 +
150 + ++BackfilledPoints;
151 + if (NowTV.tv_sec > PrevTV.tv_sec) {
152 + PrevTV = NowTV;
153 + worker_set_metric(WORKER_JOB_METRIC_POINTS_BACKFILLED, BackfilledPoints * NumCharts * NumDimsPerChart);
154 + BackfilledPoints = 0;
155 + }
156 +
157 + size_t RemainingSeconds = (CollectionTV.tv_sec >= NowTV.tv_sec) ? 0 : (NowTV.tv_sec - CollectionTV.tv_sec);
158 + worker_set_metric(WORKER_JOB_METRIC_DURATION_TO_BACKFILL, RemainingSeconds);
159 +
160 + if (CollectionTV.tv_sec >= NowTV.tv_sec) {
161 + worker_is_idle();
162 + heartbeat_next(&HB, UpdateEvery * USEC_PER_SEC);
163 + }
164 + }
165 + }
166 +
167 +private:
168 + size_t ID;
169 + size_t NumCharts;
170 + size_t NumDimsPerChart;
171 + size_t SecondsToBackfill;
172 + int UpdateEvery;
173 +
174 + Generator Gen;
175 + std::vector<RRDSET *> Charts;
176 + std::vector<RRDDIM *> Dimensions;
177 +};
178 +
179 +static void *subprofile_main(void* Arg) {
180 + Profiler *P = reinterpret_cast<Profiler *>(Arg);
181 + P->run();
182 + return nullptr;
183 +}
184 +
185 +static void profile_main_cleanup(void *ptr) {
186 + struct netdata_static_thread *static_thread = (struct netdata_static_thread *) ptr;
187 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
188 +
189 + info("cleaning up...");
190 +
191 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
192 +}
193 +
194 +extern "C" void *profile_main(void *ptr) {
195 + netdata_thread_cleanup_push(profile_main_cleanup, ptr);
196 +
197 + int UpdateEvery = (int) config_get_number(CONFIG_SECTION_PROFILE, "update every", 1);
198 + if (UpdateEvery < localhost->rrd_update_every)
199 + UpdateEvery = localhost->rrd_update_every;
200 +
201 + // pick low-default values, in case this plugin is ever enabled accidentaly.
202 + size_t NumThreads = config_get_number(CONFIG_SECTION_PROFILE, "number of threads", 2);
203 + size_t NumCharts = config_get_number(CONFIG_SECTION_PROFILE, "number of charts", 2);
204 + size_t NumDimsPerChart = config_get_number(CONFIG_SECTION_PROFILE, "number of dimensions per chart", 2);
205 + size_t SecondsToBackfill = config_get_number(CONFIG_SECTION_PROFILE, "seconds to backfill", 10 * 60);
206 +
207 + std::vector<Profiler> Profilers;
208 +
209 + for (size_t Idx = 0; Idx != NumThreads; Idx++) {
210 + Profiler P(1e8 + Idx * 1e6, NumCharts, NumDimsPerChart, SecondsToBackfill, UpdateEvery);
211 + Profilers.push_back(P);
212 + }
213 +
214 + std::vector<netdata_thread_t> Threads(NumThreads);
215 +
216 + for (size_t Idx = 0; Idx != NumThreads; Idx++) {
217 + char Tag[NETDATA_THREAD_TAG_MAX + 1];
218 +
219 + snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "PROFILER[%zu]", Idx);
220 + netdata_thread_create(&Threads[Idx], Tag, NETDATA_THREAD_OPTION_JOINABLE, subprofile_main, static_cast<void *>(&Profilers[Idx]));
221 + }
222 +
223 + for (size_t Idx = 0; Idx != NumThreads; Idx++)
224 + netdata_thread_join(Threads[Idx], nullptr);
225 +
226 + netdata_thread_cleanup_pop(1);
227 + return NULL;
228 +}
daemon/global_statistics.c
+1
@@ -3435,6 +3435,7 @@ static struct worker_utilization all_workers_utilization[] = {
3435 { .name = "RRDCONTEXT", .family = "workers contexts", .priority = 1000000 },
3436 { .name = "REPLICATION", .family = "workers replication sender", .priority = 1000000 },
3437 { .name = "SERVICE", .family = "workers service", .priority = 1000000 },
3438 + { .name = "PROFILER", .family = "workers profile", .priority = 1000000 },
3439
3440 // has to be terminated with a NULL
3441 { .name = NULL, .family = NULL }
daemon/static_threads.c
+10
@@ -13,6 +13,7 @@ void *pluginsd_main(void *ptr);
13 void *service_main(void *ptr);
14 void *statsd_main(void *ptr);
15 void *timex_main(void *ptr);
16 +void *profile_main(void *ptr);
17 void *replication_thread_main(void *ptr __maybe_unused);
18
19 extern bool global_statistics_enabled;
@@ -185,6 +186,15 @@ const struct netdata_static_thread static_threads_common[] = {
186 .init_routine = NULL,
187 .start_routine = replication_thread_main
188 },
189 + {
190 + .name = "P[PROFILE]",
191 + .config_section = CONFIG_SECTION_PLUGINS,
192 + .config_name = "profile",
193 + .enabled = 0,
194 + .thread = NULL,
195 + .init_routine = NULL,
196 + .start_routine = profile_main
197 + },
198
199 // terminator
200 {