@cryptotaxi247 / netdata-1 / commits / 56a1808d2

Exporting/send variables (#13221)

boxjan committed Jul 11, 2022 at 21:00 UTC 56a1808d2e72beb2cb08aea847d3e231d4eea823

16 files changed +145 -11

exporting/aws_kinesis/aws_kinesis.c

@@ -45,6 +45,7 @@ int init_aws_kinesis_instance(struct instance *instance)

45 instance->metric_formatting = format_dimension_stored_json_plaintext;

47 instance->end_chart_formatting = NULL;

48 + instance->variables_formatting = NULL;

49 instance->end_host_formatting = flush_host_labels;

50 instance->end_batch_formatting = NULL;

exporting/exporting.conf

@@ -45,6 +45,7 @@

45 # send names instead of ids = yes

46 # send charts matching = *

47 # send hosts matching = localhost *

48 + # send variables = no

50 # [kinesis:my_kinesis_instance]

51 # enabled = no

exporting/exporting_engine.h

+6 -1

@@ -26,7 +26,8 @@ typedef enum exporting_options {

26 EXPORTING_OPTION_SEND_AUTOMATIC_LABELS = (1 << 4),

27 EXPORTING_OPTION_USE_TLS = (1 << 5),

29 - EXPORTING_OPTION_SEND_NAMES = (1 << 16)

29 + EXPORTING_OPTION_SEND_NAMES = (1 << 16),

30 + EXPORTING_OPTION_SEND_VARIABLES = (1 << 17)

31 } EXPORTING_OPTIONS;

33 #define EXPORTING_OPTIONS_SOURCE_BITS \

@@ -45,6 +46,8 @@ extern const char *global_exporting_prefix;

46 (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \

47 label_source & RRDLABEL_SRC_AUTO))

49 +#define should_send_variables(instance) (instance->config.options & EXPORTING_OPTION_SEND_VARIABLES)

50 +

51 typedef enum exporting_connector_types {

52 EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type

53 EXPORTING_CONNECTOR_TYPE_GRAPHITE, // Send plain text to Graphite

@@ -219,6 +222,7 @@ struct instance {

222 int (*start_chart_formatting)(struct instance *instance, RRDSET *st);

223 int (*metric_formatting)(struct instance *instance, RRDDIM *rd);

224 int (*end_chart_formatting)(struct instance *instance, RRDSET *st);

225 + int (*variables_formatting)(struct instance *instance, RRDHOST *host);

226 int (*end_host_formatting)(struct instance *instance, RRDHOST *host);

227 int (*end_batch_formatting)(struct instance *instance);

228

@@ -280,6 +284,7 @@ void start_host_formatting(struct engine *engine, RRDHOST *host);

284 void start_chart_formatting(struct engine *engine, RRDSET *st);

285 void metric_formatting(struct engine *engine, RRDDIM *rd);

286 void end_chart_formatting(struct engine *engine, RRDSET *st);

287 +void variables_formatting(struct engine *engine, RRDHOST *host);

288 void end_host_formatting(struct engine *engine, RRDHOST *host);

289 void end_batch_formatting(struct engine *engine);

290 int flush_host_labels(struct instance *instance, RRDHOST *host);

exporting/graphite/graphite.c

@@ -37,6 +37,7 @@ int init_graphite_instance(struct instance *instance)

37 instance->metric_formatting = format_dimension_stored_graphite_plaintext;

39 instance->end_chart_formatting = NULL;

40 + instance->variables_formatting = NULL;

41 instance->end_host_formatting = flush_host_labels;

42 instance->end_batch_formatting = simple_connector_end_batch;

exporting/json/json.c

@@ -29,6 +29,7 @@ int init_json_instance(struct instance *instance)

29 instance->metric_formatting = format_dimension_stored_json_plaintext;

31 instance->end_chart_formatting = NULL;

32 + instance->variables_formatting = NULL;

33 instance->end_host_formatting = flush_host_labels;

34 instance->end_batch_formatting = simple_connector_end_batch;

@@ -87,6 +88,7 @@ int init_json_http_instance(struct instance *instance)

88 instance->metric_formatting = format_dimension_stored_json_plaintext;

90 instance->end_chart_formatting = NULL;

91 + instance->variables_formatting = NULL;

92 instance->end_host_formatting = flush_host_labels;

93 instance->end_batch_formatting = close_batch_json_http;

exporting/mongodb/mongodb.c

@@ -99,6 +99,7 @@ int init_mongodb_instance(struct instance *instance)

99 instance->metric_formatting = format_dimension_stored_json_plaintext;

100

101 instance->end_chart_formatting = NULL;

102 + instance->variables_formatting = NULL;

103 instance->end_host_formatting = flush_host_labels;

104 instance->end_batch_formatting = format_batch_mongodb;

105

exporting/opentsdb/opentsdb.c

@@ -38,6 +38,7 @@ int init_opentsdb_telnet_instance(struct instance *instance)

38 instance->metric_formatting = format_dimension_stored_opentsdb_telnet;

40 instance->end_chart_formatting = NULL;

41 + instance->variables_formatting = NULL;

42 instance->end_host_formatting = flush_host_labels;

43 instance->end_batch_formatting = simple_connector_end_batch;

@@ -94,6 +95,7 @@ int init_opentsdb_http_instance(struct instance *instance)

95 instance->metric_formatting = format_dimension_stored_opentsdb_http;

97 instance->end_chart_formatting = NULL;

98 + instance->variables_formatting = NULL;

99 instance->end_host_formatting = flush_host_labels;

100 instance->end_batch_formatting = close_batch_json_http;

101

exporting/process_data.c

+22 -1

@@ -258,6 +258,27 @@ void end_chart_formatting(struct engine *engine, RRDSET *st)

258 }

259 }

260

261 +/**

262 + * Format variables for every connector instance's buffer

263 + *

264 + * @param engine an engine data structure.

265 + * @param host a data collecting host.

266 + */

267 +void variables_formatting(struct engine *engine, RRDHOST *host)

268 +{

269 + for (struct instance *instance = engine->instance_root; instance; instance = instance->next) {

270 + if (instance->scheduled && !instance->skip_host && should_send_variables(instance)) {

271 + if (instance->variables_formatting && instance->variables_formatting(instance, host) != 0){

272 + error("EXPORTING: cannot format variables for %s", instance->config.name);

273 + disable_instance(instance);

274 + continue;

275 + }

276 + // sum all variables as one metrics

277 + instance->stats.buffered_metrics++;

278 + }

279 + }

280 +}

281 +

282 /**

283 * End host formatting for every connector instance's buffer

284 *

@@ -334,7 +355,7 @@ void prepare_buffers(struct engine *engine)

355 end_chart_formatting(engine, st);

356 rrdset_unlock(st);

357 }

337 -

358 + variables_formatting(engine, host);

359 end_host_formatting(engine, host);

360 rrdhost_unlock(host);

361 }

exporting/prometheus/remote_write/remote_write.c

+52 -9

@@ -97,6 +97,7 @@ int init_prometheus_remote_write_instance(struct instance *instance)

97 instance->start_chart_formatting = format_chart_prometheus_remote_write;

98 instance->metric_formatting = format_dimension_prometheus_remote_write;

99 instance->end_chart_formatting = NULL;

100 + instance->variables_formatting = format_variables_prometheus_remote_write;

101 instance->end_host_formatting = NULL;

102 instance->end_batch_formatting = format_batch_prometheus_remote_write;

103

@@ -134,14 +135,6 @@ int init_prometheus_remote_write_instance(struct instance *instance)

135 return 0;

136 }

137

137 -/**

138 - * Format host data for Prometheus Remote Write connector

139 - *

140 - * @param instance an instance data structure.

141 - * @param host a data collecting host.

142 - * @return Always returns 0.

143 - */

144 -

138 struct format_remote_write_label_callback {

139 struct instance *instance;

140 void *write_request;

@@ -161,6 +154,13 @@ static int format_remote_write_label_callback(const char *name, const char *valu

154 return 1;

155 }

156

157 +/**

158 + * Format host data for Prometheus Remote Write connector

159 + *

160 + * @param instance an instance data structure.

161 + * @param host a data collecting host.

162 + * @return Always returns 0.

163 + */

164 int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)

165 {

166 struct simple_connector_data *simple_connector_data =

@@ -177,7 +177,7 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host

177 add_host_info(

178 connector_specific_data->write_request,

179 "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);

180 -

180 +

181 if (unlikely(sending_labels_configured(instance))) {

182 struct format_remote_write_label_callback tmp = {

183 .write_request = connector_specific_data->write_request,

@@ -321,6 +321,49 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *

321 return 0;

322 }

323

324 +int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) {

325 + struct prometheus_remote_write_variables_callback_options *opts = data;

326 +

327 + if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) {

328 + RRDHOST *host = opts->host;

329 + struct instance *instance = opts->instance;

330 + struct simple_connector_data *simple_connector_data =

331 + (struct simple_connector_data *)instance->connector_specific_data;

332 + struct prometheus_remote_write_specific_data *connector_specific_data =

333 + (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data;

334 +

335 + char name[PROMETHEUS_LABELS_MAX + 1];

336 + char *suffix = "";

337 +

338 + prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX);

339 + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix);

340 +

341 + NETDATA_DOUBLE value = rrdvar2number(rv);

342 + add_variable(connector_specific_data->write_request, name,

343 + (host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS);

344 + }

345 +

346 + return 0;

347 +}

348 +

349 +/**

350 + * Format a variable for Prometheus Remote Write connector

351 + *

352 + * @param rv a variable.

353 + * @param instance an instance data structure.

354 + * @return Always returns 0.

355 + */

356 +int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host)

357 +{

358 + struct prometheus_remote_write_variables_callback_options opt = {

359 + .host = host,

360 + .instance = instance,

361 + .now = now_realtime_usec(),

362 + };

363 +

364 + return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt);

365 +}

366 +

367 /**

368 * Format a batch for Prometheus Remote Write connector

369 *

exporting/prometheus/remote_write/remote_write.h

@@ -11,12 +11,19 @@ struct prometheus_remote_write_specific_data {

11 void *write_request;

12 };

14 +struct prometheus_remote_write_variables_callback_options {

15 + RRDHOST *host;

16 + time_t now;

17 + struct instance *instance;

18 +};

19 +

20 int init_prometheus_remote_write_instance(struct instance *instance);

21 extern void clean_prometheus_remote_write(struct instance *instance);

23 int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host);

24 int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st);

25 int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd);

26 +int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host);

27 int format_batch_prometheus_remote_write(struct instance *instance);

29 void prometheus_remote_write_prepare_header(struct instance *instance);

exporting/prometheus/remote_write/remote_write_request.cc

+32

@@ -137,6 +137,38 @@ void add_metric(

137 sample->set_timestamp(timestamp);

138 }

139

140 +/**

141 + * Adds a metric to a write request

142 + *

143 + * @param write_request_p the write request

144 + * @param name the name of the metric

145 + * @param instance the name of the host, the metric belongs to

146 + * @param value the value of the metric

147 + * @param timestamp the timestamp for the metric in milliseconds

148 + */

149 +void add_variable(

150 + void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp)

151 +{

152 + WriteRequest *write_request = (WriteRequest *)write_request_p;

153 + TimeSeries *timeseries;

154 + Sample *sample;

155 + Label *label;

156 +

157 + timeseries = write_request->add_timeseries();

158 +

159 + label = timeseries->add_labels();

160 + label->set_name("__name__");

161 + label->set_value(name);

162 +

163 + label = timeseries->add_labels();

164 + label->set_name("instance");

165 + label->set_value(instance);

166 +

167 + sample = timeseries->add_samples();

168 + sample->set_value(value);

169 + sample->set_timestamp(timestamp);

170 +}

171 +

172 /**

173 * Gets the size of a write request

174 *

exporting/prometheus/remote_write/remote_write_request.h

@@ -20,6 +20,9 @@ void add_metric(

20 const char *name, const char *chart, const char *family, const char *dimension,

21 const char *instance, const double value, const int64_t timestamp);

23 +void add_variable(

24 + void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp);

25 +

26 size_t get_write_request_size(void *write_request_p);

28 int pack_and_clear_write_request(void *write_request_p, char *buffer, size_t *size);

exporting/pubsub/pubsub.c

@@ -23,6 +23,7 @@ int init_pubsub_instance(struct instance *instance)

23 instance->metric_formatting = format_dimension_stored_json_plaintext;

25 instance->end_chart_formatting = NULL;

26 + instance->variables_formatting = NULL;

27 instance->end_host_formatting = flush_host_labels;

28 instance->end_batch_formatting = NULL;

exporting/read_config.c

@@ -399,6 +399,11 @@ struct engine *read_exporting_config()

399 else

400 tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;

401

402 + if (exporter_get_boolean(instance_name, "send variables", CONFIG_BOOLEAN_YES))

403 + tmp_instance->config.options |= EXPORTING_OPTION_SEND_VARIABLES;

404 + else

405 + tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_VARIABLES;

406 +

407 if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {

408 struct prometheus_remote_write_specific_config *connector_specific_config =

409 callocz(1, sizeof(struct prometheus_remote_write_specific_config));

exporting/tests/exporting_doubles.c

@@ -156,6 +156,14 @@ int __mock_end_chart_formatting(struct instance *instance, RRDSET *st)

156 return mock_type(int);

157 }

158

159 +int __mock_variables_formatting(struct instance *instance, RRDHOST *host)

160 +{

161 + function_called();

162 + check_expected_ptr(instance);

163 + check_expected_ptr(host);

164 + return mock_type(int);

165 +}

166 +

167 int __mock_end_host_formatting(struct instance *instance, RRDHOST *host)

168 {

169 function_called();

exporting/tests/test_exporting_engine.h

@@ -120,6 +120,7 @@ int __mock_start_host_formatting(struct instance *instance, RRDHOST *host);

120 int __mock_start_chart_formatting(struct instance *instance, RRDSET *st);

121 int __mock_metric_formatting(struct instance *instance, RRDDIM *rd);

122 int __mock_end_chart_formatting(struct instance *instance, RRDSET *st);

123 +int __mock_variables_formatting(struct instance *instance, RRDHOST *host);

124 int __mock_end_host_formatting(struct instance *instance, RRDHOST *host);

125 int __mock_end_batch_formatting(struct instance *instance);

126