| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "remote_write.h" |
| 4 | |
| 5 | static int as_collected; |
| 6 | static int homogeneous; |
| 7 | char context[PROMETHEUS_ELEMENT_MAX + 1]; |
| 8 | char chart[PROMETHEUS_ELEMENT_MAX + 1]; |
| 9 | char family[PROMETHEUS_ELEMENT_MAX + 1]; |
| 10 | char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; |
| 11 | |
| 12 | /** |
| 13 | * Prepare HTTP header |
| 14 | * |
| 15 | * @param instance an instance data structure. |
| 16 | */ |
| 17 | void prometheus_remote_write_prepare_header(struct instance *instance) |
| 18 | { |
| 19 | struct prometheus_remote_write_specific_config *connector_specific_config = |
| 20 | instance->config.connector_specific_config; |
| 21 | struct simple_connector_data *simple_connector_data = instance->connector_specific_data; |
| 22 | |
| 23 | buffer_sprintf( |
| 24 | simple_connector_data->last_buffer->header, |
| 25 | "POST %s HTTP/1.1\r\n" |
| 26 | "Host: %s\r\n" |
| 27 | "Accept: */*\r\n" |
| 28 | "%s" |
| 29 | "Content-Encoding: snappy\r\n" |
| 30 | "Content-Type: application/x-protobuf\r\n" |
| 31 | "X-Prometheus-Remote-Write-Version: 0.1.0\r\n" |
| 32 | "Content-Length: %zu\r\n" |
| 33 | "\r\n", |
| 34 | connector_specific_config->remote_write_path, |
| 35 | simple_connector_data->connected_to, |
| 36 | simple_connector_data->auth_string ? simple_connector_data->auth_string : "", |
| 37 | buffer_strlen(simple_connector_data->last_buffer->buffer)); |
| 38 | } |
| 39 | |
| 40 | /** |
| 41 | * Process a response received after Prometheus remote write connector had sent data |
| 42 | * |
| 43 | * @param buffer a response from a remote service. |
| 44 | * @param instance an instance data structure. |
| 45 | * @return Returns 0 on success, 1 on failure. |
| 46 | */ |
| 47 | int process_prometheus_remote_write_response(BUFFER *buffer, struct instance *instance) |
| 48 | { |
| 49 | if (unlikely(!buffer)) |
| 50 | return 1; |
| 51 | |
| 52 | const char *s = buffer_tostring(buffer); |
| 53 | int len = buffer_strlen(buffer); |
| 54 | |
| 55 | // do nothing with HTTP responses 200 or 204 |
| 56 | |
| 57 | while (!isspace(*s) && len) { |
| 58 | s++; |
| 59 | len--; |
| 60 | } |
| 61 | s++; |
| 62 | len--; |
| 63 | |
| 64 | if (likely(len > 4 && (!strncmp(s, "200 ", 4) || !strncmp(s, "204 ", 4)))) |
| 65 | return 0; |
| 66 | else |
| 67 | return exporting_discard_response(buffer, instance); |
| 68 | } |
| 69 | |
| 70 | /** |
| 71 | * Release specific data allocated. |
| 72 | * |
| 73 | * @param instance an instance data structure. |
| 74 | */ |
| 75 | void clean_prometheus_remote_write(struct instance *instance) |
| 76 | { |
| 77 | struct simple_connector_data *simple_connector_data = instance->connector_specific_data; |
| 78 | freez(simple_connector_data->connector_specific_data); |
| 79 | |
| 80 | struct prometheus_remote_write_specific_config *connector_specific_config = |
| 81 | instance->config.connector_specific_config; |
| 82 | freez(connector_specific_config->remote_write_path); |
| 83 | } |
| 84 | |
| 85 | /** |
| 86 | * Initialize Prometheus Remote Write connector instance |
| 87 | * |
| 88 | * @param instance an instance data structure. |
| 89 | * @return Returns 0 on success, 1 on failure. |
| 90 | */ |
| 91 | int init_prometheus_remote_write_instance(struct instance *instance) |
| 92 | { |
| 93 | instance->worker = simple_connector_worker; |
| 94 | |
| 95 | instance->start_batch_formatting = NULL; |
| 96 | instance->start_host_formatting = format_host_prometheus_remote_write; |
| 97 | instance->start_chart_formatting = format_chart_prometheus_remote_write; |
| 98 | instance->metric_formatting = format_dimension_prometheus_remote_write; |
| 99 | instance->end_chart_formatting = NULL; |
| 100 | instance->variables_formatting = format_variables_prometheus_remote_write; |
| 101 | instance->end_host_formatting = NULL; |
| 102 | instance->end_batch_formatting = format_batch_prometheus_remote_write; |
| 103 | |
| 104 | instance->prepare_header = prometheus_remote_write_prepare_header; |
| 105 | instance->check_response = process_prometheus_remote_write_response; |
| 106 | |
| 107 | instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters); |
| 108 | |
| 109 | if (netdata_mutex_init(&instance->mutex)) |
| 110 | return 1; |
| 111 | if (netdata_cond_init(&instance->cond_var)) |
| 112 | return 1; |
| 113 | |
| 114 | struct simple_connector_data *simple_connector_data = callocz(1, sizeof(struct simple_connector_data)); |
| 115 | instance->connector_specific_data = simple_connector_data; |
| 116 | |
| 117 | simple_connector_data->ssl = NETDATA_SSL_UNSET_CONNECTION; |
| 118 | if (instance->config.options & EXPORTING_OPTION_USE_TLS) { |
| 119 | netdata_ssl_initialize_ctx(NETDATA_SSL_EXPORTING_CTX); |
| 120 | } |
| 121 | |
| 122 | struct prometheus_remote_write_specific_data *connector_specific_data = |
| 123 | callocz(1, sizeof(struct prometheus_remote_write_specific_data)); |
| 124 | simple_connector_data->connector_specific_data = (void *)connector_specific_data; |
| 125 | |
| 126 | simple_connector_init(instance); |
| 127 | |
| 128 | connector_specific_data->write_request = init_write_request(); |
| 129 | |
| 130 | instance->engine->protocol_buffers_initialized = 1; |
| 131 | |
| 132 | return 0; |
| 133 | } |
| 134 | |
| 135 | struct format_remote_write_label_callback { |
| 136 | struct instance *instance; |
| 137 | void *write_request; |
| 138 | }; |
| 139 | |
| 140 | static int format_remote_write_label_callback(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) |
| 141 | { |
| 142 | struct format_remote_write_label_callback *d = (struct format_remote_write_label_callback *)data; |
| 143 | |
| 144 | if (!should_send_label(d->instance, ls)) return 0; |
| 145 | char k[PROMETHEUS_ELEMENT_MAX + 1]; |
| 146 | char v[PROMETHEUS_ELEMENT_MAX + 1]; |
| 147 | |
| 148 | prometheus_name_copy(k, name, sizeof(k)); |
| 149 | prometheus_label_copy(v, value, sizeof(v)); |
| 150 | add_label(d->write_request, k, v); |
| 151 | return 1; |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * Format host data for Prometheus Remote Write connector |
| 156 | * |
| 157 | * @param instance an instance data structure. |
| 158 | * @param host a data collecting host. |
| 159 | * @return Always returns 0. |
| 160 | */ |
| 161 | int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host) |
| 162 | { |
| 163 | struct simple_connector_data *simple_connector_data = |
| 164 | (struct simple_connector_data *)instance->connector_specific_data; |
| 165 | struct prometheus_remote_write_specific_data *connector_specific_data = |
| 166 | (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data; |
| 167 | |
| 168 | char hostname[PROMETHEUS_ELEMENT_MAX + 1]; |
| 169 | prometheus_label_copy( |
| 170 | hostname, |
| 171 | (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), |
| 172 | sizeof(hostname)); |
| 173 | |
| 174 | add_host_info( |
| 175 | connector_specific_data->write_request, |
| 176 | "netdata_info", hostname, rrdhost_program_name(host), rrdhost_program_version(host), now_realtime_usec() / USEC_PER_MS); |
| 177 | |
| 178 | if (unlikely(sending_labels_configured(instance))) { |
| 179 | struct format_remote_write_label_callback tmp = { |
| 180 | .write_request = connector_specific_data->write_request, |
| 181 | .instance = instance |
| 182 | }; |
| 183 | rrdlabels_walkthrough_read(host->rrdlabels, format_remote_write_label_callback, &tmp); |
| 184 | } |
| 185 | |
| 186 | return 0; |
| 187 | } |
| 188 | |
| 189 | /** |
| 190 | * Format chart data for Prometheus Remote Write connector |
| 191 | * |
| 192 | * @param instance an instance data structure. |
| 193 | * @param st a chart. |
| 194 | * @return Always returns 0. |
| 195 | */ |
| 196 | int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st) |
| 197 | { |
| 198 | prometheus_label_copy( |
| 199 | chart, |
| 200 | (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), |
| 201 | sizeof(chart)); |
| 202 | prometheus_label_copy(family, rrdset_family(st), sizeof(family)); |
| 203 | prometheus_name_copy(context, rrdset_context(st), sizeof(context)); |
| 204 | |
| 205 | as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED); |
| 206 | homogeneous = 1; |
| 207 | if (as_collected) { |
| 208 | if (rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK)) |
| 209 | rrdset_update_heterogeneous_flag(st); |
| 210 | |
| 211 | if (rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) |
| 212 | homogeneous = 0; |
| 213 | } else { |
| 214 | if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE) |
| 215 | prometheus_units_copy(units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, 0); |
| 216 | } |
| 217 | |
| 218 | return 0; |
| 219 | } |
| 220 | |
| 221 | /** |
| 222 | * Format dimension data for Prometheus Remote Write connector |
| 223 | * |
| 224 | * @param instance an instance data structure. |
| 225 | * @param rd a dimension. |
| 226 | * @return Always returns 0. |
| 227 | */ |
| 228 | int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd) |
| 229 | { |
| 230 | struct simple_connector_data *simple_connector_data = |
| 231 | (struct simple_connector_data *)instance->connector_specific_data; |
| 232 | struct prometheus_remote_write_specific_data *connector_specific_data = |
| 233 | (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data; |
| 234 | |
| 235 | if (rd->collector.counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { |
| 236 | char name[PROMETHEUS_LABELS_MAX + 1]; |
| 237 | char dimension[PROMETHEUS_ELEMENT_MAX + 1]; |
| 238 | char *suffix = ""; |
| 239 | RRDHOST *host = rd->rrdset->rrdhost; |
| 240 | |
| 241 | if (as_collected) { |
| 242 | // we need as-collected / raw data |
| 243 | |
| 244 | if (unlikely(rd->collector.last_collected_time.tv_sec < instance->after)) { |
| 245 | netdata_log_debug( |
| 246 | D_EXPORTING, |
| 247 | "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', " |
| 248 | "its last data collection (%lu) is not within our timeframe (%lu to %lu)", |
| 249 | rrddim_id(rd), rrdset_id(rd->rrdset), |
| 250 | (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), |
| 251 | (unsigned long)rd->collector.last_collected_time.tv_sec, |
| 252 | (unsigned long)instance->after, |
| 253 | (unsigned long)instance->before); |
| 254 | return 0; |
| 255 | } |
| 256 | |
| 257 | if (rd->algorithm == RRD_ALGORITHM_INCREMENTAL || rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) { |
| 258 | if (strcmp(rrdset_module_name(rd->rrdset), "prometheus")) |
| 259 | suffix = "_total"; |
| 260 | } |
| 261 | |
| 262 | if (homogeneous) { |
| 263 | // all the dimensions of the chart, has the same algorithm, multiplier and divisor |
| 264 | // we add all dimensions as labels |
| 265 | |
| 266 | prometheus_label_copy( |
| 267 | dimension, |
| 268 | (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), |
| 269 | sizeof(dimension)); |
| 270 | snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix); |
| 271 | |
| 272 | add_metric( |
| 273 | connector_specific_data->write_request, |
| 274 | name, chart, family, dimension, |
| 275 | (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), |
| 276 | rrddim_last_collected_as_double(rd), timeval_msec(&rd->collector.last_collected_time)); |
| 277 | } else { |
| 278 | // the dimensions of the chart, do not have the same algorithm, multiplier or divisor |
| 279 | // we create a metric per dimension |
| 280 | |
| 281 | prometheus_name_copy( |
| 282 | dimension, |
| 283 | (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), |
| 284 | sizeof(dimension)); |
| 285 | snprintf( |
| 286 | name, sizeof(name), "%s_%s_%s%s", instance->config.prefix, context, dimension, |
| 287 | suffix); |
| 288 | |
| 289 | add_metric( |
| 290 | connector_specific_data->write_request, |
| 291 | name, chart, family, NULL, |
| 292 | (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), |
| 293 | rrddim_last_collected_as_double(rd), timeval_msec(&rd->collector.last_collected_time)); |
| 294 | } |
| 295 | } else { |
| 296 | // we need average or sum of the data |
| 297 | |
| 298 | time_t last_t = instance->before; |
| 299 | NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); |
| 300 | |
| 301 | if (!isnan(value) && !isinf(value)) { |
| 302 | if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE) |
| 303 | suffix = "_average"; |
| 304 | else if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM) |
| 305 | suffix = "_sum"; |
| 306 | |
| 307 | prometheus_label_copy( |
| 308 | dimension, |
| 309 | (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), |
| 310 | sizeof(dimension)); |
| 311 | snprintf( |
| 312 | name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", instance->config.prefix, context, units, suffix); |
| 313 | |
| 314 | add_metric( |
| 315 | connector_specific_data->write_request, |
| 316 | name, chart, family, dimension, |
| 317 | (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), |
| 318 | value, last_t * MSEC_PER_SEC); |
| 319 | } |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | return 0; |
| 324 | } |
| 325 | |
| 326 | static int format_variable_prometheus_remote_write_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rv_ptr __maybe_unused, void *data) { |
| 327 | const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item; |
| 328 | |
| 329 | struct prometheus_remote_write_variables_callback_options *opts = data; |
| 330 | |
| 331 | RRDHOST *host = opts->host; |
| 332 | struct instance *instance = opts->instance; |
| 333 | struct simple_connector_data *simple_connector_data = |
| 334 | (struct simple_connector_data *)instance->connector_specific_data; |
| 335 | struct prometheus_remote_write_specific_data *connector_specific_data = |
| 336 | (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data; |
| 337 | |
| 338 | char name[PROMETHEUS_LABELS_MAX + 1]; |
| 339 | char *suffix = ""; |
| 340 | |
| 341 | prometheus_name_copy(context, rrdvar_name(rv), sizeof(context)); |
| 342 | snprintf(name, sizeof(name), "%s_%s%s", instance->config.prefix, context, suffix); |
| 343 | |
| 344 | NETDATA_DOUBLE value = rrdvar2number(rv); |
| 345 | add_variable(connector_specific_data->write_request, name, |
| 346 | (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS); |
| 347 | |
| 348 | return 0; |
| 349 | } |
| 350 | |
| 351 | /** |
| 352 | * Format a variable for Prometheus Remote Write connector |
| 353 | * |
| 354 | * @param rv a variable. |
| 355 | * @param instance an instance data structure. |
| 356 | * @return Always returns 0. |
| 357 | */ |
| 358 | int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host) |
| 359 | { |
| 360 | struct prometheus_remote_write_variables_callback_options opt = { |
| 361 | .host = host, |
| 362 | .instance = instance, |
| 363 | .now = now_realtime_usec(), |
| 364 | }; |
| 365 | |
| 366 | return rrdvar_walkthrough_read(host->rrdvars, format_variable_prometheus_remote_write_callback, &opt); |
| 367 | } |
| 368 | |
| 369 | /** |
| 370 | * Format a batch for Prometheus Remote Write connector |
| 371 | * |
| 372 | * @param instance an instance data structure. |
| 373 | * @return Returns 0 on success, 1 on failure. |
| 374 | */ |
| 375 | int format_batch_prometheus_remote_write(struct instance *instance) |
| 376 | { |
| 377 | struct simple_connector_data *simple_connector_data = |
| 378 | (struct simple_connector_data *)instance->connector_specific_data; |
| 379 | struct prometheus_remote_write_specific_data *connector_specific_data = |
| 380 | (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data; |
| 381 | |
| 382 | size_t data_size = get_write_request_size(connector_specific_data->write_request); |
| 383 | |
| 384 | if (unlikely(!data_size)) { |
| 385 | netdata_log_error("EXPORTING: write request size is out of range"); |
| 386 | return 1; |
| 387 | } |
| 388 | |
| 389 | BUFFER *buffer = instance->buffer; |
| 390 | |
| 391 | buffer_need_bytes(buffer, data_size); |
| 392 | if (unlikely(pack_and_clear_write_request(connector_specific_data->write_request, buffer->buffer, &data_size))) { |
| 393 | netdata_log_error("EXPORTING: cannot pack write request"); |
| 394 | return 1; |
| 395 | } |
| 396 | buffer->len = data_size; |
| 397 | |
| 398 | simple_connector_end_batch(instance); |
| 399 | |
| 400 | return 0; |
| 401 | } |