| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #define EXPORTING_INTERNALS |
| 4 | #include "mongodb.h" |
| 5 | |
| 6 | #define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2) |
| 7 | |
| 8 | /** |
| 9 | * Initialize MongoDB connector specific data, including a ring buffer |
| 10 | * |
| 11 | * @param instance an instance data structure. |
| 12 | * @return Returns 0 on success, 1 on failure. |
| 13 | */ |
| 14 | int mongodb_init(struct instance *instance) |
| 15 | { |
| 16 | struct mongodb_specific_config *connector_specific_config = instance->config.connector_specific_config; |
| 17 | mongoc_uri_t *uri; |
| 18 | bson_error_t bson_error; |
| 19 | |
| 20 | if (unlikely(!connector_specific_config->collection || !*connector_specific_config->collection)) { |
| 21 | netdata_log_error("EXPORTING: collection name is a mandatory MongoDB parameter, but it is not configured"); |
| 22 | return 1; |
| 23 | } |
| 24 | |
| 25 | uri = mongoc_uri_new_with_error(instance->config.destination, &bson_error); |
| 26 | if (unlikely(!uri)) { |
| 27 | netdata_log_error("EXPORTING: failed to parse URI: %s. Error message: %s", |
| 28 | instance->config.destination, |
| 29 | bson_error.message); |
| 30 | return 1; |
| 31 | } |
| 32 | |
| 33 | int32_t socket_timeout = |
| 34 | mongoc_uri_get_option_as_int32(uri, MONGOC_URI_SOCKETTIMEOUTMS, instance->config.timeoutms); |
| 35 | if (!mongoc_uri_set_option_as_int32(uri, MONGOC_URI_SOCKETTIMEOUTMS, socket_timeout)) { |
| 36 | netdata_log_error("EXPORTING: failed to set %s to the value %d", MONGOC_URI_SOCKETTIMEOUTMS, socket_timeout); |
| 37 | return 1; |
| 38 | }; |
| 39 | |
| 40 | struct mongodb_specific_data *connector_specific_data = |
| 41 | (struct mongodb_specific_data *)instance->connector_specific_data; |
| 42 | |
| 43 | connector_specific_data->client = mongoc_client_new_from_uri(uri); |
| 44 | if (unlikely(!connector_specific_data->client)) { |
| 45 | netdata_log_error("EXPORTING: failed to create a new client"); |
| 46 | return 1; |
| 47 | } |
| 48 | |
| 49 | if (!mongoc_client_set_appname(connector_specific_data->client, "netdata")) { |
| 50 | netdata_log_error("EXPORTING: failed to set client appname"); |
| 51 | }; |
| 52 | |
| 53 | connector_specific_data->collection = mongoc_client_get_collection( |
| 54 | connector_specific_data->client, connector_specific_config->database, connector_specific_config->collection); |
| 55 | |
| 56 | mongoc_uri_destroy(uri); |
| 57 | |
| 58 | // create a ring buffer |
| 59 | struct bson_buffer *first_buffer = NULL; |
| 60 | |
| 61 | if (instance->config.buffer_on_failures < 2) |
| 62 | instance->config.buffer_on_failures = 1; |
| 63 | else |
| 64 | instance->config.buffer_on_failures -= 1; |
| 65 | |
| 66 | for (int i = 0; i < instance->config.buffer_on_failures; i++) { |
| 67 | struct bson_buffer *current_buffer = callocz(1, sizeof(struct bson_buffer)); |
| 68 | |
| 69 | if (!connector_specific_data->first_buffer) |
| 70 | first_buffer = current_buffer; |
| 71 | else |
| 72 | current_buffer->next = connector_specific_data->first_buffer; |
| 73 | |
| 74 | connector_specific_data->first_buffer = current_buffer; |
| 75 | } |
| 76 | |
| 77 | first_buffer->next = connector_specific_data->first_buffer; |
| 78 | connector_specific_data->last_buffer = connector_specific_data->first_buffer; |
| 79 | |
| 80 | return 0; |
| 81 | } |
| 82 | |
| 83 | /** |
| 84 | * Initialize a MongoDB connector instance |
| 85 | * |
| 86 | * @param instance an instance data structure. |
| 87 | * @return Returns 0 on success, 1 on failure. |
| 88 | */ |
| 89 | int init_mongodb_instance(struct instance *instance) |
| 90 | { |
| 91 | instance->worker = mongodb_connector_worker; |
| 92 | |
| 93 | instance->start_batch_formatting = NULL; |
| 94 | instance->start_host_formatting = format_host_labels_json_plaintext; |
| 95 | instance->start_chart_formatting = NULL; |
| 96 | |
| 97 | if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED) |
| 98 | instance->metric_formatting = format_dimension_collected_json_plaintext; |
| 99 | else |
| 100 | instance->metric_formatting = format_dimension_stored_json_plaintext; |
| 101 | |
| 102 | instance->end_chart_formatting = NULL; |
| 103 | instance->variables_formatting = NULL; |
| 104 | instance->end_host_formatting = flush_host_labels; |
| 105 | instance->end_batch_formatting = format_batch_mongodb; |
| 106 | |
| 107 | instance->prepare_header = NULL; |
| 108 | instance->check_response = NULL; |
| 109 | |
| 110 | instance->buffer = (void *)buffer_create(0, &netdata_buffers_statistics.buffers_exporters); |
| 111 | if (!instance->buffer) { |
| 112 | netdata_log_error("EXPORTING: cannot create buffer for MongoDB exporting connector instance %s", |
| 113 | instance->config.name); |
| 114 | return 1; |
| 115 | } |
| 116 | if (netdata_mutex_init(&instance->mutex)) |
| 117 | return 1; |
| 118 | if (netdata_cond_init(&instance->cond_var)) |
| 119 | return 1; |
| 120 | |
| 121 | struct mongodb_specific_data *connector_specific_data = callocz(1, sizeof(struct mongodb_specific_data)); |
| 122 | instance->connector_specific_data = (void *)connector_specific_data; |
| 123 | |
| 124 | instance->config.timeoutms = |
| 125 | (instance->config.update_every >= 2) ? (instance->engine->config.update_every * MSEC_PER_SEC - 500) : 1000; |
| 126 | |
| 127 | if (!instance->engine->mongoc_initialized) { |
| 128 | mongoc_init(); |
| 129 | instance->engine->mongoc_initialized = 1; |
| 130 | } |
| 131 | |
| 132 | if (unlikely(mongodb_init(instance))) { |
| 133 | netdata_log_error("EXPORTING: cannot initialize MongoDB exporting connector"); |
| 134 | return 1; |
| 135 | } |
| 136 | |
| 137 | return 0; |
| 138 | } |
| 139 | |
| 140 | /** |
| 141 | * Free an array of BSON structures |
| 142 | * |
| 143 | * @param insert an array of documents. |
| 144 | * @param documents_inserted the number of documents inserted. |
| 145 | */ |
| 146 | void free_bson(bson_t **insert, size_t documents_inserted) |
| 147 | { |
| 148 | size_t i; |
| 149 | |
| 150 | for (i = 0; i < documents_inserted; i++) |
| 151 | bson_destroy(insert[i]); |
| 152 | |
| 153 | freez(insert); |
| 154 | } |
| 155 | |
| 156 | /** |
| 157 | * Format a batch for the MongoDB connector |
| 158 | * |
| 159 | * @param instance an instance data structure. |
| 160 | * @return Returns 0 on success, 1 on failure. |
| 161 | */ |
| 162 | int format_batch_mongodb(struct instance *instance) |
| 163 | { |
| 164 | struct mongodb_specific_data *connector_specific_data = |
| 165 | (struct mongodb_specific_data *)instance->connector_specific_data; |
| 166 | struct stats *stats = &instance->stats; |
| 167 | |
| 168 | bson_t **insert = connector_specific_data->last_buffer->insert; |
| 169 | if (insert) { |
| 170 | // ring buffer is full, reuse the oldest element |
| 171 | connector_specific_data->first_buffer = connector_specific_data->first_buffer->next; |
| 172 | free_bson(insert, connector_specific_data->last_buffer->documents_inserted); |
| 173 | connector_specific_data->total_documents_inserted -= connector_specific_data->last_buffer->documents_inserted; |
| 174 | stats->buffered_bytes -= connector_specific_data->last_buffer->buffered_bytes; |
| 175 | } |
| 176 | insert = callocz((size_t)stats->buffered_metrics, sizeof(bson_t *)); |
| 177 | connector_specific_data->last_buffer->insert = insert; |
| 178 | |
| 179 | BUFFER *buffer = (BUFFER *)instance->buffer; |
| 180 | char *start = (char *)buffer_tostring(buffer); |
| 181 | char *end = start; |
| 182 | |
| 183 | size_t documents_inserted = 0; |
| 184 | |
| 185 | while (*end && documents_inserted <= (size_t)stats->buffered_metrics) { |
| 186 | while (*end && *end != '\n') |
| 187 | end++; |
| 188 | |
| 189 | if (likely(*end)) { |
| 190 | *end = '\0'; |
| 191 | end++; |
| 192 | } else { |
| 193 | break; |
| 194 | } |
| 195 | |
| 196 | bson_error_t bson_error; |
| 197 | insert[documents_inserted] = bson_new_from_json((const uint8_t *)start, -1, &bson_error); |
| 198 | |
| 199 | if (unlikely(!insert[documents_inserted])) { |
| 200 | netdata_log_error( |
| 201 | "EXPORTING: Failed creating a BSON document from a JSON string \"%s\" : %s", start, bson_error.message); |
| 202 | free_bson(insert, documents_inserted); |
| 203 | return 1; |
| 204 | } |
| 205 | |
| 206 | start = end; |
| 207 | |
| 208 | documents_inserted++; |
| 209 | } |
| 210 | |
| 211 | stats->buffered_bytes += connector_specific_data->last_buffer->buffered_bytes = buffer_strlen(buffer); |
| 212 | |
| 213 | buffer_flush(buffer); |
| 214 | |
| 215 | // The stats->buffered_metrics is used in the MongoDB batch formatting as a variable for the number |
| 216 | // of metrics, added in the current iteration, so we are clearing it here. We will use the |
| 217 | // connector_specific_data->total_documents_inserted in the worker to show the statistics. |
| 218 | stats->buffered_metrics = 0; |
| 219 | connector_specific_data->total_documents_inserted += documents_inserted; |
| 220 | |
| 221 | connector_specific_data->last_buffer->documents_inserted = documents_inserted; |
| 222 | connector_specific_data->last_buffer = connector_specific_data->last_buffer->next; |
| 223 | |
| 224 | return 0; |
| 225 | } |
| 226 | |
| 227 | /** |
| 228 | * Clean a MongoDB connector instance up |
| 229 | * |
| 230 | * @param instance an instance data structure. |
| 231 | */ |
| 232 | void mongodb_cleanup(struct instance *instance) |
| 233 | { |
| 234 | netdata_log_info("EXPORTING: cleaning up instance %s ...", instance->config.name); |
| 235 | |
| 236 | struct mongodb_specific_data *connector_specific_data = |
| 237 | (struct mongodb_specific_data *)instance->connector_specific_data; |
| 238 | |
| 239 | mongoc_collection_destroy(connector_specific_data->collection); |
| 240 | mongoc_client_destroy(connector_specific_data->client); |
| 241 | if (instance->engine->mongoc_initialized) { |
| 242 | mongoc_cleanup(); |
| 243 | instance->engine->mongoc_initialized = 0; |
| 244 | } |
| 245 | |
| 246 | buffer_free(instance->buffer); |
| 247 | |
| 248 | struct bson_buffer *next_buffer = connector_specific_data->first_buffer; |
| 249 | for (int i = 0; i < instance->config.buffer_on_failures; i++) { |
| 250 | struct bson_buffer *current_buffer = next_buffer; |
| 251 | next_buffer = next_buffer->next; |
| 252 | |
| 253 | if (current_buffer->insert) |
| 254 | free_bson(current_buffer->insert, current_buffer->documents_inserted); |
| 255 | freez(current_buffer); |
| 256 | } |
| 257 | |
| 258 | freez(connector_specific_data); |
| 259 | |
| 260 | struct mongodb_specific_config *connector_specific_config = |
| 261 | (struct mongodb_specific_config *)instance->config.connector_specific_config; |
| 262 | freez(connector_specific_config->database); |
| 263 | freez(connector_specific_config->collection); |
| 264 | freez(connector_specific_config); |
| 265 | |
| 266 | netdata_log_info("EXPORTING: instance %s exited", instance->config.name); |
| 267 | instance->exited = 1; |
| 268 | |
| 269 | return; |
| 270 | } |
| 271 | |
| 272 | /** |
| 273 | * MongoDB connector worker |
| 274 | * |
| 275 | * Runs in a separate thread for every instance. |
| 276 | * |
| 277 | * @param instance_p an instance data structure. |
| 278 | */ |
| 279 | void mongodb_connector_worker(void *instance_p) |
| 280 | { |
| 281 | struct instance *instance = (struct instance *)instance_p; |
| 282 | #ifdef NETDATA_INTERNAL_CHECKS |
| 283 | struct mongodb_specific_config *connector_specific_config = instance->config.connector_specific_config; |
| 284 | #endif |
| 285 | struct mongodb_specific_data *connector_specific_data = |
| 286 | (struct mongodb_specific_data *)instance->connector_specific_data; |
| 287 | |
| 288 | while (!instance->engine->exit) { |
| 289 | struct stats *stats = &instance->stats; |
| 290 | |
| 291 | netdata_mutex_lock(&instance->mutex); |
| 292 | if (!connector_specific_data->first_buffer->insert || |
| 293 | !connector_specific_data->first_buffer->documents_inserted) { |
| 294 | while (!instance->data_is_ready) |
| 295 | netdata_cond_wait(&instance->cond_var, &instance->mutex); |
| 296 | instance->data_is_ready = 0; |
| 297 | } |
| 298 | |
| 299 | if (unlikely(instance->engine->exit)) { |
| 300 | netdata_mutex_unlock(&instance->mutex); |
| 301 | break; |
| 302 | } |
| 303 | |
| 304 | // reset the monitoring chart counters |
| 305 | stats->received_bytes = |
| 306 | stats->sent_bytes = |
| 307 | stats->sent_metrics = |
| 308 | stats->lost_metrics = |
| 309 | stats->receptions = |
| 310 | stats->transmission_successes = |
| 311 | stats->transmission_failures = |
| 312 | stats->data_lost_events = |
| 313 | stats->lost_bytes = |
| 314 | stats->reconnects = 0; |
| 315 | |
| 316 | bson_t **insert = connector_specific_data->first_buffer->insert; |
| 317 | size_t documents_inserted = connector_specific_data->first_buffer->documents_inserted; |
| 318 | size_t buffered_bytes = connector_specific_data->first_buffer->buffered_bytes; |
| 319 | |
| 320 | connector_specific_data->first_buffer->insert = NULL; |
| 321 | connector_specific_data->first_buffer->documents_inserted = 0; |
| 322 | connector_specific_data->first_buffer->buffered_bytes = 0; |
| 323 | connector_specific_data->first_buffer = connector_specific_data->first_buffer->next; |
| 324 | |
| 325 | netdata_mutex_unlock(&instance->mutex); |
| 326 | |
| 327 | size_t data_size = 0; |
| 328 | for (size_t i = 0; i < documents_inserted; i++) { |
| 329 | data_size += insert[i]->len; |
| 330 | } |
| 331 | |
| 332 | netdata_log_debug( |
| 333 | D_EXPORTING, |
| 334 | "EXPORTING: mongodb_insert(): destination = %s, database = %s, collection = %s, data size = %zu", |
| 335 | instance->config.destination, |
| 336 | connector_specific_config->database, |
| 337 | connector_specific_config->collection, |
| 338 | data_size); |
| 339 | |
| 340 | if (likely(documents_inserted != 0)) { |
| 341 | bson_error_t bson_error; |
| 342 | if (likely(mongoc_collection_insert_many( |
| 343 | connector_specific_data->collection, |
| 344 | (const bson_t **)insert, |
| 345 | documents_inserted, |
| 346 | NULL, |
| 347 | NULL, |
| 348 | &bson_error))) { |
| 349 | stats->sent_metrics = documents_inserted; |
| 350 | stats->sent_bytes += data_size; |
| 351 | stats->transmission_successes++; |
| 352 | stats->receptions++; |
| 353 | } else { |
| 354 | // oops! we couldn't send (all or some of the) data |
| 355 | netdata_log_error("EXPORTING: %s", bson_error.message); |
| 356 | netdata_log_error( |
| 357 | "EXPORTING: failed to write data to the database '%s'. " |
| 358 | "Willing to write %zu bytes, wrote %zu bytes.", |
| 359 | instance->config.destination, data_size, 0UL); |
| 360 | |
| 361 | stats->transmission_failures++; |
| 362 | stats->data_lost_events++; |
| 363 | stats->lost_bytes += buffered_bytes; |
| 364 | stats->lost_metrics += documents_inserted; |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | free_bson(insert, documents_inserted); |
| 369 | |
| 370 | if (unlikely(instance->engine->exit)) |
| 371 | break; |
| 372 | |
| 373 | netdata_mutex_lock(&instance->mutex); |
| 374 | |
| 375 | stats->buffered_metrics = connector_specific_data->total_documents_inserted; |
| 376 | |
| 377 | send_internal_metrics(instance); |
| 378 | |
| 379 | connector_specific_data->total_documents_inserted -= documents_inserted; |
| 380 | |
| 381 | stats->buffered_metrics = 0; |
| 382 | stats->buffered_bytes -= buffered_bytes; |
| 383 | |
| 384 | netdata_mutex_unlock(&instance->mutex); |
| 385 | |
| 386 | #ifdef UNIT_TESTING |
| 387 | return; |
| 388 | #endif |
| 389 | } |
| 390 | |
| 391 | mongodb_cleanup(instance); |
| 392 | } |