| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "sqlite_functions.h" |
| 4 | #include "sqlite_context.h" |
| 5 | #include "sqlite_db_migration.h" |
| 6 | #include "database/contexts/rrdcontext-internal.h" |
| 7 | |
| 8 | #define DB_CONTEXT_METADATA_VERSION 1 |
| 9 | |
| 10 | const char *database_context_config[] = { |
| 11 | "CREATE TABLE IF NOT EXISTS context (host_id BLOB, id TEXT NOT NULL, version INT NOT NULL, title TEXT NOT NULL, " |
| 12 | "chart_type TEXT NOT NULL, unit TEXT NOT NULL, priority INT NOT NULL, first_time_t INT NOT NULL, " |
| 13 | "last_time_t INT NOT NULL, deleted INT NOT NULL, " |
| 14 | "family TEXT, PRIMARY KEY (host_id, id))", |
| 15 | |
| 16 | NULL |
| 17 | }; |
| 18 | |
| 19 | const char *database_context_cleanup[] = { |
| 20 | "DROP TRIGGER IF EXISTS del_context1", |
| 21 | "DROP TABLE IF EXISTS context_metadata_cleanup", |
| 22 | "VACUUM", |
| 23 | NULL |
| 24 | }; |
| 25 | |
| 26 | sqlite3 *db_context_meta = NULL; |
| 27 | /* |
| 28 | * Initialize the SQLite database |
| 29 | * Return 0 on success |
| 30 | */ |
| 31 | int sql_init_context_database(int memory) |
| 32 | { |
| 33 | char sqlite_database[FILENAME_MAX + 1]; |
| 34 | int rc; |
| 35 | |
| 36 | if (likely(!memory)) |
| 37 | snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/context-meta.db", netdata_configured_cache_dir); |
| 38 | else |
| 39 | strcpy(sqlite_database, ":memory:"); |
| 40 | |
| 41 | rc = sqlite3_open(sqlite_database, &db_context_meta); |
| 42 | if (rc != SQLITE_OK) { |
| 43 | error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); |
| 44 | sqlite3_close(db_context_meta); |
| 45 | db_context_meta = NULL; |
| 46 | return 1; |
| 47 | } |
| 48 | |
| 49 | errno_clear(); |
| 50 | netdata_log_info("SQLite database %s initialization", sqlite_database); |
| 51 | |
| 52 | char buf[1024 + 1] = ""; |
| 53 | const char *list[2] = { buf, NULL }; |
| 54 | |
| 55 | int target_version = DB_CONTEXT_METADATA_VERSION; |
| 56 | if (likely(!memory)) |
| 57 | target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION); |
| 58 | |
| 59 | if (configure_sqlite_database(db_context_meta, target_version, "context_config")) |
| 60 | return 1; |
| 61 | |
| 62 | if (likely(!memory)) |
| 63 | snprintfz(buf, sizeof(buf) - 1, "ATTACH DATABASE \"%s/netdata-meta.db\" as meta", netdata_configured_cache_dir); |
| 64 | else |
| 65 | snprintfz(buf, sizeof(buf) - 1, "ATTACH DATABASE ':memory:' as meta"); |
| 66 | |
| 67 | if(init_database_batch(db_context_meta, list, "context")) return 1; |
| 68 | |
| 69 | if (init_database_batch(db_context_meta, &database_context_config[0], "context_init")) |
| 70 | return 1; |
| 71 | |
| 72 | if (init_database_batch(db_context_meta, &database_context_cleanup[0], "context_cleanup")) |
| 73 | return 1; |
| 74 | |
| 75 | return 0; |
| 76 | } |
| 77 | |
| 78 | extern __thread sqlite3 *db_meta_thread; |
| 79 | extern __thread sqlite3 *db_context_thread; |
| 80 | // |
| 81 | // Fetching data |
| 82 | // |
| 83 | #define CTX_GET_CHART_LIST "SELECT c.chart_id, c.type||'.'||c.id, c.name, c.context, c.title, c.unit, c.priority, " \ |
| 84 | "c.update_every, c.chart_type, c.family FROM chart c WHERE c.host_id = @host_id AND c.chart_id IS NOT NULL" |
| 85 | |
| 86 | void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data) |
| 87 | { |
| 88 | sqlite3_stmt *res = NULL; |
| 89 | |
| 90 | if (unlikely(!host_uuid)) { |
| 91 | internal_error(true, "Requesting context chart list without host_id"); |
| 92 | return; |
| 93 | } |
| 94 | |
| 95 | if (!PREPARE_STATEMENT(db_meta_thread ? db_meta_thread : db_meta, CTX_GET_CHART_LIST, &res)) |
| 96 | return; |
| 97 | |
| 98 | int param = 0; |
| 99 | SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC)); |
| 100 | |
| 101 | char host_guid[UUID_STR_LEN]; |
| 102 | uuid_unparse_lower(*host_uuid, host_guid); |
| 103 | |
| 104 | param = 0; |
| 105 | SQL_CHART_DATA chart_data = { 0 }; |
| 106 | while (sqlite3_step_monitored(res) == SQLITE_ROW) { |
| 107 | if (unlikely(exit_initiated_get())) |
| 108 | break; |
| 109 | |
| 110 | if (unlikely(!sqlite3_column_uuid_copy(res, 0, chart_data.chart_id))) { |
| 111 | error_report("CTX [%s]: Got invalid chart id in column 0. Ignoring it.", host_guid); |
| 112 | continue; |
| 113 | } |
| 114 | |
| 115 | chart_data.id = (char *) sqlite3_column_text(res, 1); |
| 116 | chart_data.name = (char *) sqlite3_column_text(res, 2); |
| 117 | chart_data.context = (char *) sqlite3_column_text(res, 3); |
| 118 | chart_data.title = (char *) sqlite3_column_text(res, 4); |
| 119 | chart_data.units = (char *) sqlite3_column_text(res, 5); |
| 120 | chart_data.priority = sqlite3_column_int(res, 6); |
| 121 | chart_data.update_every = sqlite3_column_int(res, 7); |
| 122 | chart_data.chart_type = sqlite3_column_int(res, 8); |
| 123 | chart_data.family = (char *) sqlite3_column_text(res, 9); |
| 124 | dict_cb(&chart_data, data); |
| 125 | } |
| 126 | |
| 127 | done: |
| 128 | REPORT_BIND_FAIL(res, param); |
| 129 | SQLITE_FINALIZE(res); |
| 130 | } |
| 131 | |
| 132 | // Dimension list |
| 133 | #define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END, c.type||'.'||c.id, c.context, d.algorithm " \ |
| 134 | "FROM dimension d, chart c WHERE c.host_id = @host_id AND d.chart_id = c.chart_id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC" |
| 135 | void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data) |
| 136 | { |
| 137 | sqlite3_stmt *res = NULL; |
| 138 | |
| 139 | if (!PREPARE_STATEMENT(db_meta_thread ? db_meta_thread : db_meta, CTX_GET_DIMENSION_LIST, &res)) |
| 140 | return; |
| 141 | |
| 142 | int param = 0; |
| 143 | SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC)); |
| 144 | |
| 145 | char host_guid[UUID_STR_LEN]; |
| 146 | uuid_unparse_lower(*host_uuid, host_guid); |
| 147 | |
| 148 | SQL_DIMENSION_DATA dimension_data; |
| 149 | |
| 150 | param = 0; |
| 151 | while (sqlite3_step_monitored(res) == SQLITE_ROW) { |
| 152 | if (unlikely(exit_initiated_get())) |
| 153 | break; |
| 154 | |
| 155 | if (unlikely(!sqlite3_column_uuid_copy(res, 0, dimension_data.dim_id))) { |
| 156 | error_report("CTX [%s]: Got invalid dimension id in column 0. Ignoring it.", host_guid); |
| 157 | continue; |
| 158 | } |
| 159 | |
| 160 | dimension_data.id = (char *) sqlite3_column_text(res, 1); |
| 161 | dimension_data.name = (char *) sqlite3_column_text(res, 2); |
| 162 | dimension_data.hidden = sqlite3_column_int(res, 3); |
| 163 | dimension_data.chart_id = (char *) sqlite3_column_text(res, 4); |
| 164 | dimension_data.context = (char *) sqlite3_column_text(res, 5); |
| 165 | dimension_data.algorithm = sqlite3_column_int(res, 6); |
| 166 | dict_cb(&dimension_data, data); |
| 167 | } |
| 168 | |
| 169 | done: |
| 170 | REPORT_BIND_FAIL(res, param); |
| 171 | SQLITE_FINALIZE(res); |
| 172 | } |
| 173 | |
| 174 | // LABEL LIST |
| 175 | #define CTX_GET_LABEL_LIST "SELECT l.label_key, l.label_value, l.source_type FROM meta.chart_label l WHERE l.chart_id = @id" |
| 176 | |
| 177 | void ctx_get_label_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data) |
| 178 | { |
| 179 | sqlite3_stmt *res = NULL; |
| 180 | |
| 181 | if (!PREPARE_STATEMENT(db_context_meta, CTX_GET_LABEL_LIST, &res)) |
| 182 | return; |
| 183 | |
| 184 | int param = 0; |
| 185 | SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC)); |
| 186 | |
| 187 | param = 0; |
| 188 | SQL_CLABEL_DATA label_data; |
| 189 | |
| 190 | while (sqlite3_step_monitored(res) == SQLITE_ROW) { |
| 191 | label_data.label_key = (char *) sqlite3_column_text(res, 0); |
| 192 | label_data.label_value = (char *) sqlite3_column_text(res, 1); |
| 193 | label_data.label_source = sqlite3_column_int(res, 2); |
| 194 | dict_cb(&label_data, data); |
| 195 | } |
| 196 | |
| 197 | done: |
| 198 | REPORT_BIND_FAIL(res, param); |
| 199 | SQLITE_FINALIZE(res); |
| 200 | } |
| 201 | |
| 202 | // CONTEXT LIST |
| 203 | #define CTX_GET_CONTEXT_LIST "SELECT id, version, title, chart_type, unit, priority, first_time_t, " \ |
| 204 | "last_time_t, deleted, family FROM context c WHERE c.host_id = @host_id" |
| 205 | |
| 206 | void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEXT_DATA *, void *), void *data) |
| 207 | { |
| 208 | if (unlikely(!host_uuid)) |
| 209 | return; |
| 210 | |
| 211 | sqlite3_stmt *res = NULL; |
| 212 | |
| 213 | if (!PREPARE_STATEMENT(db_context_thread ? db_context_thread : db_context_meta, CTX_GET_CONTEXT_LIST, &res)) |
| 214 | return; |
| 215 | |
| 216 | VERSIONED_CONTEXT_DATA context_data = {0}; |
| 217 | |
| 218 | int param = 0; |
| 219 | SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC)); |
| 220 | param = 0; |
| 221 | |
| 222 | while (sqlite3_step_monitored(res) == SQLITE_ROW) { |
| 223 | if (unlikely(exit_initiated_get())) |
| 224 | break; |
| 225 | |
| 226 | context_data.id = (char *) sqlite3_column_text(res, 0); |
| 227 | context_data.version = sqlite3_column_int64(res, 1); |
| 228 | context_data.title = (char *) sqlite3_column_text(res, 2); |
| 229 | context_data.chart_type = (char *) sqlite3_column_text(res, 3); |
| 230 | context_data.units = (char *) sqlite3_column_text(res, 4); |
| 231 | context_data.priority = sqlite3_column_int64(res, 5); |
| 232 | context_data.first_time_s = sqlite3_column_int64(res, 6); |
| 233 | context_data.last_time_s = sqlite3_column_int64(res, 7); |
| 234 | context_data.deleted = sqlite3_column_int(res, 8); |
| 235 | context_data.family = (char *) sqlite3_column_text(res, 9); |
| 236 | dict_cb(&context_data, data); |
| 237 | } |
| 238 | |
| 239 | done: |
| 240 | REPORT_BIND_FAIL(res, param); |
| 241 | SQLITE_FINALIZE(res); |
| 242 | } |
| 243 | |
| 244 | |
| 245 | // |
| 246 | // Storing Data |
| 247 | // |
| 248 | #define CTX_STORE_CONTEXT \ |
| 249 | "INSERT OR REPLACE INTO context " \ |
| 250 | "(host_id, id, version, title, chart_type, unit, priority, first_time_t, last_time_t, deleted, family) " \ |
| 251 | "VALUES (@host_id, @context, @version, @title, @chart_type, @unit, @priority, @first_t, @last_t, @delete, @family)" |
| 252 | |
| 253 | int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) |
| 254 | { |
| 255 | int rc_stored = 1; |
| 256 | sqlite3_stmt *res = NULL; |
| 257 | |
| 258 | if (unlikely(!host_uuid || !context_data || !context_data->id)) |
| 259 | return 0; |
| 260 | |
| 261 | if (!PREPARE_STATEMENT(db_context_meta ? db_context_meta : db_meta, CTX_STORE_CONTEXT, &res)) |
| 262 | return 1; |
| 263 | |
| 264 | int param = 0; |
| 265 | SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC)); |
| 266 | SQLITE_BIND_FAIL(done, bind_text_null(res, ++param, context_data->id, 0)); |
| 267 | SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (time_t) context_data->version)); |
| 268 | SQLITE_BIND_FAIL(done, bind_text_null(res, ++param, context_data->title, 0)); |
| 269 | SQLITE_BIND_FAIL(done, bind_text_null(res, ++param, context_data->chart_type, 0)); |
| 270 | SQLITE_BIND_FAIL(done, bind_text_null(res, ++param, context_data->units, 0)); |
| 271 | SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (time_t) context_data->priority)); |
| 272 | SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (time_t) context_data->first_time_s)); |
| 273 | SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (time_t) context_data->last_time_s)); |
| 274 | SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, context_data->deleted)); |
| 275 | SQLITE_BIND_FAIL(done, bind_text_null(res, ++param, context_data->family, 1)); |
| 276 | |
| 277 | param = 0; |
| 278 | rc_stored = execute_insert(res); |
| 279 | |
| 280 | if (rc_stored != SQLITE_DONE) |
| 281 | error_report("Failed store context details for context %s, rc = %d", context_data->id, rc_stored); |
| 282 | |
| 283 | done: |
| 284 | REPORT_BIND_FAIL(res, param); |
| 285 | SQLITE_FINALIZE(res); |
| 286 | return (rc_stored != SQLITE_DONE); |
| 287 | } |
| 288 | |
| 289 | // Delete a context |
| 290 | #define CTX_DELETE_CONTEXT "DELETE FROM context WHERE host_id = @host_id AND id = @context" |
| 291 | int ctx_delete_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data) |
| 292 | { |
| 293 | int rc_stored = 1; |
| 294 | sqlite3_stmt *res = NULL; |
| 295 | |
| 296 | if (unlikely(!context_data || !context_data->id)) |
| 297 | return 0; |
| 298 | |
| 299 | if (!PREPARE_STATEMENT(db_context_meta, CTX_DELETE_CONTEXT, &res)) |
| 300 | return 1; |
| 301 | |
| 302 | metadata_queue_ctx_host_cleanup(host_uuid, context_data->id); |
| 303 | |
| 304 | int param = 0; |
| 305 | SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC)); |
| 306 | SQLITE_BIND_FAIL(done, sqlite3_bind_text(res, ++param, context_data->id, -1, SQLITE_STATIC)); |
| 307 | |
| 308 | param = 0; |
| 309 | rc_stored = execute_insert(res); |
| 310 | |
| 311 | if (rc_stored != SQLITE_DONE) |
| 312 | error_report("Failed to delete context %s, rc = %d", context_data->id, rc_stored); |
| 313 | |
| 314 | done: |
| 315 | REPORT_BIND_FAIL(res, param); |
| 316 | SQLITE_FINALIZE(res); |
| 317 | return (rc_stored != SQLITE_DONE); |
| 318 | } |
| 319 | |
| 320 | int sql_context_cache_stats(int op) |
| 321 | { |
| 322 | int count, dummy; |
| 323 | |
| 324 | if (unlikely(!db_context_meta)) |
| 325 | return 0; |
| 326 | |
| 327 | sqlite3_db_status(db_context_meta, op, &count, &dummy, 0); |
| 328 | return count; |
| 329 | } |
| 330 | |
| 331 | |
| 332 | uint64_t sqlite_get_context_space(void) |
| 333 | { |
| 334 | return sqlite_get_db_space(db_context_meta); |
| 335 | } |
| 336 | |
| 337 | // |
| 338 | // TESTING FUNCTIONS |
| 339 | // |
| 340 | |
| 341 | static void dict_ctx_get_context_list_cb(VERSIONED_CONTEXT_DATA *context_data, void *data) |
| 342 | { |
| 343 | (void)data; |
| 344 | netdata_log_info(" Context id = %s " |
| 345 | "version = %"PRIu64" " |
| 346 | "title = %s " |
| 347 | "chart_type = %s " |
| 348 | "units = %s " |
| 349 | "priority = %"PRIu64" " |
| 350 | "first time = %"PRIu64" " |
| 351 | "last time = %"PRIu64" " |
| 352 | "deleted = %d " |
| 353 | "family = %s", |
| 354 | context_data->id, |
| 355 | context_data->version, |
| 356 | context_data->title, |
| 357 | context_data->chart_type, |
| 358 | context_data->units, |
| 359 | context_data->priority, |
| 360 | context_data->first_time_s, |
| 361 | context_data->last_time_s, |
| 362 | context_data->deleted, |
| 363 | context_data->family); |
| 364 | } |
| 365 | |
| 366 | int ctx_unittest(void) |
| 367 | { |
| 368 | nd_uuid_t host_uuid; |
| 369 | uuid_generate(host_uuid); |
| 370 | |
| 371 | if (sqlite_library_init()) |
| 372 | return 1; |
| 373 | |
| 374 | int rc = sql_init_context_database(1); |
| 375 | |
| 376 | if (rc != SQLITE_OK) |
| 377 | return 1; |
| 378 | |
| 379 | // Store a context |
| 380 | VERSIONED_CONTEXT_DATA context_data; |
| 381 | |
| 382 | context_data.id = strdupz("cpu.cpu"); |
| 383 | context_data.title = strdupz("TestContextTitle"); |
| 384 | context_data.units= strdupz("TestContextUnits"); |
| 385 | context_data.chart_type = strdupz("TestContextChartType"); |
| 386 | context_data.family = strdupz("TestContextFamily"); |
| 387 | context_data.priority = 50000; |
| 388 | context_data.deleted = 0; |
| 389 | context_data.first_time_s = 1657781000; |
| 390 | context_data.last_time_s = 1657781100; |
| 391 | context_data.version = now_realtime_usec(); |
| 392 | |
| 393 | if (likely(!ctx_store_context(&host_uuid, &context_data))) |
| 394 | netdata_log_info("Entry %s inserted", context_data.id); |
| 395 | else |
| 396 | netdata_log_info("Entry %s not inserted", context_data.id); |
| 397 | |
| 398 | if (likely(!ctx_store_context(&host_uuid, &context_data))) |
| 399 | netdata_log_info("Entry %s inserted", context_data.id); |
| 400 | else |
| 401 | netdata_log_info("Entry %s not inserted", context_data.id); |
| 402 | |
| 403 | // This will change end time |
| 404 | context_data.first_time_s = 1657781000; |
| 405 | context_data.last_time_s = 1657782001; |
| 406 | if (likely(!ctx_update_context(&host_uuid, &context_data))) |
| 407 | netdata_log_info("Entry %s updated", context_data.id); |
| 408 | else |
| 409 | netdata_log_info("Entry %s not updated", context_data.id); |
| 410 | netdata_log_info("List context start after insert"); |
| 411 | ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL); |
| 412 | netdata_log_info("List context end after insert"); |
| 413 | |
| 414 | // This will change start time |
| 415 | context_data.first_time_s = 1657782000; |
| 416 | context_data.last_time_s = 1657782001; |
| 417 | if (likely(!ctx_update_context(&host_uuid, &context_data))) |
| 418 | netdata_log_info("Entry %s updated", context_data.id); |
| 419 | else |
| 420 | netdata_log_info("Entry %s not updated", context_data.id); |
| 421 | |
| 422 | // This will list one entry |
| 423 | netdata_log_info("List context start after insert"); |
| 424 | ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL); |
| 425 | netdata_log_info("List context end after insert"); |
| 426 | |
| 427 | netdata_log_info("List context start after insert"); |
| 428 | ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL); |
| 429 | netdata_log_info("List context end after insert"); |
| 430 | |
| 431 | // This will delete the entry |
| 432 | if (likely(!ctx_delete_context(&host_uuid, &context_data))) |
| 433 | netdata_log_info("Entry %s deleted", context_data.id); |
| 434 | else |
| 435 | netdata_log_info("Entry %s not deleted", context_data.id); |
| 436 | |
| 437 | freez((void *)context_data.id); |
| 438 | freez((void *)context_data.title); |
| 439 | freez((void *)context_data.chart_type); |
| 440 | freez((void *)context_data.family); |
| 441 | freez((void *)context_data.units); |
| 442 | |
| 443 | // The list should be empty |
| 444 | netdata_log_info("List context start after delete"); |
| 445 | ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL); |
| 446 | netdata_log_info("List context end after delete"); |
| 447 | |
| 448 | sql_close_database(db_context_meta, "CONTEXT"); |
| 449 | sqlite_library_shutdown(); |
| 450 | |
| 451 | return 0; |
| 452 | } |