Remove archivedcharts endpoint, optimize indices (#14296)
Remove undocumented archivedcharts endpoint. Use context endpoint instead Remove unused functions to lookup chart and dimension UUIDs Drop/Add new index for dimension and chart tables
Stelios Fragkakis committed
Jan 19, 2023 at 15:18 UTC
1934696c45fbcd35d453b5f026fc278bab40d5f7
4 files changed
+5
-337
database/sqlite/sqlite_functions.c
+5
-317
@@ -22,9 +22,8 @@ const char *database_config[] = {
22
"multiplier int, divisor int , algorithm int, options text);",
23
24
"CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);",
25
- "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);",
26
- "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);",
27
- "CREATE INDEX IF NOT EXISTS ind_c2 on chart (host_id, context);",
25
+ "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id);",
26
+ "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id);",
27
"CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "
28
"label_value text, date_created int, PRIMARY KEY (chart_id, label_key));",
29
"CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created);",
@@ -55,6 +54,9 @@ const char *database_cleanup[] = {
54
"DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);",
55
"DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host);",
56
"DROP TRIGGER IF EXISTS tr_dim_del;",
57
+ "DROP INDEX IF EXISTS ind_d1;",
58
+ "DROP INDEX IF EXISTS ind_c1;",
59
+ "DROP INDEX IF EXISTS ind_c2;",
60
NULL
61
};
62
@@ -504,211 +506,6 @@ skip:
506
return result;
507
}
508
507
-
508
-
509
-//
510
-// Support for archived charts (TO BE REMOVED)
511
-//
512
-#define SELECT_DIMENSION "select d.id, d.name from dimension d where d.chart_id = @chart_uuid;"
513
-
514
-static void sql_rrdim2json(sqlite3_stmt *res_dim, uuid_t *chart_uuid, BUFFER *wb, size_t *dimensions_count)
515
-{
516
- int rc;
517
-
518
- rc = sqlite3_bind_blob(res_dim, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
519
- if (rc != SQLITE_OK)
520
- return;
521
-
522
- int dimensions = 0;
523
- buffer_sprintf(wb, "\t\t\t\"dimensions\": {\n");
524
-
525
- while (sqlite3_step_monitored(res_dim) == SQLITE_ROW) {
526
- if (dimensions)
527
- buffer_strcat(wb, ",\n\t\t\t\t\"");
528
- else
529
- buffer_strcat(wb, "\t\t\t\t\"");
530
- buffer_strcat_jsonescape(wb, (const char *) sqlite3_column_text(res_dim, 0));
531
- buffer_strcat(wb, "\": { \"name\": \"");
532
- buffer_strcat_jsonescape(wb, (const char *) sqlite3_column_text(res_dim, 1));
533
- buffer_strcat(wb, "\" }");
534
- dimensions++;
535
- }
536
- *dimensions_count += dimensions;
537
- buffer_sprintf(wb, "\n\t\t\t}");
538
-}
539
-
540
-#define SELECT_CHART "select chart_id, id, name, type, family, context, title, priority, plugin, " \
541
- "module, unit, chart_type, update_every from chart " \
542
- "where host_id = @host_uuid and chart_id not in (select chart_id from chart_active) order by chart_id asc;"
543
-
544
-void sql_rrdset2json(RRDHOST *host, BUFFER *wb)
545
-{
546
- // time_t first_entry_t = 0; //= rrdset_first_entry_t(st);
547
- // time_t last_entry_t = 0; //rrdset_last_entry_t(st);
548
- static char *custom_dashboard_info_js_filename = NULL;
549
- int rc;
550
-
551
- sqlite3_stmt *res_chart = NULL;
552
- sqlite3_stmt *res_dim = NULL;
553
- time_t now = now_realtime_sec();
554
-
555
- rc = sqlite3_prepare_v2(db_meta, SELECT_CHART, -1, &res_chart, 0);
556
- if (unlikely(rc != SQLITE_OK)) {
557
- error_report("Failed to prepare statement to fetch host archived charts");
558
- return;
559
- }
560
-
561
- rc = sqlite3_bind_blob(res_chart, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
562
- if (unlikely(rc != SQLITE_OK)) {
563
- error_report("Failed to bind host parameter to fetch archived charts");
564
- goto failed;
565
- }
566
-
567
- rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION, -1, &res_dim, 0);
568
- if (unlikely(rc != SQLITE_OK)) {
569
- error_report("Failed to prepare statement to fetch chart archived dimensions");
570
- goto failed;
571
- };
572
-
573
- if(unlikely(!custom_dashboard_info_js_filename))
574
- custom_dashboard_info_js_filename = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", "");
575
-
576
- buffer_sprintf(wb, "{\n"
577
- "\t\"hostname\": \"%s\""
578
- ",\n\t\"version\": \"%s\""
579
- ",\n\t\"release_channel\": \"%s\""
580
- ",\n\t\"os\": \"%s\""
581
- ",\n\t\"timezone\": \"%s\""
582
- ",\n\t\"update_every\": %d"
583
- ",\n\t\"history\": %ld"
584
- ",\n\t\"memory_mode\": \"%s\""
585
- ",\n\t\"custom_info\": \"%s\""
586
- ",\n\t\"charts\": {"
587
- , rrdhost_hostname(host)
588
- , rrdhost_program_version(host)
589
- , get_release_channel()
590
- , rrdhost_os(host)
591
- , rrdhost_timezone(host)
592
- , host->rrd_update_every
593
- , host->rrd_history_entries
594
- , rrd_memory_mode_name(host->rrd_memory_mode)
595
- , custom_dashboard_info_js_filename
596
- );
597
-
598
- size_t c = 0;
599
- size_t dimensions = 0;
600
-
601
- while (sqlite3_step_monitored(res_chart) == SQLITE_ROW) {
602
- char id[512];
603
- sprintf(id, "%s.%s", sqlite3_column_text(res_chart, 3), sqlite3_column_text(res_chart, 1));
604
- RRDSET *st = rrdset_find(host, id);
605
- if (st && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))
606
- continue;
607
-
608
- if (c)
609
- buffer_strcat(wb, ",\n\t\t\"");
610
- else
611
- buffer_strcat(wb, "\n\t\t\"");
612
- c++;
613
-
614
- buffer_strcat(wb, id);
615
- buffer_strcat(wb, "\": ");
616
-
617
- buffer_sprintf(
618
- wb,
619
- "\t\t{\n"
620
- "\t\t\t\"id\": \"%s\",\n"
621
- "\t\t\t\"name\": \"%s\",\n"
622
- "\t\t\t\"type\": \"%s\",\n"
623
- "\t\t\t\"family\": \"%s\",\n"
624
- "\t\t\t\"context\": \"%s\",\n"
625
- "\t\t\t\"title\": \"%s (%s)\",\n"
626
- "\t\t\t\"priority\": %ld,\n"
627
- "\t\t\t\"plugin\": \"%s\",\n"
628
- "\t\t\t\"module\": \"%s\",\n"
629
- "\t\t\t\"enabled\": %s,\n"
630
- "\t\t\t\"units\": \"%s\",\n"
631
- "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
632
- "\t\t\t\"chart_type\": \"%s\",\n",
633
- id //sqlite3_column_text(res_chart, 1)
634
- ,
635
- id // sqlite3_column_text(res_chart, 2)
636
- ,
637
- sqlite3_column_text(res_chart, 3), sqlite3_column_text(res_chart, 4), sqlite3_column_text(res_chart, 5),
638
- sqlite3_column_text(res_chart, 6), id //sqlite3_column_text(res_chart, 2)
639
- ,
640
- (long ) sqlite3_column_int(res_chart, 7),
641
- (const char *) sqlite3_column_text(res_chart, 8) ? (const char *) sqlite3_column_text(res_chart, 8) : (char *) "",
642
- (const char *) sqlite3_column_text(res_chart, 9) ? (const char *) sqlite3_column_text(res_chart, 9) : (char *) "", (char *) "false",
643
- (const char *) sqlite3_column_text(res_chart, 10), id //sqlite3_column_text(res_chart, 2)
644
- ,
645
- rrdset_type_name(sqlite3_column_int(res_chart, 11)));
646
-
647
- sql_rrdim2json(res_dim, (uuid_t *) sqlite3_column_blob(res_chart, 0), wb, &dimensions);
648
-
649
- rc = sqlite3_reset(res_dim);
650
- if (unlikely(rc != SQLITE_OK))
651
- error_report("Failed to reset the prepared statement when reading archived chart dimensions");
652
- buffer_strcat(wb, "\n\t\t}");
653
- }
654
-
655
- buffer_sprintf(wb
656
- , "\n\t}"
657
- ",\n\t\"charts_count\": %zu"
658
- ",\n\t\"dimensions_count\": %zu"
659
- ",\n\t\"alarms_count\": %zu"
660
- ",\n\t\"rrd_memory_bytes\": %zu"
661
- ",\n\t\"hosts_count\": %zu"
662
- ",\n\t\"hosts\": ["
663
- , c
664
- , dimensions
665
- , (size_t) 0
666
- , (size_t) 0
667
- , rrdhost_hosts_available()
668
- );
669
-
670
- if(unlikely(rrdhost_hosts_available() > 1)) {
671
- rrd_rdlock();
672
-
673
- size_t found = 0;
674
- RRDHOST *h;
675
- rrdhost_foreach_read(h) {
676
- if(!rrdhost_should_be_removed(h, host, now) && !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED)) {
677
- buffer_sprintf(wb
678
- , "%s\n\t\t{"
679
- "\n\t\t\t\"hostname\": \"%s\""
680
- "\n\t\t}"
681
- , (found > 0) ? "," : ""
682
- , rrdhost_hostname(h)
683
- );
684
-
685
- found++;
686
- }
687
- }
688
-
689
- rrd_unlock();
690
- }
691
- else {
692
- buffer_sprintf(wb
693
- , "\n\t\t{"
694
- "\n\t\t\t\"hostname\": \"%s\""
695
- "\n\t\t}"
696
- , rrdhost_hostname(host)
697
- );
698
- }
699
-
700
- buffer_sprintf(wb, "\n\t]\n}\n");
701
-
702
- rc = sqlite3_finalize(res_dim);
703
- if (unlikely(rc != SQLITE_OK))
704
- error_report("Failed to finalize the prepared statement when reading archived chart dimensions");
705
-
706
-failed:
707
- rc = sqlite3_finalize(res_chart);
708
- if (unlikely(rc != SQLITE_OK))
709
- error_report("Failed to finalize the prepared statement when reading archived charts");
710
-}
711
-
509
void db_execute(const char *cmd)
510
{
511
int rc;
@@ -1268,112 +1065,3 @@ int sql_metadata_cache_stats(int op)
1065
netdata_thread_enable_cancelability();
1066
return count;
1067
}
1271
-
1272
-#define SQL_FIND_CHART_UUID \
1273
- "SELECT chart_id FROM chart WHERE host_id = @host AND type=@type AND id=@id AND (name IS NULL OR name=@name) AND chart_id IS NOT NULL;"
1274
-
1275
-#define SQL_FIND_DIMENSION_UUID \
1276
- "SELECT dim_id FROM dimension WHERE chart_id=@chart AND id=@id AND name=@name AND LENGTH(dim_id)=16;"
1277
-
1278
-
1279
-//Do a database lookup to find the UUID of a chart
1280
-//If found store it in store_uuid and return 0
1281
-int sql_find_chart_uuid(RRDHOST *host, RRDSET *st, uuid_t *store_uuid)
1282
-{
1283
- static __thread sqlite3_stmt *res = NULL;
1284
- int rc;
1285
-
1286
- const char *name = string2str(st->parts.name);
1287
-
1288
- if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1289
- return 1;
1290
-
1291
- if (unlikely(!res)) {
1292
- rc = prepare_statement(db_meta, SQL_FIND_CHART_UUID, &res);
1293
- if (rc != SQLITE_OK) {
1294
- error_report("Failed to prepare statement to lookup chart UUID in the database");
1295
- return 1;
1296
- }
1297
- }
1298
-
1299
- rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
1300
- if (unlikely(rc != SQLITE_OK))
1301
- goto bind_fail;
1302
-
1303
- rc = sqlite3_bind_text(res, 2, string2str(st->parts.type), -1, SQLITE_STATIC);
1304
- if (unlikely(rc != SQLITE_OK))
1305
- goto bind_fail;
1306
-
1307
- rc = sqlite3_bind_text(res, 3, string2str(st->parts.id), -1, SQLITE_STATIC);
1308
- if (unlikely(rc != SQLITE_OK))
1309
- goto bind_fail;
1310
-
1311
- rc = sqlite3_bind_text(res, 4, name && *name ? name : string2str(st->parts.id), -1, SQLITE_STATIC);
1312
- if (unlikely(rc != SQLITE_OK))
1313
- goto bind_fail;
1314
-
1315
- int status = 1;
1316
- rc = sqlite3_step_monitored(res);
1317
- if (likely(rc == SQLITE_ROW)) {
1318
- uuid_copy(*store_uuid, sqlite3_column_blob(res, 0));
1319
- status = 0;
1320
- }
1321
-
1322
- rc = sqlite3_reset(res);
1323
- if (unlikely(rc != SQLITE_OK))
1324
- error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc);
1325
-
1326
- return status;
1327
-
1328
-bind_fail:
1329
- error_report("Failed to bind input parameter to perform chart UUID database lookup, rc = %d", rc);
1330
- rc = sqlite3_reset(res);
1331
- if (unlikely(rc != SQLITE_OK))
1332
- error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc);
1333
- return 1;
1334
-}
1335
-
1336
-int sql_find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
1337
-{
1338
- static __thread sqlite3_stmt *res = NULL;
1339
- int rc;
1340
- int status = 1;
1341
-
1342
- if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1343
- return 1;
1344
-
1345
- if (unlikely(!res)) {
1346
- rc = prepare_statement(db_meta, SQL_FIND_DIMENSION_UUID, &res);
1347
- if (rc != SQLITE_OK) {
1348
- error_report("Failed to bind prepare statement to lookup dimension UUID in the database");
1349
- return 1;
1350
- }
1351
- }
1352
-
1353
- rc = sqlite3_bind_blob(res, 1, st->chart_uuid, sizeof(*st->chart_uuid), SQLITE_STATIC);
1354
- if (unlikely(rc != SQLITE_OK))
1355
- goto bind_fail;
1356
-
1357
- rc = sqlite3_bind_text(res, 2, rrddim_id(rd), -1, SQLITE_STATIC);
1358
- if (unlikely(rc != SQLITE_OK))
1359
- goto bind_fail;
1360
-
1361
- rc = sqlite3_bind_text(res, 3, rrddim_name(rd), -1, SQLITE_STATIC);
1362
- if (unlikely(rc != SQLITE_OK))
1363
- goto bind_fail;
1364
-
1365
- rc = sqlite3_step_monitored(res);
1366
- if (likely(rc == SQLITE_ROW)) {
1367
- uuid_copy(*store_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
1368
- status = 0;
1369
- }
1370
-
1371
- rc = sqlite3_reset(res);
1372
- if (unlikely(rc != SQLITE_OK))
1373
- error_report("Failed to reset statement find dimension uuid, rc = %d", rc);
1374
- return status;
1375
-
1376
-bind_fail:
1377
- error_report("Failed to bind input parameter to perform dimension UUID database lookup, rc = %d", rc);
1378
- return 1;
1379
-}
database/sqlite/sqlite_functions.h
-5
@@ -65,16 +65,11 @@ int get_host_id(uuid_t *node_id, uuid_t *host_id);
65
struct node_instance_list *get_node_list(void);
66
void sql_load_node_id(RRDHOST *host);
67
char *get_hostname_by_node_id(char *node_id);
68
-int sql_find_chart_uuid(RRDHOST *host, RRDSET *st, uuid_t *store_uuid);
69
-int sql_find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid);
68
69
// Help build archived hosts in memory when agent starts
70
void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);
71
DICTIONARY *sql_load_host_labels(uuid_t *host_id);
72
75
-// For queries: To be removed when context queries are implemented
76
-void sql_rrdset2json(RRDHOST *host, BUFFER *wb);
77
-
73
// TODO: move to metadata
74
int update_node_id(uuid_t *host_id, uuid_t *node_id);
75
web/api/web_api_v1.c
-13
@@ -561,18 +561,6 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
561
return HTTP_RESP_OK;
562
}
563
564
-inline int web_client_api_request_v1_archivedcharts(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
565
- (void)url;
566
-
567
- buffer_flush(w->response.data);
568
- w->response.data->contenttype = CT_APPLICATION_JSON;
569
-#ifdef ENABLE_DBENGINE
570
- if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
571
- sql_rrdset2json(host, w->response.data);
572
-#endif
573
- return HTTP_RESP_OK;
574
-}
575
-
564
inline int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url) {
565
return web_client_api_request_single_chart(host, w, url, rrd_stats_api_v1_chart);
566
}
@@ -1620,7 +1608,6 @@ static struct api_command {
1608
{ "charts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_charts },
1609
{ "context", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_context },
1610
{ "contexts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_contexts },
1623
- { "archivedcharts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_archivedcharts },
1611
1612
// registry checks the ACL by itself, so we allow everything
1613
{ "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry },
web/api/web_api_v1.h
-2
@@ -9,7 +9,6 @@
9
#include "web/api/health/health_cmdapi.h"
10
#include "web/api/queries/weights.h"
11
12
-#define MAX_CHART_LABELS_FILTER (32)
12
RRDR_OPTIONS web_client_api_request_v1_data_options(char *o);
13
void web_client_api_request_v1_data_options_to_buffer(BUFFER *wb, RRDR_OPTIONS options);
14
void web_client_api_request_v1_data_options_to_string(char *buf, size_t size, RRDR_OPTIONS options);
@@ -24,7 +23,6 @@ int web_client_api_request_single_chart(RRDHOST *host, struct web_client *w, cha
23
int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *w, char *url);
24
int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
25
int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
27
-int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url);
26
int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
27
int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
28
int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);