Added AceBase maintenance. AceBase record removal seems to be terribly slow.
Ylian Saint-Hilaire committed
Aug 14, 2022 at 13:24 UTC
0fd8b17f5e74007c357a40ca917343325589ba8c
2 files changed
+23
-25
db.js
+23
-11
@@ -114,8 +114,18 @@ module.exports.CreateDB = function (parent, func) {
114
sqlDbQuery('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))], function (doc, err) { }); // Delete events older than expirePowerSeconds
115
sqlDbQuery('DELETE FROM serverstats WHERE expire < ?', [new Date()], function (doc, err) { }); // Delete events where expiration date is in the past
116
sqlDbQuery('DELETE FROM smbios WHERE expire < ?', [new Date()], function (doc, err) { }); // Delete events where expiration date is in the past
117
+ } else if (obj.databaseType == 7) { // AceBase
118
+ //console.log('Performing AceBase maintenance');
119
+ // Delete older events. AceBase seems to be VERY slow as removing records. Here, we remove 100 records at a time only since so far, it's terribly slow.
120
+ // TODO: Fugure out what record removal in AceBase is so slow
121
+ obj.file.query('events').take(100).filter('time', '<', new Date(Date.now() - (expireEventsSeconds * 1000))).remove().then(function () {
122
+ obj.file.query('stats').take(100).filter('time', '<', new Date(Date.now() - (expireServerStatsSeconds * 1000))).remove().then(function () {
123
+ obj.file.query('power').take(100).filter('time', '<', new Date(Date.now() - (expirePowerEventsSeconds * 1000))).remove().then(function () {
124
+ //console.log('AceBase maintenance done');
125
+ });
126
+ });
127
+ });
128
}
118
-
129
obj.removeInactiveDevices();
130
}
131
@@ -237,7 +247,10 @@ module.exports.CreateDB = function (parent, func) {
247
// Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
248
if (obj.databaseType == 7) {
249
// AceBase
240
-
250
+ pendingCalls = 3;
251
+ obj.file.query('meshcentral').filter('domain', '==', domainName).remove().then(function () { if (--pendingCalls == 0) { func(); } });
252
+ obj.file.query('events').filter('domain', '==', domainName).remove().then(function () { if (--pendingCalls == 0) { func(); } });
253
+ obj.file.query('power').filter('domain', '==', domainName).remove().then(function () { if (--pendingCalls == 0) { func(); } });
254
} else if ((obj.databaseType == 4) || (obj.databaseType == 5) || (obj.databaseType == 6)) {
255
// MariaDB, MySQL or PostgreSQL
256
pendingCalls = 2;
@@ -263,10 +276,7 @@ module.exports.CreateDB = function (parent, func) {
276
// TODO: Remove all meshes that dont have any links
277
278
// Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
266
- if (obj.databaseType == 7) {
267
- // AceBase
268
-
269
- } else if ((obj.databaseType == 4) || (obj.databaseType == 5) || (obj.databaseType == 6)) {
279
+ if ((obj.databaseType == 4) || (obj.databaseType == 5) || (obj.databaseType == 6)) {
280
// MariaDB, MySQL or PostgreSQL
281
obj.RemoveAllOfType('event', function () { });
282
obj.RemoveAllOfType('power', function () { });
@@ -276,7 +286,7 @@ module.exports.CreateDB = function (parent, func) {
286
obj.file.deleteMany({ type: 'event' }, { multi: true });
287
obj.file.deleteMany({ type: 'power' }, { multi: true });
288
obj.file.deleteMany({ type: 'smbios' }, { multi: true });
279
- } else {
289
+ } else if ((obj.databaseType == 1) || (obj.databaseType == 2)) {
290
// NeDB or MongoJS
291
obj.file.remove({ type: 'event' }, { multi: true });
292
obj.file.remove({ type: 'power' }, { multi: true });
@@ -661,25 +671,27 @@ module.exports.CreateDB = function (parent, func) {
671
obj.file = new AceBase('meshcentral', { sponsor: ((typeof parent.args.acebase == 'object') && (parent.args.acebase.sponsor)), logLevel: 'error', storage: { path: parent.datapath } });
672
// Get all the databases ready
673
obj.file.ready(function () {
674
+ // Create AceBase indexes
675
obj.file.indexes.create('meshcenral', 'type', { include: ['domain', 'meshid'] });
676
obj.file.indexes.create('meshcenral', 'email');
677
obj.file.indexes.create('meshcenral', 'meshid');
678
obj.file.indexes.create('meshcenral', 'intelamt.uuid');
679
obj.file.indexes.create('events', 'userid', { include: ['action'] });
669
- obj.file.indexes.create('events', 'domain', { include: ['nodeid','time'] });
670
- obj.file.indexes.create('events', 'ids', { include: ['time'] });
680
+ obj.file.indexes.create('events', 'domain', { include: ['nodeid', 'time'] });
681
+ obj.file.indexes.create('events', 'ids', { include: ['time'] });
682
obj.file.indexes.create('events', 'time');
683
obj.file.indexes.create('power', 'nodeid', { include: ['time'] });
684
obj.file.indexes.create('power', 'time');
685
obj.file.indexes.create('stats', 'time');
686
obj.file.indexes.create('stats', 'expire');
687
+ // Completed setup of AceBase
688
setupFunctions(func);
677
- }); // Completed setup of AceBase
689
+ });
690
} else if (parent.args.mariadb || parent.args.mysql) {
691
var connectinArgs = (parent.args.mariadb) ? parent.args.mariadb : parent.args.mysql;
692
var dbname = (connectinArgs.database != null) ? connectinArgs.database : 'meshcentral';
693
682
- // Including the db name in the connection obj will cause a connection failure if it does not exist
694
+ // Including the db name in the connection obj will cause a connection faliure if it does not exist
695
var connectionObject = Clone(connectinArgs);
696
delete connectionObject.database;
697
meshcentral.js
-14
@@ -802,20 +802,6 @@ function CreateMeshCentralServer(config, args) {
802
803
require('./db.js').CreateDB(obj,
804
function (db) {
805
-
806
-
807
- //db.Get('user//admin', function (err, docs) { console.log('GetResult', err, docs); });
808
- //db.Set({ _id: 'user//admin', type: 'user', domain: 'a', test: 'this is a user' }, function () { console.log('SetResult'); });
809
- //db.Get('user//admin', function (err, docs) { console.log('GetResult', err, docs); });
810
- //db.GetAll(function (err, docs) { console.log('GetAll', err, docs); });
811
- //db.GetAllTypeNoTypeField('user', 'a', function (err, docs) { console.log('GetAllTypeNoTypeField', err, docs); });
812
- //db.isMaxType(10, 'user', 'a', function (max, count) { console.log('yy', max, count); })
813
-
814
- //db.StoreEvent({ test: "this is an event" }, function () { console.log('event stored'); });
815
- //db.GetAllEvents(function (err, docs) { console.log('events', docs); });
816
-
817
- //return;
818
-
805
obj.db = db;
806
obj.db.SetupDatabase(function (dbversion) {
807
// See if any database operations needs to be completed