Add sqlite config options (#6517)
PTR committed
Nov 10, 2024 at 15:04 UTC
777eb53476d2036ac46415a5f2e6a3b51db85e81
2 files changed
+125
-22
db.js
+91
-20
@@ -39,6 +39,17 @@ module.exports.CreateDB = function (parent, func) {
39
let databaseName = 'meshcentral';
40
let datapathParentPath = path.dirname(parent.datapath);
41
let datapathFoldername = path.basename(parent.datapath);
42
+ const SQLITE_AUTOVACUUM = ['none', 'full', 'incremental'];
43
+ const SQLITE_SYNCHRONOUS = ['off', 'normal', 'full', 'extra'];
44
+ obj.sqliteConfig = {
45
+ maintenance: '',
46
+ startupVacuum: false,
47
+ autoVacuum: 'full',
48
+ incrementalVacuum: 100,
49
+ journalMode: 'delete',
50
+ journalSize: 4096000,
51
+ synchronous: 'full',
52
+ };
53
obj.performingBackup = false;
54
const BACKUPFAIL_ZIPCREATE = 0x0001;
55
const BACKUPFAIL_ZIPMODULE = 0x0010;
@@ -119,6 +130,7 @@ module.exports.CreateDB = function (parent, func) {
130
131
// Perform database maintenance
132
obj.maintenance = function () {
133
+ parent.debug('db', 'Entering database maintenance');
134
if (obj.databaseType == DB_NEDB) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
135
obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
136
obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
@@ -138,12 +150,21 @@ module.exports.CreateDB = function (parent, func) {
150
});
151
});
152
} else if (obj.databaseType == DB_SQLITE) { // SQLite3
141
- // TODO: Combine with others?
142
- sqlDbQuery('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))], function (doc, err) { });
143
- sqlDbQuery('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))], function (doc, err) { });
144
- sqlDbQuery('DELETE FROM serverstats WHERE expire < ?', [new Date()], function (doc, err) { });
145
- sqlDbQuery('DELETE FROM smbios WHERE expire < ?', [new Date()], function (doc, err) { });
146
- obj.file.run( 'PRAGMA optimize;' ); //see https://sqlite.org/pragma.html#pragma_optimize
153
+ //sqlite does not return rows affected for INSERT, UPDATE or DELETE statements, see https://www.sqlite.org/pragma.html#pragma_count_changes
154
+ obj.file.serialize(function () {
155
+ obj.file.run('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))]);
156
+ obj.file.run('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))]);
157
+ obj.file.run('DELETE FROM serverstats WHERE expire < ?', [new Date()]);
158
+ obj.file.run('DELETE FROM smbios WHERE expire < ?', [new Date()]);
159
+ obj.file.exec(obj.sqliteConfig.maintenance, function (err) {
160
+ if (err) {console.log('Maintenance error: ' + err.message)};
161
+ if (parent.config.settings.debug) {
162
+ sqliteGetPragmas(['freelist_count', 'page_size', 'page_count', 'cache_size' ], function (pragma, pragmaValue) {
163
+ parent.debug('db', 'SQLite Maintenance: ' + pragma + '=' + pragmaValue);
164
+ });
165
+ };
166
+ });
167
+ });
168
}
169
obj.removeInactiveDevices();
170
}
@@ -742,13 +763,29 @@ module.exports.CreateDB = function (parent, func) {
763
// SQLite3 database setup
764
obj.databaseType = DB_SQLITE;
765
const sqlite3 = require('sqlite3');
745
- if (typeof parent.config.settings.sqlite3 == 'string') {databaseName = parent.config.settings.sqlite3};
746
- //use sqlite3 cache mode https://github.com/TryGhost/node-sqlite3/wiki/Caching#caching
747
- obj.file = new sqlite3.cached.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
766
+ let configParams = parent.config.settings.sqlite3;
767
+ if (typeof configParams == 'string') {databaseName = configParams} else {databaseName = configParams.name ? configParams.name : 'meshcentral';};
768
+ obj.sqliteConfig.startupVacuum = configParams.startupvacuum ? configParams.startupvacuum : false;
769
+ obj.sqliteConfig.autoVacuum = configParams.autovacuum ? configParams.autovacuum.toLowerCase() : 'incremental';
770
+ obj.sqliteConfig.incrementalVacuum = configParams.incrementalvacuum ? configParams.incrementalvacuum : 100;
771
+ obj.sqliteConfig.journalMode = configParams.journalmode ? configParams.journalmode.toLowerCase() : 'delete';
772
+ //allowed modes, 'none' excluded because not usefull for this app, maybe also remove 'memory'?
773
+ if (!(['delete', 'truncate', 'persist', 'memory', 'wal'].includes(obj.sqliteConfig.journalMode))) { obj.sqliteConfig.journalMode = 'delete'};
774
+ obj.sqliteConfig.journalSize = configParams.journalsize ? configParams.journalsize : 409600;
775
+ //wal can use the more performant 'normal' mode, see https://www.sqlite.org/pragma.html#pragma_synchronous
776
+ obj.sqliteConfig.synchronous = (obj.sqliteConfig.journalMode == 'wal') ? 'normal' : 'full';
777
+ if (obj.sqliteConfig.journalMode == 'wal') {obj.sqliteConfig.maintenance += 'PRAGMA wal_checkpoint(PASSIVE);'};
778
+ if (obj.sqliteConfig.autoVacuum == 'incremental') {obj.sqliteConfig.maintenance += 'PRAGMA incremental_vacuum(' + obj.sqliteConfig.incrementalVacuum + ');'};
779
+ obj.sqliteConfig.maintenance += 'PRAGMA optimize;';
780
+
781
+ parent.debug('db', 'SQlite config options: ' + JSON.stringify(obj.sqliteConfig, null, 4));
782
+ if (obj.sqliteConfig.journalMode == 'memory') { console.log('[WARNING] journal_mode=memory: this can lead to database corruption if there is a crash during a transaction. See https://www.sqlite.org/pragma.html#pragma_journal_mode') };
783
+ //.cached not usefull
784
+ obj.file = new sqlite3.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
785
if (err && (err.code == 'SQLITE_CANTOPEN')) {
786
// Database needs to be created
787
obj.file = new sqlite3.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), function (err) {
751
- if (err) { console.log("SQLite Error: " + err); process.exit(1);; return; }
788
+ if (err) { console.log("SQLite Error: " + err); process.exit(1); }
789
obj.file.exec(`
790
CREATE TABLE main (id VARCHAR(256) PRIMARY KEY NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON);
791
CREATE TABLE events(id INTEGER PRIMARY KEY, time TIMESTAMP, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON);
@@ -771,23 +808,18 @@ module.exports.CreateDB = function (parent, func) {
808
CREATE INDEX ndxsmbiosexpire ON smbios (expire);
809
`, function (err) {
810
// Completed DB creation of SQLite3
774
- //WAL mode instead of roll-back/delete
775
- obj.file.run( 'PRAGMA journal_mode=WAL;' );
776
- //Together with the optimize in the maintenance run, see https://sqlite.org/pragma.html#pragma_optimize
777
- obj.file.run( 'PRAGMA optimize=0x10002;' );
811
+ sqliteSetOptions(func);
812
+ //setupFunctions could be put in the sqliteSetupOptions, but left after it for clarity
813
setupFunctions(func);
814
}
815
);
816
});
817
return;
783
- } else if (err) { console.log("SQLite Error: " + err); process.exit(0); return; }
818
+ } else if (err) { console.log("SQLite Error: " + err); process.exit(0); }
819
785
- // Completed setup of SQLite3
820
//for existing db's
787
- //WAL mode instead of roll-back/delete
788
- obj.file.run( 'PRAGMA journal_mode=WAL;' );
789
- //Together with the optimize in the maintenance run, see https://sqlite.org/pragma.html#pragma_optimize
790
- obj.file.run( 'PRAGMA optimize=0x10002;' );
821
+ sqliteSetOptions();
822
+ //setupFunctions could be put in the sqliteSetupOptions, but left after it for clarity
823
setupFunctions(func);
824
});
825
} else if (parent.args.acebase) {
@@ -1277,6 +1309,45 @@ module.exports.CreateDB = function (parent, func) {
1309
setupFunctions(func); // Completed setup of NeDB
1310
}
1311
1312
+ function sqliteSetOptions(func) {
1313
+ //get current auto_vacuum mode for comparison
1314
+ obj.file.get('PRAGMA auto_vacuum;', function(err, current){
1315
+ let pragma = 'PRAGMA journal_mode=' + obj.sqliteConfig.journalMode + ';' +
1316
+ 'PRAGMA synchronous='+ obj.sqliteConfig.synchronous + ';' +
1317
+ 'PRAGMA journal_size_limit=' + obj.sqliteConfig.journalSize + ';' +
1318
+ 'PRAGMA auto_vacuum=' + obj.sqliteConfig.autoVacuum + ';' +
1319
+ 'PRAGMA incremental_vacuum=' + obj.sqliteConfig.incrementalVacuum + ';' +
1320
+ 'PRAGMA optimize=0x10002;';
1321
+ //check new autovacuum mode, if changing from or to 'none', a VACUUM needs to be done to activate it. See https://www.sqlite.org/pragma.html#pragma_auto_vacuum
1322
+ if ( obj.sqliteConfig.startupVacuum
1323
+ || (current.auto_vacuum == 0 && obj.sqliteConfig.autoVacuum !='none')
1324
+ || (current.auto_vacuum != 0 && obj.sqliteConfig.autoVacuum =='none'))
1325
+ {
1326
+ pragma += 'VACUUM;';
1327
+ };
1328
+ parent.debug ('db', 'Config statement: ' + pragma);
1329
+
1330
+ obj.file.exec( pragma,
1331
+ function (err) {
1332
+ if (err) { parent.debug('db', 'Config pragma error: ' + (err.message)) };
1333
+ sqliteGetPragmas(['journal_mode', 'journal_size_limit', 'freelist_count', 'auto_vacuum', 'page_size', 'wal_autocheckpoint', 'synchronous'], function (pragma, pragmaValue) {
1334
+ parent.debug('db', 'PRAGMA: ' + pragma + '=' + pragmaValue);
1335
+ });
1336
+ });
1337
+ });
1338
+ //setupFunctions(func);
1339
+ }
1340
+
1341
+ function sqliteGetPragmas (pragmas, func){
1342
+ //pragmas can only be gotting one by one
1343
+ pragmas.forEach (function (pragma) {
1344
+ obj.file.get('PRAGMA ' + pragma + ';', function(err, res){
1345
+ if (pragma == 'auto_vacuum') { res[pragma] = SQLITE_AUTOVACUUM[res[pragma]] };
1346
+ if (pragma == 'synchronous') { res[pragma] = SQLITE_SYNCHRONOUS[res[pragma]] };
1347
+ if (func) { func (pragma, res[pragma]); }
1348
+ });
1349
+ });
1350
+ }
1351
// Create the PostgreSQL tables
1352
function postgreSqlCreateTables(func) {
1353
// Database was created, create the tables
meshcentral-config-schema.json
+34
-2
@@ -94,9 +94,41 @@
94
}
95
},
96
"sqlite3": {
97
- "type": [ "boolean", "string" ],
97
+ "type": [ "boolean", "string", "object" ],
98
"default": false,
99
- "description": "Set boolean true to use SQLite3 as a local MeshCentral database with default db filename 'meshcentral' or enter a string for a different db filename. Extension .sqlite is appended"
99
+ "description": "Set boolean true to use SQLite3 as a local MeshCentral database with default db filename 'meshcentral' or enter a string for a different db filename. Extension .sqlite is appended",
100
+ "properties":{
101
+ "name": {
102
+ "type": "string",
103
+ "default": "meshcentral",
104
+ "description": "Database filename. '.sqlite' is appended"
105
+ },
106
+ "journalMode": {
107
+ "type": "string",
108
+ "default": "delete",
109
+ "description": "DELETE, TRUNCATE, PERSIST, MEMORY, WAL. NONE not allowed. See: https://www.sqlite.org/pragma.html#pragma_journal_mode"
110
+ },
111
+ "journalSize": {
112
+ "type": "integer",
113
+ "default": 4096000,
114
+ "description": "Maximum size of the journal file in bytes. Can grow larger if needed, but will shrink to this size. -1 is unlimited growth. See: https://www.sqlite.org/pragma.html#pragma_journal_size_limit"
115
+ },
116
+ "autoVacuum": {
117
+ "type": "string",
118
+ "default": "incremental",
119
+ "description": "none, full, incremental. Removes unused pages and shrinks databasefile during maintenance. See: https://www.sqlite.org/pragma.html#pragma_auto_vacuum"
120
+ },
121
+ "incrementalVacuum": {
122
+ "type": "integer",
123
+ "default": 100,
124
+ "description": "Maximum amount of pages to free during maintenance. Default page size is 4k, so default frees up to 400k from the databasefile. See: https://www.sqlite.org/pragma.html#pragma_incremental_vacuum"
125
+ },
126
+ "startupVacuum": {
127
+ "type": "boolean",
128
+ "default": false,
129
+ "description": "Do a full VACUUM at startup. Shrinks the db file and optimizes it. This can take some time with a large database and can temporarily take up to double the database size on disk. See: https://www.sqlite.org/lang_vacuum.html"
130
+ }
131
+ }
132
},
133
"mySQL": {
134
"type": "object",