Fix autobackup defaults and zip level for performance (#6518)
* Fix autobackup defaults and zip level for performance * Add zipcompression configuration option
PTR committed
Nov 10, 2024 at 14:49 UTC
b71c69e81dcb92abeff3aaab8236cc773f7a83a8
4 files changed
+19
-23
db.js
+10
-18
@@ -3434,7 +3434,8 @@ module.exports.CreateDB = function (parent, func) {
3434
obj.performBackup = function (func) {
3435
parent.debug('db','Entering performBackup');
3436
try {
3437
- if (obj.performingBackup) return 1;
3437
+ if (obj.performingBackup) return 'Backup alreay in progress.';
3438
+ if (parent.config.settings.autobackup.backupintervalhours == -1) { if (func) { func('Unable to create backup if backuppath is set to the data folder.'); return 'Backup aborted.' }};
3439
obj.performingBackup = true;
3440
let backupPath = parent.backuppath;
3441
let dataPath = parent.datapath;
@@ -3446,11 +3447,10 @@ module.exports.CreateDB = function (parent, func) {
3447
obj.newAutoBackupFile = path.join(backupPath, ((typeof parent.config.settings.autobackup.backupname == 'string') ? parent.config.settings.autobackup.backupname : 'meshcentral-autobackup-') + fileSuffix + '.zip');
3448
3449
if ((obj.databaseType == DB_MONGOJS) || (obj.databaseType == DB_MONGODB)) {
3449
- // Perform a MongoDump in the datadir
3450
+ // Perform a MongoDump
3451
const dbname = (parent.args.mongodbname) ? (parent.args.mongodbname) : 'meshcentral';
3452
const dburl = parent.args.mongodb;
3453
3453
- //const obj.newDBDumpFile = 'mongodump-' + fileSuffix;
3454
obj.newDBDumpFile = path.join(backupPath, (dbname + '-mongodump-' + fileSuffix + '.archive'));
3455
3456
var cmd = buildMongoDumpCommand();
@@ -3490,16 +3490,6 @@ module.exports.CreateDB = function (parent, func) {
3490
} else if (obj.databaseType == DB_SQLITE) {
3491
//.db3 suffix to escape escape backupfile glob to exclude the sqlite db files
3492
obj.newDBDumpFile = path.join(backupPath, databaseName + '-sqlitedump-' + fileSuffix + '.db3');
3493
- /*undocumented in node-sqlite3 API, check https://github.com/TryGhost/node-sqlite3/blob/593c9d498be2510d286349134537e3bf89401c4a/test/backup.test.js
3494
- var backup = obj.file.backup(obj.newDBDumpFile);
3495
- backup.step(-1, function (err) {
3496
- if (err) { console.log('SQLite start-backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP; obj.createBackupfile(func); };
3497
- backup.finish(function (err) {
3498
- if (err) { console.log('SQLite backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP;};
3499
- obj.createBackupfile(func);
3500
- });
3501
- });
3502
- */
3493
// do a VACUUM INTO in favor of the backup API to compress the export, see https://www.sqlite.org/backup.html
3494
obj.file.exec('VACUUM INTO \'' + obj.newDBDumpFile + '\'', function (err) {
3495
if (err) { console.log('SQLite start-backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP;};
@@ -3529,19 +3519,21 @@ module.exports.CreateDB = function (parent, func) {
3519
obj.createBackupfile(func);
3520
}
3521
} catch (ex) { console.log(ex); };
3532
- return(0);
3522
+ return 'Starting auto-backup...';
3523
};
3524
3525
obj.createBackupfile = function(func) {
3526
parent.debug('db', 'Entering createFileBackup');
3527
let archiver = require('archiver');
3528
let archive = null;
3529
+ let zipLevel = Math.min(Math.max(Number(parent.config.settings.autobackup.zipcompression ? parent.config.settings.autobackup.zipcompression : 5),1),9);
3530
+
3531
//if password defined, create encrypted zip
3532
if (parent.config.settings.autobackup && (typeof parent.config.settings.autobackup.zippassword == 'string')) {
3533
try {
3534
//Only register format once, otherwise it triggers an error
3535
if (archiver.isRegisteredFormat('zip-encrypted') == false) { archiver.registerFormat('zip-encrypted', require('archiver-zip-encrypted')); }
3544
- archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword });
3536
+ archive = archiver.create('zip-encrypted', { zlib: { level: zipLevel }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword });
3537
if (func) { func('Creating encrypted ZIP'); }
3538
} catch (ex) { // registering encryption failed, do not fall back to non-encrypted, fail backup and skip old backup removal as a precaution to not lose any backups
3539
obj.backupStatus |= BACKUPFAIL_ZIPMODULE;
@@ -3550,7 +3542,7 @@ module.exports.CreateDB = function (parent, func) {
3542
}
3543
} else {
3544
if (func) { func('Creating a NON-ENCRYPTED ZIP'); }
3553
- archive = archiver('zip', { zlib: { level: 9 } });
3545
+ archive = archiver('zip', { zlib: { level: zipLevel } });
3546
}
3547
3548
//original behavior, just a filebackup if dbdump fails : (obj.backupStatus == 0 || obj.backupStatus == BACKUPFAIL_DBDUMP)
@@ -3628,14 +3620,14 @@ module.exports.CreateDB = function (parent, func) {
3620
3621
let globIgnoreFiles;
3622
//slice in case exclusion gets pushed
3631
- globIgnoreFiles = parent.config.settings.autobackup.backupignorefilesglob.slice();
3623
+ globIgnoreFiles = parent.config.settings.autobackup.backupignorefilesglob ? parent.config.settings.autobackup.backupignorefilesglob.slice() : [];
3624
if (parent.config.settings.sqlite3) { globIgnoreFiles.push (datapathFoldername + '/' + databaseName + '.sqlite*'); }; //skip sqlite database file, and temp files with ext -journal, -wal & -shm
3625
//archiver.glob doesn't seem to use the third param, archivesubdir. Bug?
3626
//workaround: go up a dir and add data dir explicitly to keep the zip tidy
3627
archive.glob((datapathFoldername + '/**'), {
3628
cwd: datapathParentPath,
3629
ignore: globIgnoreFiles,
3638
- skip: parent.config.settings.autobackup.backupskipfoldersglob
3630
+ skip: (parent.config.settings.autobackup.backupskipfoldersglob ? parent.config.settings.autobackup.backupskipfoldersglob : [])
3631
});
3632
3633
if (parent.config.settings.autobackup.backupwebfolders) {
meshcentral-config-schema.json
+5
@@ -851,6 +851,11 @@
851
"default": 10,
852
"description": "How many days of backups should the autobackup keep? Default is 10 Days worth"
853
},
854
+ "zipCompression" : {
855
+ "type": "integer",
856
+ "default": "5",
857
+ "description": "Set the zip compression level, 1=fast/less small file to 9=slow/smallest file."
858
+ },
859
"zipPassword": {
860
"type": "string",
861
"default": "",
meshcentral.js
+3
-3
@@ -2089,7 +2089,7 @@ function CreateMeshCentralServer(config, args) {
2089
obj.updateServerState('state', "running");
2090
2091
// Setup auto-backup defaults
2092
- if (obj.config.settings.autobackup == null || obj.config.settings.autobackup == false || obj.config.settings.autobackup == 'false') { delete obj.config.settings.autobackup; }
2092
+ if (obj.config.settings.autobackup == null || obj.config.settings.autobackup == false || obj.config.settings.autobackup == 'false') { obj.config.settings.autobackup = {backupintervalhours: 0}; } //no schedule, but able to console autobackup
2093
else {
2094
if (obj.config.settings.autobackup === true) {obj.config.settings.autobackup = {backupintervalhours: 24, keeplastdaysbackup: 10}; };
2095
if (typeof obj.config.settings.autobackup.backupintervalhours != 'number') { obj.config.settings.autobackup.backupintervalhours = 24; };
@@ -2104,7 +2104,7 @@ function CreateMeshCentralServer(config, args) {
2104
// Check that autobackup path is not within the "meshcentral-data" folder.
2105
if ((typeof obj.config.settings.autobackup == 'object') && (typeof obj.config.settings.autobackup.backuppath == 'string') && (obj.path.normalize(obj.config.settings.autobackup.backuppath).startsWith(obj.path.normalize(obj.datapath)))) {
2106
addServerWarning("Backup path can't be set within meshcentral-data folder, backup settings ignored.", 21);
2107
- delete obj.config.settings.autobackup;
2107
+ obj.config.settings.autobackup = {backupintervalhours: -1}; //block console autobackup
2108
}
2109
2110
// Load Intel AMT passwords from the "amtactivation.log" file
@@ -2267,7 +2267,7 @@ function CreateMeshCentralServer(config, args) {
2267
2268
// Check if we need to perform an automatic backup
2269
function checkAutobackup() {
2270
- if (obj.config.settings.autobackup && (typeof obj.config.settings.autobackup.backupintervalhours == 'number')) {
2270
+ if (obj.config.settings.autobackup.backupintervalhours >= 1) {
2271
obj.db.Get('LastAutoBackupTime', function (err, docs) {
2272
if (err != null) return;
2273
var lastBackup = 0;
meshuser.js
+1
-2
@@ -7627,10 +7627,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
7627
}
7628
7629
function serverUserCommandAutoBackup(cmdData) {
7630
- var backupResult = parent.db.performBackup(function (msg) {
7630
+ cmdData.result = parent.db.performBackup(function (msg) {
7631
try { ws.send(JSON.stringify({ action: 'serverconsole', value: msg, tag: cmdData.command.tag })); } catch (ex) { }
7632
});
7633
- if (backupResult == 0) { cmdData.result = 'Starting auto-backup...'; } else { cmdData.result = 'Backup alreay in progress.'; }
7633
}
7634
7635
function serverUserCommandBackupConfig(cmdData) {