add s3 autobackup support (#6280)
Signed-off-by: si458 <simonsmith5521@gmail.com>
Simon Smith committed
Jul 29, 2024 at 14:41 UTC
6da92228718eb6a91ef0ffc81a897bc04354b3a3
4 files changed
+153
db.js
+93
@@ -3575,6 +3575,99 @@ module.exports.CreateDB = function (parent, func) {
3575
});
3576
});
3577
}
3578
+
3579
+ // S3 Backup
3580
+ if ((typeof parent.config.settings.autobackup == 'object') && (typeof parent.config.settings.autobackup.s3 == 'object')) {
3581
+ var s3folderName = 'MeshCentral-Backups';
3582
+ if (typeof parent.config.settings.autobackup.s3.foldername == 'string') { s3folderName = parent.config.settings.autobackup.s3.foldername; }
3583
+ // Construct the config object
3584
+ var accessKey = parent.config.settings.autobackup.s3.accesskey,
3585
+ secretKey = parent.config.settings.autobackup.s3.secretkey,
3586
+ endpoint = parent.config.settings.autobackup.s3.endpoint ? parent.config.settings.autobackup.s3.endpoint : 's3.amazonaws.com',
3587
+ port = parent.config.settings.autobackup.s3.port ? parent.config.settings.autobackup.s3.port : 443,
3588
+ useSsl = parent.config.settings.autobackup.s3.ssl ? parent.config.settings.autobackup.s3.ssl : true,
3589
+ bucketName = parent.config.settings.autobackup.s3.bucketname,
3590
+ pathPrefix = s3folderName,
3591
+ threshold = parent.config.settings.autobackup.s3.maxfiles ? parent.config.settings.autobackup.s3.maxfiles : 0,
3592
+ fileToUpload = filename;
3593
+ // Create a MinIO client
3594
+ const Minio = require('minio');
3595
+ var minioClient = new Minio.Client({
3596
+ endPoint: endpoint,
3597
+ port: port,
3598
+ useSSL: useSsl,
3599
+ accessKey: accessKey,
3600
+ secretKey: secretKey
3601
+ });
3602
+ // List objects in the specified bucket and path prefix
3603
+ var listObjectsPromise = new Promise(function(resolve, reject) {
3604
+ var items = [];
3605
+ var stream = minioClient.listObjects(bucketName, pathPrefix, true);
3606
+ stream.on('data', function(item) {
3607
+ if (!item.name.endsWith('/')) { // Exclude directories
3608
+ items.push(item);
3609
+ }
3610
+ });
3611
+ stream.on('end', function() {
3612
+ resolve(items);
3613
+ });
3614
+ stream.on('error', function(err) {
3615
+ reject(err);
3616
+ });
3617
+ });
3618
+ listObjectsPromise.then(function(objects) {
3619
+ // Count the number of files
3620
+ var fileCount = objects.length;
3621
+ // Return if no files to carry on uploading
3622
+ if (fileCount === 0) { return Promise.resolve(); }
3623
+ // Sort the files by LastModified date (oldest first)
3624
+ objects.sort(function(a, b) { return new Date(a.lastModified) - new Date(b.lastModified); });
3625
+ // Check if the threshold is zero and return if
3626
+ if (threshold === 0) { return Promise.resolve(); }
3627
+ // Check if the number of files exceeds the threshold (maxfiles) is 0
3628
+ if (fileCount >= threshold) {
3629
+ // Calculate how many files need to be deleted to make space for the new file
3630
+ var filesToDelete = fileCount - threshold + 1; // +1 to make space for the new file
3631
+ if (func) { func('Deleting ' + filesToDelete + ' older ' + (filesToDelete == 1 ? 'file' : 'files') + ' from S3 ...'); }
3632
+ // Create an array of promises for deleting files
3633
+ var deletePromises = objects.slice(0, filesToDelete).map(function(fileToDelete) {
3634
+ return new Promise(function(resolve, reject) {
3635
+ minioClient.removeObject(bucketName, fileToDelete.name, function(err) {
3636
+ if (err) {
3637
+ reject(err);
3638
+ } else {
3639
+ if (func) { func('Deleted file: ' + fileToDelete.name + ' from S3'); }
3640
+ resolve();
3641
+ }
3642
+ });
3643
+ });
3644
+ });
3645
+ // Wait for all deletions to complete
3646
+ return Promise.all(deletePromises);
3647
+ } else {
3648
+ return Promise.resolve(); // No deletion needed
3649
+ }
3650
+ }).then(function() {
3651
+ // Determine the upload path by combining the pathPrefix with the filename
3652
+ var fileName = require('path').basename(fileToUpload);
3653
+ var uploadPath = require('path').join(pathPrefix, fileName);
3654
+ // Upload a new file
3655
+ var uploadPromise = new Promise(function(resolve, reject) {
3656
+ if (func) { func('Uploading file ' + uploadPath + ' to S3'); }
3657
+ minioClient.fPutObject(bucketName, uploadPath, fileToUpload, function(err, etag) {
3658
+ if (err) {
3659
+ reject(err);
3660
+ } else {
3661
+ if (func) { func('Uploaded file: ' + uploadPath + ' to S3'); }
3662
+ resolve(etag);
3663
+ }
3664
+ });
3665
+ });
3666
+ return uploadPromise;
3667
+ }).catch(function(error) {
3668
+ if (func) { func('Error managing files in S3: ' + error); }
3669
+ });
3670
+ }
3671
}
3672
3673
// Transfer NeDB data into the current database
meshcentral-config-schema.json
+48
@@ -897,6 +897,54 @@
897
"description": "The maximum number of files to keep in the WebDAV folder, older files will be removed if needed."
898
}
899
}
900
+ },
901
+ "s3": {
902
+ "type": "object",
903
+ "description": "Enabled automated upload of the server backups to an S3 server.",
904
+ "required": [
905
+ "accessKey",
906
+ "secretKey",
907
+ "bucketName"
908
+ ],
909
+ "properties": {
910
+ "endpoint": {
911
+ "type": "string",
912
+ "default": "s3.amazonaws.com",
913
+ "description": "S3 Endpoint address e.g. myS3.myserver.com"
914
+ },
915
+ "accessKey": {
916
+ "type": "string",
917
+ "description": "S3 accessKey, Required"
918
+ },
919
+ "secretKey": {
920
+ "type": "string",
921
+ "description": "S3 secretKey, Required"
922
+ },
923
+ "port": {
924
+ "type": "integer",
925
+ "default": 443,
926
+ "description": "S3 Endpoint port number, Default is 443"
927
+ },
928
+ "ssl": {
929
+ "type": "boolean",
930
+ "default": true,
931
+ "description": "If \"true\", the S3 Endpoint will use \"https\", else it will use \"http\", Default is true"
932
+ },
933
+ "bucketName": {
934
+ "type": "string",
935
+ "description": "S3 Bucket Name, Required"
936
+ },
937
+ "folderName": {
938
+ "type": "string",
939
+ "default": "MeshCentral-Backups",
940
+ "description": "The name of the folder to create in the S3 Bucket. Defaults to \"MeshCentral-Backups\"."
941
+ },
942
+ "maxFiles": {
943
+ "type": "integer",
944
+ "default": 0,
945
+ "description": "The maximum number of files to keep in the S3 folder, older files will be removed if needed. Default is 0 which is keep everything."
946
+ }
947
+ }
948
}
949
}
950
},
meshcentral.js
+2
@@ -4101,6 +4101,8 @@ function mainStart() {
4101
if (typeof config.settings.autobackup.webdav == 'object') {
4102
if ((typeof config.settings.autobackup.webdav.url != 'string') || (typeof config.settings.autobackup.webdav.username != 'string') || (typeof config.settings.autobackup.webdav.password != 'string')) { addServerWarning("Missing WebDAV parameters.", 2, null, !args.launch); } else { modules.push('webdav@4.11.3'); }
4103
}
4104
+ // Enable S3 Support
4105
+ if (typeof config.settings.autobackup.s3 == 'object') { modules.push('minio@8.0.1'); }
4106
}
4107
4108
// Setup common password blocking
sample-config-advanced.json
+10
@@ -132,6 +132,16 @@
132
"password": "pass",
133
"folderName": "MeshCentral-Backups",
134
"maxFiles": 10
135
+ },
136
+ "_s3": {
137
+ "accessKey": "MYLONGACCESSKEY",
138
+ "secretKey": "MYLONGSECRETKEY",
139
+ "endpoint": "myS3.myserver.com",
140
+ "port": 9000,
141
+ "ssl": false,
142
+ "bucketName": "test",
143
+ "folderName": "MeshCentral-Backups",
144
+ "maxfiles": 10
145
}
146
},
147
"_redirects": {