Added new --nedbtodb to transfer all NeDB records into the database.
Ylian Saint-Hilaire committed
Jan 4, 2021 at 16:26 UTC
8ac8953298b774a44d2efb03e7503c1bea3336c8
2 files changed
+105
-6
db.js
+95
-4
@@ -927,10 +927,10 @@ module.exports.CreateDB = function (parent, func) {
927
928
// Database actions on the events collection
929
obj.GetAllEvents = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.events', null, func); };
930
- obj.StoreEvent = function (event) {
930
+ obj.StoreEvent = function (event, func) {
931
var batchQuery = [['INSERT INTO meshcentral.events VALUE (?, ?, ?, ?, ?, ?, ?)', [null, event.time, ((typeof event.domain == 'string') ? event.domain : null), event.action, event.nodeid ? event.nodeid : null, event.userid ? event.userid : null, JSON.stringify(event)]]];
932
for (var i in event.ids) { if (event.ids[i] != '*') { batchQuery.push(['INSERT INTO meshcentral.eventids VALUE (LAST_INSERT_ID(), ?)', [event.ids[i]]]); } }
933
- sqlDbBatchExec(batchQuery, function (err, docs) { });
933
+ sqlDbBatchExec(batchQuery, function (err, docs) { if (func != null) { func(err, docs); } });
934
};
935
obj.GetEvents = function (ids, domain, func) {
936
if (ids.indexOf('*') >= 0) {
@@ -1101,7 +1101,7 @@ module.exports.CreateDB = function (parent, func) {
1101
1102
// Database actions on the events collection
1103
obj.GetAllEvents = function (func) { obj.eventsfile.find({}).toArray(func); };
1104
- obj.StoreEvent = function (event) { obj.eventsfile.insertOne(event); };
1104
+ obj.StoreEvent = function (event, func) { obj.eventsfile.insertOne(event, func); };
1105
obj.GetEvents = function (ids, domain, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
1106
obj.GetEventsWithLimit = function (ids, domain, limit, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
1107
obj.GetUserEvents = function (ids, domain, username, func) { obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
@@ -1249,7 +1249,7 @@ module.exports.CreateDB = function (parent, func) {
1249
1250
// Database actions on the events collection
1251
obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); };
1252
- obj.StoreEvent = function (event) { obj.eventsfile.insert(event); };
1252
+ obj.StoreEvent = function (event, func) { obj.eventsfile.insert(event, func); };
1253
obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } };
1254
obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } };
1255
obj.GetUserEvents = function (ids, domain, username, func) {
@@ -1657,6 +1657,97 @@ module.exports.CreateDB = function (parent, func) {
1657
}
1658
}
1659
1660
+ // Transfer NeDB data into the current database
1661
+ obj.nedbtodb = function (func) {
1662
+ var nedbDatastore = require('nedb');
1663
+ var datastoreOptions = { filename: parent.getConfigFilePath('meshcentral.db'), autoload: true };
1664
+
1665
+ // If a DB encryption key is provided, perform database encryption
1666
+ if ((typeof parent.args.dbencryptkey == 'string') && (parent.args.dbencryptkey.length != 0)) {
1667
+ // Hash the database password into a AES256 key and setup encryption and decryption.
1668
+ var nedbKey = parent.crypto.createHash('sha384').update(parent.args.dbencryptkey).digest('raw').slice(0, 32);
1669
+ datastoreOptions.afterSerialization = function (plaintext) {
1670
+ const iv = parent.crypto.randomBytes(16);
1671
+ const aes = parent.crypto.createCipheriv('aes-256-cbc', nedbKey, iv);
1672
+ var ciphertext = aes.update(plaintext);
1673
+ ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
1674
+ return ciphertext.toString('base64');
1675
+ }
1676
+ datastoreOptions.beforeDeserialization = function (ciphertext) {
1677
+ const ciphertextBytes = Buffer.from(ciphertext, 'base64');
1678
+ const iv = ciphertextBytes.slice(0, 16);
1679
+ const data = ciphertextBytes.slice(16);
1680
+ const aes = parent.crypto.createDecipheriv('aes-256-cbc', nedbKey, iv);
1681
+ var plaintextBytes = Buffer.from(aes.update(data));
1682
+ plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
1683
+ return plaintextBytes.toString();
1684
+ }
1685
+ }
1686
+
1687
+ // Setup all NeDB collections
1688
+ var nedbfile = new nedbDatastore(datastoreOptions);
1689
+ var nedbeventsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 });
1690
+ var nedbpowerfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 });
1691
+ var nedbserverstatsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 });
1692
+
1693
+ // Transfered record counts
1694
+ var normalRecordsTransferCount = 0;
1695
+ var eventRecordsTransferCount = 0;
1696
+ var powerRecordsTransferCount = 0;
1697
+ var statsRecordsTransferCount = 0;
1698
+ var pendingTransfer = 0;
1699
+
1700
+ // Transfer the data from main database
1701
+ nedbfile.find({}, function (err, docs) {
1702
+ if ((err == null) && (docs.length > 0)) {
1703
+ performTypedRecordDecrypt(docs)
1704
+ for (var i in docs) {
1705
+ pendingTransfer++;
1706
+ normalRecordsTransferCount++;
1707
+ obj.Set(common.unEscapeLinksFieldName(docs[i]), function () { pendingTransfer--; });
1708
+ }
1709
+ }
1710
+
1711
+ // Transfer events
1712
+ nedbeventsfile.find({}, function (err, docs) {
1713
+ if ((err == null) && (docs.length > 0)) {
1714
+ for (var i in docs) {
1715
+ pendingTransfer++;
1716
+ eventRecordsTransferCount++;
1717
+ obj.StoreEvent(docs[i], function () { pendingTransfer--; });
1718
+ }
1719
+ }
1720
+
1721
+ // Transfer power events
1722
+ nedbpowerfile.find({}, function (err, docs) {
1723
+ if ((err == null) && (docs.length > 0)) {
1724
+ for (var i in docs) {
1725
+ pendingTransfer++;
1726
+ powerRecordsTransferCount++;
1727
+ obj.storePowerEvent(docs[i], null, function () { pendingTransfer--; });
1728
+ }
1729
+ }
1730
+
1731
+ // Transfer server stats
1732
+ nedbserverstatsfile.find({}, function (err, docs) {
1733
+ if ((err == null) && (docs.length > 0)) {
1734
+ for (var i in docs) {
1735
+ pendingTransfer++;
1736
+ statsRecordsTransferCount++;
1737
+ obj.SetServerStats(docs[i], function () { pendingTransfer--; });
1738
+ }
1739
+ }
1740
+
1741
+ // Only exit when all the records are stored.
1742
+ setInterval(function () {
1743
+ if (pendingTransfer == 0) { func("Done. " + normalRecordsTransferCount + " record(s), " + eventRecordsTransferCount + " event(s), " + powerRecordsTransferCount + " power change(s), " + statsRecordsTransferCount + " stat(s)."); }
1744
+ }, 200)
1745
+ });
1746
+ });
1747
+ });
1748
+ });
1749
+ }
1750
+
1751
function padNumber(number, digits) { return Array(Math.max(digits - String(number).length + 1, 0)).join(0) + number; }
1752
1753
// Called when a node has changed
meshcentral.js
+10
-2
@@ -138,7 +138,7 @@ function CreateMeshCentralServer(config, args) {
138
try { require('./pass').hash('test', function () { }, 0); } catch (e) { console.log('Old version of node, must upgrade.'); return; } // TODO: Not sure if this test works or not.
139
140
// Check for invalid arguments
141
- var validArguments = ['_', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email', 'configfile', 'maintenancemode'];
141
+ var validArguments = ['_', 'user', 'port', 'aliasport', 'mpsport', 'mpsaliasport', 'redirport', 'rediraliasport', 'cert', 'mpscert', 'deletedomain', 'deletedefaultdomain', 'showall', 'showusers', 'showitem', 'listuserids', 'showusergroups', 'shownodes', 'showallmeshes', 'showmeshes', 'showevents', 'showsmbios', 'showpower', 'clearpower', 'showiplocations', 'help', 'exactports', 'xinstall', 'xuninstall', 'install', 'uninstall', 'start', 'stop', 'restart', 'debug', 'filespath', 'datapath', 'noagentupdate', 'launch', 'noserverbackup', 'mongodb', 'mongodbcol', 'wanonly', 'lanonly', 'nousers', 'mpspass', 'ciralocalfqdn', 'dbexport', 'dbexportmin', 'dbimport', 'dbmerge', 'dbfix', 'dbencryptkey', 'selfupdate', 'tlsoffload', 'userallowedip', 'userblockedip', 'swarmallowedip', 'agentallowedip', 'agentblockedip', 'fastcert', 'swarmport', 'logintoken', 'logintokenkey', 'logintokengen', 'mailtokengen', 'admin', 'unadmin', 'sessionkey', 'sessiontime', 'minify', 'minifycore', 'dblistconfigfiles', 'dbshowconfigfile', 'dbpushconfigfiles', 'dbpullconfigfiles', 'dbdeleteconfigfiles', 'vaultpushconfigfiles', 'vaultpullconfigfiles', 'vaultdeleteconfigfiles', 'configkey', 'loadconfigfromdb', 'npmpath', 'serverid', 'recordencryptionrecode', 'vault', 'token', 'unsealkey', 'name', 'log', 'dbstats', 'translate', 'createaccount', 'resetaccount', 'pass', 'adminaccount', 'removeaccount', 'domain', 'email', 'configfile', 'maintenancemode', 'nedbtodb'];
142
for (var arg in obj.args) { obj.args[arg.toLocaleLowerCase()] = obj.args[arg]; if (validArguments.indexOf(arg.toLocaleLowerCase()) == -1) { console.log('Invalid argument "' + arg + '", use --help.'); return; } }
143
if (obj.args.mongodb == true) { console.log('Must specify: --mongodb [connectionstring] \r\nSee https://docs.mongodb.com/manual/reference/connection-string/ for MongoDB connection string.'); return; }
144
for (i in obj.config.settings) { obj.args[i] = obj.config.settings[i]; } // Place all settings into arguments, arguments have already been placed into settings so arguments take precedence.
@@ -783,6 +783,14 @@ function CreateMeshCentralServer(config, args) {
783
return;
784
}
785
786
+ // Import NeDB data into database
787
+ if (obj.args.nedbtodb) {
788
+ if (db.databaseType == 1) { console.log("NeDB is current database, can't perform transfer."); process.exit(); return; }
789
+ console.log("Transfering NeDB data into database...");
790
+ db.nedbtodb(function (msg) { console.log(msg); process.exit(); })
791
+ return;
792
+ }
793
+
794
// Show a list of all configuration files in the database
795
if (obj.args.dblistconfigfiles) {
796
obj.db.GetAllType('cfile', function (err, docs) { if (err == null) { if (docs.length == 0) { console.log("No files found."); } else { for (var i in docs) { console.log(docs[i]._id.split('/')[1] + ', ' + Buffer.from(docs[i].data, 'base64').length + ' bytes.'); } } } else { console.log('Unable to read from database.'); } process.exit(); }); return;
@@ -2891,7 +2899,7 @@ function mainStart() {
2899
}
2900
2901
// Build the list of required modules
2894
- var modules = ['ws', 'cbor@~5.2.0', 'nedb', 'https', 'yauzl', 'xmldom', 'ipcheck', 'express', 'archiver@4.0.2', 'multiparty', 'node-forge', 'express-ws', 'compression', 'body-parser', 'connect-redis', 'cookie-session', 'express-handlebars'];
2902
+ var modules = ['ws', 'cbor@5.2.0', 'nedb', 'https', 'yauzl', 'xmldom', 'ipcheck', 'express', 'archiver@4.0.2', 'multiparty', 'node-forge', 'express-ws', 'compression', 'body-parser', 'connect-redis', 'cookie-session', 'express-handlebars'];
2903
if (require('os').platform() == 'win32') { modules.push('node-windows@0.1.14'); if (sspi == true) { modules.push('node-sspi'); } } // Add Windows modules
2904
if (ldap == true) { modules.push('ldapauth-fork'); }
2905
if (mstsc == true) { modules.push('node-rdpjs-2'); }