Added MongoDB bulk remove for added performance.
Ylian Saint-Hilaire committed
Jan 12, 2021 at 14:02 UTC
394a2e2878739fdba4e0d238445b5b6bf962cf3f
2 files changed
+37
-3
db.js
+36
-2
@@ -39,9 +39,11 @@ module.exports.CreateDB = function (parent, func) {
39
obj.changeStream = false;
40
obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
41
42
- // MongoDB bulk write state
42
+ // MongoDB bulk operations state
43
obj.filePendingGet = null;
44
obj.filePendingGets = null;
45
+ obj.filePendingRemove = null;
46
+ obj.filePendingRemoves = null;
47
obj.filePendingSet = false;
48
obj.filePendingSets = null;
49
obj.filePendingCb = null;
@@ -1133,7 +1135,20 @@ module.exports.CreateDB = function (parent, func) {
1135
obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1136
obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1137
obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1136
- obj.Remove = function (id, func) { obj.file.deleteOne({ _id: id }, func); }; // TODO: May want to do bulk removes to speed up the database.
1138
+
1139
+ obj.Remove = function (id, func) { // Fast remove operation using a bulk find() to reduce round trips to the database.
1140
+ if (obj.filePendingRemoves == null) {
1141
+ // No pending gets, perform the operation now.
1142
+ obj.filePendingRemoves = {};
1143
+ obj.filePendingRemoves[id] = [func];
1144
+ obj.file.deleteOne({ _id: id }, fileBulkRemoveCompleted);
1145
+ } else {
1146
+ // Add remove to pending list.
1147
+ if (obj.filePendingRemove == null) { obj.filePendingRemove = {}; }
1148
+ if (obj.filePendingRemove[id] == null) { obj.filePendingRemove[id] = [func]; } else { obj.filePendingRemove[id].push(func); }
1149
+ }
1150
+ };
1151
+
1152
obj.RemoveAll = function (func) { obj.file.deleteMany({}, { multi: true }, func); };
1153
obj.RemoveAllOfType = function (type, func) { obj.file.deleteMany({ type: type }, { multi: true }, func); };
1154
obj.InsertMany = function (data, func) { obj.file.insertMany(data, func); };
@@ -1513,6 +1528,25 @@ module.exports.CreateDB = function (parent, func) {
1528
}
1529
}
1530
1531
+ // MongoDB pending bulk remove operation, perform fast bulk document removes.
1532
+ function fileBulkRemoveCompleted(err) {
1533
+ // Send out callbacks
1534
+ for (var i in obj.filePendingGets) {
1535
+ for (var j in obj.filePendingGets[i]) {
1536
+ obj.filePendingGets[i][j](err);
1537
+ }
1538
+ }
1539
+
1540
+ // Move on to process any more pending get operations
1541
+ obj.filePendingRemoves = obj.filePendingRemove;
1542
+ obj.filePendingRemove = null;
1543
+ if (obj.filePendingRemoves != null) {
1544
+ var findlist = [], count = 0;
1545
+ for (var i in obj.filePendingRemoves) { findlist.push(i); count++; }
1546
+ obj.file.deleteMany({ _id: { $in: findlist } }, { multi: true }, fileBulkRemoveCompleted);
1547
+ }
1548
+ }
1549
+
1550
// MongoDB pending bulk write operation, perform fast bulk document replacement.
1551
function fileBulkWriteCompleted() {
1552
// Callbacks
meshuser.js
+1
-1
@@ -3587,7 +3587,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
3587
if (db.RemoveSMBIOS) { db.RemoveSMBIOS(node._id); } // Remove SMBios data
3588
db.RemoveAllNodeEvents(node._id); // Remove all events for this node
3589
db.removeAllPowerEventsForNode(node._id); // Remove all power events for this node
3590
- db.Get('ra' + obj.dbNodeKey, function (err, nodes) {
3590
+ db.Get('ra' + node._id, function (err, nodes) {
3591
if ((nodes != null) && (nodes.length == 1)) { db.Remove('da' + nodes[0].daid); } // Remove diagnostic agent to real agent link
3592
db.Remove('ra' + node._id); // Remove real agent to diagnostic agent link
3593
});