MongoDB speed boost using bulkWrite()
Ylian Saint-Hilaire committed
Jan 10, 2021 at 04:53 UTC
e41ac082211ad8e50cb06b5c6e8b8ad5259e6e79
2 files changed
+28
-2
db.js
+27
-1
@@ -38,6 +38,8 @@ module.exports.CreateDB = function (parent, func) {
38
obj.dbRecordsDecryptKey = null;
39
obj.changeStream = false;
40
obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
41
+ obj.pendingSet = false;
42
+ obj.pendingSets = null;
43
44
obj.SetupDatabase = function (func) {
45
// Check if the database unique identifier is present
@@ -1029,7 +1031,17 @@ module.exports.CreateDB = function (parent, func) {
1031
}
1032
} else if (obj.databaseType == 3) {
1033
// Database actions on the main collection (MongoDB)
1032
- obj.Set = function (data, func) { data = common.escapeLinksFieldNameEx(data); obj.file.replaceOne({ _id: data._id }, performTypedRecordEncrypt(data), { upsert: true }, func); };
1034
+ obj.Set = function (data) { // Fast Set operation using bulkWrite(), this is much faster then using replaceOne()
1035
+ if (obj.pendingSet == false) {
1036
+ // Perform the operation now
1037
+ obj.pendingSet = true; obj.pendingSets = null;
1038
+ obj.file.bulkWrite([{ replaceOne: { filter: { _id: data._id }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(data)), upsert: true } }], bulkWriteCompleted);
1039
+ } else {
1040
+ // Add this operation to the pending list
1041
+ if (obj.pendingSets == null) { obj.pendingSets = {} }
1042
+ obj.pendingSets[data._id] = data;
1043
+ }
1044
+ };
1045
obj.Get = function (id, func) {
1046
if (arguments.length > 2) {
1047
var parms = [func];
@@ -1408,6 +1420,20 @@ module.exports.CreateDB = function (parent, func) {
1420
}
1421
}
1422
1423
+ // MongoDB pending bulk write operation, perform fast bulk document replacement.
1424
+ function bulkWriteCompleted() {
1425
+ if (obj.pendingSets != null) {
1426
+ // Perform pending operations
1427
+ var ops = [], c = 0;
1428
+ for (var i in obj.pendingSets) { c++; ops.push({ replaceOne: { filter: { _id: i }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(obj.pendingSets[i])), upsert: true } }); }
1429
+ obj.file.bulkWrite(ops, bulkWriteCompleted);
1430
+ obj.pendingSets = null;
1431
+ } else {
1432
+ // All done, no pending operations.
1433
+ obj.pendingSet = false;
1434
+ }
1435
+ }
1436
+
1437
// Perform a server backup
1438
obj.performingBackup = false;
1439
obj.performBackup = function (func) {
webserver.js
+1
-1
@@ -4972,7 +4972,7 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) {
4972
// Add HTTP security headers to all responses
4973
obj.app.use(function (req, res, next) {
4974
// Useful for debugging reverse proxy issues
4975
- parent.debug('httpheaders', req.headers);
4975
+ parent.debug('httpheaders', req.method, req.url, req.headers);
4976
4977
// Set the real IP address of the request
4978
// If a trusted reverse-proxy is sending us the remote IP address, use it.