Merged MongoDB bulk operations support.
Ylian Saint-Hilaire committed
Feb 25, 2021 at 19:13 UTC
1b4f2f6002944ca78a74a62a60cbc3def5d302db
3 files changed
+256
-2059
db-bulk.js
deleted
-2031
@@ -1,2031 +0,0 @@
1
-/**
2
-* @description MeshCentral database module
3
-* @author Ylian Saint-Hilaire
4
-* @copyright Intel Corporation 2018-2021
5
-* @license Apache-2.0
6
-* @version v0.0.2
7
-*/
8
-
9
-/*xjslint node: true */
10
-/*xjslint plusplus: true */
11
-/*xjslint maxlen: 256 */
12
-/*jshint node: true */
13
-/*jshint strict: false */
14
-/*jshint esversion: 6 */
15
-"use strict";
16
-
17
-//
18
-// Construct Meshcentral database object
19
-//
20
-// The default database is NeDB
21
-// https://github.com/louischatriot/nedb
22
-//
23
-// Alternativety, MongoDB can be used
24
-// https://www.mongodb.com/
25
-// Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/
26
-// The default collection is "meshcentral", but you can override it using --mongodbcol [collection]
27
-//
28
-module.exports.CreateDB = function (parent, func) {
29
- var obj = {};
30
- var Datastore = null;
31
- var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days (1728000). (Seconds * Minutes * Hours * Days)
32
- var expirePowerEventsSeconds = (60 * 60 * 24 * 10); // By default, expire power events after 10 days (864000). (Seconds * Minutes * Hours * Days)
33
- var expireServerStatsSeconds = (60 * 60 * 24 * 30); // By default, expire power events after 30 days (2592000). (Seconds * Minutes * Hours * Days)
34
- const common = require('./common.js');
35
- obj.identifier = null;
36
- obj.dbKey = null;
37
- obj.dbRecordsEncryptKey = null;
38
- obj.dbRecordsDecryptKey = null;
39
- obj.changeStream = false;
40
- obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
41
-
42
- // MongoDB bulk operations state
43
- obj.filePendingGet = null;
44
- obj.filePendingGets = null;
45
- obj.filePendingRemove = null;
46
- obj.filePendingRemoves = null;
47
- obj.filePendingSet = false;
48
- obj.filePendingSets = null;
49
- obj.filePendingCb = null;
50
- obj.filePendingCbs = null;
51
- obj.powerFilePendingSet = false;
52
- obj.powerFilePendingSets = null;
53
- obj.powerFilePendingCb = null;
54
- obj.powerFilePendingCbs = null;
55
- obj.eventsFilePendingSet = false;
56
- obj.eventsFilePendingSets = null;
57
- obj.eventsFilePendingCb = null;
58
- obj.eventsFilePendingCbs = null;
59
-
60
- obj.SetupDatabase = function (func) {
61
- // Check if the database unique identifier is present
62
- // This is used to check that in server peering mode, everyone is using the same database.
63
- obj.Get('DatabaseIdentifier', function (err, docs) {
64
- if (err != null) { parent.debug('db', 'ERROR (Get DatabaseIdentifier): ' + err); }
65
- if ((err == null) && (docs.length == 1) && (docs[0].value != null)) {
66
- obj.identifier = docs[0].value;
67
- } else {
68
- obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex');
69
- obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier });
70
- }
71
- });
72
-
73
- // Load database schema version and check if we need to update
74
- obj.Get('SchemaVersion', function (err, docs) {
75
- if (err != null) { parent.debug('db', 'ERROR (Get SchemaVersion): ' + err); }
76
- var ver = 0;
77
- if ((err == null) && (docs.length == 1)) { ver = docs[0].value; }
78
- if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); }
79
-
80
- // TODO: Any schema upgrades here...
81
- obj.Set({ _id: 'SchemaVersion', value: 2 });
82
-
83
- func(ver);
84
- });
85
- };
86
-
87
- // Perform database maintenance
88
- obj.maintenance = function () {
89
- if (obj.databaseType == 1) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
90
- obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
91
- obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
92
- obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
93
- }
94
- }
95
-
96
- obj.cleanup = function (func) {
97
- // TODO: Remove all mesh links to invalid users
98
- // TODO: Remove all meshes that dont have any links
99
-
100
- // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
101
- if ((obj.databaseType == 4) || (obj.databaseType == 5)) {
102
- // MariaDB or MySQL
103
- obj.RemoveAllOfType('event', function () { });
104
- obj.RemoveAllOfType('power', function () { });
105
- obj.RemoveAllOfType('smbios', function () { });
106
- } else if (obj.databaseType == 3) {
107
- // MongoDB
108
- obj.file.deleteMany({ type: 'event' }, { multi: true });
109
- obj.file.deleteMany({ type: 'power' }, { multi: true });
110
- obj.file.deleteMany({ type: 'smbios' }, { multi: true });
111
- } else {
112
- // NeDB or MongoJS
113
- obj.file.remove({ type: 'event' }, { multi: true });
114
- obj.file.remove({ type: 'power' }, { multi: true });
115
- obj.file.remove({ type: 'smbios' }, { multi: true });
116
- }
117
-
118
- // List of valid identifiers
119
- var validIdentifiers = {}
120
-
121
- // Load all user groups
122
- obj.GetAllType('ugrp', function (err, docs) {
123
- if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
124
- if ((err == null) && (docs.length > 0)) {
125
- for (var i in docs) {
126
- // Add this as a valid user identifier
127
- validIdentifiers[docs[i]._id] = 1;
128
- }
129
- }
130
-
131
- // Fix all of the creating & login to ticks by seconds, not milliseconds.
132
- obj.GetAllType('user', function (err, docs) {
133
- if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
134
- if ((err == null) && (docs.length > 0)) {
135
- for (var i in docs) {
136
- var fixed = false;
137
-
138
- // Add this as a valid user identifier
139
- validIdentifiers[docs[i]._id] = 1;
140
-
141
- // Fix email address capitalization
142
- if (docs[i].email && (docs[i].email != docs[i].email.toLowerCase())) {
143
- docs[i].email = docs[i].email.toLowerCase(); fixed = true;
144
- }
145
-
146
- // Fix account creation
147
- if (docs[i].creation) {
148
- if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; }
149
- if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; }
150
- }
151
-
152
- // Fix last account login
153
- if (docs[i].login) {
154
- if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; }
155
- if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; }
156
- }
157
-
158
- // Fix last password change
159
- if (docs[i].passchange) {
160
- if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; }
161
- if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; }
162
- }
163
-
164
- // Fix subscriptions
165
- if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; }
166
-
167
- // Save the user if needed
168
- if (fixed) { obj.Set(docs[i]); }
169
- }
170
-
171
- // Remove all objects that have a "meshid" that no longer points to a valid mesh.
172
- // Fix any incorrectly escaped user identifiers
173
- obj.GetAllType('mesh', function (err, docs) {
174
- if (err != null) { parent.debug('db', 'ERROR (GetAll mesh): ' + err); }
175
- var meshlist = [];
176
- if ((err == null) && (docs.length > 0)) {
177
- for (var i in docs) {
178
- var meshChange = false;
179
- docs[i] = common.unEscapeLinksFieldName(docs[i]);
180
- meshlist.push(docs[i]._id);
181
-
182
- // Make sure all mesh types are number type, if not, fix it.
183
- if (typeof docs[i].mtype == 'string') { docs[i].mtype = parseInt(docs[i].mtype); meshChange = true; }
184
-
185
- // Take a look at the links
186
- if (docs[i].links != null) {
187
- for (var j in docs[i].links) {
188
- if (validIdentifiers[j] == null) {
189
- // This identifier is not known, let see if we can fix it.
190
- var xid = j, xid2 = common.unEscapeFieldName(xid);
191
- while ((xid != xid2) && (validIdentifiers[xid2] == null)) { xid = xid2; xid2 = common.unEscapeFieldName(xid2); }
192
- if (validIdentifiers[xid2] == 1) {
193
- //console.log('Fixing id: ' + j + ' to ' + xid2);
194
- docs[i].links[xid2] = docs[i].links[j];
195
- delete docs[i].links[j];
196
- meshChange = true;
197
- } else {
198
- // TODO: here, we may want to clean up links to users and user groups that do not exist anymore.
199
- //console.log('Unknown id: ' + j);
200
- }
201
- }
202
- }
203
- }
204
-
205
- // Save the updated device group if needed
206
- if (meshChange) { obj.Set(docs[i]); }
207
- }
208
- }
209
- if ((obj.databaseType == 4) || (obj.databaseType == 5)) {
210
- // MariaDB
211
- sqlDbQuery('DELETE FROM MeshCentral.Main WHERE (extra LIKE ("mesh/%") AND (extra NOT IN ?)', [meshlist], func);
212
- } else if (obj.databaseType == 3) {
213
- // MongoDB
214
- obj.file.deleteMany({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
215
- } else {
216
- // NeDB or MongoJS
217
- obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
218
- }
219
-
220
- // We are done
221
- validIdentifiers = null;
222
- if (func) { func(); }
223
- });
224
- }
225
- });
226
- });
227
- };
228
-
229
- // Get encryption key
230
- obj.getEncryptDataKey = function (password) {
231
- if (typeof password != 'string') return null;
232
- return parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32);
233
- }
234
-
235
- // Encrypt data
236
- obj.encryptData = function (password, plaintext) {
237
- var key = obj.getEncryptDataKey(password);
238
- if (key == null) return null;
239
- const iv = parent.crypto.randomBytes(16);
240
- const aes = parent.crypto.createCipheriv('aes-256-cbc', key, iv);
241
- var ciphertext = aes.update(plaintext);
242
- ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
243
- return ciphertext.toString('base64');
244
- }
245
-
246
- // Decrypt data
247
- obj.decryptData = function (password, ciphertext) {
248
- try {
249
- var key = obj.getEncryptDataKey(password);
250
- if (key == null) return null;
251
- const ciphertextBytes = Buffer.from(ciphertext, 'base64');
252
- const iv = ciphertextBytes.slice(0, 16);
253
- const data = ciphertextBytes.slice(16);
254
- const aes = parent.crypto.createDecipheriv('aes-256-cbc', key, iv);
255
- var plaintextBytes = Buffer.from(aes.update(data));
256
- plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
257
- return plaintextBytes;
258
- } catch (ex) { return null; }
259
- }
260
-
261
- // Get the number of records in the database for various types, this is the slow NeDB way.
262
- // WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database.
263
- obj.getStats = function (func) {
264
- if (obj.databaseType == 3) {
265
- // MongoDB
266
- obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }]).toArray(function (err, docs) {
267
- var counters = {}, totalCount = 0;
268
- if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
269
- func(counters);
270
- });
271
- } else if (obj.databaseType == 2) {
272
- // MongoJS
273
- obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) {
274
- var counters = {}, totalCount = 0;
275
- if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
276
- func(counters);
277
- });
278
- } else if (obj.databaseType == 1) {
279
- // NeDB version
280
- obj.file.count({ type: 'node' }, function (err, nodeCount) {
281
- obj.file.count({ type: 'mesh' }, function (err, meshCount) {
282
- obj.file.count({ type: 'user' }, function (err, userCount) {
283
- obj.file.count({ type: 'sysinfo' }, function (err, sysinfoCount) {
284
- obj.file.count({ type: 'note' }, function (err, noteCount) {
285
- obj.file.count({ type: 'iploc' }, function (err, iplocCount) {
286
- obj.file.count({ type: 'ifinfo' }, function (err, ifinfoCount) {
287
- obj.file.count({ type: 'cfile' }, function (err, cfileCount) {
288
- obj.file.count({ type: 'lastconnect' }, function (err, lastconnectCount) {
289
- obj.file.count({}, function (err, totalCount) {
290
- func({ node: nodeCount, mesh: meshCount, user: userCount, sysinfo: sysinfoCount, iploc: iplocCount, note: noteCount, ifinfo: ifinfoCount, cfile: cfileCount, lastconnect: lastconnectCount, total: totalCount });
291
- });
292
- });
293
- });
294
- });
295
- });
296
- });
297
- });
298
- });
299
- });
300
- });
301
- }
302
- }
303
-
304
- // This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db.
305
- obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if ((err == null) && (docs.length == 1)) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); };
306
- obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); }
307
-
308
- // Encrypt an database object
309
- obj.performRecordEncryptionRecode = function (func) {
310
- var count = 0;
311
- obj.GetAllType('user', function (err, docs) {
312
- if (err != null) { parent.debug('db', 'ERROR (performRecordEncryptionRecode): ' + err); }
313
- if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
314
- obj.GetAllType('node', function (err, docs) {
315
- if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
316
- obj.GetAllType('mesh', function (err, docs) {
317
- if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
318
- if (obj.databaseType == 1) { // If we are using NeDB, compact the database.
319
- obj.file.persistence.compactDatafile();
320
- obj.file.on('compaction.done', function () { func(count); }); // It's important to wait for compaction to finish before exit, otherwise NeDB may corrupt.
321
- } else {
322
- func(count); // For all other databases, normal exit.
323
- }
324
- });
325
- });
326
- });
327
- }
328
-
329
- // Encrypt an database object
330
- function performTypedRecordDecrypt(data) {
331
- if ((data == null) || (obj.dbRecordsDecryptKey == null) || (typeof data != 'object')) return data;
332
- for (var i in data) {
333
- if (data[i] == null) continue;
334
- if (data[i].type == 'user') {
335
- data[i] = performPartialRecordDecrypt(data[i]);
336
- } else if ((data[i].type == 'node') && (data[i].intelamt != null)) {
337
- data[i].intelamt = performPartialRecordDecrypt(data[i].intelamt);
338
- } else if ((data[i].type == 'mesh') && (data[i].amt != null)) {
339
- data[i].amt = performPartialRecordDecrypt(data[i].amt);
340
- }
341
- }
342
- return data;
343
- }
344
-
345
- // Encrypt an database object
346
- function performTypedRecordEncrypt(data) {
347
- if (obj.dbRecordsEncryptKey == null) return data;
348
- if (data.type == 'user') { return performPartialRecordEncrypt(Clone(data), ['otpkeys', 'otphkeys', 'otpsecret', 'salt', 'hash', 'oldpasswords']); }
349
- else if ((data.type == 'node') && (data.intelamt != null)) { var xdata = Clone(data); xdata.intelamt = performPartialRecordEncrypt(xdata.intelamt, ['user', 'pass', 'mpspass']); return xdata; }
350
- else if ((data.type == 'mesh') && (data.amt != null)) { var xdata = Clone(data); xdata.amt = performPartialRecordEncrypt(xdata.amt, ['password']); return xdata; }
351
- return data;
352
- }
353
-
354
- // Encrypt an object and return a buffer.
355
- function performPartialRecordEncrypt(plainobj, encryptNames) {
356
- if (typeof plainobj != 'object') return plainobj;
357
- var enc = {}, enclen = 0;
358
- for (var i in encryptNames) { if (plainobj[encryptNames[i]] != null) { enclen++; enc[encryptNames[i]] = plainobj[encryptNames[i]]; delete plainobj[encryptNames[i]]; } }
359
- if (enclen > 0) { plainobj._CRYPT = performRecordEncrypt(enc); } else { delete plainobj._CRYPT; }
360
- return plainobj;
361
- }
362
-
363
- // Encrypt an object and return a buffer.
364
- function performPartialRecordDecrypt(plainobj) {
365
- if ((typeof plainobj != 'object') || (plainobj._CRYPT == null)) return plainobj;
366
- var enc = performRecordDecrypt(plainobj._CRYPT);
367
- if (enc != null) { for (var i in enc) { plainobj[i] = enc[i]; } }
368
- delete plainobj._CRYPT;
369
- return plainobj;
370
- }
371
-
372
- // Encrypt an object and return a base64.
373
- function performRecordEncrypt(plainobj) {
374
- if (obj.dbRecordsEncryptKey == null) return null;
375
- const iv = parent.crypto.randomBytes(12);
376
- const aes = parent.crypto.createCipheriv('aes-256-gcm', obj.dbRecordsEncryptKey, iv);
377
- var ciphertext = aes.update(JSON.stringify(plainobj));
378
- var cipherfinal = aes.final();
379
- ciphertext = Buffer.concat([iv, aes.getAuthTag(), ciphertext, cipherfinal]);
380
- return ciphertext.toString('base64');
381
- }
382
-
383
- // Takes a base64 and return an object.
384
- function performRecordDecrypt(ciphertext) {
385
- if (obj.dbRecordsDecryptKey == null) return null;
386
- const ciphertextBytes = Buffer.from(ciphertext, 'base64');
387
- const iv = ciphertextBytes.slice(0, 12);
388
- const data = ciphertextBytes.slice(28);
389
- const aes = parent.crypto.createDecipheriv('aes-256-gcm', obj.dbRecordsDecryptKey, iv);
390
- aes.setAuthTag(ciphertextBytes.slice(12, 28));
391
- var plaintextBytes, r;
392
- try {
393
- plaintextBytes = Buffer.from(aes.update(data));
394
- plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
395
- r = JSON.parse(plaintextBytes.toString());
396
- } catch (e) { throw "Incorrect DbRecordsDecryptKey/DbRecordsEncryptKey or invalid database _CRYPT data: " + e; }
397
- return r;
398
- }
399
-
400
- // Clone an object (TODO: Make this more efficient)
401
- function Clone(v) { return JSON.parse(JSON.stringify(v)); }
402
-
403
- // Read expiration time from configuration file
404
- if (typeof parent.args.dbexpire == 'object') {
405
- if (typeof parent.args.dbexpire.events == 'number') { expireEventsSeconds = parent.args.dbexpire.events; }
406
- if (typeof parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = parent.args.dbexpire.powerevents; }
407
- if (typeof parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = parent.args.dbexpire.statsevents; }
408
- }
409
-
410
- // If a DB record encryption key is provided, perform database record encryption
411
- if ((typeof parent.args.dbrecordsencryptkey == 'string') && (parent.args.dbrecordsencryptkey.length != 0)) {
412
- // Hash the database password into a AES256 key and setup encryption and decryption.
413
- obj.dbRecordsEncryptKey = obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsencryptkey).digest('raw').slice(0, 32);
414
- }
415
-
416
- // If a DB record decryption key is provided, perform database record decryption
417
- if ((typeof parent.args.dbrecordsdecryptkey == 'string') && (parent.args.dbrecordsdecryptkey.length != 0)) {
418
- // Hash the database password into a AES256 key and setup encryption and decryption.
419
- obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsdecryptkey).digest('raw').slice(0, 32);
420
- }
421
-
422
- if (parent.args.mariadb || parent.args.mysql) {
423
- if (parent.args.mariadb) {
424
- // Use MariaDB
425
- obj.databaseType = 4;
426
- Datastore = require('mariadb').createPool(parent.args.mariadb);
427
- } else if (parent.args.mysql) {
428
- // Use MySQL
429
- Datastore = require('mysql').createConnection(parent.args.mysql);
430
- obj.databaseType = 5;
431
- }
432
- //sqlDbQuery('DROP DATABASE MeshCentral', null, function (err, docs) { console.log('DROP'); }); return;
433
- sqlDbQuery('USE meshcentral', null, function (err, docs) {
434
- if (err != null) { parent.debug('db', 'ERROR: USE meshcentral: ' + err); }
435
- if (err == null) { setupFunctions(func); } else {
436
- parent.debug('db', 'Creating database...');
437
- sqlDbBatchExec([
438
- 'CREATE DATABASE meshcentral',
439
- // Main table
440
- 'CREATE TABLE meshcentral.main (id VARCHAR(256) NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
441
- 'CREATE INDEX ndxtypedomainextra ON meshcentral.main (type, domain, extra)',
442
- 'CREATE INDEX ndxextra ON meshcentral.main (extra)',
443
- 'CREATE INDEX ndxextraex ON meshcentral.main (extraex)',
444
- // Events table
445
- 'CREATE TABLE meshcentral.events(id INT NOT NULL AUTO_INCREMENT, time DATETIME, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK(json_valid(doc)))',
446
- 'CREATE INDEX ndxeventstime ON meshcentral.events(time)',
447
- 'CREATE INDEX ndxeventsusername ON meshcentral.events(domain, userid, time)',
448
- 'CREATE INDEX ndxeventsdomainnodeidtime ON meshcentral.events(domain, nodeid, time)',
449
- // Events ID table
450
- 'CREATE TABLE meshcentral.eventids(fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT)',
451
- 'CREATE INDEX ndxeventids ON meshcentral.eventids(target)',
452
- // Server stats table
453
- 'CREATE TABLE meshcentral.serverstats (time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(time), CHECK (json_valid(doc)))',
454
- 'CREATE INDEX ndxserverstattime ON meshcentral.serverstats (time)',
455
- 'CREATE INDEX ndxserverstatexpire ON meshcentral.serverstats (expire)',
456
- // Power events table
457
- 'CREATE TABLE meshcentral.power (id INT NOT NULL AUTO_INCREMENT, time DATETIME, nodeid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
458
- 'CREATE INDEX ndxpowernodeidtime ON meshcentral.power (nodeid, time)',
459
- // SMBIOS table
460
- 'CREATE TABLE meshcentral.smbios (id CHAR(255), time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
461
- 'CREATE INDEX ndxsmbiostime ON meshcentral.smbios (time)',
462
- 'CREATE INDEX ndxsmbiosexpire ON meshcentral.smbios (expire)',
463
- // Plugins table
464
- 'CREATE TABLE meshcentral.plugin (id INT NOT NULL AUTO_INCREMENT, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))'
465
- ], function (err) {
466
- if (err != null) { parent.debug('db', 'BatchSetupDb: ' + err); }
467
- setupFunctions(func);
468
- });
469
- }
470
- });
471
- } else if (parent.args.mongodb) {
472
- // Use MongoDB
473
- obj.databaseType = 3;
474
- require('mongodb').MongoClient.connect(parent.args.mongodb, { useNewUrlParser: true, useUnifiedTopology: true }, function (err, client) {
475
- if (err != null) { console.log("Unable to connect to database: " + err); process.exit(); return; }
476
- Datastore = client;
477
- parent.debug('db', 'Connected to MongoDB database...');
478
-
479
- // Get the database name and setup the database client
480
- var dbname = 'meshcentral';
481
- if (parent.args.mongodbname) { dbname = parent.args.mongodbname; }
482
- const dbcollectionname = (parent.args.mongodbcol) ? (parent.args.mongodbcol) : 'meshcentral';
483
- const db = client.db(dbname);
484
-
485
- // Check the database version
486
- db.admin().serverInfo(function (err, info) {
487
- if ((err != null) || (info == null) || (info.versionArray == null) || (Array.isArray(info.versionArray) == false) || (info.versionArray.length < 2) || (typeof info.versionArray[0] != 'number') || (typeof info.versionArray[1] != 'number')) {
488
- console.log('WARNING: Unable to check MongoDB version.');
489
- } else {
490
- if ((info.versionArray[0] < 3) || ((info.versionArray[0] == 3) && (info.versionArray[1] < 6))) {
491
- // We are running with mongoDB older than 3.6, this is not good.
492
- parent.addServerWarning("Current version of MongoDB (" + info.version + ") is too old, please upgrade to MongoDB 3.6 or better.");
493
- }
494
- }
495
- });
496
-
497
- // Setup MongoDB main collection and indexes
498
- obj.file = db.collection(dbcollectionname);
499
- obj.file.indexes(function (err, indexes) {
500
- // Check if we need to reset indexes
501
- var indexesByName = {}, indexCount = 0;
502
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
503
- if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) {
504
- console.log('Resetting main indexes...');
505
- obj.file.dropIndexes(function (err) {
506
- obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered()
507
- obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail()
508
- obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh()
509
- });
510
- }
511
- });
512
-
513
- // Setup the changeStream on the MongoDB main collection if possible
514
- if (parent.args.mongodbchangestream == true) {
515
- if (typeof obj.file.watch != 'function') {
516
- console.log('WARNING: watch() is not a function, MongoDB ChangeStream not supported.');
517
- } else {
518
- obj.fileChangeStream = obj.file.watch([{ $match: { $or: [{ 'fullDocument.type': { $in: ['node', 'mesh', 'user', 'ugrp'] } }, { 'operationType': 'delete' }] } }], { fullDocument: 'updateLookup' });
519
- obj.fileChangeStream.on('change', function (change) {
520
- if ((change.operationType == 'update') || (change.operationType == 'replace')) {
521
- switch (change.fullDocument.type) {
522
- case 'node': { dbNodeChange(change, false); break; } // A node has changed
523
- case 'mesh': { dbMeshChange(change, false); break; } // A device group has changed
524
- case 'user': { dbUserChange(change, false); break; } // A user account has changed
525
- case 'ugrp': { dbUGrpChange(change, false); break; } // A user account has changed
526
- }
527
- } else if (change.operationType == 'insert') {
528
- switch (change.fullDocument.type) {
529
- case 'node': { dbNodeChange(change, true); break; } // A node has added
530
- case 'mesh': { dbMeshChange(change, true); break; } // A device group has created
531
- case 'user': { dbUserChange(change, true); break; } // A user account has created
532
- case 'ugrp': { dbUGrpChange(change, true); break; } // A user account has created
533
- }
534
- } else if (change.operationType == 'delete') {
535
- if ((change.documentKey == null) || (change.documentKey._id == null)) return;
536
- var splitId = change.documentKey._id.split('/');
537
- switch (splitId[0]) {
538
- case 'node': {
539
- //Not Good: Problem here is that we don't know what meshid the node belonged to before the delete.
540
- //parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: 'removenode', nodeid: change.documentKey._id, domain: splitId[1] });
541
- break;
542
- }
543
- case 'mesh': {
544
- parent.DispatchEvent(['*', change.documentKey._id], obj, { etype: 'mesh', action: 'deletemesh', meshid: change.documentKey._id, domain: splitId[1] });
545
- break;
546
- }
547
- case 'user': {
548
- //Not Good: This is not a perfect user removal because we don't know what groups the user was in.
549
- //parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', action: 'accountremove', userid: change.documentKey._id, domain: splitId[1], username: splitId[2] });
550
- break;
551
- }
552
- case 'ugrp': {
553
- parent.DispatchEvent(['*', change.documentKey._id], obj, { etype: 'ugrp', action: 'deleteusergroup', ugrpid: change.documentKey._id, domain: splitId[1] });
554
- break;
555
- }
556
- }
557
- }
558
- });
559
- obj.changeStream = true;
560
- }
561
- }
562
-
563
- // Setup MongoDB events collection and indexes
564
- obj.eventsfile = db.collection('events'); // Collection containing all events
565
- obj.eventsfile.indexes(function (err, indexes) {
566
- // Check if we need to reset indexes
567
- var indexesByName = {}, indexCount = 0;
568
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
569
- if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
570
- // Reset all indexes
571
- console.log("Resetting events indexes...");
572
- obj.eventsfile.dropIndexes(function (err) {
573
- obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' });
574
- obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' });
575
- obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' });
576
- obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
577
- });
578
- } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) {
579
- // Reset the timeout index
580
- console.log("Resetting events expire index...");
581
- obj.eventsfile.dropIndex('ExpireTime1', function (err) {
582
- obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
583
- });
584
- }
585
- });
586
-
587
- // Setup MongoDB power events collection and indexes
588
- obj.powerfile = db.collection('power'); // Collection containing all power events
589
- obj.powerfile.indexes(function (err, indexes) {
590
- // Check if we need to reset indexes
591
- var indexesByName = {}, indexCount = 0;
592
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
593
- if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
594
- // Reset all indexes
595
- console.log("Resetting power events indexes...");
596
- obj.powerfile.dropIndexes(function (err) {
597
- // Create all indexes
598
- obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' });
599
- obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
600
- });
601
- } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) {
602
- // Reset the timeout index
603
- console.log("Resetting power events expire index...");
604
- obj.powerfile.dropIndex('ExpireTime1', function (err) {
605
- // Reset the expire power events index
606
- obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
607
- });
608
- }
609
- });
610
-
611
- // Setup MongoDB smbios collection, no indexes needed
612
- obj.smbiosfile = db.collection('smbios'); // Collection containing all smbios information
613
-
614
- // Setup MongoDB server stats collection
615
- obj.serverstatsfile = db.collection('serverstats'); // Collection of server stats
616
- obj.serverstatsfile.indexes(function (err, indexes) {
617
- // Check if we need to reset indexes
618
- var indexesByName = {}, indexCount = 0;
619
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
620
- if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) {
621
- // Reset all indexes
622
- console.log("Resetting server stats indexes...");
623
- obj.serverstatsfile.dropIndexes(function (err) {
624
- // Create all indexes
625
- obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
626
- obj.serverstatsfile.createIndex({ 'expire': 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' }); // Auto-expire events
627
- });
628
- } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) {
629
- // Reset the timeout index
630
- console.log("Resetting server stats expire index...");
631
- obj.serverstatsfile.dropIndex('ExpireTime1', function (err) {
632
- // Reset the expire server stats index
633
- obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
634
- });
635
- }
636
- });
637
-
638
- // Setup plugin info collection
639
- if (obj.pluginsActive) { obj.pluginsfile = db.collection('plugins'); }
640
-
641
- setupFunctions(func); // Completed setup of MongoDB
642
- });
643
- } else if (parent.args.xmongodb) {
644
- // Use MongoJS, this is the old system.
645
- obj.databaseType = 2;
646
- Datastore = require('mongojs');
647
- var db = Datastore(parent.args.xmongodb);
648
- var dbcollection = 'meshcentral';
649
- if (parent.args.mongodbcol) { dbcollection = parent.args.mongodbcol; }
650
-
651
- // Setup MongoDB main collection and indexes
652
- obj.file = db.collection(dbcollection);
653
- obj.file.getIndexes(function (err, indexes) {
654
- // Check if we need to reset indexes
655
- var indexesByName = {}, indexCount = 0;
656
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
657
- if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) {
658
- console.log("Resetting main indexes...");
659
- obj.file.dropIndexes(function (err) {
660
- obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered()
661
- obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail()
662
- obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh()
663
- });
664
- }
665
- });
666
-
667
- // Setup MongoDB events collection and indexes
668
- obj.eventsfile = db.collection('events'); // Collection containing all events
669
- obj.eventsfile.getIndexes(function (err, indexes) {
670
- // Check if we need to reset indexes
671
- var indexesByName = {}, indexCount = 0;
672
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
673
- if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
674
- // Reset all indexes
675
- console.log("Resetting events indexes...");
676
- obj.eventsfile.dropIndexes(function (err) {
677
- obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' });
678
- obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' });
679
- obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' });
680
- obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
681
- });
682
- } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) {
683
- // Reset the timeout index
684
- console.log("Resetting events expire index...");
685
- obj.eventsfile.dropIndex('ExpireTime1', function (err) {
686
- obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
687
- });
688
- }
689
- });
690
-
691
- // Setup MongoDB power events collection and indexes
692
- obj.powerfile = db.collection('power'); // Collection containing all power events
693
- obj.powerfile.getIndexes(function (err, indexes) {
694
- // Check if we need to reset indexes
695
- var indexesByName = {}, indexCount = 0;
696
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
697
- if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
698
- // Reset all indexes
699
- console.log("Resetting power events indexes...");
700
- obj.powerfile.dropIndexes(function (err) {
701
- // Create all indexes
702
- obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' });
703
- obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
704
- });
705
- } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) {
706
- // Reset the timeout index
707
- console.log("Resetting power events expire index...");
708
- obj.powerfile.dropIndex('ExpireTime1', function (err) {
709
- // Reset the expire power events index
710
- obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
711
- });
712
- }
713
- });
714
-
715
- // Setup MongoDB smbios collection, no indexes needed
716
- obj.smbiosfile = db.collection('smbios'); // Collection containing all smbios information
717
-
718
- // Setup MongoDB server stats collection
719
- obj.serverstatsfile = db.collection('serverstats'); // Collection of server stats
720
- obj.serverstatsfile.getIndexes(function (err, indexes) {
721
- // Check if we need to reset indexes
722
- var indexesByName = {}, indexCount = 0;
723
- for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
724
- if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) {
725
- // Reset all indexes
726
- console.log("Resetting server stats indexes...");
727
- obj.serverstatsfile.dropIndexes(function (err) {
728
- // Create all indexes
729
- obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
730
- obj.serverstatsfile.createIndex({ 'expire': 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' }); // Auto-expire events
731
- });
732
- } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) {
733
- // Reset the timeout index
734
- console.log("Resetting server stats expire index...");
735
- obj.serverstatsfile.dropIndex('ExpireTime1', function (err) {
736
- // Reset the expire server stats index
737
- obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
738
- });
739
- }
740
- });
741
-
742
- // Setup plugin info collection
743
- if (obj.pluginsActive) { obj.pluginsfile = db.collection('plugins'); }
744
-
745
- setupFunctions(func); // Completed setup of MongoJS
746
- } else {
747
- // Use NeDB (The default)
748
- obj.databaseType = 1;
749
- Datastore = require('nedb');
750
- var datastoreOptions = { filename: parent.getConfigFilePath('meshcentral.db'), autoload: true };
751
-
752
- // If a DB encryption key is provided, perform database encryption
753
- if ((typeof parent.args.dbencryptkey == 'string') && (parent.args.dbencryptkey.length != 0)) {
754
- // Hash the database password into a AES256 key and setup encryption and decryption.
755
- obj.dbKey = parent.crypto.createHash('sha384').update(parent.args.dbencryptkey).digest('raw').slice(0, 32);
756
- datastoreOptions.afterSerialization = function (plaintext) {
757
- const iv = parent.crypto.randomBytes(16);
758
- const aes = parent.crypto.createCipheriv('aes-256-cbc', obj.dbKey, iv);
759
- var ciphertext = aes.update(plaintext);
760
- ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
761
- return ciphertext.toString('base64');
762
- }
763
- datastoreOptions.beforeDeserialization = function (ciphertext) {
764
- const ciphertextBytes = Buffer.from(ciphertext, 'base64');
765
- const iv = ciphertextBytes.slice(0, 16);
766
- const data = ciphertextBytes.slice(16);
767
- const aes = parent.crypto.createDecipheriv('aes-256-cbc', obj.dbKey, iv);
768
- var plaintextBytes = Buffer.from(aes.update(data));
769
- plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
770
- return plaintextBytes.toString();
771
- }
772
- }
773
-
774
- // Start NeDB main collection and setup indexes
775
- obj.file = new Datastore(datastoreOptions);
776
- obj.file.persistence.setAutocompactionInterval(86400000); // Compact once a day
777
- obj.file.ensureIndex({ fieldName: 'type' });
778
- obj.file.ensureIndex({ fieldName: 'domain' });
779
- obj.file.ensureIndex({ fieldName: 'meshid', sparse: true });
780
- obj.file.ensureIndex({ fieldName: 'nodeid', sparse: true });
781
- obj.file.ensureIndex({ fieldName: 'email', sparse: true });
782
-
783
- // Setup the events collection and setup indexes
784
- obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 });
785
- obj.eventsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
786
- obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field.
787
- obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true });
788
- obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expireEventsSeconds });
789
- obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
790
-
791
- // Setup the power collection and setup indexes
792
- obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 });
793
- obj.powerfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
794
- obj.powerfile.ensureIndex({ fieldName: 'nodeid' });
795
- obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expirePowerEventsSeconds });
796
- obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
797
-
798
- // Setup the SMBIOS collection, for NeDB we don't setup SMBIOS since NeDB will corrupt the database. Remove any existing ones.
799
- //obj.smbiosfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true, corruptAlertThreshold: 1 });
800
- parent.fs.unlink(parent.getConfigFilePath('meshcentral-smbios.db'), function () { });
801
-
802
- // Setup the server stats collection and setup indexes
803
- obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 });
804
- obj.serverstatsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
805
- obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expireServerStatsSeconds });
806
- obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events
807
- obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
808
-
809
- // Setup plugin info collection
810
- if (obj.pluginsActive) {
811
- obj.pluginsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-plugins.db'), autoload: true });
812
- obj.pluginsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
813
- }
814
-
815
- setupFunctions(func); // Completed setup of NeDB
816
- }
817
-
818
- // Check the object names for a "."
819
- function checkObjectNames(r, tag) {
820
- if (typeof r != 'object') return;
821
- for (var i in r) {
822
- if (i.indexOf('.') >= 0) { throw ('BadDbName (' + tag + '): ' + JSON.stringify(r)); }
823
- checkObjectNames(r[i], tag);
824
- }
825
- }
826
-
827
- // Query the database
828
- function sqlDbQuery(query, args, func) {
829
- if (obj.databaseType == 4) { // MariaDB
830
- Datastore.getConnection()
831
- .then(function (conn) {
832
- conn.query(query, args)
833
- .then(function (rows) {
834
- conn.release();
835
- const docs = [];
836
- for (var i in rows) { if (rows[i].doc) { docs.push(performTypedRecordDecrypt((typeof rows[i].doc == 'object')? rows[i].doc : JSON.parse(rows[i].doc))); } }
837
- if (func) try { func(null, docs); } catch (ex) { console.log('SQLERR1', ex); }
838
- })
839
- .catch(function (err) { conn.release(); if (func) try { func(err); } catch (ex) { console.log('SQLERR2', ex); } });
840
- }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log('SQLERR3', ex); } } });
841
- } else if (obj.databaseType == 5) { // MySQL
842
- Datastore.query(query, args, function (error, results, fields) {
843
- if (error != null) {
844
- if (func) try { func(error); } catch (ex) { console.log('SQLERR4', ex); }
845
- } else {
846
- var docs = [];
847
- for (var i in results) { if (results[i].doc) { docs.push(JSON.parse(results[i].doc)); } }
848
- //console.log(docs);
849
- if (func) { try { func(null, docs); } catch (ex) { console.log('SQLERR5', ex); } }
850
- }
851
- });
852
- }
853
- }
854
-
855
- // Exec on the database
856
- function sqlDbExec(query, args, func) {
857
- if (obj.databaseType == 4) { // MariaDB
858
- Datastore.getConnection()
859
- .then(function (conn) {
860
- conn.query(query, args)
861
- .then(function (rows) {
862
- conn.release();
863
- if (func) try { func(null, rows[0]); } catch (ex) { console.log(ex); }
864
- })
865
- .catch(function (err) { conn.release(); if (func) try { func(err); } catch (ex) { console.log(ex); } });
866
- }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log(ex); } } });
867
- } else if (obj.databaseType == 5) { // MySQL
868
- Datastore.query(query, args, function (error, results, fields) {
869
- if (func) try { func(error, results[0]); } catch (ex) { console.log(ex); }
870
- });
871
- }
872
- }
873
-
874
- // Execute a batch of commands on the database
875
- function sqlDbBatchExec(queries, func) {
876
- if (obj.databaseType == 4) { // MariaDB
877
- Datastore.getConnection()
878
- .then(function (conn) {
879
- var Promises = [];
880
- for (var i in queries) { if (typeof queries[i] == 'string') { Promises.push(conn.query(queries[i])); } else { Promises.push(conn.query(queries[i][0], queries[i][1])); } }
881
- Promise.all(Promises)
882
- .then(function (rows) { conn.release(); if (func) { try { func(null); } catch (ex) { console.log(ex); } } })
883
- .catch(function (err) { conn.release(); if (func) { try { func(err); } catch (ex) { console.log(ex); } } });
884
- })
885
- .catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log(ex); } } });
886
- } else if (obj.databaseType == 5) { // MySQL
887
- var Promises = [];
888
- for (var i in queries) { if (typeof queries[i] == 'string') { Promises.push(Datastore.query(queries[i])); } else { Promises.push(Datastore.query(queries[i][0], queries[i][1])); } }
889
- Promise.all(Promises)
890
- .then(function (error, results, fields) { if (func) { try { func(error, results); } catch (ex) { console.log(ex); } } })
891
- .catch(function (error, results, fields) { if (func) { try { func(error); } catch (ex) { console.log(ex); } } });
892
- }
893
- }
894
-
895
- function setupFunctions(func) {
896
- if ((obj.databaseType == 4) || (obj.databaseType == 5)) {
897
- // Database actions on the main collection (MariaDB or MySQL)
898
- obj.Set = function (value, func) {
899
- var extra = null, extraex = null;
900
- value = common.escapeLinksFieldNameEx(value);
901
- if (value.meshid) { extra = value.meshid; } else if (value.email) { extra = 'email/' + value.email; } else if (value.nodeid) { extra = value.nodeid; }
902
- if ((value.type == 'node') && (value.intelamt != null) && (value.intelamt.uuid != null)) { extraex = 'uuid/' + value.intelamt.uuid; }
903
- if (value._id == null) { value._id = require('crypto').randomBytes(16).toString('hex'); }
904
- sqlDbQuery('REPLACE INTO meshcentral.main VALUE (?, ?, ?, ?, ?, ?)', [value._id, (value.type ? value.type : null), ((value.domain != null) ? value.domain : null), extra, extraex, JSON.stringify(performTypedRecordEncrypt(value))], func);
905
- }
906
- obj.Get = function (_id, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE id = ?', [_id], function (err, docs) { if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); } func(err, docs); }); }
907
- obj.GetAll = function (func) { sqlDbQuery('SELECT domain, doc FROM meshcentral.main', null, func); }
908
- obj.GetHash = function (id, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE id = ?', [id], func); }
909
- obj.GetAllTypeNoTypeField = function (type, domain, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE type = ? AND domain = ?', [type, domain], function (err, docs) { if (err == null) { for (var i in docs) { delete docs[i].type } } func(err, docs); }); };
910
- obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, extrasids, domain, type, id, func) {
911
- if (id && (id != '')) {
912
- sqlDbQuery('SELECT doc FROM meshcentral.main WHERE id = ? AND type = ? AND domain = ? AND extra IN (?)', [id, type, domain, meshes], function (err, docs) { if (err == null) { for (var i in docs) { delete docs[i].type } } func(err, docs); });
913
- } else {
914
- if (extrasids == null) {
915
- sqlDbQuery('SELECT doc FROM meshcentral.main WHERE type = ? AND domain = ? AND extra IN (?)', [type, domain, meshes], function (err, docs) { if (err == null) { for (var i in docs) { delete docs[i].type } } func(err, docs); });
916
- } else {
917
- sqlDbQuery('SELECT doc FROM meshcentral.main WHERE type = ? AND domain = ? AND (extra IN (?) OR id IN (?))', [type, domain, meshes, extrasids], function (err, docs) { if (err == null) { for (var i in docs) { delete docs[i].type } } func(err, docs); });
918
- }
919
- }
920
- };
921
- obj.GetAllTypeNodeFiltered = function (nodes, domain, type, id, func) {
922
- if (id && (id != '')) {
923
- sqlDbQuery('SELECT doc FROM meshcentral.main WHERE id = ? AND type = ? AND domain = ? AND extra IN (?)', [id, type, domain, nodes], function (err, docs) { if (err == null) { for (var i in docs) { delete docs[i].type } } func(err, docs); });
924
- } else {
925
- sqlDbQuery('SELECT doc FROM meshcentral.main WHERE type = ? AND domain = ? AND extra IN (?)', [type, domain, nodes], function (err, docs) { if (err == null) { for (var i in docs) { delete docs[i].type } } func(err, docs); });
926
- }
927
- };
928
- obj.GetAllType = function (type, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE type = ?', [type], func); }
929
- obj.GetAllIdsOfType = function (ids, domain, type, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE id IN (?) AND domain = ? AND type = ?', [ids, domain, type], func); }
930
- obj.GetUserWithEmail = function (domain, email, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE domain = ? AND extra = ?', [domain, 'email/' + email], func); }
931
- obj.GetUserWithVerifiedEmail = function (domain, email, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE domain = ? AND extra = ?', [domain, 'email/' + email], func); }
932
- obj.Remove = function (id, func) { sqlDbQuery('DELETE FROM meshcentral.main WHERE id = ?', [id], func); };
933
- obj.RemoveAll = function (func) { sqlDbQuery('DELETE FROM meshcentral.main', null, func); };
934
- obj.RemoveAllOfType = function (type, func) { sqlDbQuery('DELETE FROM meshcentral.main WHERE type = ?', [type], func); };
935
- obj.InsertMany = function (data, func) { var pendingOps = 0; for (var i in data) { pendingOps++; obj.Set(data[i], function () { if (--pendingOps == 0) { func(); } }); } };
936
- obj.RemoveMeshDocuments = function (id) { sqlDbQuery('DELETE FROM meshcentral.main WHERE extra = ?', [id], function () { sqlDbQuery('DELETE FROM meshcentral.main WHERE id = ?', ['nt' + id], func); } ); };
937
- obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if ((err == null) && (docs.length == 1)) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); };
938
- obj.DeleteDomain = function (domain, func) { sqlDbQuery('DELETE FROM meshcentral.main WHERE domain = ?', [domain], func); };
939
- obj.SetUser = function (user) { if (user.subscriptions != null) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); } else { obj.Set(user); } };
940
- obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } };
941
- obj.getLocalAmtNodes = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE (type = "node") AND (extraex IS NOT NULL)', null, function (err, docs) { var r = []; if (err == null) { for (var i in docs) { if (docs[i].host != null) { r.push(docs[i]); } } } func(err, r); }); };
942
- obj.getAmtUuidMeshNode = function (meshid, uuid, func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE meshid = ? AND extraex = ?', [meshid, 'uuid/' + uuid], func); };
943
- obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { sqlDbExec('SELECT COUNT(id) FROM meshcentral.main WHERE domain = ? AND type = ?', [domainid, type], function (err, response) { func((response['COUNT(id)'] == null) || (response['COUNT(id)'] > max), response['COUNT(id)']) }); } }
944
-
945
- // Database actions on the events collection
946
- obj.GetAllEvents = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.events', null, func); };
947
- obj.StoreEvent = function (event, func) {
948
- var batchQuery = [['INSERT INTO meshcentral.events VALUE (?, ?, ?, ?, ?, ?, ?)', [null, event.time, ((typeof event.domain == 'string') ? event.domain : null), event.action, event.nodeid ? event.nodeid : null, event.userid ? event.userid : null, JSON.stringify(event)]]];
949
- for (var i in event.ids) { if (event.ids[i] != '*') { batchQuery.push(['INSERT INTO meshcentral.eventids VALUE (LAST_INSERT_ID(), ?)', [event.ids[i]]]); } }
950
- sqlDbBatchExec(batchQuery, function (err, docs) { if (func != null) { func(err, docs); } });
951
- };
952
- obj.GetEvents = function (ids, domain, func) {
953
- if (ids.indexOf('*') >= 0) {
954
- sqlDbQuery('SELECT doc FROM meshcentral.events WHERE (domain = ?) ORDER BY time DESC', [domain], func);
955
- } else {
956
- sqlDbQuery('SELECT doc FROM meshcentral.events JOIN meshcentral.eventids ON id = fkid WHERE (domain = ? AND target IN (?)) GROUP BY id ORDER BY time DESC', [domain, ids], func);
957
- }
958
- };
959
- obj.GetEventsWithLimit = function (ids, domain, limit, func) {
960
- if (ids.indexOf('*') >= 0) {
961
- sqlDbQuery('SELECT doc FROM meshcentral.events WHERE (domain = ?) ORDER BY time DESC LIMIT ?', [domain, limit], func);
962
- } else {
963
- sqlDbQuery('SELECT doc FROM meshcentral.events JOIN meshcentral.eventids ON id = fkid WHERE (domain = ? AND target IN (?)) GROUP BY id ORDER BY time DESC LIMIT ?', [domain, ids, limit], func);
964
- }
965
- };
966
- obj.GetUserEvents = function (ids, domain, username, func) {
967
- const userid = 'user/' + domain + '/' + username.toLowerCase();
968
- if (ids.indexOf('*') >= 0) {
969
- sqlDbQuery('SELECT doc FROM meshcentral.events WHERE (domain = ? AND userid = ?) ORDER BY time DESC', [domain, userid], func);
970
- } else {
971
- sqlDbQuery('SELECT doc FROM meshcentral.events JOIN meshcentral.eventids ON id = fkid WHERE (domain = ? AND userid = ? AND target IN (?)) GROUP BY id ORDER BY time DESC', [domain, userid, ids, limit], func);
972
- }
973
- };
974
- obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) {
975
- const userid = 'user/' + domain + '/' + username.toLowerCase();
976
- if (ids.indexOf('*') >= 0) {
977
- sqlDbQuery('SELECT doc FROM meshcentral.events WHERE (domain = ? AND userid = ?) ORDER BY time DESC LIMIT ?', [domain, userid, limit], func);
978
- } else {
979
- sqlDbQuery('SELECT doc FROM meshcentral.events JOIN meshcentral.eventids ON id = fkid WHERE (domain = ? AND userid = ? AND target IN (?)) GROUP BY id ORDER BY time DESC LIMIT ?', [domain, userid, ids, limit], func);
980
- }
981
- };
982
- obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { sqlDbQuery('SELECT doc FROM meshcentral.events WHERE (nodeid = ?) AND (domain = ?) ORDER BY time DESC LIMIT ?', [nodeid, domain, limit], func); };
983
- obj.GetNodeEventsSelfWithLimit = function (nodeid, domain, userid, limit, func) { sqlDbQuery('SELECT doc FROM meshcentral.events WHERE (nodeid = ?) AND (domain = ?) AND ((userid = ?) OR (userid IS NULL)) ORDER BY time DESC LIMIT ?', [nodeid, domain, userid, limit], func); };
984
- obj.RemoveAllEvents = function (domain) { sqlDbQuery('DELETE FROM meshcentral.events', null, function (err, docs) { }); };
985
- obj.RemoveAllNodeEvents = function (domain, nodeid) { sqlDbQuery('DELETE FROM meshcentral.events WHERE domain = ? AND nodeid = ?', [domain, nodeid], function (err, docs) { }); };
986
- obj.RemoveAllUserEvents = function (domain, userid) { sqlDbQuery('DELETE FROM meshcentral.events WHERE domain = ? AND userid = ?', [domain, userid], function (err, docs) { }); };
987
- obj.GetFailedLoginCount = function (username, domainid, lastlogin, func) { sqlDbExec('SELECT COUNT(id) FROM meshcentral.events WHERE action = "authfail" AND domain = ? AND userid = ? AND time > ?', [domainid, 'user/' + domainid + '/' + username.toLowerCase(), lastlogin], function (err, response) { func(err == null ? response['COUNT(id)'] : 0); }); }
988
-
989
- // Database actions on the power collection
990
- obj.getAllPower = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.power', null, func); };
991
- obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } sqlDbQuery('INSERT INTO meshcentral.power VALUE (?, ?, ?, ?)', [null, event.time, event.nodeid ? event.nodeid : null, JSON.stringify(event)], func); };
992
- obj.getPowerTimeline = function (nodeid, func) { sqlDbQuery('SELECT doc FROM meshcentral.power WHERE ((nodeid = ?) OR (nodeid = "*")) ORDER BY time DESC', [nodeid], func); };
993
- obj.removeAllPowerEvents = function () { sqlDbQuery('DELETE FROM meshcentral.power', null, function (err, docs) { }); };
994
- obj.removeAllPowerEventsForNode = function (nodeid) { sqlDbQuery('DELETE FROM meshcentral.power WHERE nodeid = ?', [nodeid], function (err, docs) { }); };
995
-
996
- // Database actions on the SMBIOS collection
997
- obj.GetAllSMBIOS = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.smbios', null, func); };
998
- obj.SetSMBIOS = function (smbios, func) { var expire = new Date(smbios.time); expire.setMonth(expire.getMonth() + 6); sqlDbQuery('REPLACE INTO meshcentral.smbios VALUE (?, ?, ?, ?)', [smbios._id, smbios.time, expire, JSON.stringify(smbios)], func); };
999
- obj.RemoveSMBIOS = function (id) { sqlDbQuery('DELETE FROM meshcentral.smbios WHERE id = ?', [id], function (err, docs) { }); };
1000
- obj.GetSMBIOS = function (id, func) { sqlDbQuery('SELECT doc FROM meshcentral.smbios WHERE id = ?', [id], func); };
1001
-
1002
- // Database actions on the Server Stats collection
1003
- obj.SetServerStats = function (data, func) { sqlDbQuery('REPLACE INTO meshcentral.serverstats VALUE (?, ?, ?)', [data.time, data.expire, JSON.stringify(data)], func); };
1004
- obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); sqlDbQuery('SELECT doc FROM meshcentral.main WHERE time < ?', [t], func); }; // TODO: Expire old entries
1005
-
1006
- // Read a configuration file from the database
1007
- obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); }
1008
-
1009
- // Write a configuration file to the database
1010
- obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); }
1011
-
1012
- // List all configuration files
1013
- obj.listConfigFiles = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.main WHERE type = "cfile" ORDER BY id', func); }
1014
-
1015
- // Get all configuration files
1016
- obj.getAllConfigFiles = function (password, func) {
1017
- obj.file.find({ type: 'cfile' }).toArray(function (err, docs) {
1018
- if (err != null) { func(null); return; }
1019
- var r = null;
1020
- for (var i = 0; i < docs.length; i++) {
1021
- var name = docs[i]._id.split('/')[1];
1022
- var data = obj.decryptData(password, docs[i].data);
1023
- if (data != null) { if (r == null) { r = {}; } r[name] = data; }
1024
- }
1025
- func(r);
1026
- });
1027
- }
1028
-
1029
- // Get database information (TODO: Complete this)
1030
- obj.getDbStats = function (func) {
1031
- obj.stats = { c: 4 };
1032
- sqlDbExec('SELECT COUNT(id) FROM meshcentral.main', null, function (err, response) { obj.stats.meshcentral = response['COUNT(id)']; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1033
- sqlDbExec('SELECT COUNT(time) FROM meshcentral.serverstats', null, function (err, response) { obj.stats.serverstats = response['COUNT(time)']; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1034
- sqlDbExec('SELECT COUNT(id) FROM meshcentral.power', null, function (err, response) { obj.stats.power = response['COUNT(id)']; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1035
- sqlDbExec('SELECT COUNT(id) FROM meshcentral.smbios', null, function (err, response) { obj.stats.smbios = response['COUNT(id)']; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1036
- }
1037
-
1038
- // Plugin operations
1039
- if (obj.pluginsActive) {
1040
- obj.addPlugin = function (plugin, func) { sqlDbQuery('INSERT INTO meshcentral.plugin VALUE (?, ?)', [null, JSON.stringify(value)], func); }; // Add a plugin
1041
- obj.getPlugins = function (func) { sqlDbQuery('SELECT doc FROM meshcentral.plugin', null, func); }; // Get all plugins
1042
- obj.getPlugin = function (id, func) { sqlDbQuery('SELECT doc FROM meshcentral.plugin WHERE id = ?', [id], func); }; // Get plugin
1043
- obj.deletePlugin = function (id, func) { sqlDbQuery('DELETE FROM meshcentral.plugin WHERE id = ?', [id], func); }; // Delete plugin
1044
- obj.setPluginStatus = function (id, status, func) { obj.getPlugin(id, function (err, docs) { if ((err == null) && (docs.length == 1)) { docs[0].status = status; obj.updatePlugin(id, docs[0], func); } }); };
1045
- obj.updatePlugin = function (id, args, func) { delete args._id; sqlDbQuery('REPLACE INTO meshcentral.plugin VALUE (?, ?)', [id, JSON.stringify(args)], func); };
1046
- }
1047
- } else if (obj.databaseType == 3) {
1048
- // Database actions on the main collection (MongoDB)
1049
- obj.Set = function (data, func) { // Fast Set operation using bulkWrite(), this is much faster then using replaceOne()
1050
- if (obj.filePendingSet == false) {
1051
- // Perform the operation now
1052
- obj.filePendingSet = true; obj.filePendingSets = null;
1053
- if (func != null) { obj.filePendingCbs = [func]; }
1054
- obj.file.bulkWrite([{ replaceOne: { filter: { _id: data._id }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(data)), upsert: true } }], fileBulkWriteCompleted);
1055
- } else {
1056
- // Add this operation to the pending list
1057
- if (obj.filePendingSets == null) { obj.filePendingSets = {} }
1058
- obj.filePendingSets[data._id] = data;
1059
- if (func != null) { if (obj.filePendingCb == null) { obj.filePendingCb = [ func ]; } else { obj.filePendingCb.push(func); } }
1060
- }
1061
- };
1062
-
1063
- /*
1064
- obj.Get = function (id, func) {
1065
- if (arguments.length > 2) {
1066
- var parms = [func];
1067
- for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); }
1068
- var func2 = function _func2(arg1, arg2) {
1069
- var userCallback = _func2.userArgs.shift();
1070
- _func2.userArgs.unshift(arg2);
1071
- _func2.userArgs.unshift(arg1);
1072
- userCallback.apply(obj, _func2.userArgs);
1073
- };
1074
- func2.userArgs = parms;
1075
- obj.file.find({ _id: id }).toArray(function (err, docs) {
1076
- if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1077
- func2(err, performTypedRecordDecrypt(docs));
1078
- });
1079
- } else {
1080
- obj.file.find({ _id: id }).toArray(function (err, docs) {
1081
- if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1082
- func(err, performTypedRecordDecrypt(docs));
1083
- });
1084
- }
1085
- };
1086
- */
1087
-
1088
- obj.Get = function (id, func) { // Fast Get operation using a bulk find() to reduce round trips to the database.
1089
- // Encode arguments into return function if any are present.
1090
- var func2 = func;
1091
- if (arguments.length > 2) {
1092
- var parms = [func];
1093
- for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); }
1094
- var func2 = function _func2(arg1, arg2) {
1095
- var userCallback = _func2.userArgs.shift();
1096
- _func2.userArgs.unshift(arg2);
1097
- _func2.userArgs.unshift(arg1);
1098
- userCallback.apply(obj, _func2.userArgs);
1099
- };
1100
- func2.userArgs = parms;
1101
- }
1102
-
1103
- if (obj.filePendingGets == null) {
1104
- // No pending gets, perform the operation now.
1105
- obj.filePendingGets = {};
1106
- obj.filePendingGets[id] = [func2];
1107
- obj.file.find({ _id: id }).toArray(fileBulkReadCompleted);
1108
- } else {
1109
- // Add get to pending list.
1110
- if (obj.filePendingGet == null) { obj.filePendingGet = {}; }
1111
- if (obj.filePendingGet[id] == null) { obj.filePendingGet[id] = [func2]; } else { obj.filePendingGet[id].push(func2); }
1112
- }
1113
- };
1114
-
1115
- obj.GetAll = function (func) { obj.file.find({}).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1116
- obj.GetHash = function (id, func) { obj.file.find({ _id: id }).project({ _id: 0, hash: 1 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1117
- obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }).project({ type: 0 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1118
- obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, extrasids, domain, type, id, func) {
1119
- if (extrasids == null) {
1120
- var x = { type: type, domain: domain, meshid: { $in: meshes } };
1121
- if (id) { x._id = id; }
1122
- obj.file.find(x, { type: 0 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1123
- } else {
1124
- var x = { type: type, domain: domain, $or: [ { meshid: { $in: meshes } }, { _id: { $in: extrasids } } ] };
1125
- if (id) { x._id = id; }
1126
- obj.file.find(x, { type: 0 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1127
- }
1128
- };
1129
- obj.GetAllTypeNodeFiltered = function (nodes, domain, type, id, func) {
1130
- var x = { type: type, domain: domain, nodeid: { $in: nodes } };
1131
- if (id) { x._id = id; }
1132
- obj.file.find(x, { type: 0 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1133
- };
1134
- obj.GetAllType = function (type, func) { obj.file.find({ type: type }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1135
- obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1136
- obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1137
- obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1138
-
1139
- obj.Remove = function (id, func) { // Fast remove operation using a bulk find() to reduce round trips to the database.
1140
- if (obj.filePendingRemoves == null) {
1141
- // No pending gets, perform the operation now.
1142
- obj.filePendingRemoves = {};
1143
- obj.filePendingRemoves[id] = [func];
1144
- obj.file.deleteOne({ _id: id }, fileBulkRemoveCompleted);
1145
- } else {
1146
- // Add remove to pending list.
1147
- if (obj.filePendingRemove == null) { obj.filePendingRemove = {}; }
1148
- if (obj.filePendingRemove[id] == null) { obj.filePendingRemove[id] = [func]; } else { obj.filePendingRemove[id].push(func); }
1149
- }
1150
- };
1151
-
1152
- obj.RemoveAll = function (func) { obj.file.deleteMany({}, { multi: true }, func); };
1153
- obj.RemoveAllOfType = function (type, func) { obj.file.deleteMany({ type: type }, { multi: true }, func); };
1154
- obj.InsertMany = function (data, func) { obj.file.insertMany(data, func); };
1155
- obj.RemoveMeshDocuments = function (id) { obj.file.deleteMany({ meshid: id }, { multi: true }); obj.file.deleteOne({ _id: 'nt' + id }); };
1156
- obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if ((err == null) && (docs.length == 1)) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); };
1157
- obj.DeleteDomain = function (domain, func) { obj.file.deleteMany({ domain: domain }, { multi: true }, func); };
1158
- obj.SetUser = function (user) { if (user.subscriptions != null) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); } else { obj.Set(user); } };
1159
- obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } };
1160
- obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }).toArray(func); }; // TODO: This query is not optimized, but local mode only.
1161
- obj.getAmtUuidMeshNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }).toArray(func); };
1162
-
1163
- // TODO: Starting in MongoDB 4.0.3, you should use countDocuments() instead of count() that is deprecated. We should detect MongoDB version and switch.
1164
- // https://docs.mongodb.com/manual/reference/method/db.collection.countDocuments/
1165
- //obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.countDocuments({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max)); }); } }
1166
- obj.isMaxType = function (max, type, domainid, func) {
1167
- if (obj.file.countDocuments) {
1168
- if (max == null) { func(false); } else { obj.file.countDocuments({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max), count); }); }
1169
- } else {
1170
- if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max), count); }); }
1171
- }
1172
- }
1173
-
1174
- // Database actions on the events collection
1175
- obj.GetAllEvents = function (func) { obj.eventsfile.find({}).toArray(func); };
1176
- obj.StoreEvent = function (event, func) { // Fast MongoDB event store using bulkWrite()
1177
- if (obj.eventsFilePendingSet == false) {
1178
- // Perform the operation now
1179
- obj.eventsFilePendingSet = true; obj.eventsFilePendingSets = null;
1180
- if (func != null) { obj.eventsFilePendingCbs = [func]; }
1181
- obj.eventsfile.bulkWrite([{ insertOne: { document: event } }], eventsFileBulkWriteCompleted);
1182
- } else {
1183
- // Add this operation to the pending list
1184
- if (obj.eventsFilePendingSets == null) { obj.eventsFilePendingSets = [] }
1185
- obj.eventsFilePendingSets.push(event);
1186
- if (func != null) { if (obj.eventsFilePendingCb == null) { obj.eventsFilePendingCb = [func]; } else { obj.eventsFilePendingCb.push(func); } }
1187
- }
1188
- };
1189
- obj.GetEvents = function (ids, domain, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
1190
- obj.GetEventsWithLimit = function (ids, domain, limit, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
1191
- obj.GetUserEvents = function (ids, domain, username, func) { obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
1192
- obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) { obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
1193
- obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }).project({ type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
1194
- obj.GetNodeEventsSelfWithLimit = function (nodeid, domain, userid, limit, func) { obj.eventsfile.find({ domain: domain, nodeid: nodeid, userid: { $in: [userid, null] } }).project({ type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
1195
- obj.RemoveAllEvents = function (domain) { obj.eventsfile.deleteMany({ domain: domain }, { multi: true }); };
1196
- obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.deleteMany({ domain: domain, nodeid: nodeid }, { multi: true }); };
1197
- obj.RemoveAllUserEvents = function (domain, userid) { obj.eventsfile.deleteMany({ domain: domain, userid: userid }, { multi: true }); };
1198
- obj.GetFailedLoginCount = function (username, domainid, lastlogin, func) {
1199
- if (obj.eventsfile.countDocuments) {
1200
- obj.eventsfile.countDocuments({ action: 'authfail', username: username, domain: domainid, time: { "$gte": lastlogin } }, function (err, count) { func((err == null) ? count : 0); });
1201
- } else {
1202
- obj.eventsfile.count({ action: 'authfail', username: username, domain: domainid, time: { "$gte": lastlogin } }, function (err, count) { func((err == null) ? count : 0); });
1203
- }
1204
- }
1205
-
1206
- // Database actions on the power collection
1207
- obj.getAllPower = function (func) { obj.powerfile.find({}).toArray(func); };
1208
- obj.storePowerEvent = function (event, multiServer, func) { // Fast MongoDB event store using bulkWrite()
1209
- if (multiServer != null) { event.server = multiServer.serverid; }
1210
- if (obj.powerFilePendingSet == false) {
1211
- // Perform the operation now
1212
- obj.powerFilePendingSet = true; obj.powerFilePendingSets = null;
1213
- if (func != null) { obj.powerFilePendingCbs = [func]; }
1214
- obj.powerfile.bulkWrite([{ insertOne: { document: event } }], powerFileBulkWriteCompleted);
1215
- } else {
1216
- // Add this operation to the pending list
1217
- if (obj.powerFilePendingSets == null) { obj.powerFilePendingSets = [] }
1218
- obj.powerFilePendingSets.push(event);
1219
- if (func != null) { if (obj.powerFilePendingCb == null) { obj.powerFilePendingCb = [func]; } else { obj.powerFilePendingCb.push(func); } }
1220
- }
1221
- };
1222
- obj.getPowerTimeline = function (nodeid, func) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }).project({ _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).toArray(func); };
1223
- obj.removeAllPowerEvents = function () { obj.powerfile.deleteMany({}, { multi: true }); };
1224
- obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.deleteMany({ nodeid: nodeid }, { multi: true }); };
1225
-
1226
- // Database actions on the SMBIOS collection
1227
- obj.GetAllSMBIOS = function (func) { obj.smbiosfile.find({}).toArray(func); };
1228
- obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.updateOne({ _id: smbios._id }, { $set: smbios }, { upsert: true }, func); };
1229
- obj.RemoveSMBIOS = function (id) { obj.smbiosfile.deleteOne({ _id: id }); };
1230
- obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }).toArray(func); };
1231
-
1232
- // Database actions on the Server Stats collection
1233
- obj.SetServerStats = function (data, func) { obj.serverstatsfile.insertOne(data, func); };
1234
- obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }).toArray(func); };
1235
-
1236
- // Read a configuration file from the database
1237
- obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); }
1238
-
1239
- // Write a configuration file to the database
1240
- obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); }
1241
-
1242
- // List all configuration files
1243
- obj.listConfigFiles = function (func) { obj.file.find({ type: 'cfile' }).sort({ _id: 1 }).toArray(func); }
1244
-
1245
- // Get all configuration files
1246
- obj.getAllConfigFiles = function (password, func) {
1247
- obj.file.find({ type: 'cfile' }).toArray(function (err, docs) {
1248
- if (err != null) { func(null); return; }
1249
- var r = null;
1250
- for (var i = 0; i < docs.length; i++) {
1251
- var name = docs[i]._id.split('/')[1];
1252
- var data = obj.decryptData(password, docs[i].data);
1253
- if (data != null) { if (r == null) { r = {}; } r[name] = data; }
1254
- }
1255
- func(r);
1256
- });
1257
- }
1258
-
1259
- // Get database information
1260
- obj.getDbStats = function (func) {
1261
- obj.stats = { c: 6 };
1262
- obj.getStats(function (r) { obj.stats.recordTypes = r; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } })
1263
- obj.file.stats().then(function (stats) { obj.stats[stats.ns] = { size: stats.size, count: stats.count, avgObjSize: stats.avgObjSize, capped: stats.capped }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } }, function () { if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1264
- obj.eventsfile.stats().then(function (stats) { obj.stats[stats.ns] = { size: stats.size, count: stats.count, avgObjSize: stats.avgObjSize, capped: stats.capped }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } }, function () { if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1265
- obj.powerfile.stats().then(function (stats) { obj.stats[stats.ns] = { size: stats.size, count: stats.count, avgObjSize: stats.avgObjSize, capped: stats.capped }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } }, function () { if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1266
- obj.smbiosfile.stats().then(function (stats) { obj.stats[stats.ns] = { size: stats.size, count: stats.count, avgObjSize: stats.avgObjSize, capped: stats.capped }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } }, function () { if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1267
- obj.serverstatsfile.stats().then(function (stats) { obj.stats[stats.ns] = { size: stats.size, count: stats.count, avgObjSize: stats.avgObjSize, capped: stats.capped }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } }, function () { if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1268
- }
1269
-
1270
- // Plugin operations
1271
- if (obj.pluginsActive) {
1272
- obj.addPlugin = function (plugin, func) { plugin.type = 'plugin'; obj.pluginsfile.insertOne(plugin, func); }; // Add a plugin
1273
- obj.getPlugins = function (func) { obj.pluginsfile.find({ type: 'plugin' }).project({ type: 0 }).sort({ name: 1 }).toArray(func); }; // Get all plugins
1274
- obj.getPlugin = function (id, func) { id = require('mongodb').ObjectID(id); obj.pluginsfile.find({ _id: id }).sort({ name: 1 }).toArray(func); }; // Get plugin
1275
- obj.deletePlugin = function (id, func) { id = require('mongodb').ObjectID(id); obj.pluginsfile.deleteOne({ _id: id }, func); }; // Delete plugin
1276
- obj.setPluginStatus = function (id, status, func) { id = require('mongodb').ObjectID(id); obj.pluginsfile.updateOne({ _id: id }, { $set: { status: status } }, func); };
1277
- obj.updatePlugin = function (id, args, func) { delete args._id; id = require('mongodb').ObjectID(id); obj.pluginsfile.updateOne({ _id: id }, { $set: args }, func); };
1278
- }
1279
-
1280
- } else {
1281
- // Database actions on the main collection (NeDB and MongoJS)
1282
- obj.Set = function (data, func) { data = common.escapeLinksFieldNameEx(data); var xdata = performTypedRecordEncrypt(data); obj.file.update({ _id: xdata._id }, xdata, { upsert: true }, func); };
1283
- obj.Get = function (id, func) {
1284
- if (arguments.length > 2) {
1285
- var parms = [func];
1286
- for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); }
1287
- var func2 = function _func2(arg1, arg2) {
1288
- var userCallback = _func2.userArgs.shift();
1289
- _func2.userArgs.unshift(arg2);
1290
- _func2.userArgs.unshift(arg1);
1291
- userCallback.apply(obj, _func2.userArgs);
1292
- };
1293
- func2.userArgs = parms;
1294
- obj.file.find({ _id: id }, function (err, docs) {
1295
- if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1296
- func2(err, performTypedRecordDecrypt(docs));
1297
- });
1298
- } else {
1299
- obj.file.find({ _id: id }, function (err, docs) {
1300
- if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1301
- func(err, performTypedRecordDecrypt(docs));
1302
- });
1303
- }
1304
- };
1305
- obj.GetAll = function (func) { obj.file.find({}, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1306
- obj.GetHash = function (id, func) { obj.file.find({ _id: id }, { _id: 0, hash: 1 }, func); };
1307
- obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }, { type: 0 }, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1308
- //obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, domain, type, id, func) {
1309
- //var x = { type: type, domain: domain, meshid: { $in: meshes } };
1310
- //if (id) { x._id = id; }
1311
- //obj.file.find(x, { type: 0 }, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1312
- //};
1313
- obj.GetAllTypeNoTypeFieldMeshFiltered = function (meshes, extrasids, domain, type, id, func) {
1314
- if (extrasids == null) {
1315
- var x = { type: type, domain: domain, meshid: { $in: meshes } };
1316
- if (id) { x._id = id; }
1317
- obj.file.find(x, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1318
- } else {
1319
- var x = { type: type, domain: domain, $or: [{ meshid: { $in: meshes } }, { _id: { $in: extrasids } }] };
1320
- if (id) { x._id = id; }
1321
- obj.file.find(x, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1322
- }
1323
- };
1324
- obj.GetAllTypeNodeFiltered = function (nodes, domain, type, id, func) {
1325
- var x = { type: type, domain: domain, nodeid: { $in: nodes } };
1326
- if (id) { x._id = id; }
1327
- obj.file.find(x, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); });
1328
- };
1329
- obj.GetAllType = function (type, func) { obj.file.find({ type: type }, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1330
- obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1331
- obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }, { type: 0 }, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1332
- obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }, function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1333
- obj.Remove = function (id, func) { obj.file.remove({ _id: id }, func); };
1334
- obj.RemoveAll = function (func) { obj.file.remove({}, { multi: true }, func); };
1335
- obj.RemoveAllOfType = function (type, func) { obj.file.remove({ type: type }, { multi: true }, func); };
1336
- obj.InsertMany = function (data, func) { obj.file.insert(data, func); };
1337
- obj.RemoveMeshDocuments = function (id) { obj.file.remove({ meshid: id }, { multi: true }); obj.file.remove({ _id: 'nt' + id }); };
1338
- obj.MakeSiteAdmin = function (username, domain) { obj.Get('user/' + domain + '/' + username, function (err, docs) { if ((err == null) && (docs.length == 1)) { docs[0].siteadmin = 0xFFFFFFFF; obj.Set(docs[0]); } }); };
1339
- obj.DeleteDomain = function (domain, func) { obj.file.remove({ domain: domain }, { multi: true }, func); };
1340
- obj.SetUser = function (user) { if (user.subscriptions != null) { var u = Clone(user); if (u.subscriptions) { delete u.subscriptions; } obj.Set(u); } else { obj.Set(user); } };
1341
- obj.dispose = function () { for (var x in obj) { if (obj[x].close) { obj[x].close(); } delete obj[x]; } };
1342
- obj.getLocalAmtNodes = function (func) { obj.file.find({ type: 'node', host: { $exists: true, $ne: null }, intelamt: { $exists: true } }, func); };
1343
- obj.getAmtUuidMeshNode = function (meshid, uuid, func) { obj.file.find({ type: 'node', meshid: meshid, 'intelamt.uuid': uuid }, func); };
1344
- obj.isMaxType = function (max, type, domainid, func) { if (max == null) { func(false); } else { obj.file.count({ type: type, domain: domainid }, function (err, count) { func((err != null) || (count > max), count); }); } }
1345
-
1346
- // Database actions on the events collection
1347
- obj.GetAllEvents = function (func) { obj.eventsfile.find({}, func); };
1348
- obj.StoreEvent = function (event, func) { obj.eventsfile.insert(event, func); };
1349
- obj.GetEvents = function (ids, domain, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func); } };
1350
- obj.GetEventsWithLimit = function (ids, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func); } };
1351
- obj.GetUserEvents = function (ids, domain, username, func) {
1352
- if (obj.databaseType == 1) {
1353
- obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).exec(func);
1354
- } else {
1355
- obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }, func);
1356
- }
1357
- };
1358
- obj.GetUserEventsWithLimit = function (ids, domain, username, limit, func) {
1359
- if (obj.databaseType == 1) {
1360
- obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).exec(func);
1361
- } else {
1362
- obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }, { type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit, func);
1363
- }
1364
- };
1365
- obj.GetNodeEventsWithLimit = function (nodeid, domain, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } };
1366
- obj.GetNodeEventsSelfWithLimit = function (nodeid, domain, userid, limit, func) { if (obj.databaseType == 1) { obj.eventsfile.find({ domain: domain, nodeid: nodeid, userid: { $in: [userid, null] } }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit).exec(func); } else { obj.eventsfile.find({ domain: domain, nodeid: nodeid }, { type: 0, etype: 0, _id: 0, domain: 0, ids: 0, node: 0, nodeid: 0 }).sort({ time: -1 }).limit(limit, func); } };
1367
- obj.RemoveAllEvents = function (domain) { obj.eventsfile.remove({ domain: domain }, { multi: true }); };
1368
- obj.RemoveAllNodeEvents = function (domain, nodeid) { obj.eventsfile.remove({ domain: domain, nodeid: nodeid }, { multi: true }); };
1369
- obj.RemoveAllUserEvents = function (domain, userid) { obj.eventsfile.remove({ domain: domain, userid: userid }, { multi: true }); };
1370
- obj.GetFailedLoginCount = function (username, domainid, lastlogin, func) { obj.eventsfile.count({ action: 'authfail', username: username, domain: domainid, time: { "$gte": lastlogin } }, function (err, count) { func((err == null) ? count : 0); }); }
1371
-
1372
- // Database actions on the power collection
1373
- obj.getAllPower = function (func) { obj.powerfile.find({}, func); };
1374
- obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insert(event, func); };
1375
- obj.getPowerTimeline = function (nodeid, func) { if (obj.databaseType == 1) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).exec(func); } else { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }, { _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }, func); } };
1376
- obj.removeAllPowerEvents = function () { obj.powerfile.remove({}, { multi: true }); };
1377
- obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.remove({ nodeid: nodeid }, { multi: true }); };
1378
-
1379
- // Database actions on the SMBIOS collection
1380
- if (obj.smbiosfile != null) {
1381
- obj.GetAllSMBIOS = function (func) { obj.smbiosfile.find({}, func); };
1382
- obj.SetSMBIOS = function (smbios, func) { obj.smbiosfile.update({ _id: smbios._id }, smbios, { upsert: true }, func); };
1383
- obj.RemoveSMBIOS = function (id) { obj.smbiosfile.remove({ _id: id }); };
1384
- obj.GetSMBIOS = function (id, func) { obj.smbiosfile.find({ _id: id }, func); };
1385
- }
1386
-
1387
- // Database actions on the Server Stats collection
1388
- obj.SetServerStats = function (data, func) { obj.serverstatsfile.insert(data, func); };
1389
- obj.GetServerStats = function (hours, func) { var t = new Date(); t.setTime(t.getTime() - (60 * 60 * 1000 * hours)); obj.serverstatsfile.find({ time: { $gt: t } }, { _id: 0, cpu: 0 }, func); };
1390
-
1391
- // Read a configuration file from the database
1392
- obj.getConfigFile = function (path, func) { obj.Get('cfile/' + path, func); }
1393
-
1394
- // Write a configuration file to the database
1395
- obj.setConfigFile = function (path, data, func) { obj.Set({ _id: 'cfile/' + path, type: 'cfile', data: data.toString('base64') }, func); }
1396
-
1397
- // List all configuration files
1398
- obj.listConfigFiles = function (func) { obj.file.find({ type: 'cfile' }).sort({ _id: 1 }).exec(func); }
1399
-
1400
- // Get all configuration files
1401
- obj.getAllConfigFiles = function (password, func) {
1402
- obj.file.find({ type: 'cfile' }, function (err, docs) {
1403
- if (err != null) { func(null); return; }
1404
- var r = null;
1405
- for (var i = 0; i < docs.length; i++) {
1406
- var name = docs[i]._id.split('/')[1];
1407
- var data = obj.decryptData(password, docs[i].data);
1408
- if (data != null) { if (r == null) { r = {}; } r[name] = data; }
1409
- }
1410
- func(r);
1411
- });
1412
- }
1413
-
1414
- // Get database information
1415
- obj.getDbStats = function (func) {
1416
- obj.stats = { c: 5 };
1417
- obj.getStats(function (r) { obj.stats.recordTypes = r; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } })
1418
- obj.file.count({}, function (err, count) { obj.stats.meshcentral = { count: count }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1419
- obj.eventsfile.count({}, function (err, count) { obj.stats.events = { count: count }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1420
- obj.powerfile.count({}, function (err, count) { obj.stats.power = { count: count }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1421
- obj.serverstatsfile.count({}, function (err, count) { obj.stats.serverstats = { count: count }; if (--obj.stats.c == 0) { delete obj.stats.c; func(obj.stats); } });
1422
- }
1423
-
1424
- // Plugin operations
1425
- if (obj.pluginsActive) {
1426
- obj.addPlugin = function (plugin, func) { plugin.type = 'plugin'; obj.pluginsfile.insert(plugin, func); }; // Add a plugin
1427
- obj.getPlugins = function (func) { obj.pluginsfile.find({ 'type': 'plugin' }, { 'type': 0 }).sort({ name: 1 }).exec(func); }; // Get all plugins
1428
- obj.getPlugin = function (id, func) { obj.pluginsfile.find({ _id: id }).sort({ name: 1 }).exec(func); }; // Get plugin
1429
- obj.deletePlugin = function (id, func) { obj.pluginsfile.remove({ _id: id }, func); }; // Delete plugin
1430
- obj.setPluginStatus = function (id, status, func) { obj.pluginsfile.update({ _id: id }, { $set: { status: status } }, func); };
1431
- obj.updatePlugin = function (id, args, func) { delete args._id; obj.pluginsfile.update({ _id: id }, { $set: args }, func); };
1432
- }
1433
-
1434
- }
1435
-
1436
- func(obj); // Completed function setup
1437
- }
1438
-
1439
- // Return a human readable string with current backup configuration
1440
- obj.getBackupConfig = function () {
1441
- var r = '', backupPath = parent.backuppath;
1442
- if (parent.config.settings.autobackup && parent.config.settings.autobackup.backuppath) { backupPath = parent.config.settings.autobackup.backuppath; }
1443
- const dbname = (parent.args.mongodbname) ? (parent.args.mongodbname) : 'meshcentral';
1444
- const currentDate = new Date();
1445
- const fileSuffix = currentDate.getFullYear() + '-' + padNumber(currentDate.getMonth() + 1, 2) + '-' + padNumber(currentDate.getDate(), 2) + '-' + padNumber(currentDate.getHours(), 2) + '-' + padNumber(currentDate.getMinutes(), 2);
1446
- const newAutoBackupFile = 'meshcentral-autobackup-' + fileSuffix;
1447
- const newAutoBackupPath = parent.path.join(backupPath, newAutoBackupFile);
1448
-
1449
- r += 'DB Name: ' + dbname + '\r\n';
1450
- r += 'DB Type: ' + ['None', 'NeDB', 'MongoJS', 'MongoDB'][obj.databaseType] + '\r\n';
1451
- r += 'BackupPath: ' + backupPath + '\r\n';
1452
- r += 'newAutoBackupFile: ' + newAutoBackupFile + '\r\n';
1453
- r += 'newAutoBackupPath: ' + newAutoBackupPath + '\r\n';
1454
-
1455
- if (parent.config.settings.autobackup == null) {
1456
- r += 'No Settings/AutoBackup\r\n';
1457
- } else {
1458
- if (parent.config.settings.autobackup.backupintervalhours != null) {
1459
- if (typeof parent.config.settings.autobackup.backupintervalhours != 'number') { r += 'Bad backupintervalhours type\r\n'; }
1460
- else { r += 'Backup Interval (Hours): ' + parent.config.settings.autobackup.backupintervalhours + '\r\n'; }
1461
- }
1462
- if (parent.config.settings.autobackup.keeplastdaysbackup != null) {
1463
- if (typeof parent.config.settings.autobackup.keeplastdaysbackup != 'number') { r += 'Bad keeplastdaysbackup type\r\n'; }
1464
- else { r += 'Keep Last Backups (Days): ' + parent.config.settings.autobackup.keeplastdaysbackup + '\r\n'; }
1465
- }
1466
- if (parent.config.settings.autobackup.zippassword != null) {
1467
- if (typeof parent.config.settings.autobackup.zippassword != 'string') { r += 'Bad zippassword type\r\n'; }
1468
- else { r += 'ZIP Password Set\r\n'; }
1469
- }
1470
- if (parent.config.settings.autobackup.mongodumppath != null) {
1471
- if (typeof parent.config.settings.autobackup.mongodumppath != 'string') { r += 'Bad mongodumppath type\r\n'; }
1472
- else { r += 'MongoDump Path: ' + parent.config.settings.autobackup.mongodumppath + '\r\n'; }
1473
- }
1474
- }
1475
-
1476
- return r;
1477
- }
1478
-
1479
- // Check that the server is capable of performing a backup
1480
- obj.checkBackupCapability = function (func) {
1481
- if ((parent.config.settings.autobackup == null) || (parent.config.settings.autobackup == false)) { func(); }
1482
- if ((obj.databaseType == 2) || (obj.databaseType == 3)) {
1483
- // Check that we have access to MongoDump
1484
- var backupPath = parent.backuppath;
1485
- if (parent.config.settings.autobackup && parent.config.settings.autobackup.backuppath) { backupPath = parent.config.settings.autobackup.backuppath; }
1486
- var mongoDumpPath = 'mongodump';
1487
- if (parent.config.settings.autobackup && parent.config.settings.autobackup.mongodumppath) { mongoDumpPath = parent.config.settings.autobackup.mongodumppath; }
1488
- const child_process = require('child_process');
1489
- child_process.exec('"' + mongoDumpPath + '"', { cwd: backupPath }, function (error, stdout, stderr) {
1490
- try {
1491
- if ((error != null) && (error != '')) {
1492
- if (parent.platform == 'win32') {
1493
- func(1, "Unable to find mongodump.exe, MongoDB database auto-backup will not be performed.");
1494
- } else {
1495
- func(1, "Unable to find mongodump, MongoDB database auto-backup will not be performed.");
1496
- }
1497
- } else {
1498
- func();
1499
- }
1500
- } catch (ex) { console.log(ex); }
1501
- });
1502
- } else {
1503
- func();
1504
- }
1505
- }
1506
-
1507
- // MongoDB pending bulk read operation, perform fast bulk document reads.
1508
- function fileBulkReadCompleted(err, docs) {
1509
- // Send out callbacks with results
1510
- if (docs != null) {
1511
- for (var i in docs) {
1512
- if (docs[i].links != null) { docs[i] = common.unEscapeLinksFieldName(docs[i]); }
1513
- const id = docs[i]._id;
1514
- if (obj.filePendingGets[id] != null) {
1515
- for (var j in obj.filePendingGets[id]) {
1516
- if (typeof obj.filePendingGets[id][j] == 'function') { obj.filePendingGets[id][j](err, performTypedRecordDecrypt([docs[i]])); }
1517
- }
1518
- delete obj.filePendingGets[id];
1519
- }
1520
- }
1521
- }
1522
-
1523
- // If there are not results, send out a null callback
1524
- for (var i in obj.filePendingGets) { for (var j in obj.filePendingGets[i]) { obj.filePendingGets[i][j](err, []); } }
1525
-
1526
- // Move on to process any more pending get operations
1527
- obj.filePendingGets = obj.filePendingGet;
1528
- obj.filePendingGet = null;
1529
- if (obj.filePendingGets != null) {
1530
- var findlist = [];
1531
- for (var i in obj.filePendingGets) { findlist.push(i); }
1532
- obj.file.find({ _id: { $in: findlist } }).toArray(fileBulkReadCompleted);
1533
- }
1534
- }
1535
-
1536
- // MongoDB pending bulk remove operation, perform fast bulk document removes.
1537
- function fileBulkRemoveCompleted(err) {
1538
- // Send out callbacks
1539
- for (var i in obj.filePendingRemoves) {
1540
- for (var j in obj.filePendingRemoves[i]) {
1541
- if (typeof obj.filePendingRemoves[i][j] == 'function') { obj.filePendingRemoves[i][j](err); }
1542
- }
1543
- }
1544
-
1545
- // Move on to process any more pending get operations
1546
- obj.filePendingRemoves = obj.filePendingRemove;
1547
- obj.filePendingRemove = null;
1548
- if (obj.filePendingRemoves != null) {
1549
- var findlist = [], count = 0;
1550
- for (var i in obj.filePendingRemoves) { findlist.push(i); count++; }
1551
- obj.file.deleteMany({ _id: { $in: findlist } }, { multi: true }, fileBulkRemoveCompleted);
1552
- }
1553
- }
1554
-
1555
- // MongoDB pending bulk write operation, perform fast bulk document replacement.
1556
- function fileBulkWriteCompleted() {
1557
- // Callbacks
1558
- if (obj.filePendingCbs != null) {
1559
- for (var i in obj.filePendingCbs) { if (typeof obj.filePendingCbs[i] == 'function') { obj.filePendingCbs[i](); } }
1560
- obj.filePendingCbs = null;
1561
- }
1562
- if (obj.filePendingSets != null) {
1563
- // Perform pending operations
1564
- var ops = [];
1565
- obj.filePendingCbs = obj.filePendingCb;
1566
- obj.filePendingCb = null;
1567
- for (var i in obj.filePendingSets) { ops.push({ replaceOne: { filter: { _id: i }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(obj.filePendingSets[i])), upsert: true } }); }
1568
- obj.file.bulkWrite(ops, fileBulkWriteCompleted);
1569
- obj.filePendingSets = null;
1570
- } else {
1571
- // All done, no pending operations.
1572
- obj.filePendingSet = false;
1573
- }
1574
- }
1575
-
1576
- // MongoDB pending bulk write operation, perform fast bulk document replacement.
1577
- function eventsFileBulkWriteCompleted() {
1578
- // Callbacks
1579
- if (obj.eventsFilePendingCbs != null) { for (var i in obj.eventsFilePendingCbs) { obj.eventsFilePendingCbs[i](); } obj.eventsFilePendingCbs = null; }
1580
- if (obj.eventsFilePendingSets != null) {
1581
- // Perform pending operations
1582
- var ops = [];
1583
- for (var i in obj.eventsFilePendingSets) { ops.push({ document: obj.eventsFilePendingSets[i] }); }
1584
- obj.eventsFilePendingCbs = obj.eventsFilePendingCb;
1585
- obj.eventsFilePendingCb = null;
1586
- obj.eventsFilePendingSets = null;
1587
- obj.eventsfile.bulkWrite(ops, eventsFileBulkWriteCompleted);
1588
- } else {
1589
- // All done, no pending operations.
1590
- obj.eventsFilePendingSet = false;
1591
- }
1592
- }
1593
-
1594
- // MongoDB pending bulk write operation, perform fast bulk document replacement.
1595
- function powerFileBulkWriteCompleted() {
1596
- // Callbacks
1597
- if (obj.powerFilePendingCbs != null) { for (var i in obj.powerFilePendingCbs) { obj.powerFilePendingCbs[i](); } obj.powerFilePendingCbs = null; }
1598
- if (obj.powerFilePendingSets != null) {
1599
- // Perform pending operations
1600
- var ops = [];
1601
- for (var i in obj.powerFilePendingSets) { ops.push({ document: obj.powerFilePendingSets[i] }); }
1602
- obj.powerFilePendingCbs = obj.powerFilePendingCb;
1603
- obj.powerFilePendingCb = null;
1604
- obj.powerFilePendingSets = null;
1605
- obj.powerfile.bulkWrite(ops, powerFileBulkWriteCompleted);
1606
- } else {
1607
- // All done, no pending operations.
1608
- obj.powerFilePendingSet = false;
1609
- }
1610
- }
1611
-
1612
- // Perform a server backup
1613
- obj.performingBackup = false;
1614
- obj.performBackup = function (func) {
1615
- try {
1616
- if (obj.performingBackup) return 1;
1617
- obj.performingBackup = true;
1618
- //console.log('Performing backup...');
1619
-
1620
- var backupPath = parent.backuppath;
1621
- if (parent.config.settings.autobackup && parent.config.settings.autobackup.backuppath) { backupPath = parent.config.settings.autobackup.backuppath; }
1622
- try { parent.fs.mkdirSync(backupPath); } catch (e) { }
1623
- const dbname = (parent.args.mongodbname) ? (parent.args.mongodbname) : 'meshcentral';
1624
- const dburl = parent.args.mongodb;
1625
- const currentDate = new Date();
1626
- const fileSuffix = currentDate.getFullYear() + '-' + padNumber(currentDate.getMonth() + 1, 2) + '-' + padNumber(currentDate.getDate(), 2) + '-' + padNumber(currentDate.getHours(), 2) + '-' + padNumber(currentDate.getMinutes(), 2);
1627
- const newAutoBackupFile = 'meshcentral-autobackup-' + fileSuffix;
1628
- const newAutoBackupPath = parent.path.join(backupPath, newAutoBackupFile);
1629
-
1630
- if ((obj.databaseType == 2) || (obj.databaseType == 3)) {
1631
- // Perform a MongoDump backup
1632
- const newBackupFile = 'mongodump-' + fileSuffix;
1633
- var newBackupPath = parent.path.join(backupPath, newBackupFile);
1634
- var mongoDumpPath = 'mongodump';
1635
- if (parent.config.settings.autobackup && parent.config.settings.autobackup.mongodumppath) { mongoDumpPath = parent.config.settings.autobackup.mongodumppath; }
1636
- const child_process = require('child_process');
1637
- var cmd = '\"' + mongoDumpPath + '\" --db=\"' + dbname + '\" --archive=\"' + newBackupPath + '.archive\"';
1638
- if (dburl) { cmd = '\"' + mongoDumpPath + '\" --uri=\"' + dburl.replace('?', '/?') + '\" --archive=\"' + newBackupPath + '.archive\"'; }
1639
- var backupProcess = child_process.exec(cmd, { cwd: backupPath }, function (error, stdout, stderr) {
1640
- try {
1641
- var mongoDumpSuccess = true;
1642
- backupProcess = null;
1643
- if ((error != null) && (error != '')) { mongoDumpSuccess = false; console.log('ERROR: Unable to perform MongoDB backup: ' + error + '\r\n'); }
1644
-
1645
- // Perform archive compression
1646
- var archiver = require('archiver');
1647
- var output = parent.fs.createWriteStream(newAutoBackupPath + '.zip');
1648
- var archive = null;
1649
- if (parent.config.settings.autobackup && (typeof parent.config.settings.autobackup.zippassword == 'string')) {
1650
- try { archiver.registerFormat('zip-encrypted', require("archiver-zip-encrypted")); } catch (ex) { }
1651
- archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword });
1652
- } else {
1653
- archive = archiver('zip', { zlib: { level: 9 } });
1654
- }
1655
- output.on('close', function () {
1656
- obj.performingBackup = false;
1657
- if (func) { if (mongoDumpSuccess) { func('Auto-backup completed.'); } else { func('Auto-backup completed without mongodb database: ' + error); } }
1658
- obj.performCloudBackup(newAutoBackupPath + '.zip', func);
1659
- setTimeout(function () { try { parent.fs.unlink(newBackupPath + '.archive', function () { }); } catch (ex) { console.log(ex); } }, 5000);
1660
- });
1661
- output.on('end', function () { });
1662
- archive.on('warning', function (err) { console.log('Backup warning: ' + err); if (func) { func('Backup warning: ' + err); } });
1663
- archive.on('error', function (err) { console.log('Backup error: ' + err); if (func) { func('Backup error: ' + err); } });
1664
- archive.pipe(output);
1665
- if (mongoDumpSuccess == true) { archive.file(newBackupPath + '.archive', { name: newBackupFile + '.archive' }); }
1666
- archive.directory(parent.datapath, 'meshcentral-data');
1667
- archive.finalize();
1668
- } catch (ex) { console.log(ex); }
1669
- });
1670
- } else {
1671
- // Perform a NeDB backup
1672
- var archiver = require('archiver');
1673
- var output = parent.fs.createWriteStream(newAutoBackupPath + '.zip');
1674
- var archive = null;
1675
- if (parent.config.settings.autobackup && (typeof parent.config.settings.autobackup.zippassword == 'string')) {
1676
- try { archiver.registerFormat('zip-encrypted', require("archiver-zip-encrypted")); } catch (ex) { }
1677
- archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword });
1678
- } else {
1679
- archive = archiver('zip', { zlib: { level: 9 } });
1680
- }
1681
- output.on('close', function () { obj.performingBackup = false; if (func) { func('Auto-backup completed.'); } obj.performCloudBackup(newAutoBackupPath + '.zip', func); });
1682
- output.on('end', function () { });
1683
- archive.on('warning', function (err) { console.log('Backup warning: ' + err); if (func) { func('Backup warning: ' + err); } });
1684
- archive.on('error', function (err) { console.log('Backup error: ' + err); if (func) { func('Backup error: ' + err); } });
1685
- archive.pipe(output);
1686
- archive.directory(parent.datapath, 'meshcentral-data');
1687
- archive.finalize();
1688
- }
1689
-
1690
- // Remove old backups
1691
- if (parent.config.settings.autobackup && (typeof parent.config.settings.autobackup.keeplastdaysbackup == 'number')) {
1692
- var cutoffDate = new Date();
1693
- cutoffDate.setDate(cutoffDate.getDate() - parent.config.settings.autobackup.keeplastdaysbackup);
1694
- parent.fs.readdir(parent.backuppath, function (err, dir) {
1695
- try {
1696
- if ((err == null) && (dir.length > 0)) {
1697
- for (var i in dir) {
1698
- var name = dir[i];
1699
- if (name.startsWith('meshcentral-autobackup-') && name.endsWith('.zip')) {
1700
- var timex = name.substring(23, name.length - 4).split('-');
1701
- if (timex.length == 5) {
1702
- var fileDate = new Date(parseInt(timex[0]), parseInt(timex[1]) - 1, parseInt(timex[2]), parseInt(timex[3]), parseInt(timex[4]));
1703
- if (fileDate && (cutoffDate > fileDate)) { try { parent.fs.unlink(parent.path.join(parent.backuppath, name), function () { }); } catch (ex) { } }
1704
- }
1705
- }
1706
- }
1707
- }
1708
- } catch (ex) { console.log(ex); }
1709
- });
1710
- }
1711
- } catch (ex) { console.log(ex); }
1712
- return 0;
1713
- }
1714
-
1715
- // Perform cloud backup
1716
- obj.performCloudBackup = function (filename, func) {
1717
-
1718
- // WebDAV Backup
1719
- if ((typeof parent.config.settings.autobackup == 'object') && (typeof parent.config.settings.autobackup.webdav == 'object')) {
1720
- const xdateTimeSort = function (a, b) { if (a.xdate > b.xdate) return 1; if (a.xdate < b.xdate) return -1; return 0; }
1721
-
1722
- // Fetch the folder name
1723
- var webdavfolderName = 'MeshCentral-Backups';
1724
- if (typeof parent.config.settings.autobackup.webdav.foldername == 'string') { webdavfolderName = parent.config.settings.autobackup.webdav.foldername; }
1725
-
1726
- // Clean up our WebDAV folder
1727
- function performWebDavCleanup(client) {
1728
- if ((typeof parent.config.settings.autobackup.webdav.maxfiles == 'number') && (parent.config.settings.autobackup.webdav.maxfiles > 1)) {
1729
- var directoryItems = client.getDirectoryContents(webdavfolderName);
1730
- directoryItems.then(
1731
- function (files) {
1732
- for (var i in files) { files[i].xdate = new Date(files[i].lastmod); }
1733
- files.sort(xdateTimeSort);
1734
- while (files.length >= parent.config.settings.autobackup.webdav.maxfiles) {
1735
- client.deleteFile(files.shift().filename).then(function (state) {
1736
- if (func) { func('WebDAV file deleted.'); }
1737
- }).catch(function (err) {
1738
- if (func) { func('WebDAV (deleteFile) error: ' + err); }
1739
- });
1740
- }
1741
- }
1742
- ).catch(function (err) {
1743
- if (func) { func('WebDAV (getDirectoryContents) error: ' + err); }
1744
- });
1745
- }
1746
- }
1747
-
1748
- // Upload to the WebDAV folder
1749
- function performWebDavUpload(client, filepath) {
1750
- var fileStream = require('fs').createReadStream(filepath);
1751
- fileStream.on('close', function () { if (func) { func('WebDAV upload completed'); } })
1752
- fileStream.on('error', function (err) { if (func) { func('WebDAV (fileUpload) error: ' + err); } })
1753
- fileStream.pipe(client.createWriteStream('/' + webdavfolderName + '/' + require('path').basename(filepath)));
1754
- if (func) { func('Uploading using WebDAV...'); }
1755
- }
1756
-
1757
- if (func) { func('Attempting WebDAV upload...'); }
1758
- const { createClient } = require('webdav');
1759
- const client = createClient(parent.config.settings.autobackup.webdav.url, { username: parent.config.settings.autobackup.webdav.username, password: parent.config.settings.autobackup.webdav.password });
1760
- var directoryItems = client.getDirectoryContents('/');
1761
- directoryItems.then(
1762
- function (files) {
1763
- var folderFound = false;
1764
- for (var i in files) { if ((files[i].basename == webdavfolderName) && (files[i].type == 'directory')) { folderFound = true; } }
1765
- if (folderFound == false) {
1766
- client.createDirectory(webdavfolderName).then(function (a) {
1767
- if (a.statusText == 'Created') {
1768
- if (func) { func('WebDAV folder created'); }
1769
- performWebDavUpload(client, filename);
1770
- } else {
1771
- if (func) { func('WebDAV (createDirectory) status: ' + a.statusText); }
1772
- }
1773
- }).catch(function (err) {
1774
- if (func) { func('WebDAV (createDirectory) error: ' + err); }
1775
- });
1776
- } else {
1777
- performWebDavCleanup(client);
1778
- performWebDavUpload(client, filename);
1779
- }
1780
- }
1781
- ).catch(function (err) {
1782
- if (func) { func('WebDAV (getDirectoryContents) error: ' + err); }
1783
- });
1784
- }
1785
-
1786
- // Google Drive Backup
1787
- if ((typeof parent.config.settings.autobackup == 'object') && (typeof parent.config.settings.autobackup.googledrive == 'object')) {
1788
- obj.Get('GoogleDriveBackup', function (err, docs) {
1789
- if ((err != null) || (docs.length != 1) || (docs[0].state != 3)) return;
1790
- if (func) { func('Attempting Google Drive upload...'); }
1791
- const {google} = require('googleapis');
1792
- const oAuth2Client = new google.auth.OAuth2(docs[0].clientid, docs[0].clientsecret, "urn:ietf:wg:oauth:2.0:oob");
1793
- oAuth2Client.on('tokens', function (tokens) { if (tokens.refresh_token) { docs[0].token = tokens.refresh_token; parent.db.Set(docs[0]); } }); // Update the token in the database
1794
- oAuth2Client.setCredentials(docs[0].token);
1795
- const drive = google.drive({ version: 'v3', auth: oAuth2Client });
1796
- const createdTimeSort = function (a, b) { if (a.createdTime > b.createdTime) return 1; if (a.createdTime < b.createdTime) return -1; return 0; }
1797
-
1798
- // Called once we know our folder id, clean up and upload a backup.
1799
- var useGoogleDrive = function (folderid) {
1800
- // List files to see if we need to delete older ones
1801
- if (typeof parent.config.settings.autobackup.googledrive.maxfiles == 'number') {
1802
- drive.files.list({
1803
- q: 'trashed = false and \'' + folderid + '\' in parents',
1804
- fields: 'nextPageToken, files(id, name, size, createdTime)',
1805
- }, function (err, res) {
1806
- if (err) {
1807
- console.log('GoogleDrive (files.list) error: ' + err);
1808
- if (func) { func('GoogleDrive (files.list) error: ' + err); }
1809
- return;
1810
- }
1811
- // Delete any old files if more than 10 files are present in the backup folder.
1812
- res.data.files.sort(createdTimeSort);
1813
- while (res.data.files.length >= parent.config.settings.autobackup.googledrive.maxfiles) { drive.files.delete({ fileId: res.data.files.shift().id }, function (err, res) { }); }
1814
- });
1815
- }
1816
-
1817
- //console.log('Uploading...');
1818
- if (func) { func('Uploading to Google Drive...'); }
1819
-
1820
- // Upload the backup
1821
- drive.files.create({
1822
- requestBody: { name: require('path').basename(filename), mimeType: 'text/plain', parents: [folderid] },
1823
- media: { mimeType: 'application/zip', body: require('fs').createReadStream(filename) },
1824
- }, function (err, res) {
1825
- if (err) {
1826
- console.log('GoogleDrive (files.create) error: ' + err);
1827
- if (func) { func('GoogleDrive (files.create) error: ' + err); }
1828
- return;
1829
- }
1830
- //console.log('Upload done.');
1831
- if (func) { func('Google Drive upload completed.'); }
1832
- });
1833
- }
1834
-
1835
- // Fetch the folder name
1836
- var folderName = 'MeshCentral-Backups';
1837
- if (typeof parent.config.settings.autobackup.googledrive.foldername == 'string') { folderName = parent.config.settings.autobackup.googledrive.foldername; }
1838
-
1839
- // Find our backup folder, create one if needed.
1840
- drive.files.list({
1841
- q: 'mimeType = \'application/vnd.google-apps.folder\' and name=\'' + folderName + '\' and trashed = false',
1842
- fields: 'nextPageToken, files(id, name)',
1843
- }, function (err, res) {
1844
- if (err) {
1845
- console.log('GoogleDrive error: ' + err);
1846
- if (func) { func('GoogleDrive error: ' + err); }
1847
- return;
1848
- }
1849
- if (res.data.files.length == 0) {
1850
- // Create a folder
1851
- drive.files.create({ resource: { 'name': folderName, 'mimeType': 'application/vnd.google-apps.folder' }, fields: 'id' }, function (err, file) {
1852
- if (err) {
1853
- console.log('GoogleDrive (folder.create) error: ' + err);
1854
- if (func) { func('GoogleDrive (folder.create) error: ' + err); }
1855
- return;
1856
- }
1857
- useGoogleDrive(file.data.id);
1858
- });
1859
- } else { useGoogleDrive(res.data.files[0].id); }
1860
- });
1861
- });
1862
- }
1863
- }
1864
-
1865
- // Transfer NeDB data into the current database
1866
- obj.nedbtodb = function (func) {
1867
- var nedbDatastore = require('nedb');
1868
- var datastoreOptions = { filename: parent.getConfigFilePath('meshcentral.db'), autoload: true };
1869
-
1870
- // If a DB encryption key is provided, perform database encryption
1871
- if ((typeof parent.args.dbencryptkey == 'string') && (parent.args.dbencryptkey.length != 0)) {
1872
- // Hash the database password into a AES256 key and setup encryption and decryption.
1873
- var nedbKey = parent.crypto.createHash('sha384').update(parent.args.dbencryptkey).digest('raw').slice(0, 32);
1874
- datastoreOptions.afterSerialization = function (plaintext) {
1875
- const iv = parent.crypto.randomBytes(16);
1876
- const aes = parent.crypto.createCipheriv('aes-256-cbc', nedbKey, iv);
1877
- var ciphertext = aes.update(plaintext);
1878
- ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
1879
- return ciphertext.toString('base64');
1880
- }
1881
- datastoreOptions.beforeDeserialization = function (ciphertext) {
1882
- const ciphertextBytes = Buffer.from(ciphertext, 'base64');
1883
- const iv = ciphertextBytes.slice(0, 16);
1884
- const data = ciphertextBytes.slice(16);
1885
- const aes = parent.crypto.createDecipheriv('aes-256-cbc', nedbKey, iv);
1886
- var plaintextBytes = Buffer.from(aes.update(data));
1887
- plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
1888
- return plaintextBytes.toString();
1889
- }
1890
- }
1891
-
1892
- // Setup all NeDB collections
1893
- var nedbfile = new nedbDatastore(datastoreOptions);
1894
- var nedbeventsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 });
1895
- var nedbpowerfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 });
1896
- var nedbserverstatsfile = new nedbDatastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 });
1897
-
1898
- // Transfered record counts
1899
- var normalRecordsTransferCount = 0;
1900
- var eventRecordsTransferCount = 0;
1901
- var powerRecordsTransferCount = 0;
1902
- var statsRecordsTransferCount = 0;
1903
- var pendingTransfer = 0;
1904
-
1905
- // Transfer the data from main database
1906
- nedbfile.find({}, function (err, docs) {
1907
- if ((err == null) && (docs.length > 0)) {
1908
- performTypedRecordDecrypt(docs)
1909
- for (var i in docs) {
1910
- pendingTransfer++;
1911
- normalRecordsTransferCount++;
1912
- obj.Set(common.unEscapeLinksFieldName(docs[i]), function () { pendingTransfer--; });
1913
- }
1914
- }
1915
-
1916
- // Transfer events
1917
- nedbeventsfile.find({}, function (err, docs) {
1918
- if ((err == null) && (docs.length > 0)) {
1919
- for (var i in docs) {
1920
- pendingTransfer++;
1921
- eventRecordsTransferCount++;
1922
- obj.StoreEvent(docs[i], function () { pendingTransfer--; });
1923
- }
1924
- }
1925
-
1926
- // Transfer power events
1927
- nedbpowerfile.find({}, function (err, docs) {
1928
- if ((err == null) && (docs.length > 0)) {
1929
- for (var i in docs) {
1930
- pendingTransfer++;
1931
- powerRecordsTransferCount++;
1932
- obj.storePowerEvent(docs[i], null, function () { pendingTransfer--; });
1933
- }
1934
- }
1935
-
1936
- // Transfer server stats
1937
- nedbserverstatsfile.find({}, function (err, docs) {
1938
- if ((err == null) && (docs.length > 0)) {
1939
- for (var i in docs) {
1940
- pendingTransfer++;
1941
- statsRecordsTransferCount++;
1942
- obj.SetServerStats(docs[i], function () { pendingTransfer--; });
1943
- }
1944
- }
1945
-
1946
- // Only exit when all the records are stored.
1947
- setInterval(function () {
1948
- if (pendingTransfer == 0) { func("Done. " + normalRecordsTransferCount + " record(s), " + eventRecordsTransferCount + " event(s), " + powerRecordsTransferCount + " power change(s), " + statsRecordsTransferCount + " stat(s)."); }
1949
- }, 200)
1950
- });
1951
- });
1952
- });
1953
- });
1954
- }
1955
-
1956
- function padNumber(number, digits) { return Array(Math.max(digits - String(number).length + 1, 0)).join(0) + number; }
1957
-
1958
- // Called when a node has changed
1959
- function dbNodeChange(nodeChange, added) {
1960
- common.unEscapeLinksFieldName(nodeChange.fullDocument);
1961
- const node = performTypedRecordDecrypt([nodeChange.fullDocument])[0];
1962
- if (node.intelamt != null) { // Remove the Intel AMT password and MPS password before eventing this.
1963
- if (node.intelamt.pass != null) { node.intelamt.pass = 1; }
1964
- if (node.intelamt.mpspass != null) { node.intelamt.mpspass = 1; }
1965
- }
1966
- parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: (added ? 'addnode' : 'changenode'), node: node, nodeid: node._id, domain: node.domain, nolog: 1 });
1967
- }
1968
-
1969
- // Called when a device group has changed
1970
- function dbMeshChange(meshChange, added) {
1971
- if (parent.webserver == null) return;
1972
- common.unEscapeLinksFieldName(meshChange.fullDocument);
1973
- const mesh = performTypedRecordDecrypt([meshChange.fullDocument])[0];
1974
-
1975
- // Update the mesh object in memory
1976
- const mmesh = parent.webserver.meshes[mesh._id];
1977
- for (var i in mesh) { mmesh[i] = mesh[i]; }
1978
- for (var i in mmesh) { if (mesh[i] == null) { delete mmesh[i]; } }
1979
-
1980
- // Send the mesh update
1981
- if (mesh.deleted) { mesh.action = 'deletemesh'; } else { mesh.action = (added ? 'createmesh' : 'meshchange'); }
1982
- mesh.meshid = mesh._id;
1983
- mesh.nolog = 1;
1984
- delete mesh.type;
1985
- delete mesh._id;
1986
- if ((mesh.amt != null) && (mesh.amt.password != null)) {
1987
- mesh.amt = Object.assign({}, mesh.amt); // Shallow clone
1988
- if (mesh.amt.password != null) { mesh.amt.password = 1; } // Remove the Intel AMT password if present
1989
- }
1990
- parent.DispatchEvent(['*', mesh.meshid], obj, mesh);
1991
- }
1992
-
1993
- // Called when a user account has changed
1994
- function dbUserChange(userChange, added) {
1995
- if (parent.webserver == null) return;
1996
- common.unEscapeLinksFieldName(userChange.fullDocument);
1997
- const user = performTypedRecordDecrypt([userChange.fullDocument])[0];
1998
-
1999
- // Update the user object in memory
2000
- const muser = parent.webserver.users[user._id];
2001
- for (var i in user) { muser[i] = user[i]; }
2002
- for (var i in muser) { if (user[i] == null) { delete muser[i]; } }
2003
-
2004
- // Send the user update
2005
- var targets = ['*', 'server-users', user._id];
2006
- if (user.groups) { for (var i in user.groups) { targets.push('server-users:' + i); } }
2007
- parent.DispatchEvent(targets, obj, { etype: 'user', username: user.name, account: parent.webserver.CloneSafeUser(user), action: (added ? 'accountcreate' : 'accountchange'), domain: user.domain, nolog: 1 });
2008
- }
2009
-
2010
- // Called when a user group has changed
2011
- function dbUGrpChange(ugrpChange, added) {
2012
- if (parent.webserver == null) return;
2013
- common.unEscapeLinksFieldName(ugrpChange.fullDocument);
2014
- const usergroup = ugrpChange.fullDocument;
2015
-
2016
- // Update the user group object in memory
2017
- const uusergroup = parent.webserver.userGroups[usergroup._id];
2018
- for (var i in usergroup) { uusergroup[i] = usergroup[i]; }
2019
- for (var i in uusergroup) { if (usergroup[i] == null) { delete uusergroup[i]; } }
2020
-
2021
- // Send the user group update
2022
- usergroup.action = (added ? 'createusergroup' : 'usergroupchange');
2023
- usergroup.ugrpid = usergroup._id;
2024
- usergroup.nolog = 1;
2025
- delete usergroup.type;
2026
- delete usergroup._id;
2027
- parent.DispatchEvent(['*', usergroup.ugrpid], obj, usergroup);
2028
- }
2029
-
2030
- return obj;
2031
-};
db.js
+253
-26
@@ -39,6 +39,26 @@ module.exports.CreateDB = function (parent, func) {
39
obj.changeStream = false;
40
obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
41
42
+ // MongoDB bulk operations state
43
+ if (parent.config.settings.mongodbbulkoperations) {
44
+ obj.filePendingGet = null;
45
+ obj.filePendingGets = null;
46
+ obj.filePendingRemove = null;
47
+ obj.filePendingRemoves = null;
48
+ obj.filePendingSet = false;
49
+ obj.filePendingSets = null;
50
+ obj.filePendingCb = null;
51
+ obj.filePendingCbs = null;
52
+ obj.powerFilePendingSet = false;
53
+ obj.powerFilePendingSets = null;
54
+ obj.powerFilePendingCb = null;
55
+ obj.powerFilePendingCbs = null;
56
+ obj.eventsFilePendingSet = false;
57
+ obj.eventsFilePendingSets = null;
58
+ obj.eventsFilePendingCb = null;
59
+ obj.eventsFilePendingCbs = null;
60
+ }
61
+
62
obj.SetupDatabase = function (func) {
63
// Check if the database unique identifier is present
64
// This is used to check that in server peering mode, everyone is using the same database.
@@ -1028,29 +1048,74 @@ module.exports.CreateDB = function (parent, func) {
1048
}
1049
} else if (obj.databaseType == 3) {
1050
// Database actions on the main collection (MongoDB)
1031
- obj.Set = function (data, func) { data = common.escapeLinksFieldNameEx(data); obj.file.replaceOne({ _id: data._id }, performTypedRecordEncrypt(data), { upsert: true }, func); };
1032
- obj.Get = function (id, func) {
1033
- if (arguments.length > 2) {
1034
- var parms = [func];
1035
- for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); }
1036
- var func2 = function _func2(arg1, arg2) {
1037
- var userCallback = _func2.userArgs.shift();
1038
- _func2.userArgs.unshift(arg2);
1039
- _func2.userArgs.unshift(arg1);
1040
- userCallback.apply(obj, _func2.userArgs);
1041
- };
1042
- func2.userArgs = parms;
1043
- obj.file.find({ _id: id }).toArray(function (err, docs) {
1044
- if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1045
- func2(err, performTypedRecordDecrypt(docs));
1046
- });
1047
- } else {
1048
- obj.file.find({ _id: id }).toArray(function (err, docs) {
1049
- if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1050
- func(err, performTypedRecordDecrypt(docs));
1051
- });
1052
- }
1053
- };
1051
+
1052
+ // Bulk operations
1053
+ if (parent.config.settings.mongodbbulkoperations) {
1054
+ obj.Set = function (data, func) { // Fast Set operation using bulkWrite(), this is much faster then using replaceOne()
1055
+ if (obj.filePendingSet == false) {
1056
+ // Perform the operation now
1057
+ obj.filePendingSet = true; obj.filePendingSets = null;
1058
+ if (func != null) { obj.filePendingCbs = [func]; }
1059
+ obj.file.bulkWrite([{ replaceOne: { filter: { _id: data._id }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(data)), upsert: true } }], fileBulkWriteCompleted);
1060
+ } else {
1061
+ // Add this operation to the pending list
1062
+ if (obj.filePendingSets == null) { obj.filePendingSets = {} }
1063
+ obj.filePendingSets[data._id] = data;
1064
+ if (func != null) { if (obj.filePendingCb == null) { obj.filePendingCb = [func]; } else { obj.filePendingCb.push(func); } }
1065
+ }
1066
+ };
1067
+
1068
+ obj.Get = function (id, func) { // Fast Get operation using a bulk find() to reduce round trips to the database.
1069
+ // Encode arguments into return function if any are present.
1070
+ var func2 = func;
1071
+ if (arguments.length > 2) {
1072
+ var parms = [func];
1073
+ for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); }
1074
+ var func2 = function _func2(arg1, arg2) {
1075
+ var userCallback = _func2.userArgs.shift();
1076
+ _func2.userArgs.unshift(arg2);
1077
+ _func2.userArgs.unshift(arg1);
1078
+ userCallback.apply(obj, _func2.userArgs);
1079
+ };
1080
+ func2.userArgs = parms;
1081
+ }
1082
+
1083
+ if (obj.filePendingGets == null) {
1084
+ // No pending gets, perform the operation now.
1085
+ obj.filePendingGets = {};
1086
+ obj.filePendingGets[id] = [func2];
1087
+ obj.file.find({ _id: id }).toArray(fileBulkReadCompleted);
1088
+ } else {
1089
+ // Add get to pending list.
1090
+ if (obj.filePendingGet == null) { obj.filePendingGet = {}; }
1091
+ if (obj.filePendingGet[id] == null) { obj.filePendingGet[id] = [func2]; } else { obj.filePendingGet[id].push(func2); }
1092
+ }
1093
+ };
1094
+ } else {
1095
+ obj.Set = function (data, func) { data = common.escapeLinksFieldNameEx(data); obj.file.replaceOne({ _id: data._id }, performTypedRecordEncrypt(data), { upsert: true }, func); };
1096
+ obj.Get = function (id, func) {
1097
+ if (arguments.length > 2) {
1098
+ var parms = [func];
1099
+ for (var parmx = 2; parmx < arguments.length; ++parmx) { parms.push(arguments[parmx]); }
1100
+ var func2 = function _func2(arg1, arg2) {
1101
+ var userCallback = _func2.userArgs.shift();
1102
+ _func2.userArgs.unshift(arg2);
1103
+ _func2.userArgs.unshift(arg1);
1104
+ userCallback.apply(obj, _func2.userArgs);
1105
+ };
1106
+ func2.userArgs = parms;
1107
+ obj.file.find({ _id: id }).toArray(function (err, docs) {
1108
+ if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1109
+ func2(err, performTypedRecordDecrypt(docs));
1110
+ });
1111
+ } else {
1112
+ obj.file.find({ _id: id }).toArray(function (err, docs) {
1113
+ if ((docs != null) && (docs.length > 0) && (docs[0].links != null)) { docs[0] = common.unEscapeLinksFieldName(docs[0]); }
1114
+ func(err, performTypedRecordDecrypt(docs));
1115
+ });
1116
+ }
1117
+ };
1118
+ }
1119
obj.GetAll = function (func) { obj.file.find({}).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1120
obj.GetHash = function (id, func) { obj.file.find({ _id: id }).project({ _id: 0, hash: 1 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1121
obj.GetAllTypeNoTypeField = function (type, domain, func) { obj.file.find({ type: type, domain: domain }).project({ type: 0 }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
@@ -1074,7 +1139,25 @@ module.exports.CreateDB = function (parent, func) {
1139
obj.GetAllIdsOfType = function (ids, domain, type, func) { obj.file.find({ type: type, domain: domain, _id: { $in: ids } }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1140
obj.GetUserWithEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1141
obj.GetUserWithVerifiedEmail = function (domain, email, func) { obj.file.find({ type: 'user', domain: domain, email: email, emailVerified: true }).toArray(function (err, docs) { func(err, performTypedRecordDecrypt(docs)); }); };
1077
- obj.Remove = function (id, func) { obj.file.deleteOne({ _id: id }, func); };
1142
+
1143
+ // Bulk operations
1144
+ if (parent.config.settings.mongodbbulkoperations) {
1145
+ obj.Remove = function (id, func) { // Fast remove operation using a bulk find() to reduce round trips to the database.
1146
+ if (obj.filePendingRemoves == null) {
1147
+ // No pending gets, perform the operation now.
1148
+ obj.filePendingRemoves = {};
1149
+ obj.filePendingRemoves[id] = [func];
1150
+ obj.file.deleteOne({ _id: id }, fileBulkRemoveCompleted);
1151
+ } else {
1152
+ // Add remove to pending list.
1153
+ if (obj.filePendingRemove == null) { obj.filePendingRemove = {}; }
1154
+ if (obj.filePendingRemove[id] == null) { obj.filePendingRemove[id] = [func]; } else { obj.filePendingRemove[id].push(func); }
1155
+ }
1156
+ };
1157
+ } else {
1158
+ obj.Remove = function (id, func) { obj.file.deleteOne({ _id: id }, func); };
1159
+ }
1160
+
1161
obj.RemoveAll = function (func) { obj.file.deleteMany({}, { multi: true }, func); };
1162
obj.RemoveAllOfType = function (type, func) { obj.file.deleteMany({ type: type }, { multi: true }, func); };
1163
obj.InsertMany = function (data, func) { obj.file.insertMany(data, func); };
@@ -1099,7 +1182,26 @@ module.exports.CreateDB = function (parent, func) {
1182
1183
// Database actions on the events collection
1184
obj.GetAllEvents = function (func) { obj.eventsfile.find({}).toArray(func); };
1102
- obj.StoreEvent = function (event, func) { obj.eventsfile.insertOne(event, func); };
1185
+
1186
+ // Bulk operations
1187
+ if (parent.config.settings.mongodbbulkoperations) {
1188
+ obj.StoreEvent = function (event, func) { // Fast MongoDB event store using bulkWrite()
1189
+ if (obj.eventsFilePendingSet == false) {
1190
+ // Perform the operation now
1191
+ obj.eventsFilePendingSet = true; obj.eventsFilePendingSets = null;
1192
+ if (func != null) { obj.eventsFilePendingCbs = [func]; }
1193
+ obj.eventsfile.bulkWrite([{ insertOne: { document: event } }], eventsFileBulkWriteCompleted);
1194
+ } else {
1195
+ // Add this operation to the pending list
1196
+ if (obj.eventsFilePendingSets == null) { obj.eventsFilePendingSets = [] }
1197
+ obj.eventsFilePendingSets.push(event);
1198
+ if (func != null) { if (obj.eventsFilePendingCb == null) { obj.eventsFilePendingCb = [func]; } else { obj.eventsFilePendingCb.push(func); } }
1199
+ }
1200
+ };
1201
+ } else {
1202
+ obj.StoreEvent = function (event, func) { obj.eventsfile.insertOne(event, func); };
1203
+ }
1204
+
1205
obj.GetEvents = function (ids, domain, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
1206
obj.GetEventsWithLimit = function (ids, domain, limit, func) { obj.eventsfile.find({ domain: domain, ids: { $in: ids } }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).limit(limit).toArray(func); };
1207
obj.GetUserEvents = function (ids, domain, username, func) { obj.eventsfile.find({ domain: domain, $or: [{ ids: { $in: ids } }, { username: username }] }).project({ type: 0, _id: 0, domain: 0, ids: 0, node: 0 }).sort({ time: -1 }).toArray(func); };
@@ -1119,7 +1221,27 @@ module.exports.CreateDB = function (parent, func) {
1221
1222
// Database actions on the power collection
1223
obj.getAllPower = function (func) { obj.powerfile.find({}).toArray(func); };
1122
- obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insertOne(event, func); };
1224
+
1225
+ // Bulk operations
1226
+ if (parent.config.settings.mongodbbulkoperations) {
1227
+ obj.storePowerEvent = function (event, multiServer, func) { // Fast MongoDB event store using bulkWrite()
1228
+ if (multiServer != null) { event.server = multiServer.serverid; }
1229
+ if (obj.powerFilePendingSet == false) {
1230
+ // Perform the operation now
1231
+ obj.powerFilePendingSet = true; obj.powerFilePendingSets = null;
1232
+ if (func != null) { obj.powerFilePendingCbs = [func]; }
1233
+ obj.powerfile.bulkWrite([{ insertOne: { document: event } }], powerFileBulkWriteCompleted);
1234
+ } else {
1235
+ // Add this operation to the pending list
1236
+ if (obj.powerFilePendingSets == null) { obj.powerFilePendingSets = [] }
1237
+ obj.powerFilePendingSets.push(event);
1238
+ if (func != null) { if (obj.powerFilePendingCb == null) { obj.powerFilePendingCb = [func]; } else { obj.powerFilePendingCb.push(func); } }
1239
+ }
1240
+ };
1241
+ } else {
1242
+ obj.storePowerEvent = function (event, multiServer, func) { if (multiServer != null) { event.server = multiServer.serverid; } obj.powerfile.insertOne(event, func); };
1243
+ }
1244
+
1245
obj.getPowerTimeline = function (nodeid, func) { obj.powerfile.find({ nodeid: { $in: ['*', nodeid] } }).project({ _id: 0, nodeid: 0, s: 0 }).sort({ time: 1 }).toArray(func); };
1246
obj.removeAllPowerEvents = function () { obj.powerfile.deleteMany({}, { multi: true }); };
1247
obj.removeAllPowerEventsForNode = function (nodeid) { obj.powerfile.deleteMany({ nodeid: nodeid }, { multi: true }); };
@@ -1405,6 +1527,111 @@ module.exports.CreateDB = function (parent, func) {
1527
}
1528
}
1529
1530
+ // MongoDB pending bulk read operation, perform fast bulk document reads.
1531
+ function fileBulkReadCompleted(err, docs) {
1532
+ // Send out callbacks with results
1533
+ if (docs != null) {
1534
+ for (var i in docs) {
1535
+ if (docs[i].links != null) { docs[i] = common.unEscapeLinksFieldName(docs[i]); }
1536
+ const id = docs[i]._id;
1537
+ if (obj.filePendingGets[id] != null) {
1538
+ for (var j in obj.filePendingGets[id]) {
1539
+ if (typeof obj.filePendingGets[id][j] == 'function') { obj.filePendingGets[id][j](err, performTypedRecordDecrypt([docs[i]])); }
1540
+ }
1541
+ delete obj.filePendingGets[id];
1542
+ }
1543
+ }
1544
+ }
1545
+
1546
+ // If there are not results, send out a null callback
1547
+ for (var i in obj.filePendingGets) { for (var j in obj.filePendingGets[i]) { obj.filePendingGets[i][j](err, []); } }
1548
+
1549
+ // Move on to process any more pending get operations
1550
+ obj.filePendingGets = obj.filePendingGet;
1551
+ obj.filePendingGet = null;
1552
+ if (obj.filePendingGets != null) {
1553
+ var findlist = [];
1554
+ for (var i in obj.filePendingGets) { findlist.push(i); }
1555
+ obj.file.find({ _id: { $in: findlist } }).toArray(fileBulkReadCompleted);
1556
+ }
1557
+ }
1558
+
1559
+ // MongoDB pending bulk remove operation, perform fast bulk document removes.
1560
+ function fileBulkRemoveCompleted(err) {
1561
+ // Send out callbacks
1562
+ for (var i in obj.filePendingRemoves) {
1563
+ for (var j in obj.filePendingRemoves[i]) {
1564
+ if (typeof obj.filePendingRemoves[i][j] == 'function') { obj.filePendingRemoves[i][j](err); }
1565
+ }
1566
+ }
1567
+
1568
+ // Move on to process any more pending get operations
1569
+ obj.filePendingRemoves = obj.filePendingRemove;
1570
+ obj.filePendingRemove = null;
1571
+ if (obj.filePendingRemoves != null) {
1572
+ var findlist = [], count = 0;
1573
+ for (var i in obj.filePendingRemoves) { findlist.push(i); count++; }
1574
+ obj.file.deleteMany({ _id: { $in: findlist } }, { multi: true }, fileBulkRemoveCompleted);
1575
+ }
1576
+ }
1577
+
1578
+ // MongoDB pending bulk write operation, perform fast bulk document replacement.
1579
+ function fileBulkWriteCompleted() {
1580
+ // Callbacks
1581
+ if (obj.filePendingCbs != null) {
1582
+ for (var i in obj.filePendingCbs) { if (typeof obj.filePendingCbs[i] == 'function') { obj.filePendingCbs[i](); } }
1583
+ obj.filePendingCbs = null;
1584
+ }
1585
+ if (obj.filePendingSets != null) {
1586
+ // Perform pending operations
1587
+ var ops = [];
1588
+ obj.filePendingCbs = obj.filePendingCb;
1589
+ obj.filePendingCb = null;
1590
+ for (var i in obj.filePendingSets) { ops.push({ replaceOne: { filter: { _id: i }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(obj.filePendingSets[i])), upsert: true } }); }
1591
+ obj.file.bulkWrite(ops, fileBulkWriteCompleted);
1592
+ obj.filePendingSets = null;
1593
+ } else {
1594
+ // All done, no pending operations.
1595
+ obj.filePendingSet = false;
1596
+ }
1597
+ }
1598
+
1599
+ // MongoDB pending bulk write operation, perform fast bulk document replacement.
1600
+ function eventsFileBulkWriteCompleted() {
1601
+ // Callbacks
1602
+ if (obj.eventsFilePendingCbs != null) { for (var i in obj.eventsFilePendingCbs) { obj.eventsFilePendingCbs[i](); } obj.eventsFilePendingCbs = null; }
1603
+ if (obj.eventsFilePendingSets != null) {
1604
+ // Perform pending operations
1605
+ var ops = [];
1606
+ for (var i in obj.eventsFilePendingSets) { ops.push({ document: obj.eventsFilePendingSets[i] }); }
1607
+ obj.eventsFilePendingCbs = obj.eventsFilePendingCb;
1608
+ obj.eventsFilePendingCb = null;
1609
+ obj.eventsFilePendingSets = null;
1610
+ obj.eventsfile.bulkWrite(ops, eventsFileBulkWriteCompleted);
1611
+ } else {
1612
+ // All done, no pending operations.
1613
+ obj.eventsFilePendingSet = false;
1614
+ }
1615
+ }
1616
+
1617
+ // MongoDB pending bulk write operation, perform fast bulk document replacement.
1618
+ function powerFileBulkWriteCompleted() {
1619
+ // Callbacks
1620
+ if (obj.powerFilePendingCbs != null) { for (var i in obj.powerFilePendingCbs) { obj.powerFilePendingCbs[i](); } obj.powerFilePendingCbs = null; }
1621
+ if (obj.powerFilePendingSets != null) {
1622
+ // Perform pending operations
1623
+ var ops = [];
1624
+ for (var i in obj.powerFilePendingSets) { ops.push({ document: obj.powerFilePendingSets[i] }); }
1625
+ obj.powerFilePendingCbs = obj.powerFilePendingCb;
1626
+ obj.powerFilePendingCb = null;
1627
+ obj.powerFilePendingSets = null;
1628
+ obj.powerfile.bulkWrite(ops, powerFileBulkWriteCompleted);
1629
+ } else {
1630
+ // All done, no pending operations.
1631
+ obj.powerFilePendingSet = false;
1632
+ }
1633
+ }
1634
+
1635
// Perform a server backup
1636
obj.performingBackup = false;
1637
obj.performBackup = function (func) {
meshcentral-config-schema.json
+3
-2
@@ -8,9 +8,10 @@
8
"type": "object",
9
"properties": {
10
"cert": { "type": "string", "description": "Set this to the primary DNS name of this MeshCentral server." },
11
- "mongoDb": { "type": "string" },
11
+ "mongoDb": { "type": "string", "default": null },
12
"mongoDbName": { "type": "string" },
13
- "mongoDbChangeStream": { "type": "boolean" },
13
+ "mongoDbChangeStream": { "type": "boolean", "default": false },
14
+ "mongoDbBulkOperations": { "type": "boolean", "default": false },
15
"WANonly": { "type": "boolean", "default": false, "description": "When enabled, only MeshCentral WAN features are enabled and agents will connect to the server using a well known DNS name." },
16
"LANonly": { "type": "boolean", "default": false, "description": "When enabled, only MeshCentral LAN features are enabled and agents will find the server using multicast LAN packets." },
17
"maintenanceMode": { "type": "boolean", "default": false, "description": "When enabled the server is in maintenance mode, only administrators can login. Use the maintenance command in server console to change." },