Added server watchdog feature.
Ylian Saint-Hilaire committed
Jul 3, 2020 at 11:03 UTC
fbabddee7902350546cb9ca94fb9995702ba806c
4 files changed
+42
-20
meshcentral.js
+29
-1
@@ -435,7 +435,7 @@ function CreateMeshCentralServer(config, args) {
435
try {
436
var errlogpath = null;
437
if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); }
438
- obj.fs.appendFileSync(obj.getConfigFilePath('mesherrors.txt'), '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n');
438
+ obj.fs.appendFileSync(errlogpath, '-------- ' + new Date().toLocaleString() + ' ---- ' + getCurrentVerion() + ' --------\r\n\r\n' + data + '\r\n\r\n\r\n');
439
} catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); }
440
});
441
childProcess.on('close', function (code) { if ((code != 0) && (code != 123)) { /* console.log("Exited with code " + code); */ } });
@@ -1431,6 +1431,34 @@ function CreateMeshCentralServer(config, args) {
1431
}
1432
}
1433
}
1434
+
1435
+ // Start watchdog timer if needed
1436
+ // This is used to monitor if NodeJS is servicing IO correctly or getting held up a lot. Add this line to the settings section of config.json
1437
+ // "watchDog": { "interval": 100, "timeout": 150 }
1438
+ // This will check every 100ms, if the timer is more than 150ms late, it will warn.
1439
+ if ((typeof config.settings.watchdog == 'object') && (typeof config.settings.watchdog.interval == 'number') && (typeof config.settings.watchdog.timeout == 'number') && (config.settings.watchdog.interval >= 50) && (config.settings.watchdog.timeout >= 50)) {
1440
+ obj.watchdogtime = Date.now();
1441
+ obj.watchdogmax = 0;
1442
+ obj.watchdogmaxtime = null;
1443
+ obj.watchdogtable = [];
1444
+ obj.watchdog = setInterval(function () {
1445
+ var now = Date.now(), delta = now - obj.watchdogtime - config.settings.watchdog.interval;
1446
+ if (delta > obj.watchdogmax) { obj.watchdogmax = delta; obj.watchdogmaxtime = new Date().toLocaleString(); }
1447
+ if (delta > config.settings.watchdog.timeout) {
1448
+ const msg = obj.common.format("Watchdog timer timeout, {0}ms.", delta);
1449
+ obj.watchdogtable.push(new Date().toLocaleString() + ', ' + delta + 'ms');
1450
+ while (obj.watchdogtable.length > 10) { obj.watchdogtable.shift(); }
1451
+ obj.debug('main', msg);
1452
+ try {
1453
+ var errlogpath = null;
1454
+ if (typeof obj.args.mesherrorlogpath == 'string') { errlogpath = obj.path.join(obj.args.mesherrorlogpath, 'mesherrors.txt'); } else { errlogpath = obj.getConfigFilePath('mesherrors.txt'); }
1455
+ obj.fs.appendFileSync(errlogpath, new Date().toLocaleString() + ': ' + msg + '\r\n');
1456
+ } catch (ex) { console.log('ERROR: Unable to write to mesherrors.txt.'); }
1457
+ }
1458
+ obj.watchdogtime = now;
1459
+ }, config.settings.watchdog.interval);
1460
+ obj.debug('main', "Started watchdog timer.");
1461
+ }
1462
});
1463
});
1464
};
meshuser.js
+11
-18
@@ -775,7 +775,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
775
776
switch (cmd) {
777
case 'help': {
778
- var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes';
778
+ var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents,dbstats,sms,amtacm,certhashes,watchdog';
779
if (parent.parent.config.settings.heapdump === true) { availcommands += ',heapdump'; }
780
availcommands = availcommands.split(',').sort();
781
while (availcommands.length > 0) { if (f.length > 80) { fin += (f + ',\r\n'); f = ''; } f += (((f != '') ? ', ' : ' ') + availcommands.shift()); }
@@ -931,6 +931,16 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
931
}
932
break;
933
}
934
+ case 'watchdog': {
935
+ if (parent.parent.watchdog == null) {
936
+ r = 'Server watchdog not active.';
937
+ } else {
938
+ r = 'Server watchdog active.\r\n';
939
+ if (parent.parent.watchdogmaxtime != null) { r += 'Largest timeout was ' + parent.parent.watchdogmax + 'ms on ' + parent.parent.watchdogmaxtime + '\r\n'; }
940
+ for (var i in parent.parent.watchdogtable) { r += parent.parent.watchdogtable[i] + '\r\n'; }
941
+ }
942
+ break;
943
+ }
944
case 'acceleratorsstats': {
945
var stats = parent.parent.certificateOperations.getAcceleratorStats();
946
for (var i in stats) {
@@ -1091,23 +1101,6 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
1101
if (parent.parent.webViewsOverridePath) { r += 'XWebViews: ' + parent.parent.webPublicOverridePath + '\r\n'; }
1102
break;
1103
}
1094
- case 'showconfig': {
1095
- // Make a copy of the configuration and hide any secrets
1096
- var config = common.Clone(parent.parent.config);
1097
- if (config.settings) {
1098
- if (config.settings.configkey) { config.settings.configkey = '(present)'; }
1099
- if (config.settings.sessionkey) { config.settings.sessionkey = '(present)'; }
1100
- if (config.settings.dbencryptkey) { config.settings.dbencryptkey = '(present)'; }
1101
- }
1102
- if (config.domains) {
1103
- for (var i in config.domains) {
1104
- if (config.domains[i].yubikey && config.domains[i].yubikey.secret) { config.domains[i].yubikey.secret = '(present)'; }
1105
- }
1106
- }
1107
-
1108
- r = JSON.stringify(removeAllUnderScore(config), null, 4);
1109
- break;
1110
- }
1104
case 'migrationagents': {
1105
if (parent.parent.swarmserver == null) {
1106
r = 'Swarm server not running.';
public/x.pdf
Binary files /dev/null and b/public/x.pdf differ
sample-config-advanced.json
+2
-1
@@ -96,6 +96,7 @@
96
"count": 10,
97
"coolofftime": 10
98
},
99
+ "watchDog": { "interval": 100, "timeout": 400 },
100
"_plugins": { "enabled": true }
101
},
102
"_domaindefaults": {
@@ -194,7 +195,7 @@
195
"x-frame-options": "SAMEORIGIN",
196
"Content-Security-Policy": "default-src 'none'; script-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-src 'self'; media-src 'self'"
197
},
197
- "_agentConfig": [ "webSocketMaskOverride=1" ],
198
+ "_agentConfig": [ "webSocketMaskOverride=1", "coreDumpEnabled=1" ],
199
"_sessionRecording": {
200
"_filepath": "C:\\temp",
201
"_index": true,