Improved agent core dump collection system.
Ylian Saint-Hilaire committed
Jul 8, 2020 at 11:59 UTC
d3825eb4965981df1464e6b4c4d7de3968384975
3 files changed
+31
-11
db.js
+5
-5
@@ -669,7 +669,7 @@ module.exports.CreateDB = function (parent, func) {
669
670
// Start NeDB main collection and setup indexes
671
obj.file = new Datastore(datastoreOptions);
672
- obj.file.persistence.setAutocompactionInterval(36000);
672
+ obj.file.persistence.setAutocompactionInterval(86400000); // Compact once a day
673
obj.file.ensureIndex({ fieldName: 'type' });
674
obj.file.ensureIndex({ fieldName: 'domain' });
675
obj.file.ensureIndex({ fieldName: 'meshid', sparse: true });
@@ -678,14 +678,14 @@ module.exports.CreateDB = function (parent, func) {
678
679
// Setup the events collection and setup indexes
680
obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true });
681
- obj.eventsfile.persistence.setAutocompactionInterval(36000);
681
+ obj.eventsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
682
obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field.
683
obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true });
684
obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 20 }); // Limit the power event log to 20 days (Seconds * Minutes * Hours * Days)
685
686
// Setup the power collection and setup indexes
687
obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true });
688
- obj.powerfile.persistence.setAutocompactionInterval(36000);
688
+ obj.powerfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
689
obj.powerfile.ensureIndex({ fieldName: 'nodeid' });
690
obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 10 }); // Limit the power event log to 10 days (Seconds * Minutes * Hours * Days)
691
@@ -694,14 +694,14 @@ module.exports.CreateDB = function (parent, func) {
694
695
// Setup the server stats collection and setup indexes
696
obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true });
697
- obj.serverstatsfile.persistence.setAutocompactionInterval(36000);
697
+ obj.serverstatsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
698
obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days)
699
obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events
700
701
// Setup plugin info collection
702
if (parent.config.settings != null) {
703
obj.pluginsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-plugins.db'), autoload: true });
704
- obj.pluginsfile.persistence.setAutocompactionInterval(36000);
704
+ obj.pluginsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
705
}
706
707
setupFunctions(func); // Completed setup of NeDB
meshagent.js
+24
-6
@@ -978,7 +978,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
978
// Set agent core dump
979
if ((parent.parent.config.settings != null) && ((parent.parent.config.settings.agentcoredump === true) || (parent.parent.config.settings.agentcoredump === false))) {
980
obj.send(JSON.stringify({ action: 'coredump', value: parent.parent.config.settings.agentcoredump }));
981
- if (parent.parent.config.settings.agentcoredump === true) { obj.send(JSON.stringify({ action: 'getcoredump' })); }
981
+ if (parent.parent.config.settings.agentcoredump === true) {
982
+ // Check if we requested a core dump file in the last minute, if not, ask if one is present.
983
+ if ((parent.lastCoreDumpRequest == null) || ((Date.now() - parent.lastCoreDumpRequest) >= 60000)) { obj.send(JSON.stringify({ action: 'getcoredump' })); }
984
+ }
985
}
986
987
// Do this if IP location is enabled on this domain TODO: Set IP location per device group?
@@ -1370,12 +1373,27 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
1373
break;
1374
}
1375
case 'getcoredump': {
1376
+ // Check if we requested a core dump file in the last minute, if so, ignore this.
1377
+ if ((parent.lastCoreDumpRequest != null) && ((Date.now() - parent.lastCoreDumpRequest) < 60000)) break;
1378
+
1379
// Indicates if the agent has a coredump available
1374
- if (command.exists === true) {
1375
- //console.log('CoreDump for agent ' + obj.remoteaddrport);
1376
- obj.coreDumpPresent = true;
1377
- // TODO: We need to look at getting the dump uploaded to the server.
1378
- if (typeof command.agenthashhex == 'string') { obj.RequestCoreDump(command.agenthashhex); }
1380
+ if ((command.exists === true) && (typeof command.agenthashhex == 'string') && (command.agenthashhex.length == 96)) {
1381
+ // Check if we already have this exact dump file
1382
+ const coreDumpFile = parent.path.join(parent.parent.datapath, 'coredumps', obj.agentInfo.agentId + '-' + command.agenthashhex + '-' + obj.nodeid + '.dmp');
1383
+ parent.fs.stat(coreDumpFile, function (err, stats) {
1384
+ if (stats != null) return;
1385
+ obj.coreDumpPresent = true;
1386
+
1387
+ // Check how many files are in the coredumps folder
1388
+ const coreDumpPath = parent.path.join(parent.parent.datapath, 'coredumps');
1389
+ parent.fs.readdir(coreDumpPath, function (err, files) {
1390
+ if ((files != null) && (files.length >= 20)) return; // Don't get more than 20 core dump files.
1391
+
1392
+ // Get the core dump uploaded to the server.
1393
+ parent.lastCoreDumpRequest = Date.now();
1394
+ obj.RequestCoreDump(command.agenthashhex);
1395
+ });
1396
+ });
1397
}
1398
break;
1399
}
meshcentral-config-schema.json
+2
@@ -39,6 +39,7 @@
39
"agentAliasPort": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "When set, indicates the actual publically visible agent-only port. If not set, the AgentPort value is used." },
40
"agentAliasDNS": { "type": "string", "format": "hostname", "description": "When set, specified the DNS name used by agents to connect to the agent-only port." },
41
"agentPortTls": { "type": "boolean", "default": true, "description": "Indicates if the agent-only port must perform TLS, this should be set to false if TLS is performed in front of this server." },
42
+ "agentCoreDump": { "type": "boolean", "default": false, "description": "Automatically activates and transfers any agent crash dump files to the server in meshcentral-data/coredumps." },
43
"exactPorts": { "type": "boolean", "default": false },
44
"allowLoginToken": { "type": "boolean", "default": false },
45
"allowFraming": { "type": "boolean", "default": false },
@@ -53,6 +54,7 @@
54
"agentPing": { "type": "integer", "minimum": 1, "description": "When specified, sends data to the agent at x seconds interval and expects a response from the agent." },
55
"agentPong": { "type": "integer", "minimum": 1, "description": "When specified, sends data to the agent at x seconds interval." },
56
"agentIdleTimeout": { "type": "integer", "minimum": 1 },
57
+ "compression": { "type": "boolean", "default": true, "description": "Enables GZIP compression for web requests." },
58
"meshErrorLogPath": { "type": "string" },
59
"npmPath": { "type": "string" },
60
"npmProxy": { "type": "string", "format": "uri" },