Added server instrumentation
Ylian Saint-Hilaire committed
May 1, 2019 at 15:02 UTC
f8c310d39ffc0744f237f187abeeb8b4a5b1705a
7 files changed
+258
-27
certoperations.js
+49
-10
@@ -549,14 +549,47 @@ module.exports.CertificateOperations = function (parent) {
549
// Accelerators, used to dispatch work to other processes
550
const fork = require("child_process").fork;
551
const program = require("path").join(__dirname, "meshaccelerator.js");
552
- const acceleratorTotalCount = require("os").cpus().length;
552
+ const acceleratorTotalCount = 1; //require("os").cpus().length; // TODO: Check if this accelerator can scale.
553
var acceleratorCreateCount = acceleratorTotalCount;
554
var freeAccelerators = [];
555
var pendingAccelerator = [];
556
obj.acceleratorCertStore = null;
557
558
+ // Accelerator Stats
559
+ var getAcceleratorFuncCalls = 0;
560
+ var acceleratorStartFuncCall = 0;
561
+ var acceleratorPerformSignatureFuncCall = 0;
562
+ var acceleratorPerformSignaturePushFuncCall = 0;
563
+ var acceleratorPerformSignatureRunFuncCall = 0;
564
+ var acceleratorMessage = 0;
565
+ var acceleratorMessageException = 0;
566
+ var acceleratorMessageLastException = null;
567
+ var acceleratorException = 0;
568
+ var acceleratorLastException = null;
569
+
570
+ // Get stats about the accelerators
571
+ obj.getAcceleratorStats = function () {
572
+ return {
573
+ acceleratorTotalCount: acceleratorTotalCount,
574
+ acceleratorCreateCount: acceleratorCreateCount,
575
+ freeAccelerators: freeAccelerators.length,
576
+ pendingAccelerator: pendingAccelerator.length,
577
+ getAcceleratorFuncCalls: getAcceleratorFuncCalls,
578
+ startFuncCall: acceleratorStartFuncCall,
579
+ performSignatureFuncCall: acceleratorPerformSignatureFuncCall,
580
+ performSignaturePushFuncCall: acceleratorPerformSignaturePushFuncCall,
581
+ performSignatureRunFuncCall: acceleratorPerformSignatureRunFuncCall,
582
+ message: acceleratorMessage,
583
+ messageException: acceleratorMessageException,
584
+ messageLastException: acceleratorMessageLastException,
585
+ exception: acceleratorException,
586
+ lastException: acceleratorLastException
587
+ };
588
+ }
589
+
590
// Create a new accelerator module
591
obj.getAccelerator = function () {
592
+ getAcceleratorFuncCalls++;
593
if (obj.acceleratorCertStore == null) { return null; }
594
if (freeAccelerators.length > 0) { return freeAccelerators.pop(); }
595
if (acceleratorCreateCount > 0) {
@@ -564,23 +597,26 @@ module.exports.CertificateOperations = function (parent) {
597
var accelerator = fork(program, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] });
598
accelerator.accid = acceleratorCreateCount;
599
accelerator.on("message", function (message) {
567
- this.func(this.tag, message);
568
- delete this.tag;
569
- if (pendingAccelerator.length > 0) {
570
- var x = pendingAccelerator.shift();
571
- if (x.tag) { this.tag = x.tag; delete x.tag; }
572
- accelerator.send(x);
573
- } else { freeAccelerators.push(this); }
600
+ acceleratorMessage++;
601
+ try { this.func(this.tag, message); } catch (ex) { acceleratorMessageException++; acceleratorMessageLastException = ex; }
602
+ try {
603
+ delete this.tag;
604
+ if (pendingAccelerator.length > 0) {
605
+ var x = pendingAccelerator.shift();
606
+ if (x.tag) { this.tag = x.tag; delete x.tag; }
607
+ accelerator.send(x);
608
+ } else { freeAccelerators.push(this); }
609
+ } catch (ex) { acceleratorException++; acceleratorLastException = ex; }
610
});
611
accelerator.send({ action: "setState", certs: obj.acceleratorCertStore });
612
return accelerator;
577
-
613
}
614
return null;
615
};
616
617
// Set the state of the accelerators. This way, we don"t have to send certificate & keys to them each time.
618
obj.acceleratorStart = function (certificates) {
619
+ acceleratorStartFuncCall++;
620
if (obj.acceleratorCertStore != null) { console.error("ERROR: Accelerators can only be started once."); return; }
621
obj.acceleratorCertStore = [{ cert: certificates.agent.cert, key: certificates.agent.key }];
622
if (certificates.swarmserver != null) { obj.acceleratorCertStore.push({ cert: certificates.swarmserver.cert, key: certificates.swarmserver.key }); }
@@ -588,19 +624,22 @@ module.exports.CertificateOperations = function (parent) {
624
625
// Perform any RSA signature, just pass in the private key and data.
626
obj.acceleratorPerformSignature = function (privatekey, data, tag, func) {
627
+ acceleratorPerformSignatureFuncCall++;
628
if (acceleratorTotalCount <= 1) {
629
// No accelerators available
630
if (typeof privatekey == "number") { privatekey = obj.acceleratorCertStore[privatekey].key; }
631
const sign = obj.crypto.createSign("SHA384");
632
sign.end(Buffer.from(data, "binary"));
596
- func(tag, sign.sign(privatekey).toString("binary"));
633
+ try { func(tag, sign.sign(privatekey).toString("binary")); } catch (ex) { acceleratorMessageException++; acceleratorMessageLastException = ex; }
634
} else {
635
var acc = obj.getAccelerator();
636
if (acc == null) {
637
// Add to pending accelerator workload
638
+ acceleratorPerformSignaturePushFuncCall++;
639
pendingAccelerator.push({ action: "sign", key: privatekey, data: data, tag: tag });
640
} else {
641
// Send to accelerator now
642
+ acceleratorPerformSignatureRunFuncCall++;
643
acc.func = func;
644
acc.tag = tag;
645
acc.send({ action: "sign", key: privatekey, data: data });
meshagent.js
+43
-8
@@ -19,6 +19,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
19
const forge = parent.parent.certificateOperations.forge;
20
const common = parent.parent.common;
21
const agentUpdateBlockSize = 65531;
22
+ parent.agentStats.createMeshAgentCount++;
23
24
var obj = {};
25
obj.domain = domain;
@@ -155,6 +156,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
156
if (meshcorehash == null) {
157
// Clear the core
158
obj.send(common.ShortToStr(10) + common.ShortToStr(0)); // MeshCommand_CoreModule, ask mesh agent to clear the core
159
+ parent.agentStats.clearingCoreCount++;
160
parent.parent.debug(1, 'Clearing core');
161
} else {
162
// Update new core
@@ -168,7 +170,8 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
170
// Send the updated code.
171
delete obj.agentCoreUpdatePending;
172
obj.send(common.ShortToStr(10) + common.ShortToStr(0) + argument.hash + argument.core, function () { parent.parent.taskLimiter.completed(taskid); }); // MeshCommand_CoreModule, start core update
171
- parent.parent.debug(1, 'Updating code ' + argument.name);
173
+ parent.agentStats.updatingCoreCount++;
174
+ parent.parent.debug(1, 'Updating core ' + argument.name);
175
agentCoreIsStable();
176
} else {
177
// This agent is probably disconnected, nothing to do.
@@ -359,6 +362,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
362
} else {
363
// Check that the server hash matches our own web certificate hash (SHA384)
364
if ((getWebCertHash(domain) != msg.substring(2, 50)) && (getWebCertFullHash(domain) != msg.substring(2, 50))) {
365
+ parent.agentStats.agentBadWebCertHashCount++;
366
console.log('Agent bad web cert hash (Agent:' + (Buffer.from(msg.substring(2, 50), 'binary').toString('hex').substring(0, 10)) + ' != Server:' + (Buffer.from(getWebCertHash(domain), 'binary').toString('hex').substring(0, 10)) + ' or ' + (new Buffer(getWebCertFullHash(domain), 'binary').toString('hex').substring(0, 10)) + '), holding connection (' + obj.remoteaddrport + ').');
367
console.log('Agent reported web cert hash:' + (Buffer.from(msg.substring(2, 50), 'binary').toString('hex')) + '.');
368
return;
@@ -388,7 +392,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
392
393
// Check the agent signature if we can
394
if (obj.unauthsign != null) {
391
- if (processAgentSignature(obj.unauthsign) == false) { console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; } else { completeAgentConnection(); }
395
+ if (processAgentSignature(obj.unauthsign) == false) {
396
+ parent.agentStats.agentBadSignature1Count++;
397
+ console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return;
398
+ } else { completeAgentConnection(); }
399
}
400
}
401
else if (cmd == 2) {
@@ -403,7 +410,12 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
410
obj.unauth.nodeCertPem = '-----BEGIN CERTIFICATE-----\r\n' + Buffer.from(msg.substring(4, 4 + certlen), 'binary').toString('base64') + '\r\n-----END CERTIFICATE-----';
411
412
// Check the agent signature if we can
406
- if (obj.agentnonce == null) { obj.unauthsign = msg.substring(4 + certlen); } else { if (processAgentSignature(msg.substring(4 + certlen)) == false) { console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return; } }
413
+ if (obj.agentnonce == null) { obj.unauthsign = msg.substring(4 + certlen); } else {
414
+ if (processAgentSignature(msg.substring(4 + certlen)) == false) {
415
+ parent.agentStats.agentBadSignature2Count++;
416
+ console.log('Agent connected with bad signature, holding connection (' + obj.remoteaddrport + ').'); return;
417
+ }
418
+ }
419
completeAgentConnection();
420
}
421
else if (cmd == 3) {
@@ -541,7 +553,11 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
553
for (var i in parent.wsagents) { if (parent.wsagents[i].domain.id == domain.id) { domainAgentSessionCount++; } }
554
555
// Check if we have too many user sessions
544
- if (domainAgentSessionCount >= domain.limits.maxagentsessions) { return; } // Too many, hold the connection.
556
+ if (domainAgentSessionCount >= domain.limits.maxagentsessions) {
557
+ // Too many, hold the connection.
558
+ parent.agentStats.agentMaxSessionHoldCount++;
559
+ return;
560
+ }
561
}
562
563
/*
@@ -593,6 +609,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
609
// Check if the mesh exists
610
if (mesh == null) {
611
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
612
+ parent.agentStats.invalidDomainMeshCount++;
613
console.log('Agent connected with invalid domain/mesh, holding connection (' + obj.remoteaddrport + ', ' + obj.dbMeshKey + ').');
614
return;
615
}
@@ -600,6 +617,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
617
// Check if the mesh is the right type
618
if (mesh.mtype != 2) {
619
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
620
+ parent.agentStats.invalidMeshTypeCount++;
621
console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').');
622
return;
623
}
@@ -634,6 +652,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
652
// Check if the mesh exists
653
if (mesh == null) {
654
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
655
+ parent.agentStats.invalidDomainMesh2Count++;
656
console.log('Agent connected with invalid domain/mesh, holding connection (' + obj.remoteaddrport + ', ' + obj.dbMeshKey + ').');
657
return;
658
}
@@ -641,6 +660,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
660
// Check if the mesh is the right type
661
if (mesh.mtype != 2) {
662
// If we disconnect, the agent will just reconnect. We need to log this or tell agent to connect in a few hours.
663
+ parent.agentStats.invalidMeshType2Count++;
664
console.log('Agent connected with invalid mesh type, holding connection (' + obj.remoteaddrport + ').');
665
return;
666
}
@@ -686,6 +706,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
706
parent.wsagents[obj.dbNodeKey] = obj;
707
if (dupAgent) {
708
// Close the duplicate agent
709
+ parent.agentStats.duplicateAgentCount++;
710
if (obj.nodeid != null) { parent.parent.debug(1, 'Duplicate agent ' + obj.nodeid + ' (' + obj.remoteaddrport + ')'); }
711
dupAgent.close(3);
712
} else {
@@ -769,6 +790,8 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
790
}
791
792
function recoveryAgentCoreIsStable(mesh) {
793
+ parent.agentStats.recoveryCoreIsStableCount++;
794
+
795
// Recovery agent is doing ok, lets perform main agent checking.
796
//console.log('recoveryAgentCoreIsStable()');
797
@@ -793,9 +816,12 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
816
}
817
818
function agentCoreIsStable() {
819
+ parent.agentStats.coreIsStableCount++;
820
+
821
// Check that the mesh exists
822
const mesh = parent.meshes[obj.dbMeshKey];
823
if (mesh == null) {
824
+ parent.agentStats.meshDoesNotExistCount++;
825
// TODO: Mark this agent as part of a mesh that does not exists.
826
return; // Probably not worth doing anything else. Hold this agent.
827
}
@@ -806,7 +832,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
832
return;
833
}
834
809
- // Fetch the the real agent nodeid
835
+ // Fetch the the diagnostic agent nodeid
836
db.Get('ra' + obj.dbNodeKey, function (err, nodes) {
837
if (nodes.length == 1) {
838
obj.diagnosticNodeKey = nodes[0].daid;
@@ -902,7 +928,11 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
928
verifier.update(buf);
929
verified = verifier.verify(obj.unauth.nodeCertPem, sig, 'binary');
930
}
905
- if (verified == false) { return false; } // Not a valid signature
931
+ if (verified == false) {
932
+ // Not a valid signature
933
+ parent.agentStats.invalidPkcsSignatureCount++;
934
+ return false;
935
+ }
936
} catch (ex) { };
937
}
938
}
@@ -914,7 +944,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
944
if (verify.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) {
945
const verify2 = parent.crypto.createVerify('SHA384');
946
verify2.end(Buffer.from(getWebCertFullHash(domain) + obj.nonce + obj.agentnonce, 'binary')); // Test using the full cert hash
917
- if (verify2.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) { return false; }
947
+ if (verify2.verify(obj.unauth.nodeCertPem, Buffer.from(msg, 'binary')) !== true) {
948
+ parent.agentStats.invalidRsaSignatureCount++;
949
+ return false;
950
+ }
951
}
952
}
953
}
@@ -927,6 +960,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
960
delete obj.unauth;
961
delete obj.receivedCommands;
962
if (obj.unauthsign) delete obj.unauthsign;
963
+ parent.agentStats.verifiedAgentConnectionCount++;
964
parent.parent.debug(1, 'Verified agent connection to ' + obj.nodeid + ' (' + obj.remoteaddrport + ').');
965
obj.authenticated = 1;
966
return true;
@@ -936,7 +970,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
970
function processAgentData(msg) {
971
var i, str = msg.toString('utf8'), command = null;
972
if (str[0] == '{') {
939
- try { command = JSON.parse(str); } catch (ex) { console.log('Unable to parse agent JSON (' + obj.remoteaddrport + '): ' + str, ex); return; } // If the command can't be parsed, ignore it.
973
+ try { command = JSON.parse(str); } catch (ex) { parent.agentStats.invalidJsonCount++; console.log('Unable to parse agent JSON (' + obj.remoteaddrport + '): ' + str, ex); return; } // If the command can't be parsed, ignore it.
974
if (typeof command != 'object') { return; }
975
switch (command.action) {
976
case 'msg':
@@ -1159,6 +1193,7 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) {
1193
break;
1194
}
1195
default: {
1196
+ parent.agentStats.unknownAgentActionCount++;
1197
console.log('Unknown agent action (' + obj.remoteaddrport + '): ' + command.action + '.');
1198
break;
1199
}
meshcentral.js
+1
-1
@@ -216,7 +216,7 @@ function CreateMeshCentralServer(config, args) {
216
if (code == 0) { try { latestVer = xprocess.data.split(' ').join('').split('\r').join('').split('\n').join(''); } catch (e) { } }
217
callback(obj.currentVer, latestVer);
218
});
219
- } catch (ex) { callback(obj.currentVer, null); } // If the system is running out of memory, an exception here can easily happen.
219
+ } catch (ex) { callback(obj.currentVer, null, ex); } // If the system is running out of memory, an exception here can easily happen.
220
};
221
222
// Initiate server self-update
meshuser.js
+46
-3
@@ -520,7 +520,50 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
520
switch (cmd) {
521
case 'help': {
522
r = 'Available commands: help, info, versions, args, resetserver, showconfig, usersessions, tasklimiter, setmaxtasks, cores,\r\n'
523
- r += 'migrationagents, swarmstats, nodeconfig, heapdump, relays.';
523
+ r += 'migrationagents, agentstats, webstats, mpsstats, swarmstats, acceleratorsstats, updatecheck, serverupdate, nodeconfig, heapdump, relays.';
524
+ break;
525
+ }
526
+ case 'agentstats': {
527
+ var stats = parent.getAgentStats();
528
+ for (var i in stats) {
529
+ if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
530
+ }
531
+ break;
532
+ }
533
+ case 'webstats': {
534
+ var stats = parent.getStats();
535
+ for (var i in stats) {
536
+ if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
537
+ }
538
+ break;
539
+ }
540
+ case 'acceleratorsstats': {
541
+ var stats = parent.parent.certificateOperations.getAcceleratorStats();
542
+ for (var i in stats) {
543
+ if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
544
+ }
545
+ break;
546
+ }
547
+ case 'mpsstats': {
548
+ var stats = parent.parent.mpsserver.getStats();
549
+ for (var i in stats) {
550
+ if (typeof stats[i] == 'object') { r += (i + ': ' + JSON.stringify(stats[i]) + '\r\n'); } else { r += (i + ': ' + stats[i] + '\r\n'); }
551
+ }
552
+ break;
553
+ }
554
+ case 'serverupdate': {
555
+ r = 'Performing server update...';
556
+ parent.parent.performServerUpdate();
557
+ break;
558
+ }
559
+ case 'updatecheck': {
560
+ parent.parent.getLatestServerVersion(function (currentVer, newVer, error) {
561
+ var r2 = 'Current Version: ' + currentVer + '\r\n';
562
+ if (newVer != null) { r2 += 'Available Version: ' + newVer + '\r\n'; }
563
+ if (error != null) { r2 += 'Exception: ' + ex + '\r\n'; }
564
+ try { ws.send(JSON.stringify({ action: 'serverconsole', value: r2, tag: command.tag })); } catch (ex) { }
565
+ });
566
+ r = 'Checking server update...';
567
break;
568
}
569
case 'info': {
@@ -622,9 +665,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
665
} else {
666
for (var i in parent.parent.swarmserver.stats) {
667
if (typeof parent.parent.swarmserver.stats[i] == 'object') {
625
- r += i + ' ' + JSON.stringify(parent.parent.swarmserver.stats[i]) + '<br />';
668
+ r += i + ': ' + JSON.stringify(parent.parent.swarmserver.stats[i]) + '\r\n';
669
} else {
627
- r += i + ' ' + parent.parent.swarmserver.stats[i] + '<br />';
670
+ r += i + ': ' + parent.parent.swarmserver.stats[i] + '\r\n';
671
}
672
}
673
}
mpsserver.js
+70
-4
@@ -106,7 +106,58 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
106
ResourceShortage: 4,
107
};
108
109
+ // Stat counters
110
+ var connectionCount = 0;
111
+ var userAuthRequestCount = 0;
112
+ var incorrectPasswordCount = 0;
113
+ var meshNotFoundCount = 0;
114
+ var unknownTlsNodeCount = 0;
115
+ var unknownTlsMeshIdCount = 0;
116
+ var addedTlsDeviceCount = 0;
117
+ var unknownNodeCount = 0;
118
+ var unknownMeshIdCount = 0;
119
+ var addedDeviceCount = 0;
120
+ var ciraTimeoutCount = 0;
121
+ var protocolVersionCount = 0;
122
+ var badUserNameLengthCount = 0;
123
+ var channelOpenCount = 0;
124
+ var channelOpenConfirmCount = 0;
125
+ var channelOpenFailCount = 0;
126
+ var channelCloseCount = 0;
127
+ var disconnectCommandCount = 0;
128
+ var socketClosedCount = 0;
129
+ var socketErrorCount = 0;
130
+
131
+ // Return statistics about this MPS server
132
+ obj.getStats = function () {
133
+ return {
134
+ ciraConnections: Object.keys(obj.ciraConnections).length,
135
+ tlsSessionStore: Object.keys(tlsSessionStore).length,
136
+ connectionCount: connectionCount,
137
+ userAuthRequestCount: userAuthRequestCount,
138
+ incorrectPasswordCount: incorrectPasswordCount,
139
+ meshNotFoundCount: meshNotFoundCount,
140
+ unknownTlsNodeCount: unknownTlsNodeCount,
141
+ unknownTlsMeshIdCount: unknownTlsMeshIdCount,
142
+ addedTlsDeviceCount: addedTlsDeviceCount,
143
+ unknownNodeCount: unknownNodeCount,
144
+ unknownMeshIdCount: unknownMeshIdCount,
145
+ addedDeviceCount: addedDeviceCount,
146
+ ciraTimeoutCount: ciraTimeoutCount,
147
+ protocolVersionCount: protocolVersionCount,
148
+ badUserNameLengthCount: badUserNameLengthCount,
149
+ channelOpenCount: channelOpenCount,
150
+ channelOpenConfirmCount: channelOpenConfirmCount,
151
+ channelOpenFailCount: channelOpenFailCount,
152
+ channelCloseCount: channelCloseCount,
153
+ disconnectCommandCount: disconnectCommandCount,
154
+ socketClosedCount: socketClosedCount,
155
+ socketErrorCount: socketErrorCount
156
+ };
157
+ }
158
+
159
function onConnection(socket) {
160
+ connectionCount++;
161
if (obj.args.mpstlsoffload) {
162
socket.tag = { first: true, clientCert: null, accumulator: "", activetunnels: 0, boundPorts: [], socket: socket, host: null, nextchannelid: 4, channels: {}, nextsourceport: 0 };
163
} else {
@@ -117,7 +168,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
168
169
// Setup the CIRA keep alive timer
170
socket.setTimeout(MAX_IDLE);
120
- socket.on("timeout", () => { Debug(1, "MPS:CIRA timeout, disconnecting."); try { socket.end(); } catch (e) { } });
171
+ socket.on("timeout", () => { ciraTimeoutCount++; Debug(1, "MPS:CIRA timeout, disconnecting."); try { socket.end(); } catch (e) { } });
172
173
socket.addListener("data", function (data) {
174
if (args.mpsdebug) { var buf = Buffer.from(data, "binary"); console.log("MPS <-- (" + buf.length + "):" + buf.toString('hex')); } // Print out received bytes
@@ -156,12 +207,14 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
207
obj.db.Set(device);
208
209
// Event the new node
210
+ addedTlsDeviceCount++;
211
var device2 = common.Clone(device);
212
if (device2.intelamt.pass != null) delete device2.intelamt.pass; // Remove the Intel AMT password before eventing this.
213
var change = 'CIRA added device ' + socket.tag.name + ' to mesh ' + mesh.name;
214
obj.parent.DispatchEvent(['*', socket.tag.meshid], obj, { etype: 'node', action: 'addnode', node: device2, msg: change, domain: domainid });
215
} else {
216
// New CIRA connection for unknown node, disconnect.
217
+ unknownTlsNodeCount++;
218
console.log('CIRA connection for unknown node with incorrect group type. meshid: ' + socket.tag.meshid);
219
socket.end();
220
return;
@@ -177,6 +230,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
230
obj.parent.SetConnectivityState(socket.tag.meshid, socket.tag.nodeid, socket.tag.connectTime, 2, 7); // TODO: Right now report power state as "present" (7) until we can poll.
231
});
232
} else {
233
+ unknownTlsMeshIdCount++;
234
console.log('ERROR: Intel AMT CIRA connected with unknown groupid: ' + socket.tag.meshid);
235
socket.end();
236
return;
@@ -219,6 +273,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
273
}
274
case APFProtocol.PROTOCOLVERSION: {
275
if (len < 93) return 0;
276
+ protocolVersionCount++;
277
socket.tag.MajorVersion = common.ReadInt(data, 1);
278
socket.tag.MinorVersion = common.ReadInt(data, 5);
279
socket.tag.SystemId = guidToStr(common.rstr2hex(data.substring(13, 29))).toLowerCase();
@@ -227,6 +282,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
282
}
283
case APFProtocol.USERAUTH_REQUEST: {
284
if (len < 13) return 0;
285
+ userAuthRequestCount++;
286
var usernameLen = common.ReadInt(data, 1);
287
var username = data.substring(5, 5 + usernameLen);
288
var serviceNameLen = common.ReadInt(data, 5 + usernameLen);
@@ -242,13 +298,13 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
298
Debug(3, 'MPS:USERAUTH_REQUEST user=' + username + ', service=' + serviceName + ', method=' + methodName + ', password=' + password);
299
300
// Check the CIRA password
245
- if ((args.mpspass != null) && (password != args.mpspass)) { Debug(1, 'MPS:Incorrect password', username, password); SendUserAuthFail(socket); return -1; }
301
+ if ((args.mpspass != null) && (password != args.mpspass)) { incorrectPasswordCount++; Debug(1, 'MPS:Incorrect password', username, password); SendUserAuthFail(socket); return -1; }
302
303
// Check the CIRA username, which should be the start of the MeshID.
248
- if (usernameLen != 16) { Debug(1, 'MPS:Username length not 16', username, password); SendUserAuthFail(socket); return -1; }
304
+ if (usernameLen != 16) { badUserNameLengthCount++; Debug(1, 'MPS:Username length not 16', username, password); SendUserAuthFail(socket); return -1; }
305
var meshIdStart = '/' + username, mesh = null;
306
if (obj.parent.webserver.meshes) { for (var i in obj.parent.webserver.meshes) { if (obj.parent.webserver.meshes[i]._id.replace(/\@/g, 'X').replace(/\$/g, 'X').indexOf(meshIdStart) > 0) { mesh = obj.parent.webserver.meshes[i]; break; } } }
251
- if (mesh == null) { Debug(1, 'MPS:Mesh not found', username, password); SendUserAuthFail(socket); return -1; }
307
+ if (mesh == null) { meshNotFoundCount++; Debug(1, 'MPS:Mesh not found', username, password); SendUserAuthFail(socket); return -1; }
308
309
// If this is a agent-less mesh, use the device guid 3 times as ID.
310
if (mesh.mtype == 1) {
@@ -267,6 +323,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
323
obj.db.Set(device);
324
325
// Event the new node
326
+ addedDeviceCount++;
327
var device2 = common.Clone(device);
328
if (device2.intelamt.pass != null) delete device2.intelamt.pass; // Remove the Intel AMT password before eventing this.
329
var change = 'CIRA added device ' + socket.tag.name + ' to group ' + mesh.name;
@@ -287,6 +344,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
344
obj.db.getAmtUuidNode(mesh._id, socket.tag.SystemId, function (err, nodes) { // TODO: May need to optimize this request with indexes
345
if (nodes.length !== 1) {
346
// New CIRA connection for unknown node, disconnect.
347
+ unknownNodeCount++;
348
console.log('CIRA connection for unknown node. groupid: ' + mesh._id + ', uuid: ' + socket.tag.SystemId);
349
socket.end();
350
return;
@@ -306,6 +364,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
364
});
365
} else { // Unknown mesh type
366
// New CIRA connection for unknown node, disconnect.
367
+ unknownMeshIdCount++;
368
console.log('CIRA connection to a unknown group type. groupid: ' + socket.tag.meshid);
369
socket.end();
370
return;
@@ -393,6 +452,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
452
var Source = data.substring(29 + ChannelTypeLength + TargetLen, 29 + ChannelTypeLength + TargetLen + SourceLen);
453
var SourcePort = common.ReadInt(data, 29 + ChannelTypeLength + TargetLen + SourceLen);
454
455
+ channelOpenCount++;
456
Debug(3, 'MPS:CHANNEL_OPEN', ChannelType, SenderChannel, WindowSize, Target + ':' + TargetPort, Source + ':' + SourcePort);
457
458
// Check if we understand this channel type
@@ -423,6 +483,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
483
if (cirachannel == null) { /*console.log("MPS Error in CHANNEL_OPEN_CONFIRMATION: Unable to find channelid " + RecipientChannel);*/ return 17; }
484
cirachannel.amtchannelid = SenderChannel;
485
cirachannel.sendcredits = cirachannel.amtCiraWindow = WindowSize;
486
+ channelOpenConfirmCount++;
487
Debug(3, 'MPS:CHANNEL_OPEN_CONFIRMATION', RecipientChannel, SenderChannel, WindowSize);
488
if (cirachannel.closing == 1) {
489
// Close this channel
@@ -454,6 +515,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
515
if (len < 17) return 0;
516
var RecipientChannel = common.ReadInt(data, 1);
517
var ReasonCode = common.ReadInt(data, 5);
518
+ channelOpenFailCount++;
519
Debug(3, 'MPS:CHANNEL_OPEN_FAILURE', RecipientChannel, ReasonCode);
520
var cirachannel = socket.tag.channels[RecipientChannel];
521
if (cirachannel == null) { console.log("MPS Error in CHANNEL_OPEN_FAILURE: Unable to find channelid " + RecipientChannel); return 17; }
@@ -468,6 +530,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
530
{
531
if (len < 5) return 0;
532
var RecipientChannel = common.ReadInt(data, 1);
533
+ channelCloseCount++;
534
Debug(3, 'MPS:CHANNEL_CLOSE', RecipientChannel);
535
var cirachannel = socket.tag.channels[RecipientChannel];
536
if (cirachannel == null) { console.log("MPS Error in CHANNEL_CLOSE: Unable to find channelid " + RecipientChannel); return 5; }
@@ -526,6 +589,7 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
589
{
590
if (len < 7) return 0;
591
var ReasonCode = common.ReadInt(data, 1);
592
+ disconnectCommandCount++;
593
Debug(3, 'MPS:DISCONNECT', ReasonCode);
594
try { delete obj.ciraConnections[socket.tag.nodeid]; } catch (e) { }
595
obj.parent.ClearConnectivityState(socket.tag.meshid, socket.tag.nodeid, 2);
@@ -540,12 +604,14 @@ module.exports.CreateMpsServer = function (parent, db, args, certificates) {
604
}
605
606
socket.addListener("close", function () {
607
+ socketClosedCount++;
608
Debug(1, 'MPS:CIRA connection closed');
609
try { delete obj.ciraConnections[socket.tag.nodeid]; } catch (e) { }
610
obj.parent.ClearConnectivityState(socket.tag.meshid, socket.tag.nodeid, 2);
611
});
612
613
socket.addListener("error", function () {
614
+ socketErrorCount++;
615
//console.log("MPS Error: " + socket.remoteAddress);
616
});
617
package.json
+1
-1
@@ -1,6 +1,6 @@
1
{
2
"name": "meshcentral",
3
- "version": "0.3.3-n",
3
+ "version": "0.3.3-o",
4
"keywords": [
5
"Remote Management",
6
"Intel AMT",
webserver.js
+48
@@ -214,6 +214,54 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) {
214
});
215
});
216
217
+ // Return statistics about this web server
218
+ obj.getStats = function () {
219
+ return {
220
+ users: Object.keys(obj.users).length,
221
+ meshes: Object.keys(obj.meshes).length,
222
+ dnsDomains: Object.keys(obj.dnsDomains).length,
223
+ relaySessionCount: obj.relaySessionCount,
224
+ relaySessionErrorCount: obj.relaySessionErrorCount,
225
+ wsagents: Object.keys(obj.wsagents).length,
226
+ wsagentsDisconnections: Object.keys(obj.wsagentsDisconnections).length,
227
+ wsagentsDisconnectionsTimer: Object.keys(obj.wsagentsDisconnectionsTimer).length,
228
+ wssessions: Object.keys(obj.wssessions).length,
229
+ wssessions2: Object.keys(obj.wssessions2).length,
230
+ wsPeerSessions: Object.keys(obj.wsPeerSessions).length,
231
+ wsPeerSessions2: Object.keys(obj.wsPeerSessions2).length,
232
+ wsPeerSessions3: Object.keys(obj.wsPeerSessions3).length,
233
+ sessionsCount: Object.keys(obj.sessionsCount).length,
234
+ wsrelays: Object.keys(obj.wsrelays).length,
235
+ wsPeerRelays: Object.keys(obj.wsPeerRelays).length,
236
+ tlsSessionStore: Object.keys(tlsSessionStore).length
237
+ };
238
+ }
239
+
240
+ // Agent counters
241
+ obj.agentStats = {
242
+ createMeshAgentCount: 0,
243
+ coreIsStableCount: 0,
244
+ verifiedAgentConnectionCount: 0,
245
+ clearingCoreCount: 0,
246
+ updatingCoreCount: 0,
247
+ recoveryCoreIsStableCount: 0,
248
+ meshDoesNotExistCount: 0,
249
+ invalidPkcsSignatureCount: 0,
250
+ invalidRsaSignatureCount: 0,
251
+ invalidJsonCount: 0,
252
+ unknownAgentActionCount: 0,
253
+ agentBadWebCertHashCount: 0,
254
+ agentBadSignature1Count: 0,
255
+ agentBadSignature2Count: 0,
256
+ agentMaxSessionHoldCount: 0,
257
+ invalidDomainMeshCount: 0,
258
+ invalidMeshTypeCount: 0,
259
+ invalidDomainMesh2Count: 0,
260
+ invalidMeshType2Count: 0,
261
+ duplicateAgentCount: 0
262
+ }
263
+ obj.getAgentStats = function () { return obj.agentStats; }
264
+
265
// Authenticate the user
266
obj.authenticate = function (name, pass, domain, fn) {
267
if ((typeof (name) != 'string') || (typeof (pass) != 'string') || (typeof (domain) != 'object')) { fn(new Error('invalid fields')); return; }