add prometheus metrics (#6654)
Signed-off-by: si458 <simonsmith5521@gmail.com>
Simon Smith committed
Jan 2, 2025 at 23:38 UTC
61d3487f8ae8332d15ec1eb04f0228113d1d19c4
3 files changed
+126
meshcentral-config-schema.json
+8
@@ -767,6 +767,14 @@
767
"default": false,
768
"description": "When set to true, the MPS server will only accept TLS 1.2 and 1.3 connections. Older Intel AMT devices will not be able to connect."
769
},
770
+ "prometheus": {
771
+ "type": [
772
+ "boolean",
773
+ "number"
774
+ ],
775
+ "default": false,
776
+ "description": "When set to true, a prometheus metrics endpoint will be available \"0.0.0.0:9464/metrics\". If you specify a number instead, the prometheus metrics will listen on this port instead of the default 9464."
777
+ },
778
"no2FactorAuth": {
779
"type": "boolean",
780
"default": false
meshcentral.js
+4
@@ -2011,6 +2011,9 @@ function CreateMeshCentralServer(config, args) {
2011
obj.firebase = require('./firebase').CreateFirebaseRelay(obj, 'https://alt.meshcentral.com/firebaserelay.aspx');
2012
}
2013
2014
+ // Setup monitoring
2015
+ obj.monitoring = require('./monitoring.js').CreateMonitoring(obj, obj.args);
2016
+
2017
// Start periodic maintenance
2018
obj.maintenanceTimer = setInterval(obj.maintenanceActions, 1000 * 60 * 60); // Run this every hour
2019
@@ -4260,6 +4263,7 @@ function mainStart() {
4263
if (sendgrid || (config.sendgrid != null)) { modules.push('@sendgrid/mail'); } // Add SendGrid support
4264
if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier@4.0.0'); } // Translation support
4265
if (typeof config.settings.crowdsec == 'object') { modules.push('@crowdsec/express-bouncer@0.1.0'); } // Add CrowdSec bounser module (https://www.npmjs.com/package/@crowdsec/express-bouncer)
4266
+ if (config.settings.prometheus != null) { modules.push('prom-client'); } // Add Prometheus Metrics support
4267
4268
if (typeof config.settings.autobackup == 'object') {
4269
// Setup encrypted zip support if needed
monitoring.js
new
+114
@@ -0,0 +1,114 @@
1
+/**
2
+* @description MeshCentral monitoring module
3
+* @author Simon Smith
4
+* @license Apache-2.0
5
+* @version v0.0.1
6
+*/
7
+
8
+"use strict";
9
+
10
+module.exports.CreateMonitoring = function (parent, args) {
11
+ var obj = {};
12
+ obj.args = args;
13
+ obj.parent = parent;
14
+ obj.express = require('express');
15
+ obj.app = obj.express();
16
+ obj.prometheus = null;
17
+ if (args.compression !== false) { obj.app.use(require('compression')()); }
18
+ obj.app.disable('x-powered-by');
19
+ obj.counterMetrics = { // Counter Metrics always start at 0 and increase but never decrease
20
+ RelayErrors: { description: "Relay Errors" }, // parent.webserver.relaySessionErrorCount
21
+ UnknownGroup: { description: "Unknown Group" }, // meshDoesNotExistCount
22
+ InvalidPKCSsignature: { description: "Invalid PKCS signature" }, // invalidPkcsSignatureCount
23
+ InvalidRSAsignature: { description: "Invalid RSA signature" }, // invalidRsaSignatureCount
24
+ InvalidJSON: { description: "Invalid JSON" }, // invalidJsonCount
25
+ UnknownAction: { description: "Unknown Action" }, // unknownAgentActionCount
26
+ BadWebCertificate: { description: "Bad Web Certificate" }, // agentBadWebCertHashCount
27
+ BadSignature: { description: "Bad Signature" }, // (agentBadSignature1Count + agentBadSignature2Count)
28
+ MaxSessionsReached: { description: "Max Sessions Reached" }, // agentMaxSessionHoldCount
29
+ UnknownDeviceGroup: { description: "Unknown Device Group" }, // (invalidDomainMeshCount + invalidDomainMesh2Count)
30
+ InvalidDeviceGroupType: { description: "Invalid Device Group Type" }, // invalidMeshTypeCount
31
+ DuplicateAgent: { description: "Duplicate Agent" }, // duplicateAgentCount
32
+ blockedUsers: { description: "Blocked Users" }, // blockedUsers
33
+ blockedAgents: { description: "Blocked Agents" }, // blockedAgents
34
+ };
35
+ obj.guageMetrics = { // Guage Metrics always start at 0 and can increase and decrease
36
+ ConnectedIntelAMT: { description: "Connected Intel AMT" }, // parent.mpsserver.ciraConnections[i].length
37
+ UserAccounts: { description: "User Accounts" }, // Object.keys(parent.webserver.users).length
38
+ DeviceGroups: { description: "Device Groups" }, // parent.webserver.meshes (ONLY WHERE deleted=null)
39
+ AgentSessions: { description: "Agent Sessions" }, // Object.keys(parent.webserver.wsagents).length
40
+ ConnectedUsers: { description: "Connected Users" }, // Object.keys(parent.webserver.wssessions).length
41
+ UsersSessions: { description: "Users Sessions" }, // Object.keys(parent.webserver.wssessions2).length
42
+ RelaySessions: { description: "Relay Sessions" }, // parent.webserver.relaySessionCount
43
+ RelayCount: { description: "Relay Count" } // Object.keys(parent.webserver.wsrelays).length30bb4fb74dfb758d36be52a7
44
+ }
45
+ if (parent.config.settings.prometheus != null) { // Create Prometheus Monitoring Endpoint
46
+ if ((typeof parent.config.settings.prometheus == 'number') && ((parent.config.settings.prometheus < 1) || (parent.config.settings.prometheus > 65535))) {
47
+ console.log('Promethus port number is invalid, Prometheus metrics endpoint has be disabled');
48
+ delete parent.config.settings.prometheus;
49
+ } else {
50
+ const port = ((typeof parent.config.settings.prometheus == 'number') ? parent.config.settings.prometheus : 9464);
51
+ obj.prometheus = require('prom-client');
52
+ const collectDefaultMetrics = obj.prometheus.collectDefaultMetrics;
53
+ collectDefaultMetrics();
54
+ for (const key in obj.guageMetrics) {
55
+ obj.guageMetrics[key].prometheus = new obj.prometheus.Gauge({ name: 'meshcentral_' + String(key).toLowerCase(), help: obj.guageMetrics[key].description });
56
+ }
57
+ for (const key in obj.counterMetrics) {
58
+ obj.counterMetrics[key].prometheus = new obj.prometheus.Counter({ name: 'meshcentral_' + String(key).toLowerCase(), help: obj.counterMetrics[key].description });
59
+ }
60
+ obj.app.get('/', function (req, res) { res.send('MeshCentral Prometheus server.'); });
61
+ obj.app.listen(port, function () {
62
+ console.log('MeshCentral Prometheus server running on port ' + port + '.');
63
+ obj.parent.updateServerState('prometheus-port', port);
64
+ });
65
+ obj.app.get('/metrics', async (req, res) => {
66
+ try {
67
+ // Count the number of device groups that are not deleted
68
+ var activeDeviceGroups = 0;
69
+ for (var i in parent.webserver.meshes) { if (parent.webserver.meshes[i].deleted == null) { activeDeviceGroups++; } } // This is not ideal for performance, we want to dome something better.
70
+ var guages = {
71
+ UserAccounts: Object.keys(parent.webserver.users).length,
72
+ DeviceGroups: activeDeviceGroups,
73
+ AgentSessions: Object.keys(parent.webserver.wsagents).length,
74
+ ConnectedUsers: Object.keys(parent.webserver.wssessions).length,
75
+ UsersSessions: Object.keys(parent.webserver.wssessions2).length,
76
+ RelaySessions: parent.webserver.relaySessionCount,
77
+ RelayCount: Object.keys(parent.webserver.wsrelays).length,
78
+ ConnectedIntelAMT: 0
79
+ };
80
+ if (parent.mpsserver != null) {
81
+ for (var i in parent.mpsserver.ciraConnections) {
82
+ guages.ConnectedIntelAMT += parent.mpsserver.ciraConnections[i].length;
83
+ }
84
+ }
85
+ for (const key in guages) { obj.guageMetrics[key].prometheus.set(guages[key]); }
86
+ // Take a look at agent errors
87
+ var agentstats = parent.webserver.getAgentStats();
88
+ const counters = {
89
+ RelayErrors: parent.webserver.relaySessionErrorCount,
90
+ UnknownGroup: agentstats.meshDoesNotExistCount,
91
+ InvalidPKCSsignature: agentstats.invalidPkcsSignatureCount,
92
+ InvalidRSAsignature: agentstats.invalidRsaSignatureCount,
93
+ InvalidJSON: agentstats.invalidJsonCount,
94
+ UnknownAction: agentstats.unknownAgentActionCount,
95
+ BadWebCertificate: agentstats.agentBadWebCertHashCount,
96
+ BadSignature: (agentstats.agentBadSignature1Count + agentstats.agentBadSignature2Count),
97
+ MaxSessionsReached: agentstats.agentMaxSessionHoldCount,
98
+ UnknownDeviceGroup: (agentstats.invalidDomainMeshCount + agentstats.invalidDomainMesh2Count),
99
+ InvalidDeviceGroupType: (agentstats.invalidMeshTypeCount + agentstats.invalidMeshType2Count),
100
+ DuplicateAgent: agentstats.duplicateAgentCount,
101
+ blockedUsers: parent.webserver.blockedUsers,
102
+ blockedAgents: parent.webserver.blockedAgents
103
+ };
104
+ for (const key in counters) { obj.counterMetrics[key].prometheus.reset(); obj.counterMetrics[key].prometheus.inc(counters[key]); }
105
+ res.set('Content-Type', obj.prometheus.register.contentType);
106
+ res.end(await obj.prometheus.register.metrics());
107
+ } catch (ex) {
108
+ console.log(ex);
109
+ res.status(500).end();
110
+ }
111
+ });
112
+ }
113
+ }
114
+}
\ No newline at end of file