Added retry logic to self-update
Bryan Roe committed
Jan 18, 2021 at 22:08 UTC
05573d9bb96e613985b80a6996fef9b6cbbbd747
2 files changed
+85
-32
agents/meshcore.js
+21
-6
@@ -3892,13 +3892,14 @@ function createMeshCore(agent) {
3892
// If this value is null
3893
var sessionid = (updateoptions != null) ? updateoptions.sessionid : null; // If this is null, messages will be broadcast. Otherwise they will be unicasted
3894
3895
- if (this._selfupdate != null)
3895
+ if (agentUpdate_Start._selfupdate != null)
3896
{
3897
// We were already called, so we will ignore this duplicate request
3898
if (sessionid != null) { sendConsoleText('Self update already in progress...', sessionid); }
3899
}
3900
else
3901
{
3902
+ if (agentUpdate_Start._retryCount == null) { agentUpdate_Start._retryCount = 0; }
3903
if (require('MeshAgent').ARCHID == null && updateurl == null)
3904
{
3905
// This agent doesn't have the ability to tell us which ARCHID it is, so we don't know which agent to pull
@@ -3956,13 +3957,13 @@ function createMeshCore(agent) {
3957
}
3958
}
3959
options.checkServerIdentity.servertlshash = (updateoptions != null ? updateoptions.tlshash : null);
3959
- this._selfupdate = require('https').get(options);
3960
- this._selfupdate.on('error', function (e)
3960
+ agentUpdate_Start._selfupdate = require('https').get(options);
3961
+ agentUpdate_Start._selfupdate.on('error', function (e)
3962
{
3963
sendConsoleText('Self Update failed, because there was a problem trying to download the update', sessionid);
3964
sendAgentMessage('Self Update failed, because there was a problem trying to download the update', 3);
3965
});
3965
- this._selfupdate.on('response', function (img)
3966
+ agentUpdate_Start._selfupdate.on('response', function (img)
3967
{
3968
this._file = require('fs').createWriteStream(agentfilename + '.update', { flags: 'wb' });
3969
this._filehash = require('SHA384Stream').create();
@@ -3976,8 +3977,22 @@ function createMeshCore(agent) {
3977
}
3978
else
3979
{
3979
- sendConsoleText('Self Update FAILED because the downloaded agent FAILED hash check', sessionid);
3980
- sendAgentMessage('Self Update FAILED because the downloaded agent FAILED hash check', 3);
3980
+ agentUpdate_Start._retryCount++;
3981
+ sendConsoleText('Self Update FAILED because the downloaded agent FAILED hash check (' + agentUpdate_Start._retryCount + ')', sessionid);
3982
+ sendAgentMessage('Self Update FAILED because the downloaded agent FAILED hash check (' + agentUpdate_Start._retryCount + ')', 3);
3983
+ agentUpdate_Start._selfupdate = null;
3984
+
3985
+ if (agentUpdate_Start._retryCount < 4)
3986
+ {
3987
+ // Retry the download again
3988
+ sendConsoleText('Self Update will try again in 60 seconds...', sessionid);
3989
+ agentUpdate_Start._timeout = setTimeout(agentUpdate_Start, 60000, updateurl, updateoptions);
3990
+ }
3991
+ else
3992
+ {
3993
+ sendConsoleText('Self Update giving up, too many failures...', sessionid);
3994
+ sendAgentMessage('Self Update giving up, too many failures...', 3);
3995
+ }
3996
return;
3997
}
3998
}
agents/recoverycore.js
+64
-26
@@ -228,33 +228,42 @@ function windows_execve(name, agentfilename, sessionid) {
228
}
229
230
// Start a JavaScript based Agent Self-Update
231
-function agentUpdate_Start(updateurl, updateoptions) {
231
+function agentUpdate_Start(updateurl, updateoptions)
232
+{
233
// If this value is null
234
var sessionid = (updateoptions != null) ? updateoptions.sessionid : null; // If this is null, messages will be broadcast. Otherwise they will be unicasted
235
235
- if (this._selfupdate != null) {
236
+ if (agentUpdate_Start._selfupdate != null)
237
+ {
238
// We were already called, so we will ignore this duplicate request
239
if (sessionid != null) { sendConsoleText('Self update already in progress...', sessionid); }
240
}
239
- else {
240
- if (require('MeshAgent').ARCHID == null && updateurl == null) {
241
+ else
242
+ {
243
+ if (agentUpdate_Start._retryCount == null) { agentUpdate_Start._retryCount = 0; }
244
+ if (require('MeshAgent').ARCHID == null && updateurl == null)
245
+ {
246
// This agent doesn't have the ability to tell us which ARCHID it is, so we don't know which agent to pull
247
sendConsoleText('Unable to initiate update, agent ARCHID is not defined', sessionid);
248
}
244
- else {
249
+ else
250
+ {
251
var agentfilename = process.execPath.split(process.platform == 'win32' ? '\\' : '/').pop(); // Local File Name, ie: MeshAgent.exe
252
var name = require('MeshAgent').serviceName;
253
if (name == null) { name = process.platform == 'win32' ? 'Mesh Agent' : 'meshagent'; } // This is an older agent that doesn't expose the service name, so use the default
248
- try {
254
+ try
255
+ {
256
var s = require('service-manager').manager.getService(name);
250
- if (!s.isMe()) {
257
+ if (!s.isMe())
258
+ {
259
if (process.platform == 'win32') { s.close(); }
260
sendConsoleText('Self Update cannot continue, this agent is not an instance of (' + name + ')', sessionid);
261
return;
262
}
263
if (process.platform == 'win32') { s.close(); }
264
}
257
- catch (zz) {
265
+ catch (zz)
266
+ {
267
sendConsoleText('Self Update Failed because this agent is not an instance of (' + name + ')', sessionid);
268
sendAgentMessage('Self Update Failed because this agent is not an instance of (' + name + ')', 3);
269
return;
@@ -265,13 +274,15 @@ function agentUpdate_Start(updateurl, updateoptions) {
274
options.protocol = 'https:';
275
if (updateurl == null) { options.path = ('/meshagents?id=' + require('MeshAgent').ARCHID); }
276
options.rejectUnauthorized = false;
268
- options.checkServerIdentity = function checkServerIdentity(certs) {
277
+ options.checkServerIdentity = function checkServerIdentity(certs)
278
+ {
279
// If the tunnel certificate matches the control channel certificate, accept the connection
280
try { if (require('MeshAgent').ServerInfo.ControlChannelCertificate.digest == certs[0].digest) return; } catch (ex) { }
281
try { if (require('MeshAgent').ServerInfo.ControlChannelCertificate.fingerprint == certs[0].fingerprint) return; } catch (ex) { }
282
283
// Check that the certificate is the one expected by the server, fail if not.
274
- if (checkServerIdentity.servertlshash == null) {
284
+ if (checkServerIdentity.servertlshash == null)
285
+ {
286
if (require('MeshAgent').ServerInfo == null || require('MeshAgent').ServerInfo.ControlChannelCertificate == null) { return; }
287
288
sendConsoleText('Self Update failed, because the url cannot be verified', sessionid);
@@ -279,33 +290,55 @@ function agentUpdate_Start(updateurl, updateoptions) {
290
throw new Error('BadCert');
291
}
292
if (certs[0].digest == null) { return; }
282
- if ((checkServerIdentity.servertlshash != null) && (checkServerIdentity.servertlshash.toLowerCase() != certs[0].digest.split(':').join('').toLowerCase())) {
293
+ if ((checkServerIdentity.servertlshash != null) && (checkServerIdentity.servertlshash.toLowerCase() != certs[0].digest.split(':').join('').toLowerCase()))
294
+ {
295
sendConsoleText('Self Update failed, because the supplied certificate does not match', sessionid);
296
sendAgentMessage('Self Update failed, because the supplied certificate does not match', 3);
297
throw new Error('BadCert')
298
}
299
}
300
options.checkServerIdentity.servertlshash = (updateoptions != null ? updateoptions.tlshash : null);
289
- this._selfupdate = require('https').get(options);
290
- this._selfupdate.on('error', function (e) {
301
+ agentUpdate_Start._selfupdate = require('https').get(options);
302
+ agentUpdate_Start._selfupdate.on('error', function (e)
303
+ {
304
sendConsoleText('Self Update failed, because there was a problem trying to download the update', sessionid);
305
sendAgentMessage('Self Update failed, because there was a problem trying to download the update', 3);
306
});
294
- this._selfupdate.on('response', function (img) {
307
+ agentUpdate_Start._selfupdate.on('response', function (img)
308
+ {
309
this._file = require('fs').createWriteStream(agentfilename + '.update', { flags: 'wb' });
310
this._filehash = require('SHA384Stream').create();
297
- this._filehash.on('hash', function (h) {
298
- if (updateoptions != null && updateoptions.hash != null) {
299
- if (updateoptions.hash.toLowerCase() == h.toString('hex').toLowerCase()) {
311
+ this._filehash.on('hash', function (h)
312
+ {
313
+ if (updateoptions != null && updateoptions.hash != null)
314
+ {
315
+ if (updateoptions.hash.toLowerCase() == h.toString('hex').toLowerCase())
316
+ {
317
if (sessionid != null) { sendConsoleText('Download complete. HASH verified.', sessionid); }
318
}
302
- else {
303
- sendConsoleText('Self Update FAILED because the downloaded agent FAILED hash check', sessionid);
304
- sendAgentMessage('Self Update FAILED because the downloaded agent FAILED hash check', 3);
319
+ else
320
+ {
321
+ agentUpdate_Start._retryCount++;
322
+ sendConsoleText('Self Update FAILED because the downloaded agent FAILED hash check (' + agentUpdate_Start._retryCount + ')', sessionid);
323
+ sendAgentMessage('Self Update FAILED because the downloaded agent FAILED hash check (' + agentUpdate_Start._retryCount + ')', 3);
324
+ agentUpdate_Start._selfupdate = null;
325
+
326
+ if (agentUpdate_Start._retryCount < 4)
327
+ {
328
+ // Retry the download again
329
+ sendConsoleText('Self Update will try again in 60 seconds...', sessionid);
330
+ agentUpdate_Start._timeout = setTimeout(agentUpdate_Start, 60000, updateurl, updateoptions);
331
+ }
332
+ else
333
+ {
334
+ sendConsoleText('Self Update giving up, too many failures...', sessionid);
335
+ sendAgentMessage('Self Update giving up, too many failures...', 3);
336
+ }
337
return;
338
}
339
}
308
- else {
340
+ else
341
+ {
342
sendConsoleText('Download complete. HASH=' + h.toString('hex'), sessionid);
343
}
344
@@ -313,11 +346,13 @@ function agentUpdate_Start(updateurl, updateoptions) {
346
try { require('MeshAgent').SendCommand({ action: 'agentupdatedownloaded' }); } catch (e) { }
347
348
if (sessionid != null) { sendConsoleText('Updating and restarting agent...', sessionid); }
316
- if (process.platform == 'win32') {
349
+ if (process.platform == 'win32')
350
+ {
351
// Use _wexecve() equivalent to perform the update
352
windows_execve(name, agentfilename, sessionid);
353
}
320
- else {
354
+ else
355
+ {
356
var m = require('fs').statSync(process.execPath).mode;
357
require('fs').chmodSync(process.cwd() + agentfilename + '.update', m);
358
@@ -331,7 +366,8 @@ function agentUpdate_Start(updateurl, updateoptions) {
366
// erase update
367
require('fs').unlinkSync(process.cwd() + agentfilename + '.update');
368
334
- switch (process.platform) {
369
+ switch (process.platform)
370
+ {
371
case 'freebsd':
372
bsd_execv(name, agentfilename, sessionid);
373
break;
@@ -339,12 +375,14 @@ function agentUpdate_Start(updateurl, updateoptions) {
375
linux_execv(name, agentfilename, sessionid);
376
break;
377
default:
342
- try {
378
+ try
379
+ {
380
// restart service
381
var s = require('service-manager').manager.getService(name);
382
s.restart();
383
}
347
- catch (zz) {
384
+ catch (zz)
385
+ {
386
sendConsoleText('Self Update encountered an error trying to restart service', sessionid);
387
sendAgentMessage('Self Update encountered an error trying to restart service', 3);
388
}