Improved alternate Let's Encrypt support.

Ylian Saint-Hilaire committed Mar 5, 2020 at 11:18 UTC 7df7576acb00c429fd29e0d36fc4f869043de857
5 files changed +96 -54
letsEncrypt.js
+47 -29
@@ -341,6 +341,17 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
341 obj.challenges = {};
342 obj.runAsProduction = false;
343 obj.redirWebServerHooked = false;
344 + obj.configErr = null;
345 + obj.configOk = false;
346 +
347 + // Let's Encrypt debug logging
348 + obj.log = function (str) {
349 + parent.debug('cert', 'LE: ' + str);
350 + var d = new Date();
351 + obj.events.push(d.toLocaleDateString() + ' ' + d.toLocaleTimeString() + ' - ' + str);
352 + while (obj.events.length > 200) { obj.events.shift(); } // Keep only 200 last events.
353 + }
354 + obj.events = [];
355
356 // Setup the certificate storage paths
357 obj.certPath = obj.path.join(obj.parent.datapath, 'letsencrypt-certs');
@@ -353,18 +364,20 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
364 // Deal with HTTP challenges
365 function challengeCreateFn(authz, challenge, keyAuthorization) { if (challenge.type === 'http-01') { obj.challenges[challenge.token] = keyAuthorization; } }
366 function challengeRemoveFn(authz, challenge, keyAuthorization) { if (challenge.type === 'http-01') { delete obj.challenges[challenge.token]; } }
356 - obj.challenge = function (token, hostname, func) { func(obj.challenges[token]); }
367 + obj.challenge = function (token, hostname, func) { obj.log((obj.challenges[token] != null)?"Succesful response to challenge.":"Failed to respond to challenge."); func(obj.challenges[token]); }
368
369 // Get the current certificate
370 obj.getCertificate = function(certs, func) {
371 obj.runAsProduction = (obj.parent.config.letsencrypt.production === true);
361 - parent.debug('cert', "LE: Getting certs from local store (" + (obj.runAsProduction ? "Production" : "Staging") + ")");
362 - if (certs.CommonName.indexOf('.') == -1) { console.log("ERROR: Use --cert to setup the default server name before using Let's Encrypt."); func(certs); return; }
363 - if (obj.parent.config.letsencrypt == null) { func(certs); return; }
364 - if (obj.parent.config.letsencrypt.email == null) { console.log("ERROR: Let's Encrypt email address not specified."); func(certs); return; }
365 - if ((obj.parent.redirserver == null) || ((typeof obj.parent.config.settings.rediraliasport === 'number') && (obj.parent.config.settings.rediraliasport !== 80)) || ((obj.parent.config.settings.rediraliasport == null) && (obj.parent.redirserver.port !== 80))) { console.log("ERROR: Redirection web server must be active on port 80 for Let's Encrypt to work."); func(certs); return; }
366 - if (obj.redirWebServerHooked !== true) { console.log("ERROR: Redirection web server not setup for Let's Encrypt to work."); func(certs); return; }
367 - if ((obj.parent.config.letsencrypt.rsakeysize != null) && (obj.parent.config.letsencrypt.rsakeysize !== 2048) && (obj.parent.config.letsencrypt.rsakeysize !== 3072)) { console.log("ERROR: Invalid Let's Encrypt certificate key size, must be 2048 or 3072."); func(certs); return; }
372 + obj.log("Getting certs from local store (" + (obj.runAsProduction ? "Production" : "Staging") + ")");
373 + if (certs.CommonName.indexOf('.') == -1) { obj.configErr = "ERROR: Use --cert to setup the default server name before using Let's Encrypt."; obj.log(obj.configErr); console.log(obj.configErr); func(certs); return; }
374 + if (obj.parent.config.letsencrypt == null) { obj.configErr = "No Let's Encrypt configuration"; obj.log(obj.configErr); console.log(obj.configErr); func(certs); return; }
375 + if (obj.parent.config.letsencrypt.email == null) { obj.configErr = "ERROR: Let's Encrypt email address not specified."; obj.log(obj.configErr); console.log(obj.configErr); func(certs); return; }
376 + if ((obj.parent.redirserver == null) || ((typeof obj.parent.config.settings.rediraliasport === 'number') && (obj.parent.config.settings.rediraliasport !== 80)) || ((obj.parent.config.settings.rediraliasport == null) && (obj.parent.redirserver.port !== 80))) { obj.configErr = "ERROR: Redirection web server must be active on port 80 for Let's Encrypt to work."; obj.log(obj.configErr); console.log(obj.configErr); func(certs); return; }
377 + if (obj.redirWebServerHooked !== true) { obj.configErr = "ERROR: Redirection web server not setup for Let's Encrypt to work."; obj.log(obj.configErr); console.log(obj.configErr); func(certs); return; }
378 + if ((obj.parent.config.letsencrypt.rsakeysize != null) && (obj.parent.config.letsencrypt.rsakeysize !== 2048) && (obj.parent.config.letsencrypt.rsakeysize !== 3072)) { obj.configErr = "ERROR: Invalid Let's Encrypt certificate key size, must be 2048 or 3072."; obj.log(obj.configErr); console.log(obj.configErr); func(certs); return; }
379 + if (obj.checkInterval == null) { obj.checkInterval = setInterval(obj.checkRenewCertificate, 86400000); } // Call certificate check every 24 hours.
380 + obj.configOk = true;
381
382 // Get the list of domains
383 obj.leDomains = [ certs.CommonName ];
@@ -379,7 +392,7 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
392 var certFile = obj.path.join(obj.certPath, (obj.runAsProduction ? 'production.crt' : 'staging.crt'));
393 var keyFile = obj.path.join(obj.certPath, (obj.runAsProduction ? 'production.key' : 'staging.key'));
394 if (obj.fs.existsSync(certFile) && obj.fs.existsSync(keyFile)) {
382 - parent.debug('cert', "LE: Reading certificate files");
395 + obj.log("Reading certificate files");
396
397 // Read the certificate and private key
398 var certPem = obj.fs.readFileSync(certFile).toString('utf8');
@@ -397,54 +410,55 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
410
411 // Use this certificate when possible on any domain
412 if (obj.certNames.indexOf(certs.CommonName) >= 0) {
413 + obj.log("Setting LE cert for default domain.");
414 certs.web.cert = certPem;
415 certs.web.key = keyPem;
416 //certs.web.ca = [results.pems.chain];
417 }
418 for (var i in obj.parent.config.domains) {
419 if ((obj.parent.config.domains[i].dns != null) && (obj.parent.certificateOperations.compareCertificateNames(obj.certNames, obj.parent.config.domains[i].dns))) {
420 + obj.log("Setting LE cert for domain " + i + ".");
421 certs.dns[i].cert = certPem;
422 certs.dns[i].key = keyPem;
423 //certs.dns[i].ca = [results.pems.chain];
424 }
425 }
426 } else {
412 - parent.debug('cert', "LE: No certificate files found");
427 + obj.log("No certificate files found");
428 }
429 func(certs);
415 - obj.checkRenewCertificate();
430 + setTimeout(obj.checkRenewCertificate, 5000); // Hold 5 seconds and check if we need to request a certificate.
431 }
432
433 // Check if we need to get a new certificate
434 // Return 0 = CertOK, 1 = Request:NoCert, 2 = Request:Expire, 3 = Request:MissingNames
435 obj.checkRenewCertificate = function () {
421 - parent.debug('cert', "LE: Checking certificate");
436 if (obj.certNames == null) {
423 - parent.debug('cert', "LE: Got no certificates, asking for one now.");
437 + obj.log("Got no certificates, asking for one now.");
438 obj.requestCertificate();
439 return 1;
440 } else {
441 // Look at the existing certificate to see if we need to renew it
442 var daysLeft = Math.floor((obj.certExpire - new Date()) / 86400000);
429 - parent.debug('cert', "LE: Certificate has " + daysLeft + " day(s) left.");
443 + obj.log("Certificate has " + daysLeft + " day(s) left.");
444 if (daysLeft < 45) {
431 - parent.debug('cert', "LE: Asking for new certificate because of expire time.");
445 + obj.log("Asking for new certificate because of expire time.");
446 obj.requestCertificate();
447 return 2;
448 } else {
449 var missingDomain = false;
450 for (var i in obj.leDomains) {
451 if (obj.parent.certificateOperations.compareCertificateNames(obj.certNames, obj.leDomains[i]) == false) {
438 - parent.debug('cert', "LE: Missing name " + obj.leDomains[i] + ".");
452 + obj.log("Missing name \"" + obj.leDomains[i] + "\".");
453 missingDomain = true;
454 }
455 }
456 if (missingDomain) {
443 - parent.debug('cert', "LE: Asking for new certificate because of missing names.");
457 + obj.log("Asking for new certificate because of missing names.");
458 obj.requestCertificate();
459 return 3;
460 } else {
447 - parent.debug('cert', "LE: Certificate is ok.");
461 + obj.log("Certificate is ok.");
462 }
463 }
464 }
@@ -452,25 +466,27 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
466 }
467
468 obj.requestCertificate = function () {
469 + if (obj.configOk == false) { obj.log("Can't request cert, invalid configuration.");return; }
470 +
471 // Create a private key
456 - parent.debug('cert', "LE: Generating private key...");
472 + obj.log("Generating private key...");
473 acme.forge.createPrivateKey().then(function (accountKey) {
474 // Create the ACME client
459 - parent.debug('cert', "LE: Setting up ACME client...");
475 + obj.log("Setting up ACME client...");
476 obj.client = new acme.Client({
477 directoryUrl: obj.runAsProduction ? acme.directory.letsencrypt.production : acme.directory.letsencrypt.staging,
478 accountKey: accountKey
479 });
480
481 // Create Certificate Request (CSR)
466 - parent.debug('cert', "LE: Creating certificate request...");
482 + obj.log("Creating certificate request...");
483 acme.forge.createCsr({
484 commonName: obj.leDomains[0],
485 altNames: obj.leDomains
486 }).then(function (r) {
487 var csr = r[1];
488 obj.tempPrivateKey = r[0];
473 - parent.debug('cert', "LE: Requesting certificate from Let's Encrypt...");
489 + obj.log("Requesting certificate from Let's Encrypt...");
490 obj.client.auto({
491 csr,
492 email: obj.parent.config.letsencrypt.email,
@@ -478,7 +494,7 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
494 challengeCreateFn,
495 challengeRemoveFn
496 }).then(function (cert) {
481 - parent.debug('cert', "LE: Got certificate.");
497 + obj.log("Got certificate.");
498
499 // Save certificate and private key to PEM files
500 var certFile = obj.path.join(obj.certPath, (obj.runAsProduction ? 'production.crt' : 'staging.crt'));
@@ -488,16 +504,16 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
504 delete obj.tempPrivateKey;
505
506 // Cause a server restart
491 - parent.debug('cert', "LE: Performing server restart...");
507 + obj.log("Performing server restart...");
508 obj.parent.performServerCertUpdate();
509 }, function (err) {
494 - parent.debug('cert', "LE: Failed to obtain certificate: " + err.message);
510 + obj.log("Failed to obtain certificate: " + err.message);
511 });
512 }, function (err) {
497 - parent.debug('cert', "LE: Failed to generate certificate request: " + err.message);
513 + obj.log("Failed to generate certificate request: " + err.message);
514 });
515 }, function (err) {
500 - parent.debug('cert', "LE: Failed to generate private key: " + err.message);
516 + obj.log("Failed to generate private key: " + err.message);
517 });
518 }
519
@@ -505,13 +521,15 @@ module.exports.CreateLetsEncrypt2 = function (parent) {
521 obj.getStats = function () {
522 var r = {
523 lib: 'acme-client',
524 + configOk: obj.configOk,
525 leDomains: obj.leDomains,
526 challenges: obj.challenges,
527 production: obj.runAsProduction,
528 webServer: obj.redirWebServerHooked,
512 - certPath: obj.certPath,
529 + certPath: obj.certPath
530 };
514 - if (obj.certExpire) { r.daysLeft = Math.floor((obj.certExpire - new Date()) / 86400000); }
531 + if (obj.configErr) { r.error = obj.configErr; }
532 + if (obj.certExpire) { r.cert = 'Present'; r.daysLeft = Math.floor((obj.certExpire - new Date()) / 86400000); } else { r.cert = 'None'; }
533 return r;
534 }
535
meshcentral.js
+7 -1
@@ -2389,7 +2389,13 @@ function mainStart() {
2389 if (require('os').platform() == 'win32') { modules.push('node-windows'); if (sspi == true) { modules.push('node-sspi'); } } // Add Windows modules
2390 if (ldap == true) { modules.push('ldapauth-fork'); }
2391 if (recordingIndex == true) { modules.push('image-size'); } // Need to get the remote desktop JPEG sizes to index the recodring file.
2392 - if (config.letsencrypt != null) { if ((nodeVersion < 10) || (require('crypto').generateKeyPair == null)) { addServerWarning("Let's Encrypt support requires Node v10.12 or higher.", !args.launch); } else { modules.push((config.letsencrypt.lib == 'acme-client') ? 'acme-client' : 'greenlock@4.0.4'); } } // Add Greenlock Module or acme-client module
2392 + if (config.letsencrypt != null) {
2393 + if (config.letsencrypt.lib == 'acme-client') {
2394 + if (nodeVersion < 8) { addServerWarning("Let's Encrypt support requires Node v8.x or higher.", !args.launch); } else { modules.push('acme-client'); }
2395 + } else {
2396 + if ((nodeVersion < 10) || (require('crypto').generateKeyPair == null)) { addServerWarning("Let's Encrypt support requires Node v10.12 or higher.", !args.launch); } else { modules.push('greenlock@4.0.4'); }
2397 + }
2398 + } // Add Greenlock Module or acme-client module
2399 if (config.settings.mqtt != null) { modules.push('aedes'); } // Add MQTT Modules
2400 if (config.settings.mysql != null) { modules.push('mysql'); } // Add MySQL, official driver.
2401 if (config.settings.mongodb != null) { modules.push('mongodb'); } // Add MongoDB, official driver.
meshuser.js
+14 -3
@@ -689,7 +689,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
689
690 switch (cmd) {
691 case 'help': {
692 - var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,letsencrypt';
692 + var fin = '', f = '', availcommands = 'help,info,versions,args,resetserver,showconfig,usersessions,closeusersessions,tasklimiter,setmaxtasks,cores,migrationagents,agentstats,webstats,mpsstats,swarmstats,acceleratorsstats,updatecheck,serverupdate,nodeconfig,heapdump,relays,autobackup,backupconfig,dupagents,dispatchtable,badlogins,showpaths,le,lecheck,leevents';
693 availcommands = availcommands.split(',').sort();
694 while (availcommands.length > 0) {
695 if (f.length > 80) { fin += (f + ',\r\n'); f = ''; }
@@ -699,8 +699,7 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
699 r = 'Available commands: \r\n' + fin + '.';
700 break;
701 }
702 - case 'le':
703 - case 'letsencrypt': {
702 + case 'le': {
703 if (parent.parent.letsencrypt == null) {
704 r = "Let's Encrypt not in use.";
705 } else {
@@ -742,6 +741,18 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
741 }
742 break;
743 }
744 + case 'leevents': {
745 + if (parent.parent.letsencrypt == null) {
746 + r = "Let's Encrypt not in use.";
747 + } else {
748 + if (parent.parent.letsencrypt.lib == 'acme-client') {
749 + r = parent.parent.letsencrypt.events.join('\r\n');
750 + } else {
751 + r = 'Not supported';
752 + }
753 + }
754 + break;
755 + }
756 case 'badlogins': {
757 if (parent.parent.config.settings.maxinvalidlogin == false) {
758 r = 'Bad login filter is disabled.';
translate/translate.json
+25 -19
@@ -3031,7 +3031,7 @@
3031 "nl": "Weet u zeker dat u de plug-in {0} wilt gebruiken: {1}",
3032 "ru": "Вы уверенны, что {0} плагин: {1}",
3033 "xloc": [
3034 - "default.handlebars->27->1510"
3034 + "default.handlebars->27->1511"
3035 ]
3036 },
3037 {
@@ -3552,7 +3552,7 @@
3552 "pt": "Servidor CIRA",
3553 "ru": "CIRA Сервер",
3554 "xloc": [
3555 - "default.handlebars->27->1501"
3555 + "default.handlebars->27->1502"
3556 ]
3557 },
3558 {
@@ -3566,7 +3566,7 @@
3566 "pt": "Comandos do servidor CIRA",
3567 "ru": "CIRA Сервер команды",
3568 "xloc": [
3569 - "default.handlebars->27->1502"
3569 + "default.handlebars->27->1503"
3570 ]
3571 },
3572 {
@@ -3665,7 +3665,7 @@
3665 "pt": "Erro de chamada",
3666 "ru": "Ошибка вызова",
3667 "xloc": [
3668 - "default.handlebars->27->1511"
3668 + "default.handlebars->27->1512"
3669 ]
3670 },
3671 {
@@ -4000,7 +4000,7 @@
4000 "pt": "Verificando ...",
4001 "ru": "Проверка...",
4002 "xloc": [
4003 - "default.handlebars->27->1507",
4003 + "default.handlebars->27->1508",
4004 "default.handlebars->27->769"
4005 ]
4006 },
@@ -4663,7 +4663,7 @@
4663 "pt": "Encaminhador de conexão",
4664 "ru": "Ретранслятор подключения",
4665 "xloc": [
4666 - "default.handlebars->27->1500"
4666 + "default.handlebars->27->1501"
4667 ]
4668 },
4669 {
@@ -9089,7 +9089,7 @@
9089 "default.handlebars->27->1194",
9090 "default.handlebars->27->1200",
9091 "default.handlebars->27->1479",
9092 - "default.handlebars->27->1499"
9092 + "default.handlebars->27->1500"
9093 ]
9094 },
9095 {
@@ -10500,7 +10500,7 @@
10500 "pt": "Menos",
10501 "ru": "Меньше",
10502 "xloc": [
10503 - "default.handlebars->27->1513"
10503 + "default.handlebars->27->1514"
10504 ]
10505 },
10506 {
@@ -12153,7 +12153,7 @@
12153 "pt": "Mais",
12154 "ru": "Еще",
12155 "xloc": [
12156 - "default.handlebars->27->1512"
12156 + "default.handlebars->27->1513"
12157 ]
12158 },
12159 {
@@ -14314,7 +14314,7 @@
14314 "nl": "Plugin Actie",
14315 "ru": "Действие плагина",
14316 "xloc": [
14317 - "default.handlebars->27->1509",
14317 + "default.handlebars->27->1510",
14318 "default.handlebars->27->159"
14319 ]
14320 },
@@ -16345,6 +16345,12 @@
16345 "default.handlebars->27->1494"
16346 ]
16347 },
16348 + {
16349 + "en": "Server Database",
16350 + "xloc": [
16351 + "default.handlebars->27->1495"
16352 + ]
16353 + },
16354 {
16355 "cs": "Soubory serveru",
16356 "de": "Server-Dateien",
@@ -16455,7 +16461,7 @@
16461 "pt": "Rastreamento de servidor",
16462 "ru": "Трассировка сервера",
16463 "xloc": [
16458 - "default.handlebars->27->1503"
16464 + "default.handlebars->27->1504"
16465 ]
16466 },
16467 {
@@ -19152,7 +19158,7 @@
19158 "nl": "Bijgewerkt",
19159 "ru": "Актуально",
19160 "xloc": [
19155 - "default.handlebars->27->1508"
19161 + "default.handlebars->27->1509"
19162 ]
19163 },
19164 {
@@ -19991,8 +19997,8 @@
19997 "pt": "Servidor web",
19998 "ru": "Веб-сервер",
19999 "xloc": [
19994 - "default.handlebars->27->1495",
19995 - "default.handlebars->27->1496"
20000 + "default.handlebars->27->1496",
20001 + "default.handlebars->27->1497"
20002 ]
20003 },
20004 {
@@ -20006,7 +20012,7 @@
20012 "pt": "Solicitações de servidor Web",
20013 "ru": "Запросы веб-сервера",
20014 "xloc": [
20009 - "default.handlebars->27->1497"
20015 + "default.handlebars->27->1498"
20016 ]
20017 },
20018 {
@@ -20020,7 +20026,7 @@
20026 "pt": "Encaminhador de soquete da Web",
20027 "ru": "Ретранслятор Web Socket",
20028 "xloc": [
20023 - "default.handlebars->27->1498"
20029 + "default.handlebars->27->1499"
20030 ]
20031 },
20032 {
@@ -20663,7 +20669,7 @@
20669 "pt": "\\\\'",
20670 "ru": "\\\\'",
20671 "xloc": [
20666 - "default.handlebars->27->1506"
20672 + "default.handlebars->27->1507"
20673 ]
20674 },
20675 {
@@ -21088,7 +21094,7 @@
21094 "pt": "servertrace.csv",
21095 "ru": "servertrace.csv",
21096 "xloc": [
21091 - "default.handlebars->27->1505"
21097 + "default.handlebars->27->1506"
21098 ]
21099 },
21100 {
@@ -21137,7 +21143,7 @@
21143 "pt": "hora, fonte, mensagem",
21144 "ru": "time, source, message",
21145 "xloc": [
21140 - "default.handlebars->27->1504"
21146 + "default.handlebars->27->1505"
21147 ]
21148 },
21149 {
views/default.handlebars
+3 -2
@@ -10674,6 +10674,7 @@
10674 x += '<div><label><input type=checkbox id=p41c15 ' + ((serverTraceSources.indexOf('agent') >= 0) ? 'checked' : '') + '>' + "MeshAgent traffic" + '</label></div>';
10675 x += '<div><label><input type=checkbox id=p41c14 ' + ((serverTraceSources.indexOf('agentupdate') >= 0) ? 'checked' : '') + '>' + "MeshAgent update" + '</label></div>';
10676 x += '<div><label><input type=checkbox id=p41c16 ' + ((serverTraceSources.indexOf('cert') >= 0) ? 'checked' : '') + '>' + "Server Certificate" + '</label></div>';
10677 + x += '<div><label><input type=checkbox id=p41c17 ' + ((serverTraceSources.indexOf('db') >= 0) ? 'checked' : '') + '>' + "Server Database" + '</label></div>';
10678 x += '<div style="width:100%;border-bottom:1px solid gray;margin-bottom:5px;margin-top:5px"><b>' + "Web Server" + '</b></div>';
10679 x += '<div><label><input type=checkbox id=p41c5 ' + ((serverTraceSources.indexOf('web') >= 0) ? 'checked' : '') + '>' + "Web Server" + '</label></div>';
10680 x += '<div><label><input type=checkbox id=p41c6 ' + ((serverTraceSources.indexOf('webrequest') >= 0) ? 'checked' : '') + '>' + "Web Server Requests" + '</label></div>';
@@ -10690,8 +10691,8 @@
10691 }
10692
10693 function setServerTracingEx(b) {
10693 - var sources = [], allsources = ['cookie', 'dispatch', 'main', 'peer', 'web', 'webrequest', 'relay', 'webrelaydata', 'webrelay', 'mps', 'mpscmd', 'swarm', 'swarmcmd', 'agentupdate', 'agent', 'cert'];
10694 - if (b == 1) { for (var i = 1; i < 17; i++) { try { if (Q('p41c' + i).checked) { sources.push(allsources[i - 1]); } } catch (ex) { } } }
10694 + var sources = [], allsources = ['cookie', 'dispatch', 'main', 'peer', 'web', 'webrequest', 'relay', 'webrelaydata', 'webrelay', 'mps', 'mpscmd', 'swarm', 'swarmcmd', 'agentupdate', 'agent', 'cert', 'db'];
10695 + if (b == 1) { for (var i = 1; i < 18; i++) { try { if (Q('p41c' + i).checked) { sources.push(allsources[i - 1]); } } catch (ex) { } } }
10696 meshserver.send({ action: 'traceinfo', traceSources: sources });
10697 }
10698