use html-minifier-terser instead for translate and fix translate multi-threaded
Signed-off-by: si458 <simonsmith5521@gmail.com>
si458 committed
Jun 8, 2025 at 22:50 UTC
28b31b2476a8a8bb2d3a433bdf3453df44d8a2d9
3 files changed
+50
-37
docker/Dockerfile
+1
-1
@@ -15,7 +15,7 @@ COPY ./ /opt/meshcentral/meshcentral/
15
RUN if [ -n "$DISABLE_MINIFY" ] || [ -n "$DISABLE_TRANSLATE" ]; then \
16
echo -e "----------\nPREPARING ENVIRONMENT...\n----------"; \
17
cd meshcentral && \
18
- npm install html-minifier@4.0.0 jsdom@26.0.0 esprima@4.0.1 && \
18
+ npm install html-minifier-terser@7.2.0 jsdom@26.0.0 esprima@4.0.1 && \
19
cd translate && \
20
echo -e "----------\nSTARTING THE EXTRACTING PROCESS...\n----------"; \
21
node translate.js extractall && \
meshcentral.js
+1
-1
@@ -4310,7 +4310,7 @@ function mainStart() {
4310
else if (config.settings.xmongodb != null) { modules.push('mongojs@3.1.0'); } // Add MongoJS, old driver.
4311
if (nodemailer || ((config.smtp != null) && (config.smtp.name != 'console')) || (config.sendmail != null)) { modules.push('nodemailer@6.9.16'); } // Add SMTP support
4312
if (sendgrid || (config.sendgrid != null)) { modules.push('@sendgrid/mail'); } // Add SendGrid support
4313
- if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier@4.0.0'); } // Translation support
4313
+ if ((args.translate || args.dev) && (Number(process.version.match(/^v(\d+\.\d+)/)[1]) >= 16)) { modules.push('jsdom@22.1.0'); modules.push('esprima@4.0.1'); modules.push('html-minifier-terser@7.2.0'); } // Translation support
4314
if (typeof config.settings.crowdsec == 'object') { modules.push('@crowdsec/express-bouncer@0.1.0'); } // Add CrowdSec bounser module (https://www.npmjs.com/package/@crowdsec/express-bouncer)
4315
if (config.settings.prometheus != null) { modules.push('prom-client'); } // Add Prometheus Metrics support
4316
translate/translate.js
+48
-35
@@ -143,8 +143,8 @@ if (directRun && (NodeJSVer >= 12)) {
143
// Get things setup
144
jsdom = require('jsdom');
145
esprima = require('esprima'); // https://www.npmjs.com/package/esprima
146
- if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; }
147
- if (minifyLib == 2) { minify = require('html-minifier').minify; } // https://www.npmjs.com/package/html-minifier
146
+ if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; }
147
+ if (minifyLib == 2) { minify = require('html-minifier-terser').minify; } // https://www.npmjs.com/package/html-minifier
148
149
switch (op) {
150
case 'translate': {
@@ -161,8 +161,8 @@ if (directRun) { setup(); }
161
162
function setup() {
163
var libs = ['jsdom@22.1.0', 'esprima@4.0.1'];
164
- if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; }
165
- if (minifyLib == 2) { libs.push('html-minifier@4.0.0'); }
164
+ if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; }
165
+ if (minifyLib == 2) { libs.push('html-minifier-terser@7.2.0'); }
166
InstallModules(libs, start);
167
}
168
@@ -172,8 +172,8 @@ function startEx(argv) {
172
// Load dependencies
173
jsdom = require('jsdom');
174
esprima = require('esprima'); // https://www.npmjs.com/package/esprima
175
- if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier\""); process.exit(); return; }
176
- if (minifyLib == 2) { minify = require('html-minifier').minify; } // https://www.npmjs.com/package/html-minifier
175
+ if (minifyLib == 1) { log("minify-js is no longer used, please switch to \"html-minifier-terser\""); process.exit(); return; }
176
+ if (minifyLib == 2) { minify = require('html-minifier-terser').minify; } // https://www.npmjs.com/package/html-minifier-terser
177
178
var command = null;
179
if (argv.length > 2) { command = argv[2].toLowerCase(); }
@@ -366,7 +366,7 @@ function startEx(argv) {
366
if (sourceFile.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); }
367
if (sourceFile.endsWith('.js')) { inFile = '<script>' + inFile + '</script>'; }
368
369
- var minifiedOut = minify(inFile, {
369
+ minify(inFile, {
370
collapseBooleanAttributes: true,
371
collapseInlineTagWhitespace: false, // This is not good.
372
collapseWhitespace: true,
@@ -381,12 +381,12 @@ function startEx(argv) {
381
removeTagWhitespace: true,
382
preserveLineBreaks: false,
383
useShortDoctype: true
384
+ }).then(function (minifiedOut) {
385
+ // Perform minification post-processing
386
+ if (sourceFile.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
387
+ if (sourceFile.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
388
+ fs.writeFileSync(destinationFile, minifiedOut, { flag: 'w+' });
389
});
385
-
386
- // Perform minification post-processing
387
- if (sourceFile.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
388
- if (sourceFile.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
389
- fs.writeFileSync(destinationFile, minifiedOut, { flag: 'w+' });
390
}
391
} else if (sourceFiles[i].endsWith('.json')) {
392
// Minify the file .json file
@@ -440,7 +440,7 @@ function startEx(argv) {
440
441
var minifiedOut = null;
442
try {
443
- minifiedOut = minify(inFile, {
443
+ minify(inFile, {
444
collapseBooleanAttributes: true,
445
collapseInlineTagWhitespace: false, // This is not good.
446
collapseWhitespace: true,
@@ -456,17 +456,16 @@ function startEx(argv) {
456
preserveLineBreaks: false,
457
useShortDoctype: true,
458
log: function(a) { if (typeof a !== 'string') { console.log(a); } } // Log errors from UglifyJS to console output
459
+ }).then(function (minifiedOut) {
460
+ // Perform minification post-processing
461
+ if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
462
+ if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
463
+ fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
464
});
465
} catch (ex) {
466
console.log(ex);
467
}
468
464
- // Perform minification post-processing
465
- if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
466
- if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
467
-
468
- fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
469
-
469
/*
470
if (outname.endsWith('.js')) {
471
var compressHandler = function compressHandlerFunc(err, buffer, outnamemin2) {
@@ -509,7 +508,7 @@ function startEx(argv) {
508
if (outname.endsWith('.handlebars') >= 0) { inFile = inFile.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); }
509
if (outname.endsWith('.js')) { inFile = '<script>' + inFile + '</script>'; }
510
512
- var minifiedOut = minify(inFile, {
511
+ minify(inFile, {
512
collapseBooleanAttributes: true,
513
collapseInlineTagWhitespace: false, // This is not good.
514
collapseWhitespace: true,
@@ -524,12 +523,12 @@ function startEx(argv) {
523
removeTagWhitespace: true,
524
preserveLineBreaks: false,
525
useShortDoctype: true
526
+ }).then(function (minifiedOut) {
527
+ // Perform minification post-processing
528
+ if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
529
+ if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
530
+ fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
531
});
528
-
529
- // Perform minification post-processing
530
- if (outname.endsWith('.js')) { minifiedOut = minifiedOut.substring(8, minifiedOut.length - 9); }
531
- if (outname.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
532
- fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
532
}
533
}
534
}
@@ -656,14 +655,27 @@ function translate(lang, langFile, sources, createSubDir) {
655
656
langs = {};
657
for (var i in langFileData.strings) { var entry = langFileData.strings[i]; for (var j in entry) { if ((j != 'en') && (j != 'xloc') && (j != '*')) { langs[j.toLowerCase()] = true; } } }
659
- for (var i in langs) {
660
- const { Worker } = require('worker_threads')
661
- const worker = new Worker('./translate.js', { stdout: true, workerData: { op: 'translate', args: [i, langFile, sources, createSubDir] } });
662
- worker.stdout.on('data', function (msg) { console.log('wstdio:', msg.toString()); });
663
- worker.on('message', function (message) { console.log(message.msg); });
664
- worker.on('error', function (error) { console.log('error', error); });
665
- worker.on('exit', function (code) { /*console.log('exit', code);*/ })
658
+ var Worker = require('worker_threads').Worker;
659
+ var MAX_WORKERS = os.cpus().length; // limit to the number of CPU cores for now
660
+ var activeWorkers = 0;
661
+ var taskQueue = [];
662
+ function processNextTask() {
663
+ if (activeWorkers < MAX_WORKERS && taskQueue.length > 0) {
664
+ var nextTask = taskQueue.shift();
665
+ activeWorkers++;
666
+ var worker = new Worker('./translate.js', { stdout: true, workerData: { op: 'translate', args: [nextTask.lang, nextTask.langFile, nextTask.sources, nextTask.createSubDir] } });
667
+ worker.stdout.on('data', function (msg) { console.log('wstdio:', msg.toString()); });
668
+ worker.on('message', function (message) { console.log(message.msg); });
669
+ worker.on('error', function (error) { console.log('error', error); activeWorkers--; processNextTask(); });
670
+ worker.on('exit', function (code) { /*console.log('exit', code);*/ activeWorkers--; processNextTask(); });
671
+ }
672
+ }
673
+ for (var lang in langs) {
674
+ if (langs.hasOwnProperty(lang)) {
675
+ taskQueue.push({ lang: lang, langFile: langFile, sources: sources, createSubDir: createSubDir});
676
+ }
677
}
678
+ for (var i = 0; i < Math.min(MAX_WORKERS, taskQueue.length); i++) { processNextTask(); }
679
} else {
680
// Single threaded translation
681
translateSingleThreaded(lang, langFile, sources, createSubDir);
@@ -976,7 +988,7 @@ function translateFromHtml(lang, file, createSubDir) {
988
// Minify the file
989
if (minifyLib = 2) {
990
if (outnamemin.endsWith('.handlebars') >= 0) { out = out.split('{{{pluginHandler}}}').join('"{{{pluginHandler}}}"'); }
979
- var minifiedOut = minify(out, {
991
+ minify(out, {
992
collapseBooleanAttributes: true,
993
collapseInlineTagWhitespace: false, // This is not good.
994
collapseWhitespace: true,
@@ -991,9 +1003,10 @@ function translateFromHtml(lang, file, createSubDir) {
1003
removeTagWhitespace: true,
1004
preserveLineBreaks: false,
1005
useShortDoctype: true
1006
+ }).then(function (minifiedOut) {
1007
+ if (outnamemin.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
1008
+ fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
1009
});
995
- if (outnamemin.endsWith('.handlebars') >= 0) { minifiedOut = minifiedOut.split('"{{{pluginHandler}}}"').join('{{{pluginHandler}}}'); }
996
- fs.writeFileSync(outnamemin, minifiedOut, { flag: 'w+' });
1010
}
1011
}
1012