chainlint.pl: validate test scripts in parallel
Although chainlint.pl has undergone a good deal of optimization during its development -- increasing in speed significantly -- parsing and validating 1050+ scripts and 16500+ tests via Perl is not exactly instantaneous. However, perceived performance can be improved by taking advantage of the fact that there is no interdependence between test scripts or test definitions, thus parsing and validating can be done in parallel. The number of available cores is determined automatically but can be overridden via the --jobs option. Signed-off-by: Eric Sunshine <sunshine@sunshineco.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Eric Sunshine committed
Sep 1, 2022 at 00:29 UTC
29fb2ec384a867ca577335a12f4b45c184e7b642
1 file changed
+49
-1
t/chainlint.pl
+49
-1
@@ -15,9 +15,11 @@
15
16
use warnings;
17
use strict;
18
+use Config;
19
use File::Glob;
20
use Getopt::Long;
21
22
+my $jobs = -1;
23
my $show_stats;
24
my $emit_all;
25
@@ -569,6 +571,16 @@ if (eval {require Time::HiRes; Time::HiRes->import(); 1;}) {
571
$interval = sub { return Time::HiRes::tv_interval(shift); };
572
}
573
574
+sub ncores {
575
+ # Windows
576
+ return $ENV{NUMBER_OF_PROCESSORS} if exists($ENV{NUMBER_OF_PROCESSORS});
577
+ # Linux / MSYS2 / Cygwin / WSL
578
+ do { local @ARGV='/proc/cpuinfo'; return scalar(grep(/^processor\s*:/, <>)); } if -r '/proc/cpuinfo';
579
+ # macOS & BSD
580
+ return qx/sysctl -n hw.ncpu/ if $^O =~ /(?:^darwin$|bsd)/;
581
+ return 1;
582
+}
583
+
584
sub show_stats {
585
my ($start_time, $stats) = @_;
586
my $walltime = $interval->($start_time);
@@ -621,7 +633,9 @@ sub exit_code {
633
Getopt::Long::Configure(qw{bundling});
634
GetOptions(
635
"emit-all!" => \$emit_all,
636
+ "jobs|j=i" => \$jobs,
637
"stats|show-stats!" => \$show_stats) or die("option error\n");
638
+$jobs = ncores() if $jobs < 1;
639
640
my $start_time = $getnow->();
641
my @stats;
@@ -633,6 +647,40 @@ unless (@scripts) {
647
exit;
648
}
649
636
-push(@stats, check_script(1, sub { shift(@scripts); }, sub { print(@_); }));
650
+unless ($Config{useithreads} && eval {
651
+ require threads; threads->import();
652
+ require Thread::Queue; Thread::Queue->import();
653
+ 1;
654
+ }) {
655
+ push(@stats, check_script(1, sub { shift(@scripts); }, sub { print(@_); }));
656
+ show_stats($start_time, \@stats) if $show_stats;
657
+ exit(exit_code(\@stats));
658
+}
659
+
660
+my $script_queue = Thread::Queue->new();
661
+my $output_queue = Thread::Queue->new();
662
+
663
+sub next_script { return $script_queue->dequeue(); }
664
+sub emit { $output_queue->enqueue(@_); }
665
+
666
+sub monitor {
667
+ while (my $s = $output_queue->dequeue()) {
668
+ print($s);
669
+ }
670
+}
671
+
672
+my $mon = threads->create({'context' => 'void'}, \&monitor);
673
+threads->create({'context' => 'list'}, \&check_script, $_, \&next_script, \&emit) for 1..$jobs;
674
+
675
+$script_queue->enqueue(@scripts);
676
+$script_queue->end();
677
+
678
+for (threads->list()) {
679
+ push(@stats, $_->join()) unless $_ == $mon;
680
+}
681
+
682
+$output_queue->end();
683
+$mon->join();
684
+
685
show_stats($start_time, \@stats) if $show_stats;
686
exit(exit_code(\@stats));