chainlint.pl: add POSIX shell lexical analyzer
Begin fleshing out chainlint.pl by adding a lexical analyzer for the POSIX shell command language. The sole entry point Lexer::scan_token() returns the next token from the input. It will be called by the upcoming shell language parser. Signed-off-by: Eric Sunshine <sunshine@sunshineco.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Eric Sunshine committed
Sep 1, 2022 at 00:29 UTC
7d4804731ed642b92b516908fb93397b08e986bf
1 file changed
+177
t/chainlint.pl
+177
@@ -21,6 +21,183 @@ use Getopt::Long;
21
my $show_stats;
22
my $emit_all;
23
24
+# Lexer tokenizes POSIX shell scripts. It is roughly modeled after section 2.3
25
+# "Token Recognition" of POSIX chapter 2 "Shell Command Language". Although
26
+# similar to lexical analyzers for other languages, this one differs in a few
27
+# substantial ways due to quirks of the shell command language.
28
+#
29
+# For instance, in many languages, newline is just whitespace like space or
30
+# TAB, but in shell a newline is a command separator, thus a distinct lexical
31
+# token. A newline is significant and returned as a distinct token even at the
32
+# end of a shell comment.
33
+#
34
+# In other languages, `1+2` would typically be scanned as three tokens
35
+# (`1`, `+`, and `2`), but in shell it is a single token. However, the similar
36
+# `1 + 2`, which embeds whitepace, is scanned as three token in shell, as well.
37
+# In shell, several characters with special meaning lose that meaning when not
38
+# surrounded by whitespace. For instance, the negation operator `!` is special
39
+# when standing alone surrounded by whitespace; whereas in `foo!uucp` it is
40
+# just a plain character in the longer token "foo!uucp". In many other
41
+# languages, `"string"/foo:'string'` might be scanned as five tokens ("string",
42
+# `/`, `foo`, `:`, and 'string'), but in shell, it is just a single token.
43
+#
44
+# The lexical analyzer for the shell command language is also somewhat unusual
45
+# in that it recursively invokes the parser to handle the body of `$(...)`
46
+# expressions which can contain arbitrary shell code. Such expressions may be
47
+# encountered both inside and outside of double-quoted strings.
48
+#
49
+# The lexical analyzer is responsible for consuming shell here-doc bodies which
50
+# extend from the line following a `<<TAG` operator until a line consisting
51
+# solely of `TAG`. Here-doc consumption begins when a newline is encountered.
52
+# It is legal for multiple here-doc `<<TAG` operators to be present on a single
53
+# line, in which case their bodies must be present one following the next, and
54
+# are consumed in the (left-to-right) order the `<<TAG` operators appear on the
55
+# line. A special complication is that the bodies of all here-docs must be
56
+# consumed when the newline is encountered even if the parse context depth has
57
+# changed. For instance, in `cat <<A && x=$(cat <<B &&\n`, bodies of here-docs
58
+# "A" and "B" must be consumed even though "A" was introduced outside the
59
+# recursive parse context in which "B" was introduced and in which the newline
60
+# is encountered.
61
+package Lexer;
62
+
63
+sub new {
64
+ my ($class, $parser, $s) = @_;
65
+ bless {
66
+ parser => $parser,
67
+ buff => $s,
68
+ heretags => []
69
+ } => $class;
70
+}
71
+
72
+sub scan_heredoc_tag {
73
+ my $self = shift @_;
74
+ ${$self->{buff}} =~ /\G(-?)/gc;
75
+ my $indented = $1;
76
+ my $tag = $self->scan_token();
77
+ $tag =~ s/['"\\]//g;
78
+ push(@{$self->{heretags}}, $indented ? "\t$tag" : "$tag");
79
+ return "<<$indented$tag";
80
+}
81
+
82
+sub scan_op {
83
+ my ($self, $c) = @_;
84
+ my $b = $self->{buff};
85
+ return $c unless $$b =~ /\G(.)/sgc;
86
+ my $cc = $c . $1;
87
+ return scan_heredoc_tag($self) if $cc eq '<<';
88
+ return $cc if $cc =~ /^(?:&&|\|\||>>|;;|<&|>&|<>|>\|)$/;
89
+ pos($$b)--;
90
+ return $c;
91
+}
92
+
93
+sub scan_sqstring {
94
+ my $self = shift @_;
95
+ ${$self->{buff}} =~ /\G([^']*'|.*\z)/sgc;
96
+ return "'" . $1;
97
+}
98
+
99
+sub scan_dqstring {
100
+ my $self = shift @_;
101
+ my $b = $self->{buff};
102
+ my $s = '"';
103
+ while (1) {
104
+ # slurp up non-special characters
105
+ $s .= $1 if $$b =~ /\G([^"\$\\]+)/gc;
106
+ # handle special characters
107
+ last unless $$b =~ /\G(.)/sgc;
108
+ my $c = $1;
109
+ $s .= '"', last if $c eq '"';
110
+ $s .= '$' . $self->scan_dollar(), next if $c eq '$';
111
+ if ($c eq '\\') {
112
+ $s .= '\\', last unless $$b =~ /\G(.)/sgc;
113
+ $c = $1;
114
+ next if $c eq "\n"; # line splice
115
+ # backslash escapes only $, `, ", \ in dq-string
116
+ $s .= '\\' unless $c =~ /^[\$`"\\]$/;
117
+ $s .= $c;
118
+ next;
119
+ }
120
+ die("internal error scanning dq-string '$c'\n");
121
+ }
122
+ return $s;
123
+}
124
+
125
+sub scan_balanced {
126
+ my ($self, $c1, $c2) = @_;
127
+ my $b = $self->{buff};
128
+ my $depth = 1;
129
+ my $s = $c1;
130
+ while ($$b =~ /\G([^\Q$c1$c2\E]*(?:[\Q$c1$c2\E]|\z))/gc) {
131
+ $s .= $1;
132
+ $depth++, next if $s =~ /\Q$c1\E$/;
133
+ $depth--;
134
+ last if $depth == 0;
135
+ }
136
+ return $s;
137
+}
138
+
139
+sub scan_subst {
140
+ my $self = shift @_;
141
+ my @tokens = $self->{parser}->parse(qr/^\)$/);
142
+ $self->{parser}->next_token(); # closing ")"
143
+ return @tokens;
144
+}
145
+
146
+sub scan_dollar {
147
+ my $self = shift @_;
148
+ my $b = $self->{buff};
149
+ return $self->scan_balanced('(', ')') if $$b =~ /\G\((?=\()/gc; # $((...))
150
+ return '(' . join(' ', $self->scan_subst()) . ')' if $$b =~ /\G\(/gc; # $(...)
151
+ return $self->scan_balanced('{', '}') if $$b =~ /\G\{/gc; # ${...}
152
+ return $1 if $$b =~ /\G(\w+)/gc; # $var
153
+ return $1 if $$b =~ /\G([@*#?$!0-9-])/gc; # $*, $1, $$, etc.
154
+ return '';
155
+}
156
+
157
+sub swallow_heredocs {
158
+ my $self = shift @_;
159
+ my $b = $self->{buff};
160
+ my $tags = $self->{heretags};
161
+ while (my $tag = shift @$tags) {
162
+ my $indent = $tag =~ s/^\t// ? '\\s*' : '';
163
+ $$b =~ /(?:\G|\n)$indent\Q$tag\E(?:\n|\z)/gc;
164
+ }
165
+}
166
+
167
+sub scan_token {
168
+ my $self = shift @_;
169
+ my $b = $self->{buff};
170
+ my $token = '';
171
+RESTART:
172
+ $$b =~ /\G[ \t]+/gc; # skip whitespace (but not newline)
173
+ return "\n" if $$b =~ /\G#[^\n]*(?:\n|\z)/gc; # comment
174
+ while (1) {
175
+ # slurp up non-special characters
176
+ $token .= $1 if $$b =~ /\G([^\\;&|<>(){}'"\$\s]+)/gc;
177
+ # handle special characters
178
+ last unless $$b =~ /\G(.)/sgc;
179
+ my $c = $1;
180
+ last if $c =~ /^[ \t]$/; # whitespace ends token
181
+ pos($$b)--, last if length($token) && $c =~ /^[;&|<>(){}\n]$/;
182
+ $token .= $self->scan_sqstring(), next if $c eq "'";
183
+ $token .= $self->scan_dqstring(), next if $c eq '"';
184
+ $token .= $c . $self->scan_dollar(), next if $c eq '$';
185
+ $self->swallow_heredocs(), $token = $c, last if $c eq "\n";
186
+ $token = $self->scan_op($c), last if $c =~ /^[;&|<>]$/;
187
+ $token = $c, last if $c =~ /^[(){}]$/;
188
+ if ($c eq '\\') {
189
+ $token .= '\\', last unless $$b =~ /\G(.)/sgc;
190
+ $c = $1;
191
+ next if $c eq "\n" && length($token); # line splice
192
+ goto RESTART if $c eq "\n"; # line splice
193
+ $token .= '\\' . $c;
194
+ next;
195
+ }
196
+ die("internal error scanning character '$c'\n");
197
+ }
198
+ return length($token) ? $token : undef;
199
+}
200
+
201
package ScriptParser;
202
203
sub new {