@matheusbsbs / matheus-bagliano / commits / eafff6e41e
Fork 0

t/lib-httpd: avoid using macOS' sed

Among other differences relative to GNU sed, macOS' sed always ends its output with a trailing newline, even if the input did not have such a trailing newline. Surprisingly, this makes three httpd-based tests fail on macOS: t5616, t5702 and t5703. ("Surprisingly" because those tests have been around for some time, but apparently nobody runs them on macOS with a working Apache2 setup.) The reason is that we use `sed` in those tests to filter the response of the web server. Apart from the fact that we use GNU constructs (such as using a space after the `c` command instead of a backslash and a newline), we have another problem: macOS' sed LF-only newlines while webservers are supposed to use CR/LF ones. Even worse, t5616 uses `sed` to replace a binary part of the response with a new binary part (kind of hoping that the replaced binary part does not contain a 0x0a byte which would be interpreted as a newline). To that end, it calls on Perl to read the binary pack file and hex-encode it, then calls on `sed` to prefix every hex digit pair with a `\x` in order to construct the text that the `c` statement of the `sed` invocation is supposed to insert. So we call Perl and sed to construct a sed statement. The final nail in the coffin is that macOS' sed does not even interpret those `\x<hex>` constructs. Let's just replace all of that by Perl snippets. With Perl, at least, we do not have to deal with GNU vs macOS semantics, we do not have to worry about unwanted trailing newlines, and we do not have to spawn commands to construct arguments for other commands to be spawned (i.e. we can avoid a whole lot of shell scripting complexity). The upshot is that this fixes t5616, t5702 and t5703 on macOS with Apache2. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Johannes Schindelin committed Feb 27, 2020 at 13:23 UTC eafff6e41e6a16c23764dcd31373bedd61572cf1
8 files changed +66 -59
t/lib-httpd.sh
+1 -1
@@ -132,7 +132,7 @@ prepare_httpd() {
132 install_script broken-smart-http.sh
133 install_script error-smart-http.sh
134 install_script error.sh
135 - install_script apply-one-time-sed.sh
135 + install_script apply-one-time-perl.sh
136
137 ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules"
138
t/lib-httpd/apache.conf
+3 -3
@@ -113,7 +113,7 @@ Alias /auth/dumb/ www/auth/dumb/
113 SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
114 SetEnv GIT_HTTP_EXPORT_ALL
115 </LocationMatch>
116 -<LocationMatch /one_time_sed/>
116 +<LocationMatch /one_time_perl/>
117 SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
118 SetEnv GIT_HTTP_EXPORT_ALL
119 </LocationMatch>
@@ -122,7 +122,7 @@ ScriptAliasMatch /smart_*[^/]*/(.*) ${GIT_EXEC_PATH}/git-http-backend/$1
122 ScriptAlias /broken_smart/ broken-smart-http.sh/
123 ScriptAlias /error_smart/ error-smart-http.sh/
124 ScriptAlias /error/ error.sh/
125 -ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1
125 +ScriptAliasMatch /one_time_perl/(.*) apply-one-time-perl.sh/$1
126 <Directory ${GIT_EXEC_PATH}>
127 Options FollowSymlinks
128 </Directory>
@@ -135,7 +135,7 @@ ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1
135 <Files error.sh>
136 Options ExecCGI
137 </Files>
138 -<Files apply-one-time-sed.sh>
138 +<Files apply-one-time-perl.sh>
139 Options ExecCGI
140 </Files>
141 <Files ${GIT_EXEC_PATH}/git-http-backend>
t/lib-httpd/apply-one-time-perl.sh new
+27
@@ -0,0 +1,27 @@
1 +#!/bin/sh
2 +
3 +# If "one-time-perl" exists in $HTTPD_ROOT_PATH, run perl on the HTTP response,
4 +# using the contents of "one-time-perl" as the perl command to be run. If the
5 +# response was modified as a result, delete "one-time-perl" so that subsequent
6 +# HTTP responses are no longer modified.
7 +#
8 +# This can be used to simulate the effects of the repository changing in
9 +# between HTTP request-response pairs.
10 +if test -f one-time-perl
11 +then
12 + LC_ALL=C
13 + export LC_ALL
14 +
15 + "$GIT_EXEC_PATH/git-http-backend" >out
16 + perl -pe "$(cat one-time-perl)" out >out_modified
17 +
18 + if cmp -s out out_modified
19 + then
20 + cat out
21 + else
22 + cat out_modified
23 + rm one-time-perl
24 + fi
25 +else
26 + "$GIT_EXEC_PATH/git-http-backend"
27 +fi
t/lib-httpd/apply-one-time-sed.sh deleted
-24
@@ -1,24 +0,0 @@
1 -#!/bin/sh
2 -
3 -# If "one-time-sed" exists in $HTTPD_ROOT_PATH, run sed on the HTTP response,
4 -# using the contents of "one-time-sed" as the sed command to be run. If the
5 -# response was modified as a result, delete "one-time-sed" so that subsequent
6 -# HTTP responses are no longer modified.
7 -#
8 -# This can be used to simulate the effects of the repository changing in
9 -# between HTTP request-response pairs.
10 -if test -f one-time-sed
11 -then
12 - "$GIT_EXEC_PATH/git-http-backend" >out
13 - sed "$(cat one-time-sed)" out >out_modified
14 -
15 - if cmp -s out out_modified
16 - then
17 - cat out
18 - else
19 - cat out_modified
20 - rm one-time-sed
21 - fi
22 -else
23 - "$GIT_EXEC_PATH/git-http-backend"
24 -fi
t/t5537-fetch-shallow.sh
+5 -5
@@ -233,7 +233,7 @@ test_expect_success 'shallow fetches check connectivity before writing shallow f
233 git -C "$REPO" config protocol.version 2 &&
234 git -C client config protocol.version 2 &&
235
236 - git -C client fetch --depth=2 "$HTTPD_URL/one_time_sed/repo" master:a_branch &&
236 + git -C client fetch --depth=2 "$HTTPD_URL/one_time_perl/repo" master:a_branch &&
237
238 # Craft a situation in which the server sends back an unshallow request
239 # with an empty packfile. This is done by refetching with a shorter
@@ -242,13 +242,13 @@ test_expect_success 'shallow fetches check connectivity before writing shallow f
242 printf "s/0034shallow %s/0036unshallow %s/" \
243 "$(git -C "$REPO" rev-parse HEAD)" \
244 "$(git -C "$REPO" rev-parse HEAD^)" \
245 - >"$HTTPD_ROOT_PATH/one-time-sed" &&
245 + >"$HTTPD_ROOT_PATH/one-time-perl" &&
246 test_must_fail env GIT_TEST_SIDEBAND_ALL=0 git -C client \
247 - fetch --depth=1 "$HTTPD_URL/one_time_sed/repo" \
247 + fetch --depth=1 "$HTTPD_URL/one_time_perl/repo" \
248 master:a_branch &&
249
250 - # Ensure that the one-time-sed script was used.
251 - ! test -e "$HTTPD_ROOT_PATH/one-time-sed" &&
250 + # Ensure that the one-time-perl script was used.
251 + ! test -e "$HTTPD_ROOT_PATH/one-time-perl" &&
252
253 # Ensure that the resulting repo is consistent, despite our failure to
254 # fetch.
t/t5616-partial-clone.sh
+21 -17
@@ -398,14 +398,18 @@ intersperse () {
398 sed 's/\(..\)/'$1'\1/g'
399 }
400
401 -# Create a one-time-sed command to replace the existing packfile with $1.
401 +# Create a one-time-perl command to replace the existing packfile with $1.
402 replace_packfile () {
403 # The protocol requires that the packfile be sent in sideband 1, hence
404 # the extra \x01 byte at the beginning.
405 - printf "1,/packfile/!c %04x\\\\x01%s0000" \
406 - "$(($(wc -c <$1) + 5))" \
407 - "$(hex_unpack <$1 | intersperse '\\x')" \
408 - >"$HTTPD_ROOT_PATH/one-time-sed"
405 + cp $1 "$HTTPD_ROOT_PATH/one-time-pack" &&
406 + echo 'if (/packfile/) {
407 + print;
408 + my $length = -s "one-time-pack";
409 + printf "%04x\x01", $length + 5;
410 + print `cat one-time-pack` . "0000";
411 + last
412 + }' >"$HTTPD_ROOT_PATH/one-time-perl"
413 }
414
415 test_expect_success 'upon cloning, check that all refs point to objects' '
@@ -429,16 +433,16 @@ test_expect_success 'upon cloning, check that all refs point to objects' '
433 # \x01 byte at the beginning.
434 replace_packfile incomplete.pack &&
435
432 - # Use protocol v2 because the sed command looks for the "packfile"
436 + # Use protocol v2 because the perl command looks for the "packfile"
437 # section header.
438 test_config -C "$SERVER" protocol.version 2 &&
439 test_must_fail git -c protocol.version=2 clone \
436 - --filter=blob:none $HTTPD_URL/one_time_sed/server repo 2>err &&
440 + --filter=blob:none $HTTPD_URL/one_time_perl/server repo 2>err &&
441
442 test_i18ngrep "did not send all necessary objects" err &&
443
440 - # Ensure that the one-time-sed script was used.
441 - ! test -e "$HTTPD_ROOT_PATH/one-time-sed"
444 + # Ensure that the one-time-perl script was used.
445 + ! test -e "$HTTPD_ROOT_PATH/one-time-perl"
446 '
447
448 test_expect_success 'when partial cloning, tolerate server not sending target of tag' '
@@ -469,17 +473,17 @@ test_expect_success 'when partial cloning, tolerate server not sending target of
473 # \x01 byte at the beginning.
474 replace_packfile incomplete.pack &&
475
472 - # Use protocol v2 because the sed command looks for the "packfile"
476 + # Use protocol v2 because the perl command looks for the "packfile"
477 # section header.
478 test_config -C "$SERVER" protocol.version 2 &&
479
480 # Exercise to make sure it works.
481 git -c protocol.version=2 clone \
478 - --filter=blob:none $HTTPD_URL/one_time_sed/server repo 2> err &&
482 + --filter=blob:none $HTTPD_URL/one_time_perl/server repo 2> err &&
483 ! grep "missing object referenced by" err &&
484
481 - # Ensure that the one-time-sed script was used.
482 - ! test -e "$HTTPD_ROOT_PATH/one-time-sed"
485 + # Ensure that the one-time-perl script was used.
486 + ! test -e "$HTTPD_ROOT_PATH/one-time-perl"
487 '
488
489 test_expect_success 'tolerate server sending REF_DELTA against missing promisor objects' '
@@ -502,7 +506,7 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor
506
507 # Clone. The client has deltabase_have but not deltabase_missing.
508 git -c protocol.version=2 clone --no-checkout \
505 - --filter=blob:none $HTTPD_URL/one_time_sed/server repo &&
509 + --filter=blob:none $HTTPD_URL/one_time_perl/server repo &&
510 git -C repo hash-object -w -- "$SERVER/have.txt" &&
511
512 # Sanity check to ensure that the client does not have
@@ -543,7 +547,7 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor
547
548 replace_packfile thin.pack &&
549
546 - # Use protocol v2 because the sed command looks for the "packfile"
550 + # Use protocol v2 because the perl command looks for the "packfile"
551 # section header.
552 test_config -C "$SERVER" protocol.version 2 &&
553
@@ -556,8 +560,8 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor
560 grep "want $(cat deltabase_missing)" trace &&
561 ! grep "want $(cat deltabase_have)" trace &&
562
559 - # Ensure that the one-time-sed script was used.
560 - ! test -e "$HTTPD_ROOT_PATH/one-time-sed"
563 + # Ensure that the one-time-perl script was used.
564 + ! test -e "$HTTPD_ROOT_PATH/one-time-perl"
565 '
566
567 # DO NOT add non-httpd-specific tests here, because the last part of this
t/t5702-protocol-v2.sh
+6 -6
@@ -712,11 +712,11 @@ test_expect_success 'when server sends "ready", expect DELIM' '
712
713 # After "ready" in the acknowledgments section, pretend that a FLUSH
714 # (0000) was sent instead of a DELIM (0001).
715 - printf "/ready/,$ s/0001/0000/" \
716 - >"$HTTPD_ROOT_PATH/one-time-sed" &&
715 + printf "\$ready = 1 if /ready/; \$ready && s/0001/0000/" \
716 + >"$HTTPD_ROOT_PATH/one-time-perl" &&
717
718 test_must_fail git -C http_child -c protocol.version=2 \
719 - fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err &&
719 + fetch "$HTTPD_URL/one_time_perl/http_parent" 2> err &&
720 test_i18ngrep "expected packfile to be sent after .ready." err
721 '
722
@@ -737,12 +737,12 @@ test_expect_success 'when server does not send "ready", expect FLUSH' '
737
738 # After the acknowledgments section, pretend that a DELIM
739 # (0001) was sent instead of a FLUSH (0000).
740 - printf "/acknowledgments/,$ s/0000/0001/" \
741 - >"$HTTPD_ROOT_PATH/one-time-sed" &&
740 + printf "\$ack = 1 if /acknowledgments/; \$ack && s/0000/0001/" \
741 + >"$HTTPD_ROOT_PATH/one-time-perl" &&
742
743 test_must_fail env GIT_TRACE_PACKET="$(pwd)/log" git -C http_child \
744 -c protocol.version=2 \
745 - fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err &&
745 + fetch "$HTTPD_URL/one_time_perl/http_parent" 2> err &&
746 grep "fetch< .*acknowledgments" log &&
747 ! grep "fetch< .*ready" log &&
748 test_i18ngrep "expected no other sections to be sent after no .ready." err
t/t5703-upload-pack-ref-in-want.sh
+3 -3
@@ -313,7 +313,7 @@ test_expect_success 'setup repos for change-while-negotiating test' '
313 test_commit m3 &&
314 git tag -d m2 m3
315 ) &&
316 - git -C "$LOCAL_PRISTINE" remote set-url origin "http://127.0.0.1:$LIB_HTTPD_PORT/one_time_sed/repo" &&
316 + git -C "$LOCAL_PRISTINE" remote set-url origin "http://127.0.0.1:$LIB_HTTPD_PORT/one_time_perl/repo" &&
317 git -C "$LOCAL_PRISTINE" config protocol.version 2
318 '
319
@@ -326,7 +326,7 @@ inconsistency () {
326 # RPCs during a single negotiation.
327 oid1=$(git -C "$REPO" rev-parse $1) &&
328 oid2=$(git -C "$REPO" rev-parse $2) &&
329 - echo "s/$oid1/$oid2/" >"$HTTPD_ROOT_PATH/one-time-sed"
329 + echo "s/$oid1/$oid2/" >"$HTTPD_ROOT_PATH/one-time-perl"
330 }
331
332 test_expect_success 'server is initially ahead - no ref in want' '
@@ -378,7 +378,7 @@ test_expect_success 'server loses a ref - ref in want' '
378 git -C "$REPO" config uploadpack.allowRefInWant true &&
379 rm -rf local &&
380 cp -r "$LOCAL_PRISTINE" local &&
381 - echo "s/master/raster/" >"$HTTPD_ROOT_PATH/one-time-sed" &&
381 + echo "s/master/raster/" >"$HTTPD_ROOT_PATH/one-time-perl" &&
382 test_must_fail git -C local fetch 2>err &&
383
384 test_i18ngrep "fatal: remote error: unknown ref refs/heads/raster" err