git-cvsserver: add ability to guess -kb from contents
If "gitcvs.allbinary" is set to "guess", then any file that has not been explicitly marked as binary or text using the "crlf" attribute and the "gitcvs.usecrlfattr" config will guess binary based on the contents of the file. Signed-off-by: Matthew Ogilvie <mmogilvi_git@miniinfo.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Matthew Ogilvie committed
May 14, 2008 at 22:35 UTC
90948a42892779734f77d62f20326c868392fd8f
4 files changed
+354
-25
Documentation/config.txt
+8
-5
@@ -670,11 +670,14 @@ gitcvs.usecrlfattr
670
then 'gitcvs.allbinary' is used. See linkgit:gitattribute[5].
671
672
gitcvs.allbinary::
673
- If true, all files not otherwise specified using
674
- 'gitcvs.usecrlfattr' and an explicitly set or unset `crlf`
675
- attribute are sent to the client in mode '-kb'. This
676
- causes the client to treat them as binary files which
677
- suppresses any newline munging it otherwise might do.
673
+ This is used if 'gitcvs.usecrlfattr' does not resolve
674
+ the correct '-kb' mode to use. If true, all
675
+ unresolved files are sent to the client in
676
+ mode '-kb'. This causes the client to treat them
677
+ as binary files, which suppresses any newline munging it
678
+ otherwise might do. Alternatively, if it is set to "guess",
679
+ then the contents of the file are examined to decide if
680
+ it is binary, similar to 'core.autocrlf'.
681
682
gitcvs.dbname::
683
Database used by git-cvsserver to cache revision information
Documentation/git-cvsserver.txt
+10
-4
@@ -311,17 +311,23 @@ to crlf conversion on some platforms.
311
You can make the server use `crlf` attributes to set the '-k' modes
312
for files by setting the `gitcvs.usecrlfattr` config variable.
313
In this case, if `crlf` is explicitly unset ('-crlf'), then the
314
-will set '-kb' mode, for binary files. If it `crlf` is set,
314
+server will set '-kb' mode for binary files. If `crlf` is set,
315
then the '-k' mode will explicitly be left blank. See
316
also linkgit:gitattributes[5] for more information about the `crlf`
317
attribute.
318
319
Alternatively, if `gitcvs.usecrlfattr` config is not enabled
320
or if the `crlf` attribute is unspecified for a filename, then
321
-the server uses the `gitcvs.allbinary` for the default setting.
322
-If `gitcvs.allbinary` is set, then the files not otherwise
321
+the server uses the `gitcvs.allbinary` config for the default setting.
322
+If `gitcvs.allbinary` is set, then file not otherwise
323
specified will default to '-kb' mode. Otherwise the '-k' mode
324
-is left blank.
324
+is left blank. But if `gitcvs.allbinary` is set to "guess", then
325
+the correct '-k' mode will be guessed based on the contents of
326
+the file.
327
+
328
+For best consistency with cvs, it is probably best to override the
329
+defaults by setting `gitcvs.usecrlfattr` to true,
330
+and `gitcvs.allbinary` to "guess".
331
332
Dependencies
333
------------
git-cvsserver.perl
+177
-16
@@ -502,7 +502,7 @@ sub req_add
502
print $state->{CVSROOT} . "/$state->{module}/$filename\n";
503
504
# this is an "entries" line
505
- my $kopts = kopts_from_path($filename);
505
+ my $kopts = kopts_from_path($filename,"sha1",$meta->{filehash});
506
$log->debug("/$filepart/1.$meta->{revision}//$kopts/");
507
print "/$filepart/1.$meta->{revision}//$kopts/\n";
508
# permissions
@@ -533,7 +533,8 @@ sub req_add
533
534
print "Checked-in $dirpart\n";
535
print "$filename\n";
536
- my $kopts = kopts_from_path($filename);
536
+ my $kopts = kopts_from_path($filename,"file",
537
+ $state->{entries}{$filename}{modified_filename});
538
print "/$filepart/0//$kopts/\n";
539
540
my $requestedKopts = $state->{opt}{k};
@@ -631,7 +632,7 @@ sub req_remove
632
633
print "Checked-in $dirpart\n";
634
print "$filename\n";
634
- my $kopts = kopts_from_path($filename);
635
+ my $kopts = kopts_from_path($filename,"sha1",$meta->{filehash});
636
print "/$filepart/-1.$wrev//$kopts/\n";
637
638
$rmcount++;
@@ -910,7 +911,7 @@ sub req_co
911
print $state->{CVSROOT} . "/$module/" . ( defined ( $git->{dir} ) and $git->{dir} ne "./" ? $git->{dir} . "/" : "" ) . "$git->{name}\n";
912
913
# this is an "entries" line
913
- my $kopts = kopts_from_path($fullName);
914
+ my $kopts = kopts_from_path($fullName,"sha1",$git->{filehash});
915
print "/$git->{name}/1.$git->{revision}//$kopts/\n";
916
# permissions
917
print "u=$git->{mode},g=$git->{mode},o=$git->{mode}\n";
@@ -1119,7 +1120,7 @@ sub req_update
1120
print $state->{CVSROOT} . "/$state->{module}/$filename\n";
1121
1122
# this is an "entries" line
1122
- my $kopts = kopts_from_path($filename);
1123
+ my $kopts = kopts_from_path($filename,"sha1",$meta->{filehash});
1124
$log->debug("/$filepart/1.$meta->{revision}//$kopts/");
1125
print "/$filepart/1.$meta->{revision}//$kopts/\n";
1126
@@ -1167,7 +1168,8 @@ sub req_update
1168
print "Merged $dirpart\n";
1169
$log->debug($state->{CVSROOT} . "/$state->{module}/$filename");
1170
print $state->{CVSROOT} . "/$state->{module}/$filename\n";
1170
- my $kopts = kopts_from_path("$dirpart/$filepart");
1171
+ my $kopts = kopts_from_path("$dirpart/$filepart",
1172
+ "file",$mergedFile);
1173
$log->debug("/$filepart/1.$meta->{revision}//$kopts/");
1174
print "/$filepart/1.$meta->{revision}//$kopts/\n";
1175
}
@@ -1183,7 +1185,8 @@ sub req_update
1185
{
1186
print "Merged $dirpart\n";
1187
print $state->{CVSROOT} . "/$state->{module}/$filename\n";
1186
- my $kopts = kopts_from_path("$dirpart/$filepart");
1188
+ my $kopts = kopts_from_path("$dirpart/$filepart",
1189
+ "file",$mergedFile);
1190
print "/$filepart/1.$meta->{revision}/+/$kopts/\n";
1191
}
1192
}
@@ -1434,7 +1437,7 @@ sub req_ci
1437
}
1438
print "Checked-in $dirpart\n";
1439
print "$filename\n";
1437
- my $kopts = kopts_from_path($filename);
1440
+ my $kopts = kopts_from_path($filename,"sha1",$meta->{filehash});
1441
print "/$filepart/1.$meta->{revision}//$kopts/\n";
1442
}
1443
}
@@ -2312,7 +2315,7 @@ sub cleanupTmpDir
2315
# file should get -kb.
2316
sub kopts_from_path
2317
{
2315
- my ($path) = @_;
2318
+ my ($path, $srcType, $name) = @_;
2319
2320
if ( defined ( $cfg->{gitcvs}{usecrlfattr} ) and
2321
$cfg->{gitcvs}{usecrlfattr} =~ /\s*(1|true|yes)\s*$/i )
@@ -2332,15 +2335,55 @@ sub kopts_from_path
2335
}
2336
}
2337
2335
- unless ( defined ( $cfg->{gitcvs}{allbinary} ) and $cfg->{gitcvs}{allbinary} =~ /^\s*(1|true|yes)\s*$/i )
2338
+ if ( defined ( $cfg->{gitcvs}{allbinary} ) )
2339
{
2337
- # Return "" to give no special treatment to any path
2338
- return "";
2339
- } else {
2340
- # Alternatively, to have all files treated as if they are binary (which
2341
- # is more like git itself), always return the "-kb" option
2342
- return "-kb";
2340
+ if( ($cfg->{gitcvs}{allbinary} =~ /^\s*(1|true|yes)\s*$/i) )
2341
+ {
2342
+ return "-kb";
2343
+ }
2344
+ elsif( ($cfg->{gitcvs}{allbinary} =~ /^\s*guess\s*$/i) )
2345
+ {
2346
+ if( $srcType eq "sha1Or-k" &&
2347
+ !defined($name) )
2348
+ {
2349
+ my ($ret)=$state->{entries}{$path}{options};
2350
+ if( !defined($ret) )
2351
+ {
2352
+ $ret=$state->{opt}{k};
2353
+ if(defined($ret))
2354
+ {
2355
+ $ret="-k$ret";
2356
+ }
2357
+ else
2358
+ {
2359
+ $ret="";
2360
+ }
2361
+ }
2362
+ if( ! ($ret=~/^(|-kb|-kkv|-kkvl|-kk|-ko|-kv)$/) )
2363
+ {
2364
+ print "E Bad -k option\n";
2365
+ $log->warn("Bad -k option: $ret");
2366
+ die "Error: Bad -k option: $ret\n";
2367
+ }
2368
+
2369
+ return $ret;
2370
+ }
2371
+ else
2372
+ {
2373
+ if( is_binary($srcType,$name) )
2374
+ {
2375
+ $log->debug("... as binary");
2376
+ return "-kb";
2377
+ }
2378
+ else
2379
+ {
2380
+ $log->debug("... as text");
2381
+ }
2382
+ }
2383
+ }
2384
}
2385
+ # Return "" to give no special treatment to any path
2386
+ return "";
2387
}
2388
2389
sub check_attr
@@ -2360,6 +2403,124 @@ sub check_attr
2403
}
2404
}
2405
2406
+# This should have the same heuristics as convert.c:is_binary() and related.
2407
+# Note that the bare CR test is done by callers in convert.c.
2408
+sub is_binary
2409
+{
2410
+ my ($srcType,$name) = @_;
2411
+ $log->debug("is_binary($srcType,$name)");
2412
+
2413
+ # Minimize amount of interpreted code run in the inner per-character
2414
+ # loop for large files, by totalling each character value and
2415
+ # then analyzing the totals.
2416
+ my @counts;
2417
+ my $i;
2418
+ for($i=0;$i<256;$i++)
2419
+ {
2420
+ $counts[$i]=0;
2421
+ }
2422
+
2423
+ my $fh = open_blob_or_die($srcType,$name);
2424
+ my $line;
2425
+ while( defined($line=<$fh>) )
2426
+ {
2427
+ # Any '\0' and bare CR are considered binary.
2428
+ if( $line =~ /\0|(\r[^\n])/ )
2429
+ {
2430
+ close($fh);
2431
+ return 1;
2432
+ }
2433
+
2434
+ # Count up each character in the line:
2435
+ my $len=length($line);
2436
+ for($i=0;$i<$len;$i++)
2437
+ {
2438
+ $counts[ord(substr($line,$i,1))]++;
2439
+ }
2440
+ }
2441
+ close $fh;
2442
+
2443
+ # Don't count CR and LF as either printable/nonprintable
2444
+ $counts[ord("\n")]=0;
2445
+ $counts[ord("\r")]=0;
2446
+
2447
+ # Categorize individual character count into printable and nonprintable:
2448
+ my $printable=0;
2449
+ my $nonprintable=0;
2450
+ for($i=0;$i<256;$i++)
2451
+ {
2452
+ if( $i < 32 &&
2453
+ $i != ord("\b") &&
2454
+ $i != ord("\t") &&
2455
+ $i != 033 && # ESC
2456
+ $i != 014 ) # FF
2457
+ {
2458
+ $nonprintable+=$counts[$i];
2459
+ }
2460
+ elsif( $i==127 ) # DEL
2461
+ {
2462
+ $nonprintable+=$counts[$i];
2463
+ }
2464
+ else
2465
+ {
2466
+ $printable+=$counts[$i];
2467
+ }
2468
+ }
2469
+
2470
+ return ($printable >> 7) < $nonprintable;
2471
+}
2472
+
2473
+# Returns open file handle. Possible invocations:
2474
+# - open_blob_or_die("file",$filename);
2475
+# - open_blob_or_die("sha1",$filehash);
2476
+sub open_blob_or_die
2477
+{
2478
+ my ($srcType,$name) = @_;
2479
+ my ($fh);
2480
+ if( $srcType eq "file" )
2481
+ {
2482
+ if( !open $fh,"<",$name )
2483
+ {
2484
+ $log->warn("Unable to open file $name: $!");
2485
+ die "Unable to open file $name: $!\n";
2486
+ }
2487
+ }
2488
+ elsif( $srcType eq "sha1" || $srcType eq "sha1Or-k" )
2489
+ {
2490
+ unless ( defined ( $name ) and $name =~ /^[a-zA-Z0-9]{40}$/ )
2491
+ {
2492
+ $log->warn("Need filehash");
2493
+ die "Need filehash\n";
2494
+ }
2495
+
2496
+ my $type = `git cat-file -t $name`;
2497
+ chomp $type;
2498
+
2499
+ unless ( defined ( $type ) and $type eq "blob" )
2500
+ {
2501
+ $log->warn("Invalid type '$type' for '$name'");
2502
+ die ( "Invalid type '$type' (expected 'blob')" )
2503
+ }
2504
+
2505
+ my $size = `git cat-file -s $name`;
2506
+ chomp $size;
2507
+
2508
+ $log->debug("open_blob_or_die($name) size=$size, type=$type");
2509
+
2510
+ unless( open $fh, '-|', "git", "cat-file", "blob", $name )
2511
+ {
2512
+ $log->warn("Unable to open sha1 $name");
2513
+ die "Unable to open sha1 $name\n";
2514
+ }
2515
+ }
2516
+ else
2517
+ {
2518
+ $log->warn("Unknown type of blob source: $srcType");
2519
+ die "Unknown type of blob source: $srcType\n";
2520
+ }
2521
+ return $fh;
2522
+}
2523
+
2524
# Generate a CVS author name from Git author information, by taking
2525
# the first eight characters of the user part of the email address.
2526
sub cvs_author
t/t9401-git-cvsserver-crlf.sh
+159
@@ -175,4 +175,163 @@ test_expect_success 'updating' '
175
cmp cvswork/binfile.bin tmpExpect1
176
'
177
178
+rm -rf cvswork
179
+test_expect_success 'cvs co (use attributes/guess)' '
180
+ GIT_DIR="$SERVERDIR" git config gitcvs.allbinary guess &&
181
+ GIT_CONFIG="$git_config" cvs -Q co -d cvswork master >cvs.log 2>&1 &&
182
+ marked_as cvswork textfile.c "" &&
183
+ marked_as cvswork binfile.bin -kb &&
184
+ marked_as cvswork .gitattributes "" &&
185
+ marked_as cvswork mixedUp.c "" &&
186
+ marked_as cvswork/subdir withCr.bin -kb &&
187
+ marked_as cvswork/subdir file.h "" &&
188
+ marked_as cvswork/subdir unspecified.other "" &&
189
+ marked_as cvswork/subdir newfile.bin -kb &&
190
+ marked_as cvswork/subdir newfile.c ""
191
+'
192
+
193
+test_expect_success 'setup multi-line files' '
194
+ ( echo "line 1" &&
195
+ echo "line 2" &&
196
+ echo "line 3" &&
197
+ echo "line 4 with NUL: Q <-" ) | q_to_nul > multiline.c &&
198
+ git add multiline.c &&
199
+ ( echo "line 1" &&
200
+ echo "line 2" &&
201
+ echo "line 3" &&
202
+ echo "line 4" ) | q_to_nul > multilineTxt.c &&
203
+ git add multilineTxt.c &&
204
+ git commit -q -m "multiline files" &&
205
+ git push gitcvs.git >/dev/null
206
+'
207
+
208
+rm -rf cvswork
209
+test_expect_success 'cvs co (guess)' '
210
+ GIT_DIR="$SERVERDIR" git config --bool gitcvs.usecrlfattr false &&
211
+ GIT_CONFIG="$git_config" cvs -Q co -d cvswork master >cvs.log 2>&1 &&
212
+ marked_as cvswork textfile.c "" &&
213
+ marked_as cvswork binfile.bin -kb &&
214
+ marked_as cvswork .gitattributes "" &&
215
+ marked_as cvswork mixedUp.c -kb &&
216
+ marked_as cvswork multiline.c -kb &&
217
+ marked_as cvswork multilineTxt.c "" &&
218
+ marked_as cvswork/subdir withCr.bin -kb &&
219
+ marked_as cvswork/subdir file.h "" &&
220
+ marked_as cvswork/subdir unspecified.other "" &&
221
+ marked_as cvswork/subdir newfile.bin "" &&
222
+ marked_as cvswork/subdir newfile.c ""
223
+'
224
+
225
+test_expect_success 'cvs co another copy (guess)' '
226
+ GIT_CONFIG="$git_config" cvs -Q co -d cvswork2 master >cvs.log 2>&1 &&
227
+ marked_as cvswork2 textfile.c "" &&
228
+ marked_as cvswork2 binfile.bin -kb &&
229
+ marked_as cvswork2 .gitattributes "" &&
230
+ marked_as cvswork2 mixedUp.c -kb &&
231
+ marked_as cvswork2 multiline.c -kb &&
232
+ marked_as cvswork2 multilineTxt.c "" &&
233
+ marked_as cvswork2/subdir withCr.bin -kb &&
234
+ marked_as cvswork2/subdir file.h "" &&
235
+ marked_as cvswork2/subdir unspecified.other "" &&
236
+ marked_as cvswork2/subdir newfile.bin "" &&
237
+ marked_as cvswork2/subdir newfile.c ""
238
+'
239
+
240
+test_expect_success 'add text (guess)' '
241
+ cd cvswork &&
242
+ echo "simpleText" > simpleText.c &&
243
+ GIT_CONFIG="$git_config" cvs -Q add simpleText.c &&
244
+ cd .. &&
245
+ marked_as cvswork simpleText.c ""
246
+'
247
+
248
+test_expect_success 'add bin (guess)' '
249
+ cd cvswork &&
250
+ echo "simpleBin: NUL: Q <- there" | q_to_nul > simpleBin.bin &&
251
+ GIT_CONFIG="$git_config" cvs -Q add simpleBin.bin &&
252
+ cd .. &&
253
+ marked_as cvswork simpleBin.bin -kb
254
+'
255
+
256
+test_expect_success 'remove files (guess)' '
257
+ cd cvswork &&
258
+ GIT_CONFIG="$git_config" cvs -Q rm -f subdir/file.h &&
259
+ cd subdir &&
260
+ GIT_CONFIG="$git_config" cvs -Q rm -f withCr.bin &&
261
+ cd ../.. &&
262
+ marked_as cvswork/subdir withCr.bin -kb &&
263
+ marked_as cvswork/subdir file.h ""
264
+'
265
+
266
+test_expect_success 'cvs ci (guess)' '
267
+ cd cvswork &&
268
+ GIT_CONFIG="$git_config" cvs -Q ci -m "add/rm files" >cvs.log 2>&1 &&
269
+ cd .. &&
270
+ marked_as cvswork textfile.c "" &&
271
+ marked_as cvswork binfile.bin -kb &&
272
+ marked_as cvswork .gitattributes "" &&
273
+ marked_as cvswork mixedUp.c -kb &&
274
+ marked_as cvswork multiline.c -kb &&
275
+ marked_as cvswork multilineTxt.c "" &&
276
+ not_present cvswork/subdir withCr.bin &&
277
+ not_present cvswork/subdir file.h &&
278
+ marked_as cvswork/subdir unspecified.other "" &&
279
+ marked_as cvswork/subdir newfile.bin "" &&
280
+ marked_as cvswork/subdir newfile.c "" &&
281
+ marked_as cvswork simpleBin.bin -kb &&
282
+ marked_as cvswork simpleText.c ""
283
+'
284
+
285
+test_expect_success 'update subdir of other copy (guess)' '
286
+ cd cvswork2/subdir &&
287
+ GIT_CONFIG="$git_config" cvs -Q update &&
288
+ cd ../.. &&
289
+ marked_as cvswork2 textfile.c "" &&
290
+ marked_as cvswork2 binfile.bin -kb &&
291
+ marked_as cvswork2 .gitattributes "" &&
292
+ marked_as cvswork2 mixedUp.c -kb &&
293
+ marked_as cvswork2 multiline.c -kb &&
294
+ marked_as cvswork2 multilineTxt.c "" &&
295
+ not_present cvswork2/subdir withCr.bin &&
296
+ not_present cvswork2/subdir file.h &&
297
+ marked_as cvswork2/subdir unspecified.other "" &&
298
+ marked_as cvswork2/subdir newfile.bin "" &&
299
+ marked_as cvswork2/subdir newfile.c "" &&
300
+ not_present cvswork2 simpleBin.bin &&
301
+ not_present cvswork2 simpleText.c
302
+'
303
+
304
+echo "starting update/merge" >> "${WORKDIR}/marked.log"
305
+test_expect_success 'update/merge full other copy (guess)' '
306
+ git pull gitcvs.git master &&
307
+ sed "s/3/replaced_3/" < multilineTxt.c > ml.temp &&
308
+ mv ml.temp multilineTxt.c &&
309
+ git add multilineTxt.c &&
310
+ git commit -q -m "modify multiline file" >> "${WORKDIR}/marked.log" &&
311
+ git push gitcvs.git >/dev/null &&
312
+ cd cvswork2 &&
313
+ sed "s/1/replaced_1/" < multilineTxt.c > ml.temp &&
314
+ mv ml.temp multilineTxt.c &&
315
+ GIT_CONFIG="$git_config" cvs update > cvs.log 2>&1 &&
316
+ cd .. &&
317
+ marked_as cvswork2 textfile.c "" &&
318
+ marked_as cvswork2 binfile.bin -kb &&
319
+ marked_as cvswork2 .gitattributes "" &&
320
+ marked_as cvswork2 mixedUp.c -kb &&
321
+ marked_as cvswork2 multiline.c -kb &&
322
+ marked_as cvswork2 multilineTxt.c "" &&
323
+ not_present cvswork2/subdir withCr.bin &&
324
+ not_present cvswork2/subdir file.h &&
325
+ marked_as cvswork2/subdir unspecified.other "" &&
326
+ marked_as cvswork2/subdir newfile.bin "" &&
327
+ marked_as cvswork2/subdir newfile.c "" &&
328
+ marked_as cvswork2 simpleBin.bin -kb &&
329
+ marked_as cvswork2 simpleText.c "" &&
330
+ echo "line replaced_1" > tmpExpect2 &&
331
+ echo "line 2" >> tmpExpect2 &&
332
+ echo "line replaced_3" >> tmpExpect2 &&
333
+ echo "line 4" | q_to_nul >> tmpExpect2 &&
334
+ cmp cvswork2/multilineTxt.c tmpExpect2
335
+'
336
+
337
test_done