Raw
1 #!/bin/sh
2
3 test_description='git pack-objects using object filtering'
4
5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
7
8 . ./test-lib.sh
9
10 # Test blob:none filter.
11
12 test_expect_success 'setup r1' '
13 git init r1 &&
14 for n in 1 2 3 4 5
15 do
16 echo "This is file: $n" > r1/file.$n &&
17 git -C r1 add file.$n &&
18 git -C r1 commit -m "$n" || return 1
19 done
20 '
21
22 parse_verify_pack_blob_oid () {
23 awk '{print $1}' -
24 }
25
26 test_expect_success 'verify blob count in normal packfile' '
27 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
28 >ls_files_result &&
29 test_parse_ls_files_stage_oids <ls_files_result |
30 sort >expected &&
31
32 git -C r1 pack-objects --revs --stdout >all.pack <<-EOF &&
33 HEAD
34 EOF
35 git -C r1 index-pack ../all.pack &&
36
37 git -C r1 verify-pack -v ../all.pack >verify_result &&
38 grep blob verify_result |
39 parse_verify_pack_blob_oid |
40 sort >observed &&
41
42 test_cmp expected observed
43 '
44
45 test_expect_success 'verify blob:none packfile has no blobs' '
46 git -C r1 pack-objects --revs --stdout --filter=blob:none >filter.pack <<-EOF &&
47 HEAD
48 EOF
49 git -C r1 index-pack ../filter.pack &&
50
51 git -C r1 verify-pack -v ../filter.pack >verify_result &&
52 ! grep blob verify_result
53 '
54
55 test_expect_success 'verify blob:none packfile without --stdout' '
56 git -C r1 pack-objects --revs --filter=blob:none mypackname >packhash <<-EOF &&
57 HEAD
58 EOF
59 git -C r1 verify-pack -v "mypackname-$(cat packhash).pack" >verify_result &&
60 ! grep blob verify_result
61 '
62
63 test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
64 git -C r1 verify-pack -v ../all.pack >verify_result &&
65 grep -E "commit|tree" verify_result |
66 parse_verify_pack_blob_oid |
67 sort >expected &&
68
69 git -C r1 verify-pack -v ../filter.pack >verify_result &&
70 grep -E "commit|tree" verify_result |
71 parse_verify_pack_blob_oid |
72 sort >observed &&
73
74 test_cmp expected observed
75 '
76
77 test_expect_success 'get an error for missing tree object' '
78 git init r5 &&
79 echo foo >r5/foo &&
80 git -C r5 add foo &&
81 git -C r5 commit -m "foo" &&
82 git -C r5 rev-parse HEAD^{tree} >tree &&
83 del=$(sed "s|..|&/|" tree) &&
84 rm r5/.git/objects/$del &&
85 test_must_fail git -C r5 pack-objects --revs --stdout 2>bad_tree <<-EOF &&
86 HEAD
87 EOF
88 grep "bad tree object" bad_tree
89 '
90
91 test_expect_success 'setup for tests of tree:0' '
92 mkdir r1/subtree &&
93 echo "This is a file in a subtree" >r1/subtree/file &&
94 git -C r1 add subtree/file &&
95 git -C r1 commit -m subtree
96 '
97
98 test_expect_success 'verify tree:0 packfile has no blobs or trees' '
99 git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
100 HEAD
101 EOF
102 git -C r1 index-pack ../commitsonly.pack &&
103 git -C r1 verify-pack -v ../commitsonly.pack >objs &&
104 ! grep -E "tree|blob" objs
105 '
106
107 test_expect_success 'grab tree directly when using tree:0' '
108 # We should get the tree specified directly but not its blobs or subtrees.
109 git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
110 HEAD:
111 EOF
112 git -C r1 index-pack ../commitsonly.pack &&
113 git -C r1 verify-pack -v ../commitsonly.pack >objs &&
114 awk "/tree|blob/{print \$1}" objs >trees_and_blobs &&
115 git -C r1 rev-parse HEAD: >expected &&
116 test_cmp expected trees_and_blobs
117 '
118
119 # Test blob:limit=<n>[kmg] filter.
120 # We boundary test around the size parameter. The filter is strictly less than
121 # the value, so size 500 and 1000 should have the same results, but 1001 should
122 # filter more.
123
124 test_expect_success 'setup r2' '
125 git init r2 &&
126 for n in 1000 10000
127 do
128 printf "%"$n"s" X > r2/large.$n &&
129 git -C r2 add large.$n &&
130 git -C r2 commit -m "$n" || return 1
131 done
132 '
133
134 test_expect_success 'verify blob count in normal packfile' '
135 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
136 test_parse_ls_files_stage_oids <ls_files_result |
137 sort >expected &&
138
139 git -C r2 pack-objects --revs --stdout >all.pack <<-EOF &&
140 HEAD
141 EOF
142 git -C r2 index-pack ../all.pack &&
143
144 git -C r2 verify-pack -v ../all.pack >verify_result &&
145 grep blob verify_result |
146 parse_verify_pack_blob_oid |
147 sort >observed &&
148
149 test_cmp expected observed
150 '
151
152 test_expect_success 'verify blob:limit=500 omits all blobs' '
153 git -C r2 pack-objects --revs --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
154 HEAD
155 EOF
156 git -C r2 index-pack ../filter.pack &&
157
158 git -C r2 verify-pack -v ../filter.pack >verify_result &&
159 ! grep blob verify_result
160 '
161
162 test_expect_success 'verify blob:limit=1000' '
163 git -C r2 pack-objects --revs --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
164 HEAD
165 EOF
166 git -C r2 index-pack ../filter.pack &&
167
168 git -C r2 verify-pack -v ../filter.pack >verify_result &&
169 ! grep blob verify_result
170 '
171
172 test_expect_success 'verify blob:limit=1001' '
173 git -C r2 ls-files -s large.1000 >ls_files_result &&
174 test_parse_ls_files_stage_oids <ls_files_result |
175 sort >expected &&
176
177 git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
178 HEAD
179 EOF
180 git -C r2 index-pack ../filter.pack &&
181
182 git -C r2 verify-pack -v ../filter.pack >verify_result &&
183 grep blob verify_result |
184 parse_verify_pack_blob_oid |
185 sort >observed &&
186
187 test_cmp expected observed
188 '
189
190 test_expect_success 'verify blob:limit=10001' '
191 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
192 test_parse_ls_files_stage_oids <ls_files_result |
193 sort >expected &&
194
195 git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
196 HEAD
197 EOF
198 git -C r2 index-pack ../filter.pack &&
199
200 git -C r2 verify-pack -v ../filter.pack >verify_result &&
201 grep blob verify_result |
202 parse_verify_pack_blob_oid |
203 sort >observed &&
204
205 test_cmp expected observed
206 '
207
208 test_expect_success 'verify blob:limit=1k' '
209 git -C r2 ls-files -s large.1000 >ls_files_result &&
210 test_parse_ls_files_stage_oids <ls_files_result |
211 sort >expected &&
212
213 git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
214 HEAD
215 EOF
216 git -C r2 index-pack ../filter.pack &&
217
218 git -C r2 verify-pack -v ../filter.pack >verify_result &&
219 grep blob verify_result |
220 parse_verify_pack_blob_oid |
221 sort >observed &&
222
223 test_cmp expected observed
224 '
225
226 test_expect_success 'verify explicitly specifying oversized blob in input' '
227 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
228 test_parse_ls_files_stage_oids <ls_files_result |
229 sort >expected &&
230
231 echo HEAD >objects &&
232 git -C r2 rev-parse HEAD:large.10000 >>objects &&
233 git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k <objects >filter.pack &&
234 git -C r2 index-pack ../filter.pack &&
235
236 git -C r2 verify-pack -v ../filter.pack >verify_result &&
237 grep blob verify_result |
238 parse_verify_pack_blob_oid |
239 sort >observed &&
240
241 test_cmp expected observed
242 '
243
244 test_expect_success 'verify blob:limit=1m' '
245 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
246 test_parse_ls_files_stage_oids <ls_files_result |
247 sort >expected &&
248
249 git -C r2 pack-objects --revs --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
250 HEAD
251 EOF
252 git -C r2 index-pack ../filter.pack &&
253
254 git -C r2 verify-pack -v ../filter.pack >verify_result &&
255 grep blob verify_result |
256 parse_verify_pack_blob_oid |
257 sort >observed &&
258
259 test_cmp expected observed
260 '
261
262 test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
263 git -C r2 verify-pack -v ../all.pack >verify_result &&
264 grep -E "commit|tree" verify_result |
265 parse_verify_pack_blob_oid |
266 sort >expected &&
267
268 git -C r2 verify-pack -v ../filter.pack >verify_result &&
269 grep -E "commit|tree" verify_result |
270 parse_verify_pack_blob_oid |
271 sort >observed &&
272
273 test_cmp expected observed
274 '
275
276 test_expect_success 'verify small limit and big limit results in small limit' '
277 git -C r2 ls-files -s large.1000 >ls_files_result &&
278 test_parse_ls_files_stage_oids <ls_files_result |
279 sort >expected &&
280
281 git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 \
282 --filter=blob:limit=10001 >filter.pack <<-EOF &&
283 HEAD
284 EOF
285 git -C r2 index-pack ../filter.pack &&
286
287 git -C r2 verify-pack -v ../filter.pack >verify_result &&
288 grep blob verify_result |
289 parse_verify_pack_blob_oid |
290 sort >observed &&
291
292 test_cmp expected observed
293 '
294
295 test_expect_success 'verify big limit and small limit results in small limit' '
296 git -C r2 ls-files -s large.1000 >ls_files_result &&
297 test_parse_ls_files_stage_oids <ls_files_result |
298 sort >expected &&
299
300 git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 \
301 --filter=blob:limit=1001 >filter.pack <<-EOF &&
302 HEAD
303 EOF
304 git -C r2 index-pack ../filter.pack &&
305
306 git -C r2 verify-pack -v ../filter.pack >verify_result &&
307 grep blob verify_result |
308 parse_verify_pack_blob_oid |
309 sort >observed &&
310
311 test_cmp expected observed
312 '
313
314 # Test sparse:path=<path> filter.
315 # !!!!
316 # NOTE: sparse:path filter support has been dropped for security reasons,
317 # so the tests have been changed to make sure that using it fails.
318 # !!!!
319 # Use a local file containing a sparse-checkout specification to filter
320 # out blobs not required for the corresponding sparse-checkout. We do not
321 # require sparse-checkout to actually be enabled.
322
323 test_expect_success 'setup r3' '
324 git init r3 &&
325 mkdir r3/dir1 &&
326 for n in sparse1 sparse2
327 do
328 echo "This is file: $n" > r3/$n &&
329 git -C r3 add $n &&
330 echo "This is file: dir1/$n" > r3/dir1/$n &&
331 git -C r3 add dir1/$n || return 1
332 done &&
333 git -C r3 commit -m "sparse" &&
334 echo dir1/ >pattern1 &&
335 echo sparse1 >pattern2
336 '
337
338 test_expect_success 'verify blob count in normal packfile' '
339 git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
340 >ls_files_result &&
341 test_parse_ls_files_stage_oids <ls_files_result |
342 sort >expected &&
343
344 git -C r3 pack-objects --revs --stdout >all.pack <<-EOF &&
345 HEAD
346 EOF
347 git -C r3 index-pack ../all.pack &&
348
349 git -C r3 verify-pack -v ../all.pack >verify_result &&
350 grep blob verify_result |
351 parse_verify_pack_blob_oid |
352 sort >observed &&
353
354 test_cmp expected observed
355 '
356
357 test_expect_success 'verify sparse:path=pattern1 fails' '
358 test_must_fail git -C r3 pack-objects --revs --stdout \
359 --filter=sparse:path=../pattern1 <<-EOF
360 HEAD
361 EOF
362 '
363
364 test_expect_success 'verify sparse:path=pattern2 fails' '
365 test_must_fail git -C r3 pack-objects --revs --stdout \
366 --filter=sparse:path=../pattern2 <<-EOF
367 HEAD
368 EOF
369 '
370
371 # Test sparse:oid=<oid-ish> filter.
372 # Use a blob containing a sparse-checkout specification to filter
373 # out blobs not required for the corresponding sparse-checkout. We do not
374 # require sparse-checkout to actually be enabled.
375
376 test_expect_success 'setup r4' '
377 git init r4 &&
378 mkdir r4/dir1 &&
379 for n in sparse1 sparse2
380 do
381 echo "This is file: $n" > r4/$n &&
382 git -C r4 add $n &&
383 echo "This is file: dir1/$n" > r4/dir1/$n &&
384 git -C r4 add dir1/$n || return 1
385 done &&
386 echo dir1/ >r4/pattern &&
387 git -C r4 add pattern &&
388 git -C r4 commit -m "pattern"
389 '
390
391 test_expect_success 'verify blob count in normal packfile' '
392 git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
393 >ls_files_result &&
394 test_parse_ls_files_stage_oids <ls_files_result |
395 sort >expected &&
396
397 git -C r4 pack-objects --revs --stdout >all.pack <<-EOF &&
398 HEAD
399 EOF
400 git -C r4 index-pack ../all.pack &&
401
402 git -C r4 verify-pack -v ../all.pack >verify_result &&
403 grep blob verify_result |
404 parse_verify_pack_blob_oid |
405 sort >observed &&
406
407 test_cmp expected observed
408 '
409
410 test_expect_success 'verify sparse:oid=OID' '
411 git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
412 test_parse_ls_files_stage_oids <ls_files_result |
413 sort >expected &&
414
415 git -C r4 ls-files -s pattern >staged &&
416 oid=$(test_parse_ls_files_stage_oids <staged) &&
417 git -C r4 pack-objects --revs --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
418 HEAD
419 EOF
420 git -C r4 index-pack ../filter.pack &&
421
422 git -C r4 verify-pack -v ../filter.pack >verify_result &&
423 grep blob verify_result |
424 parse_verify_pack_blob_oid |
425 sort >observed &&
426
427 test_cmp expected observed
428 '
429
430 test_expect_success 'verify sparse:oid=oid-ish' '
431 git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
432 test_parse_ls_files_stage_oids <ls_files_result |
433 sort >expected &&
434
435 git -C r4 pack-objects --revs --stdout --filter=sparse:oid=main:pattern >filter.pack <<-EOF &&
436 HEAD
437 EOF
438 git -C r4 index-pack ../filter.pack &&
439
440 git -C r4 verify-pack -v ../filter.pack >verify_result &&
441 grep blob verify_result |
442 parse_verify_pack_blob_oid |
443 sort >observed &&
444
445 test_cmp expected observed
446 '
447
448 # Delete some loose objects and use pack-objects, but WITHOUT any filtering.
449 # This models previously omitted objects that we did not receive.
450
451 test_expect_success 'setup r1 - delete loose blobs' '
452 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
453 >ls_files_result &&
454 test_parse_ls_files_stage_oids <ls_files_result |
455 sort >expected &&
456
457 for id in `sed "s|..|&/|" expected`
458 do
459 rm r1/.git/objects/$id || return 1
460 done
461 '
462
463 test_expect_success 'verify pack-objects fails w/ missing objects' '
464 test_must_fail git -C r1 pack-objects --revs --stdout >miss.pack <<-EOF
465 HEAD
466 EOF
467 '
468
469 test_expect_success 'verify pack-objects fails w/ --missing=error' '
470 test_must_fail git -C r1 pack-objects --revs --stdout --missing=error >miss.pack <<-EOF
471 HEAD
472 EOF
473 '
474
475 test_expect_success 'verify pack-objects w/ --missing=allow-any' '
476 git -C r1 pack-objects --revs --stdout --missing=allow-any >miss.pack <<-EOF
477 HEAD
478 EOF
479 '
480
481 # Test that --path-walk produces the same object set as standard traversal
482 # when using sparse:oid filters with cone-mode patterns.
483 #
484 # The sparse:oid filter restricts only blobs, not trees. Both standard
485 # and path-walk should produce identical sets of blobs, commits, and trees.
486
487 test_expect_success 'setup pw_sparse for path-walk comparison' '
488 git init pw_sparse &&
489 mkdir -p pw_sparse/inc/sub pw_sparse/exc/sub &&
490
491 for n in 1 2
492 do
493 echo "inc $n" >pw_sparse/inc/file$n &&
494 echo "inc sub $n" >pw_sparse/inc/sub/file$n &&
495 echo "exc $n" >pw_sparse/exc/file$n &&
496 echo "exc sub $n" >pw_sparse/exc/sub/file$n &&
497 echo "root $n" >pw_sparse/root$n || return 1
498 done &&
499
500 git -C pw_sparse add . &&
501 git -C pw_sparse commit -m "first" &&
502
503 echo "inc 1 modified" >pw_sparse/inc/file1 &&
504 echo "exc 1 modified" >pw_sparse/exc/file1 &&
505 echo "root 1 modified" >pw_sparse/root1 &&
506 git -C pw_sparse add . &&
507 git -C pw_sparse commit -m "second" &&
508
509 # Cone-mode sparse pattern: include root + inc/
510 printf "/*\n!/*/\n/inc/\n" |
511 git -C pw_sparse hash-object -w --stdin >sparse_oid
512 '
513
514 test_expect_success 'sparse:oid with --path-walk produces same blobs' '
515 oid=$(cat sparse_oid) &&
516
517 git -C pw_sparse pack-objects --revs --stdout \
518 --filter=sparse:oid=$oid >standard.pack <<-EOF &&
519 HEAD
520 EOF
521 git -C pw_sparse index-pack ../standard.pack &&
522 git -C pw_sparse verify-pack -v ../standard.pack >standard_verify &&
523
524 git -C pw_sparse pack-objects --revs --stdout \
525 --path-walk --filter=sparse:oid=$oid >pathwalk.pack <<-EOF &&
526 HEAD
527 EOF
528 git -C pw_sparse index-pack ../pathwalk.pack &&
529 git -C pw_sparse verify-pack -v ../pathwalk.pack >pathwalk_verify &&
530
531 # Blobs must match exactly
532 grep -E "^[0-9a-f]{40} blob" standard_verify |
533 awk "{print \$1}" | sort >standard_blobs &&
534 grep -E "^[0-9a-f]{40} blob" pathwalk_verify |
535 awk "{print \$1}" | sort >pathwalk_blobs &&
536 test_cmp standard_blobs pathwalk_blobs &&
537
538 # Commits must match exactly
539 grep -E "^[0-9a-f]{40} commit" standard_verify |
540 awk "{print \$1}" | sort >standard_commits &&
541 grep -E "^[0-9a-f]{40} commit" pathwalk_verify |
542 awk "{print \$1}" | sort >pathwalk_commits &&
543 test_cmp standard_commits pathwalk_commits
544 '
545
546 test_expect_success 'sparse:oid with --path-walk includes all trees' '
547 # The sparse:oid filter restricts only blobs, not trees.
548 # Both standard and path-walk should include the same trees.
549 grep -E "^[0-9a-f]{40} tree" standard_verify |
550 awk "{print \$1}" | sort >standard_trees &&
551 grep -E "^[0-9a-f]{40} tree" pathwalk_verify |
552 awk "{print \$1}" | sort >pathwalk_trees &&
553
554 test_cmp standard_trees pathwalk_trees
555 '
556
557 # Test the edge case where the same tree/blob OID appears at both an
558 # in-cone and out-of-cone path. When sibling directories have identical
559 # contents, they share a tree OID. The path-walk defers marking objects
560 # SEEN until after checking sparse patterns, so an object at an out-of-cone
561 # path can still be discovered at an in-cone path.
562
563 test_expect_success 'setup pw_shared for shared OID across cone boundary' '
564 git init pw_shared &&
565 mkdir pw_shared/aaa pw_shared/zzz &&
566 echo "shared content" >pw_shared/aaa/file &&
567 echo "shared content" >pw_shared/zzz/file &&
568 echo "root file" >pw_shared/rootfile &&
569 git -C pw_shared add . &&
570 git -C pw_shared commit -m "aaa and zzz share tree OID" &&
571
572 # Verify they share a tree OID
573 aaa_tree=$(git -C pw_shared rev-parse HEAD:aaa) &&
574 zzz_tree=$(git -C pw_shared rev-parse HEAD:zzz) &&
575 test "$aaa_tree" = "$zzz_tree" &&
576
577 # Cone pattern: include root + zzz/ (not aaa/)
578 printf "/*\n!/*/\n/zzz/\n" |
579 git -C pw_shared hash-object -w --stdin >shared_sparse_oid
580 '
581
582 test_expect_success 'shared tree OID: --path-walk blobs match standard' '
583 oid=$(cat shared_sparse_oid) &&
584
585 git -C pw_shared pack-objects --revs --stdout \
586 --filter=sparse:oid=$oid >shared_std.pack <<-EOF &&
587 HEAD
588 EOF
589 git -C pw_shared index-pack ../shared_std.pack &&
590 git -C pw_shared verify-pack -v ../shared_std.pack >shared_std_verify &&
591
592 git -C pw_shared pack-objects --revs --stdout \
593 --path-walk --filter=sparse:oid=$oid >shared_pw.pack <<-EOF &&
594 HEAD
595 EOF
596 git -C pw_shared index-pack ../shared_pw.pack &&
597 git -C pw_shared verify-pack -v ../shared_pw.pack >shared_pw_verify &&
598
599 grep -E "^[0-9a-f]{40} blob" shared_std_verify |
600 awk "{print \$1}" | sort >shared_std_blobs &&
601 grep -E "^[0-9a-f]{40} blob" shared_pw_verify |
602 awk "{print \$1}" | sort >shared_pw_blobs &&
603 test_cmp shared_std_blobs shared_pw_blobs
604 '
605
606 test_done