@matheusbsbs / matheus-bagliano / commits / ca11b212eb
Fork 0

let pack-objects do the writing of unreachable objects as loose objects

Commit ccc1297226b184c40459e9d373cc9eebfb7bd898 changed the behavior of 'git repack -A' so unreachable objects are stored as loose objects. However it did so in a naive and inn efficient way by making packs about to be deleted inaccessible and feeding their content through 'git unpack-objects'. While this works, there are major flaws with this approach: - It is unacceptably sloooooooooooooow. In the Linux kernel repository with no actual unreachable objects, doing 'git repack -A -d' before: real 2m33.220s user 2m21.675s sys 0m3.510s And with this change: real 0m36.849s user 0m24.365s sys 0m1.950s For reference, here's the timing for 'git repack -a -d': real 0m35.816s user 0m22.571s sys 0m2.011s This is explained by the fact that 'git unpack-objects' was used to unpack _every_ objects even if (almost) 100% of them were thrown away. - There is a black out period. Between the removal of the .idx file for the redundant pack and the completion of its unpacking, the unreachable objects become completely unaccessible. This is not a big issue as we're talking about unreachable objects, but some consistency is always good. - There is no way to easily set a sensible mtime for the newly created unreachable loose objects. So, while having a command called "pack-objects" to perform object unpacking looks really odd, this is probably the best compromize to be able to solve the above issues in an efficient way. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Nicolas Pitre committed May 14, 2008 at 01:33 UTC ca11b212eb3e31d6fee12e9974c67dc774c1bc7c
2 files changed +45 -17
builtin-pack-objects.c
+38 -2
@@ -28,7 +28,8 @@ git-pack-objects [{ -q | --progress | --all-progress }] \n\
28 [--window=N] [--window-memory=N] [--depth=N] \n\
29 [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
30 [--threads=N] [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
31 - [--stdout | base-name] [--include-tag] [--keep-unreachable] \n\
31 + [--stdout | base-name] [--include-tag] \n\
32 + [--keep-unreachable | --unpack-unreachable] \n\
33 [<ref-list | <object-list]";
34
35 struct object_entry {
@@ -65,7 +66,7 @@ static struct pack_idx_entry **written_list;
66 static uint32_t nr_objects, nr_alloc, nr_result, nr_written;
67
68 static int non_empty;
68 -static int no_reuse_delta, no_reuse_object, keep_unreachable, include_tag;
69 +static int no_reuse_delta, no_reuse_object, keep_unreachable, unpack_unreachable, include_tag;
70 static int local;
71 static int incremental;
72 static int allow_ofs_delta;
@@ -1905,6 +1906,32 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
1906 free(in_pack.array);
1907 }
1908
1909 +static void loosen_unused_packed_objects(struct rev_info *revs)
1910 +{
1911 + struct packed_git *p;
1912 + uint32_t i;
1913 + const unsigned char *sha1;
1914 +
1915 + for (p = packed_git; p; p = p->next) {
1916 + for (i = 0; i < revs->num_ignore_packed; i++) {
1917 + if (matches_pack_name(p, revs->ignore_packed[i]))
1918 + break;
1919 + }
1920 + if (revs->num_ignore_packed <= i)
1921 + continue;
1922 +
1923 + if (open_pack_index(p))
1924 + die("cannot open pack index");
1925 +
1926 + for (i = 0; i < p->num_objects; i++) {
1927 + sha1 = nth_packed_object_sha1(p, i);
1928 + if (!locate_object_entry(sha1))
1929 + if (force_object_loose(sha1, p->mtime))
1930 + die("unable to force loose object");
1931 + }
1932 + }
1933 +}
1934 +
1935 static void get_object_list(int ac, const char **av)
1936 {
1937 struct rev_info revs;
@@ -1939,6 +1966,8 @@ static void get_object_list(int ac, const char **av)
1966
1967 if (keep_unreachable)
1968 add_objects_in_unpacked_packs(&revs);
1969 + if (unpack_unreachable)
1970 + loosen_unused_packed_objects(&revs);
1971 }
1972
1973 static int adjust_perm(const char *path, mode_t mode)
@@ -2073,6 +2102,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
2102 keep_unreachable = 1;
2103 continue;
2104 }
2105 + if (!strcmp("--unpack-unreachable", arg)) {
2106 + unpack_unreachable = 1;
2107 + continue;
2108 + }
2109 if (!strcmp("--include-tag", arg)) {
2110 include_tag = 1;
2111 continue;
@@ -2138,6 +2171,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
2171 if (!pack_to_stdout && thin)
2172 die("--thin cannot be used to build an indexable pack.");
2173
2174 + if (keep_unreachable && unpack_unreachable)
2175 + die("--keep-unreachable and --unpack-unreachable are incompatible.");
2176 +
2177 #ifdef THREADED_DELTA_SEARCH
2178 if (!delta_search_threads) /* --threads=0 means autodetect */
2179 delta_search_threads = online_cpus();
git-repack.sh
+7 -15
@@ -8,7 +8,7 @@ OPTIONS_SPEC="\
8 git-repack [options]
9 --
10 a pack everything in a single pack
11 -A same as -a, and keep unreachable objects too
11 +A same as -a, and turn unreachable objects loose
12 d remove redundant packs, and run git-prune-packed
13 f pass --no-reuse-delta to git-pack-objects
14 q,quiet be quiet
@@ -22,7 +22,7 @@ max-pack-size= maximum size of each packfile
22 SUBDIRECTORY_OK='Yes'
23 . git-sh-setup
24
25 -no_update_info= all_into_one= remove_redundant= keep_unreachable=
25 +no_update_info= all_into_one= remove_redundant= unpack_unreachable=
26 local= quiet= no_reuse= extra=
27 while test $# != 0
28 do
@@ -30,7 +30,7 @@ do
30 -n) no_update_info=t ;;
31 -a) all_into_one=t ;;
32 -A) all_into_one=t
33 - keep_unreachable=t ;;
33 + unpack_unreachable=--unpack-unreachable ;;
34 -d) remove_redundant=t ;;
35 -q) quiet=-q ;;
36 -f) no_reuse=--no-reuse-object ;;
@@ -78,6 +78,9 @@ case ",$all_into_one," in
78 if test -z "$args"
79 then
80 args='--unpacked --incremental'
81 + elif test -n "$unpack_unreachable"
82 + then
83 + args="$args $unpack_unreachable"
84 fi
85 ;;
86 esac
@@ -127,18 +130,7 @@ then
130 do
131 case " $fullbases " in
132 *" $e "*) ;;
130 - *)
131 - rm -f "$e.idx" "$e.keep"
132 - if test -n "$keep_unreachable" &&
133 - test -f "$e.pack"
134 - then
135 - git unpack-objects < "$e.pack" || {
136 - echo >&2 "Failed unpacking unreachable objects from redundant pack file $e.pack"
137 - exit 1
138 - }
139 - fi
140 - rm -f "$e.pack"
141 - ;;
133 + *) rm -f "$e.pack" "$e.idx" "$e.keep" ;;
134 esac
135 done
136 )