repack: add --keep-pack option
We allow to keep existing packs by having companion .keep files. This is helpful when a pack is permanently kept. In the next patch, git-gc just wants to keep a pack temporarily, for one pack-objects run. git-gc can use --keep-pack for this use case. A note about why the pack_keep field cannot be reused and pack_keep_in_core has to be added. This is about the case when --keep-pack is specified together with either --keep-unreachable or --unpack-unreachable, but --honor-pack-keep is NOT specified. In this case, we want to exclude objects from the packs specified on command line, not from ones with .keep files. If only one bit flag is used, we have to clear pack_keep on pack files with the .keep file. But we can't make any assumption about unreachable objects in .keep packs. If "pack_keep" field is false for .keep packs, we could potentially pull lots of unreachable objects into the new pack, or unpack them loose. The safer approach is ignore all packs with either .keep file or --keep-pack. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Nguyễn Thái Ngọc Duy committed
Apr 15, 2018 at 17:36 UTC
ed7e5fc3a2562e3d4fb4fe25e8dca0e9daa0ed13
6 files changed
+110
-18
Documentation/git-pack-objects.txt
+8
-1
@@ -12,7 +12,7 @@ SYNOPSIS
12
'git pack-objects' [-q | --progress | --all-progress] [--all-progress-implied]
13
[--no-reuse-delta] [--delta-base-offset] [--non-empty]
14
[--local] [--incremental] [--window=<n>] [--depth=<n>]
15
- [--revs [--unpacked | --all]]
15
+ [--revs [--unpacked | --all]] [--keep-pack=<pack-name>]
16
[--stdout [--filter=<filter-spec>] | base-name]
17
[--shallow] [--keep-true-parents] < object-list
18
@@ -126,6 +126,13 @@ base-name::
126
has a .keep file to be ignored, even if it would have
127
otherwise been packed.
128
129
+--keep-pack=<pack-name>::
130
+ This flag causes an object already in the given pack to be
131
+ ignored, even if it would have otherwise been
132
+ packed. `<pack-name>` is the the pack file name without
133
+ leading directory (e.g. `pack-123.pack`). The option could be
134
+ specified multiple times to keep multiple packs.
135
+
136
--incremental::
137
This flag causes an object already in a pack to be ignored
138
even if it would have otherwise been packed.
Documentation/git-repack.txt
+8
-1
@@ -9,7 +9,7 @@ git-repack - Pack unpacked objects in a repository
9
SYNOPSIS
10
--------
11
[verse]
12
-'git repack' [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [--window=<n>] [--depth=<n>] [--threads=<n>]
12
+'git repack' [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [--window=<n>] [--depth=<n>] [--threads=<n>] [--keep-pack=<pack-name>]
13
14
DESCRIPTION
15
-----------
@@ -133,6 +133,13 @@ other objects in that pack they already have locally.
133
with `-b` or `repack.writeBitmaps`, as it ensures that the
134
bitmapped packfile has the necessary objects.
135
136
+--keep-pack=<pack-name>::
137
+ Exclude the given pack from repacking. This is the equivalent
138
+ of having `.keep` file on the pack. `<pack-name>` is the the
139
+ pack file name without leading directory (e.g. `pack-123.pack`).
140
+ The option could be specified multiple times to keep multiple
141
+ packs.
142
+
143
--unpack-unreachable=<when>::
144
When loosening unreachable objects, do not bother loosening any
145
objects older than `<when>`. This can be used to optimize out
builtin/pack-objects.c
+50
-13
@@ -30,6 +30,7 @@
30
#include "list.h"
31
#include "packfile.h"
32
#include "object-store.h"
33
+#include "dir.h"
34
35
static const char *pack_usage[] = {
36
N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"),
@@ -55,7 +56,8 @@ static int pack_loose_unreachable;
56
static int local;
57
static int have_non_local_packs;
58
static int incremental;
58
-static int ignore_packed_keep;
59
+static int ignore_packed_keep_on_disk;
60
+static int ignore_packed_keep_in_core;
61
static int allow_ofs_delta;
62
static struct pack_idx_option pack_idx_opts;
63
static const char *base_name;
@@ -982,13 +984,16 @@ static int want_found_object(int exclude, struct packed_git *p)
984
* Otherwise, we signal "-1" at the end to tell the caller that we do
985
* not know either way, and it needs to check more packs.
986
*/
985
- if (!ignore_packed_keep &&
987
+ if (!ignore_packed_keep_on_disk &&
988
+ !ignore_packed_keep_in_core &&
989
(!local || !have_non_local_packs))
990
return 1;
991
992
if (local && !p->pack_local)
993
return 0;
991
- if (ignore_packed_keep && p->pack_local && p->pack_keep)
994
+ if (p->pack_local &&
995
+ ((ignore_packed_keep_on_disk && p->pack_keep) ||
996
+ (ignore_packed_keep_in_core && p->pack_keep_in_core)))
997
return 0;
998
999
/* we don't know yet; keep looking for more packs */
@@ -2675,7 +2680,7 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
2680
struct object_id oid;
2681
struct object *o;
2682
2678
- if (!p->pack_local || p->pack_keep)
2683
+ if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
2684
continue;
2685
if (open_pack_index(p))
2686
die("cannot open pack index");
@@ -2738,7 +2743,8 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
2743
get_packed_git(the_repository);
2744
2745
while (p) {
2741
- if ((!p->pack_local || p->pack_keep) &&
2746
+ if ((!p->pack_local || p->pack_keep ||
2747
+ p->pack_keep_in_core) &&
2748
find_pack_entry_one(oid->hash, p)) {
2749
last_found = p;
2750
return 1;
@@ -2781,7 +2787,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
2787
struct object_id oid;
2788
2789
for (p = get_packed_git(the_repository); p; p = p->next) {
2784
- if (!p->pack_local || p->pack_keep)
2790
+ if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
2791
continue;
2792
2793
if (open_pack_index(p))
@@ -2807,7 +2813,8 @@ static int pack_options_allow_reuse(void)
2813
{
2814
return pack_to_stdout &&
2815
allow_ofs_delta &&
2810
- !ignore_packed_keep &&
2816
+ !ignore_packed_keep_on_disk &&
2817
+ !ignore_packed_keep_in_core &&
2818
(!local || !have_non_local_packs) &&
2819
!incremental;
2820
}
@@ -2916,6 +2923,32 @@ static void get_object_list(int ac, const char **av)
2923
oid_array_clear(&recent_objects);
2924
}
2925
2926
+static void add_extra_kept_packs(const struct string_list *names)
2927
+{
2928
+ struct packed_git *p;
2929
+
2930
+ if (!names->nr)
2931
+ return;
2932
+
2933
+ for (p = get_packed_git(the_repository); p; p = p->next) {
2934
+ const char *name = basename(p->pack_name);
2935
+ int i;
2936
+
2937
+ if (!p->pack_local)
2938
+ continue;
2939
+
2940
+ for (i = 0; i < names->nr; i++)
2941
+ if (!fspathcmp(name, names->items[i].string))
2942
+ break;
2943
+
2944
+ if (i < names->nr) {
2945
+ p->pack_keep_in_core = 1;
2946
+ ignore_packed_keep_in_core = 1;
2947
+ continue;
2948
+ }
2949
+ }
2950
+}
2951
+
2952
static int option_parse_index_version(const struct option *opt,
2953
const char *arg, int unset)
2954
{
@@ -2955,6 +2988,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
2988
struct argv_array rp = ARGV_ARRAY_INIT;
2989
int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;
2990
int rev_list_index = 0;
2991
+ struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
2992
struct option pack_objects_options[] = {
2993
OPT_SET_INT('q', "quiet", &progress,
2994
N_("do not show progress meter"), 0),
@@ -3019,8 +3053,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
3053
N_("create thin packs")),
3054
OPT_BOOL(0, "shallow", &shallow,
3055
N_("create packs suitable for shallow fetches")),
3022
- OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep,
3056
+ OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep_on_disk,
3057
N_("ignore packs that have companion .keep file")),
3058
+ OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
3059
+ N_("ignore this pack")),
3060
OPT_INTEGER(0, "compression", &pack_compression_level,
3061
N_("pack compression level")),
3062
OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents,
@@ -3148,19 +3184,20 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
3184
if (progress && all_progress_implied)
3185
progress = 2;
3186
3151
- if (ignore_packed_keep) {
3187
+ add_extra_kept_packs(&keep_pack_list);
3188
+ if (ignore_packed_keep_on_disk) {
3189
struct packed_git *p;
3190
for (p = get_packed_git(the_repository); p; p = p->next)
3191
if (p->pack_local && p->pack_keep)
3192
break;
3193
if (!p) /* no keep-able packs found */
3157
- ignore_packed_keep = 0;
3194
+ ignore_packed_keep_on_disk = 0;
3195
}
3196
if (local) {
3197
/*
3161
- * unlike ignore_packed_keep above, we do not want to
3162
- * unset "local" based on looking at packs, as it
3163
- * also covers non-local objects
3198
+ * unlike ignore_packed_keep_on_disk above, we do not
3199
+ * want to unset "local" based on looking at packs, as
3200
+ * it also covers non-local objects
3201
*/
3202
struct packed_git *p;
3203
for (p = get_packed_git(the_repository); p; p = p->next) {
builtin/repack.c
+18
-3
@@ -86,7 +86,8 @@ static void remove_pack_on_signal(int signo)
86
* have a corresponding .keep or .promisor file. These packs are not to
87
* be kept if we are going to pack everything into one file.
88
*/
89
-static void get_non_kept_pack_filenames(struct string_list *fname_list)
89
+static void get_non_kept_pack_filenames(struct string_list *fname_list,
90
+ const struct string_list *extra_keep)
91
{
92
DIR *dir;
93
struct dirent *e;
@@ -97,6 +98,14 @@ static void get_non_kept_pack_filenames(struct string_list *fname_list)
98
99
while ((e = readdir(dir)) != NULL) {
100
size_t len;
101
+ int i;
102
+
103
+ for (i = 0; i < extra_keep->nr; i++)
104
+ if (!fspathcmp(e->d_name, extra_keep->items[i].string))
105
+ break;
106
+ if (extra_keep->nr > 0 && i < extra_keep->nr)
107
+ continue;
108
+
109
if (!strip_suffix(e->d_name, ".pack", &len))
110
continue;
111
@@ -148,7 +157,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
157
struct string_list rollback = STRING_LIST_INIT_NODUP;
158
struct string_list existing_packs = STRING_LIST_INIT_DUP;
159
struct strbuf line = STRBUF_INIT;
151
- int ext, ret, failed;
160
+ int i, ext, ret, failed;
161
FILE *out;
162
163
/* variables to be filled by option parsing */
@@ -160,6 +169,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
169
const char *depth = NULL;
170
const char *threads = NULL;
171
const char *max_pack_size = NULL;
172
+ struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
173
int no_reuse_delta = 0, no_reuse_object = 0;
174
int no_update_server_info = 0;
175
int quiet = 0;
@@ -200,6 +210,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
210
N_("maximum size of each packfile")),
211
OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects,
212
N_("repack objects in packs marked with .keep")),
213
+ OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
214
+ N_("do not repack this pack")),
215
OPT_END()
216
};
217
@@ -230,6 +242,9 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
242
argv_array_push(&cmd.args, "--keep-true-parents");
243
if (!pack_kept_objects)
244
argv_array_push(&cmd.args, "--honor-pack-keep");
245
+ for (i = 0; i < keep_pack_list.nr; i++)
246
+ argv_array_pushf(&cmd.args, "--keep-pack=%s",
247
+ keep_pack_list.items[i].string);
248
argv_array_push(&cmd.args, "--non-empty");
249
argv_array_push(&cmd.args, "--all");
250
argv_array_push(&cmd.args, "--reflog");
@@ -254,7 +269,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
269
argv_array_push(&cmd.args, "--write-bitmap-index");
270
271
if (pack_everything & ALL_INTO_ONE) {
257
- get_non_kept_pack_filenames(&existing_packs);
272
+ get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
273
274
if (existing_packs.nr && delete_redundant) {
275
if (unpack_unreachable) {
object-store.h
+1
@@ -71,6 +71,7 @@ struct packed_git {
71
int pack_fd;
72
unsigned pack_local:1,
73
pack_keep:1,
74
+ pack_keep_in_core:1,
75
freshened:1,
76
do_not_close:1,
77
pack_promisor:1;
t/t7700-repack.sh
+25
@@ -4,6 +4,12 @@ test_description='git repack works correctly'
4
5
. ./test-lib.sh
6
7
+commit_and_pack() {
8
+ test_commit "$@" >/dev/null &&
9
+ SHA1=$(git pack-objects --all --unpacked --incremental .git/objects/pack/pack </dev/null) &&
10
+ echo pack-${SHA1}.pack
11
+}
12
+
13
test_expect_success 'objects in packs marked .keep are not repacked' '
14
echo content1 > file1 &&
15
echo content2 > file2 &&
@@ -196,5 +202,24 @@ test_expect_success 'objects made unreachable by grafts only are kept' '
202
git cat-file -t $H1
203
'
204
205
+test_expect_success 'repack --keep-pack' '
206
+ test_create_repo keep-pack &&
207
+ (
208
+ cd keep-pack &&
209
+ P1=$(commit_and_pack 1) &&
210
+ P2=$(commit_and_pack 2) &&
211
+ P3=$(commit_and_pack 3) &&
212
+ P4=$(commit_and_pack 4) &&
213
+ ls .git/objects/pack/*.pack >old-counts &&
214
+ test_line_count = 4 old-counts &&
215
+ git repack -a -d --keep-pack $P1 --keep-pack $P4 &&
216
+ ls .git/objects/pack/*.pack >new-counts &&
217
+ grep -q $P1 new-counts &&
218
+ grep -q $P4 new-counts &&
219
+ test_line_count = 3 new-counts &&
220
+ git fsck
221
+ )
222
+'
223
+
224
test_done
225