@leroysheep / rag-system
Code Commits Pulls CI/CD 82,037
test
bisect jch maint master next seen test todo
c 188 lines 4.91 KB
Raw
1 /* We need this macro to access core_apply_sparse_checkout */
2 #define USE_THE_REPOSITORY_VARIABLE
3
4 #include "builtin.h"
5 #include "git-compat-util.h"
6 #include "config.h"
7 #include "parse-options.h"
8 #include "repository.h"
9 #include "commit.h"
10 #include "dir.h"
11 #include "environment.h"
12 #include "hex.h"
13 #include "tree.h"
14 #include "tree-walk.h"
15 #include "object.h"
16 #include "odb.h"
17 #include "oid-array.h"
18 #include "oidset.h"
19 #include "promisor-remote.h"
20 #include "strmap.h"
21 #include "string-list.h"
22 #include "revision.h"
23 #include "trace2.h"
24 #include "progress.h"
25 #include "packfile.h"
26 #include "path-walk.h"
27
28 static const char * const builtin_backfill_usage[] = {
29 N_("git backfill [--min-batch-size=<n>] [--[no-]sparse] [--[no-]include-edges] [<revision-range>]"),
30 NULL
31 };
32
33 struct backfill_context {
34 struct repository *repo;
35 struct oid_array current_batch;
36 size_t min_batch_size;
37 int sparse;
38 int include_edges;
39 struct rev_info revs;
40 };
41
42 static void backfill_context_clear(struct backfill_context *ctx)
43 {
44 oid_array_clear(&ctx->current_batch);
45 }
46
47 static void download_batch(struct backfill_context *ctx)
48 {
49 promisor_remote_get_direct(ctx->repo,
50 ctx->current_batch.oid,
51 ctx->current_batch.nr);
52 oid_array_clear(&ctx->current_batch);
53
54 /*
55 * We likely have a new packfile. Add it to the packed list to
56 * avoid possible duplicate downloads of the same objects.
57 */
58 odb_reprepare(ctx->repo->objects);
59 }
60
61 static int fill_missing_blobs(const char *path UNUSED,
62 struct oid_array *list,
63 enum object_type type,
64 void *data)
65 {
66 struct backfill_context *ctx = data;
67
68 if (type != OBJ_BLOB)
69 return 0;
70
71 for (size_t i = 0; i < list->nr; i++) {
72 if (!odb_has_object(ctx->repo->objects, &list->oid[i], 0))
73 oid_array_append(&ctx->current_batch, &list->oid[i]);
74 }
75
76 if (ctx->current_batch.nr >= ctx->min_batch_size)
77 download_batch(ctx);
78
79 return 0;
80 }
81
82 static void reject_unsupported_rev_list_options(struct rev_info *revs)
83 {
84 if (revs->diffopt.pickaxe)
85 die(_("'%s' cannot be used with 'git backfill'"),
86 (revs->diffopt.pickaxe_opts & DIFF_PICKAXE_REGEX) ? "-G" : "-S");
87 if (revs->diffopt.filter || revs->diffopt.filter_not)
88 die(_("'%s' cannot be used with 'git backfill'"),
89 "--diff-filter");
90 if (revs->diffopt.flags.follow_renames)
91 die(_("'%s' cannot be used with 'git backfill'"),
92 "--follow");
93 if (revs->line_level_traverse)
94 die(_("'%s' cannot be used with 'git backfill'"),
95 "-L");
96 if (revs->explicit_diff_merges)
97 die(_("'%s' cannot be used with 'git backfill'"),
98 "--diff-merges");
99 if (!path_walk_filter_compatible(&revs->filter))
100 die(_("cannot backfill with these filter options"));
101 if (revs->filter.blob_limit_value)
102 die(_("cannot backfill with blob size limits"));
103 }
104
105 static int do_backfill(struct backfill_context *ctx)
106 {
107 struct path_walk_info info = PATH_WALK_INFO_INIT;
108 int ret;
109
110 if (ctx->sparse) {
111 CALLOC_ARRAY(info.pl, 1);
112 info.pl_sparse_trees = 1;
113 if (get_sparse_checkout_patterns(info.pl)) {
114 path_walk_info_clear(&info);
115 return error(_("problem loading sparse-checkout"));
116 }
117 }
118
119 /* Walk from HEAD if otherwise unspecified. */
120 if (!ctx->revs.pending.nr)
121 add_head_to_pending(&ctx->revs);
122 if (ctx->include_edges)
123 ctx->revs.edge_hint = 1;
124
125 info.blobs = 1;
126 info.tags = info.commits = info.trees = 0;
127
128 info.revs = &ctx->revs;
129 info.path_fn = fill_missing_blobs;
130 info.path_fn_data = ctx;
131
132 ret = walk_objects_by_path(&info);
133
134 /* Download the objects that did not fill a batch. */
135 if (!ret)
136 download_batch(ctx);
137
138 path_walk_info_clear(&info);
139 return ret;
140 }
141
142 int cmd_backfill(int argc, const char **argv, const char *prefix, struct repository *repo)
143 {
144 int result;
145 struct backfill_context ctx = {
146 .repo = repo,
147 .current_batch = OID_ARRAY_INIT,
148 .min_batch_size = 50000,
149 .sparse = -1,
150 .revs = REV_INFO_INIT,
151 .include_edges = 1,
152 };
153 struct option options[] = {
154 OPT_UNSIGNED(0, "min-batch-size", &ctx.min_batch_size,
155 N_("Minimum number of objects to request at a time")),
156 OPT_BOOL(0, "sparse", &ctx.sparse,
157 N_("Restrict the missing objects to the current sparse-checkout")),
158 OPT_BOOL(0, "include-edges", &ctx.include_edges,
159 N_("Include blobs from boundary commits in the backfill")),
160 OPT_END(),
161 };
162 struct repo_config_values *cfg = repo_config_values(the_repository);
163
164 show_usage_with_options_if_asked(argc, argv,
165 builtin_backfill_usage, options);
166
167 argc = parse_options(argc, argv, prefix, options, builtin_backfill_usage,
168 PARSE_OPT_KEEP_UNKNOWN_OPT |
169 PARSE_OPT_KEEP_ARGV0 |
170 PARSE_OPT_KEEP_DASHDASH);
171
172 repo_init_revisions(repo, &ctx.revs, prefix);
173 argc = setup_revisions(argc, argv, &ctx.revs, NULL);
174
175 if (argc > 1)
176 die(_("unrecognized argument: %s"), argv[1]);
177 reject_unsupported_rev_list_options(&ctx.revs);
178
179 repo_config(repo, git_default_config, NULL);
180
181 if (ctx.sparse < 0)
182 ctx.sparse = cfg->apply_sparse_checkout;
183
184 result = do_backfill(&ctx);
185 backfill_context_clear(&ctx);
186 release_revisions(&ctx.revs);
187 return result;
188 }