Raw
1 #ifndef ODB_H
2 #define ODB_H
3
4 #include "object.h"
5 #include "oidset.h"
6 #include "oidmap.h"
7 #include "string-list.h"
8 #include "thread-utils.h"
9
10 struct cached_object_entry;
11 struct odb_source_inmemory;
12 struct packed_git;
13 struct repository;
14 struct strbuf;
15 struct strvec;
16
17 /*
18 * Set this to 0 to prevent odb_read_object_info_extended() from fetching missing
19 * blobs. This has a difference only if extensions.partialClone is set.
20 *
21 * Its default value is 1.
22 */
23 extern int fetch_if_missing;
24
25 /*
26 * Compute the exact path an alternate is at and returns it. In case of
27 * error NULL is returned and the human readable error is added to `err`
28 * `path` may be relative and should point to $GIT_DIR.
29 * `err` must not be null.
30 */
31 char *compute_alternate_path(const char *path, struct strbuf *err);
32
33 /*
34 * The object database encapsulates access to objects in a repository. It
35 * manages one or more sources that store the actual objects which are
36 * configured via alternates.
37 */
38 struct object_database {
39 /* Repository that owns this database. */
40 struct repository *repo;
41
42 /*
43 * State of current object database transaction. Only one
44 * transaction may be pending at a time. Is NULL when no transaction is
45 * configured.
46 */
47 struct odb_transaction *transaction;
48
49 /*
50 * Set of all object directories; the main directory is first (and
51 * cannot be NULL after initialization). Subsequent directories are
52 * alternates.
53 */
54 struct odb_source *sources;
55 struct odb_source **sources_tail;
56 struct kh_odb_path_map *source_by_path;
57
58 int loaded_alternates;
59
60 /*
61 * A list of alternate object directories loaded from the environment;
62 * this should not generally need to be accessed directly, but will
63 * populate the "sources" list when odb_prepare_alternates() is run.
64 */
65 char *alternate_db;
66
67 /*
68 * Objects that should be substituted by other objects
69 * (see git-replace(1)).
70 */
71 struct oidmap replace_map;
72 unsigned replace_map_initialized : 1;
73 pthread_mutex_t replace_mutex; /* protect object replace functions */
74
75 struct commit_graph *commit_graph;
76 unsigned commit_graph_attempted : 1; /* if loading has been attempted */
77
78 /*
79 * This is meant to hold a *small* number of objects that you would
80 * want odb_read_object() to be able to return, but yet you do not want
81 * to write them into the object store (e.g. a browse-only
82 * application).
83 */
84 struct odb_source *inmemory_objects;
85
86 /*
87 * A fast, rough count of the number of objects in the repository.
88 * These two fields are not meant for direct access. Use
89 * odb_count_objects() instead.
90 */
91 unsigned long object_count;
92 unsigned object_count_flags;
93 unsigned object_count_valid : 1;
94
95 /*
96 * Submodule source paths that will be added as additional sources to
97 * allow lookup of submodule objects via the main object database.
98 */
99 struct string_list submodule_source_paths;
100 };
101
102 /*
103 * Create a new object database for the given repository.
104 *
105 * If the primary source parameter is set it will override the usual primary
106 * object directory derived from the repository's common directory. The
107 * alternate sources are expected to be a PATH_SEP-separated list of secondary
108 * sources. Note that these alternate sources will be added in addition to, not
109 * instead of, the alternates identified by the primary source.
110 *
111 * Returns the newly created object database.
112 */
113 struct object_database *odb_new(struct repository *repo,
114 const char *primary_source,
115 const char *alternate_sources);
116
117 /* Free the object database and release all resources. */
118 void odb_free(struct object_database *o);
119
120 /*
121 * Close the object database and all of its sources so that any held resources
122 * will be released. The database can still be used after closing it, in which
123 * case these resources may be reallocated.
124 */
125 void odb_close(struct object_database *o);
126
127 /*
128 * Clear caches, reload alternates and then reload object sources so that new
129 * objects may become accessible.
130 */
131 void odb_reprepare(struct object_database *o);
132
133 /*
134 * Find source by its object directory path. Returns a `NULL` pointer in case
135 * the source could not be found.
136 */
137 struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir);
138
139 /* Same as `odb_find_source()`, but dies in case the source doesn't exist. */
140 struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir);
141
142 /*
143 * Replace the current writable object directory with the specified temporary
144 * object directory; returns the former primary source.
145 */
146 struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
147 const char *dir, int will_destroy);
148
149 /*
150 * Restore the primary source that was previously replaced by
151 * `odb_set_temporary_primary_source()`.
152 */
153 void odb_restore_primary_source(struct object_database *odb,
154 struct odb_source *restore_source,
155 const char *old_path);
156
157 /*
158 * Call odb_add_submodule_source_by_path() to add the submodule at the given
159 * path to a list. The object stores of all submodules in that list will be
160 * added as additional sources in the object store when looking up objects.
161 */
162 void odb_add_submodule_source_by_path(struct object_database *odb,
163 const char *path);
164
165 /*
166 * Iterate through all alternates of the database and execute the provided
167 * callback function for each of them. Stop iterating once the callback
168 * function returns a non-zero value, in which case the value is bubbled up
169 * from the callback.
170 */
171 typedef int odb_for_each_alternate_fn(struct odb_source *, void *);
172 int odb_for_each_alternate(struct object_database *odb,
173 odb_for_each_alternate_fn cb, void *payload);
174
175 /*
176 * Iterate through all alternates of the database and yield their respective
177 * references.
178 */
179 typedef void odb_for_each_alternate_ref_fn(const struct object_id *oid, void *);
180 void odb_for_each_alternate_ref(struct object_database *odb,
181 odb_for_each_alternate_ref_fn cb, void *payload);
182
183 /*
184 * Create a temporary file rooted in the primary alternate's directory, or die
185 * on failure. The filename is taken from "pattern", which should have the
186 * usual "XXXXXX" trailer, and the resulting filename is written into the
187 * "template" buffer. Returns the open descriptor.
188 */
189 int odb_mkstemp(struct object_database *odb,
190 struct strbuf *temp_filename, const char *pattern);
191
192 /*
193 * Prepare alternate object sources for the given database by reading
194 * "objects/info/alternates" and opening the respective sources.
195 */
196 void odb_prepare_alternates(struct object_database *odb);
197
198 /*
199 * Check whether the object database has any alternates. The primary object
200 * source does not count as alternate.
201 */
202 int odb_has_alternates(struct object_database *odb);
203
204 /*
205 * Add the directory to the on-disk alternates file; the new entry will also
206 * take effect in the current process.
207 */
208 void odb_add_to_alternates_file(struct object_database *odb,
209 const char *dir);
210
211 /*
212 * Add the directory to the in-memory list of alternate sources (along with any
213 * recursive alternates it points to), but do not modify the on-disk alternates
214 * file.
215 */
216 struct odb_source *odb_add_to_alternates_memory(struct object_database *odb,
217 const char *dir);
218
219 /*
220 * Read an object from the database. Returns the object data and assigns object
221 * type and size to the `type` and `size` pointers, if these pointers are
222 * non-NULL. Returns a `NULL` pointer in case the object does not exist.
223 *
224 * This function dies on corrupt objects; the callers who want to deal with
225 * them should arrange to call odb_read_object_info_extended() and give error
226 * messages themselves.
227 */
228 void *odb_read_object(struct object_database *odb,
229 const struct object_id *oid,
230 enum object_type *type,
231 size_t *size);
232
233 void *odb_read_object_peeled(struct object_database *odb,
234 const struct object_id *oid,
235 enum object_type required_type,
236 size_t *size,
237 struct object_id *oid_ret);
238
239 /*
240 * Add an object file to the in-memory object store, without writing it
241 * to disk.
242 *
243 * Callers are responsible for calling write_object_file to record the
244 * object in persistent storage before writing any other new objects
245 * that reference it.
246 */
247 int odb_pretend_object(struct object_database *odb,
248 void *buf, size_t len, enum object_type type,
249 struct object_id *oid);
250
251 struct object_info {
252 /* Request */
253 enum object_type *typep;
254 size_t *sizep;
255 off_t *disk_sizep;
256 struct object_id *delta_base_oid;
257 void **contentp;
258
259 /*
260 * The time the given looked-up object has been last modified.
261 *
262 * Note: the mtime may be ambiguous in case the object exists multiple
263 * times in the object database. It is thus _not_ recommended to use
264 * this field outside of contexts where you would read every instance
265 * of the object, like for example with `odb_for_each_object()`. As it
266 * is impossible to say at the ODB level what the intent of the caller
267 * is (e.g. whether to find the oldest or newest object), it is the
268 * responsibility of the caller to disambiguate the mtimes.
269 */
270 time_t *mtimep;
271
272 /* Response */
273 enum {
274 OI_CACHED,
275 OI_LOOSE,
276 OI_PACKED,
277 } whence;
278 union {
279 /*
280 * struct {
281 * ... Nothing to expose in this case
282 * } cached;
283 * struct {
284 * ... Nothing to expose in this case
285 * } loose;
286 */
287 struct {
288 struct packed_git *pack;
289 off_t offset;
290 enum packed_object_type {
291 PACKED_OBJECT_TYPE_UNKNOWN,
292 PACKED_OBJECT_TYPE_FULL,
293 PACKED_OBJECT_TYPE_OFS_DELTA,
294 PACKED_OBJECT_TYPE_REF_DELTA,
295 } type;
296 } packed;
297 } u;
298 };
299
300 /*
301 * Initializer for a "struct object_info" that wants no items. You may
302 * also memset() the memory to all-zeroes.
303 */
304 #define OBJECT_INFO_INIT { 0 }
305
306 /* Flags that can be passed to `odb_read_object_info_extended()`. */
307 enum object_info_flags {
308 /* Invoke lookup_replace_object() on the given hash. */
309 OBJECT_INFO_LOOKUP_REPLACE = (1 << 0),
310
311 /* Do not reprepare object sources when the first lookup has failed. */
312 OBJECT_INFO_QUICK = (1 << 1),
313
314 /*
315 * Do not attempt to fetch the object if missing (even if fetch_is_missing is
316 * nonzero).
317 */
318 OBJECT_INFO_SKIP_FETCH_OBJECT = (1 << 2),
319
320 /* Die if object corruption (not just an object being missing) was detected. */
321 OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3),
322
323 /*
324 * We have already tried reading the object, but it couldn't be found
325 * via any of the attached sources, and are now doing a second read.
326 * This second read asks the individual sources to also evaluate
327 * whether any on-disk state may have changed that may have caused the
328 * object to appear.
329 *
330 * This flag is for internal use, only. The second read only occurs
331 * when `OBJECT_INFO_QUICK` was not passed.
332 */
333 OBJECT_INFO_SECOND_READ = (1 << 4),
334
335 /*
336 * This is meant for bulk prefetching of missing blobs in a partial
337 * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK.
338 */
339 OBJECT_INFO_FOR_PREFETCH = (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK),
340 };
341
342 /*
343 * Read object info from the object database and populate the `object_info`
344 * structure. Returns 0 on success, a negative error code otherwise.
345 */
346 int odb_read_object_info_extended(struct object_database *odb,
347 const struct object_id *oid,
348 struct object_info *oi,
349 enum object_info_flags flags);
350
351 /*
352 * Read a subset of object info for the given object ID. Returns an `enum
353 * object_type` on success, a negative error code otherwise. If successful and
354 * `sizep` is non-NULL, then the size of the object will be written to the
355 * pointer.
356 */
357 int odb_read_object_info(struct object_database *odb,
358 const struct object_id *oid,
359 size_t *sizep);
360
361 enum odb_has_object_flags {
362 /* Retry packed storage after checking packed and loose storage */
363 ODB_HAS_OBJECT_RECHECK_PACKED = (1 << 0),
364 /* Allow fetching the object in case the repository has a promisor remote. */
365 ODB_HAS_OBJECT_FETCH_PROMISOR = (1 << 1),
366 };
367
368 /*
369 * Returns 1 if the object exists. This function will not lazily fetch objects
370 * in a partial clone by default.
371 */
372 int odb_has_object(struct object_database *odb,
373 const struct object_id *oid,
374 enum odb_has_object_flags flags);
375
376 int odb_freshen_object(struct object_database *odb,
377 const struct object_id *oid);
378
379 void odb_assert_oid_type(struct object_database *odb,
380 const struct object_id *oid, enum object_type expect);
381
382 /*
383 * Enabling the object read lock allows multiple threads to safely call the
384 * following functions in parallel: odb_read_object(),
385 * odb_read_object_peeled(), odb_read_object_info() and odb().
386 *
387 * obj_read_lock() and obj_read_unlock() may also be used to protect other
388 * section which cannot execute in parallel with object reading. Since the used
389 * lock is a recursive mutex, these sections can even contain calls to object
390 * reading functions. However, beware that in these cases zlib inflation won't
391 * be performed in parallel, losing performance.
392 *
393 * TODO: odb_read_object_info_extended()'s call stack has a recursive behavior. If
394 * any of its callees end up calling it, this recursive call won't benefit from
395 * parallel inflation.
396 */
397 void enable_obj_read_lock(void);
398 void disable_obj_read_lock(void);
399
400 extern int obj_read_use_lock;
401 extern pthread_mutex_t obj_read_mutex;
402
403 static inline void obj_read_lock(void)
404 {
405 if(obj_read_use_lock)
406 pthread_mutex_lock(&obj_read_mutex);
407 }
408
409 static inline void obj_read_unlock(void)
410 {
411 if(obj_read_use_lock)
412 pthread_mutex_unlock(&obj_read_mutex);
413 }
414
415 /* Flags for for_each_*_object(). */
416 enum odb_for_each_object_flags {
417 /* Iterate only over local objects, not alternates. */
418 ODB_FOR_EACH_OBJECT_LOCAL_ONLY = (1<<0),
419
420 /* Only iterate over packs obtained from the promisor remote. */
421 ODB_FOR_EACH_OBJECT_PROMISOR_ONLY = (1<<1),
422
423 /*
424 * Visit objects within a pack in packfile order rather than .idx order
425 */
426 ODB_FOR_EACH_OBJECT_PACK_ORDER = (1<<2),
427
428 /* Only iterate over packs that are not marked as kept in-core. */
429 ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS = (1<<3),
430
431 /* Only iterate over packs that do not have .keep files. */
432 ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4),
433 };
434
435 /*
436 * A callback function that can be used to iterate through objects. If given,
437 * the optional `oi` parameter will be populated the same as if you would call
438 * `odb_read_object_info()`.
439 *
440 * Returning a non-zero error code will cause iteration to abort. The error
441 * code will be propagated.
442 */
443 typedef int (*odb_for_each_object_cb)(const struct object_id *oid,
444 struct object_info *oi,
445 void *cb_data);
446
447 /*
448 * Options that can be passed to `odb_for_each_object()` and its
449 * backend-specific implementations.
450 */
451 struct odb_for_each_object_options {
452 /* A bitfield of `odb_for_each_object_flags`. */
453 enum odb_for_each_object_flags flags;
454
455 /*
456 * If set, only iterate through objects whose first `prefix_hex_len`
457 * hex characters matches the given prefix.
458 */
459 const struct object_id *prefix;
460 size_t prefix_hex_len;
461 };
462
463 /*
464 * Iterate through all objects contained in the object database. Note that
465 * objects may be iterated over multiple times in case they are either stored
466 * in different backends or in case they are stored in multiple sources.
467 * If an object info request is given, then the object info will be read and
468 * passed to the callback as if `odb_read_object_info()` was called for the
469 * object.
470 *
471 * Returning a non-zero error code from the callback function will cause
472 * iteration to abort. The error code will be propagated.
473 *
474 * Returns 0 on success, a negative error code in case a failure occurred, or
475 * an arbitrary non-zero error code returned by the callback itself.
476 */
477 int odb_for_each_object_ext(struct object_database *odb,
478 const struct object_info *request,
479 odb_for_each_object_cb cb,
480 void *cb_data,
481 const struct odb_for_each_object_options *opts);
482
483 /* Same as `odb_for_each_object_ext()` with `opts.flags` set to the given flags. */
484 int odb_for_each_object(struct object_database *odb,
485 const struct object_info *request,
486 odb_for_each_object_cb cb,
487 void *cb_data,
488 enum odb_for_each_object_flags flags);
489
490 enum odb_count_objects_flags {
491 /*
492 * Instead of providing an accurate count, allow the number of objects
493 * to be approximated. Details of how this approximation works are
494 * subject to the specific source's implementation.
495 */
496 ODB_COUNT_OBJECTS_APPROXIMATE = (1 << 0),
497 };
498
499 /*
500 * Count the number of objects in the given object database. This object count
501 * may double-count objects that are stored in multiple backends, or which are
502 * stored multiple times in a single backend.
503 *
504 * Returns 0 on success, a negative error code otherwise. The number of objects
505 * will be assigned to the `out` pointer on success.
506 */
507 int odb_count_objects(struct object_database *odb,
508 enum odb_count_objects_flags flags,
509 unsigned long *out);
510
511 /*
512 * Given an object ID, find the minimum required length required to make the
513 * object ID unique across the whole object database.
514 *
515 * The `min_len` determines the minimum abbreviated length that'll be returned
516 * by this function. If `min_len < 0`, then the function will set a sensible
517 * default minimum abbreviation length.
518 *
519 * Returns 0 on success, a negative error code otherwise. The computed length
520 * will be assigned to `*out`.
521 */
522 int odb_find_abbrev_len(struct object_database *odb,
523 const struct object_id *oid,
524 int min_len,
525 unsigned *out);
526
527 enum odb_write_object_flags {
528 /*
529 * By default, `odb_write_object()` does not actually write anything
530 * into the object store, but only computes the object ID. This flag
531 * changes that so that the object will be written as a loose object
532 * and persisted.
533 */
534 ODB_WRITE_OBJECT_PERSIST = (1 << 0),
535
536 /*
537 * Do not print an error in case something goes wrong.
538 */
539 ODB_WRITE_OBJECT_SILENT = (1 << 1),
540 };
541
542 /*
543 * Write an object into the object database. The object is being written into
544 * the local alternate of the repository. If provided, the converted object ID
545 * as well as the compatibility object ID are written to the respective
546 * pointers.
547 *
548 * Returns 0 on success, a negative error code otherwise.
549 */
550 int odb_write_object_ext(struct object_database *odb,
551 const void *buf, unsigned long len,
552 enum object_type type,
553 struct object_id *oid,
554 struct object_id *compat_oid,
555 enum odb_write_object_flags flags);
556
557 static inline int odb_write_object(struct object_database *odb,
558 const void *buf, unsigned long len,
559 enum object_type type,
560 struct object_id *oid)
561 {
562 return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0);
563 }
564
565 struct odb_write_stream;
566
567 int odb_write_object_stream(struct object_database *odb,
568 struct odb_write_stream *stream, size_t len,
569 struct object_id *oid);
570
571 void parse_alternates(const char *string,
572 int sep,
573 const char *relative_base,
574 struct strvec *out);
575
576 #endif /* ODB_H */