Raw
1 #ifndef ODB_H
2 #define ODB_H
3
4 #include "object.h"
5 #include "oidset.h"
6 #include "oidmap.h"
7 #include "string-list.h"
8 #include "thread-utils.h"
9
10 struct cached_object_entry;
11 struct list_objects_filter_options;
12 struct odb_source_inmemory;
13 struct packed_git;
14 struct repository;
15 struct strbuf;
16 struct strvec;
17
18 /*
19 * Set this to 0 to prevent odb_read_object_info_extended() from fetching missing
20 * blobs. This has a difference only if extensions.partialClone is set.
21 *
22 * Its default value is 1.
23 */
24 extern int fetch_if_missing;
25
26 /*
27 * Compute the exact path an alternate is at and returns it. In case of
28 * error NULL is returned and the human readable error is added to `err`
29 * `path` may be relative and should point to $GIT_DIR.
30 * `err` must not be null.
31 */
32 char *compute_alternate_path(const char *path, struct strbuf *err);
33
34 /*
35 * The object database encapsulates access to objects in a repository. It
36 * manages one or more sources that store the actual objects which are
37 * configured via alternates.
38 */
39 struct object_database {
40 /* Repository that owns this database. */
41 struct repository *repo;
42
43 /*
44 * State of current object database transaction. Only one
45 * transaction may be pending at a time. Is NULL when no transaction is
46 * configured.
47 */
48 struct odb_transaction *transaction;
49
50 /*
51 * Set of all object directories; the main directory is first (and
52 * cannot be NULL after initialization). Subsequent directories are
53 * alternates.
54 */
55 struct odb_source *sources;
56 struct odb_source **sources_tail;
57 struct kh_odb_path_map *source_by_path;
58
59 int loaded_alternates;
60
61 /*
62 * A list of alternate object directories loaded from the environment;
63 * this should not generally need to be accessed directly, but will
64 * populate the "sources" list when odb_prepare_alternates() is run.
65 */
66 char *alternate_db;
67
68 /*
69 * Objects that should be substituted by other objects
70 * (see git-replace(1)).
71 */
72 struct oidmap replace_map;
73 unsigned replace_map_initialized : 1;
74 pthread_mutex_t replace_mutex; /* protect object replace functions */
75
76 struct commit_graph *commit_graph;
77 unsigned commit_graph_attempted : 1; /* if loading has been attempted */
78
79 /*
80 * This is meant to hold a *small* number of objects that you would
81 * want odb_read_object() to be able to return, but yet you do not want
82 * to write them into the object store (e.g. a browse-only
83 * application).
84 */
85 struct odb_source *inmemory_objects;
86
87 /*
88 * A fast, rough count of the number of objects in the repository.
89 * These two fields are not meant for direct access. Use
90 * odb_count_objects() instead.
91 */
92 unsigned long object_count;
93 unsigned object_count_flags;
94 unsigned object_count_valid : 1;
95
96 /*
97 * Submodule source paths that will be added as additional sources to
98 * allow lookup of submodule objects via the main object database.
99 */
100 struct string_list submodule_source_paths;
101 };
102
103 /*
104 * Create a new object database for the given repository.
105 *
106 * If the primary source parameter is set it will override the usual primary
107 * object directory derived from the repository's common directory. The
108 * alternate sources are expected to be a PATH_SEP-separated list of secondary
109 * sources. Note that these alternate sources will be added in addition to, not
110 * instead of, the alternates identified by the primary source.
111 *
112 * Returns the newly created object database.
113 */
114 struct object_database *odb_new(struct repository *repo,
115 const char *primary_source,
116 const char *alternate_sources);
117
118 /* Free the object database and release all resources. */
119 void odb_free(struct object_database *o);
120
121 /*
122 * Close the object database and all of its sources so that any held resources
123 * will be released. The database can still be used after closing it, in which
124 * case these resources may be reallocated.
125 */
126 void odb_close(struct object_database *o);
127
128 enum odb_prepare_flags {
129 /*
130 * Flush caches, reload alternates and then re-prepare each object
131 * source so that new objects may become accessible.
132 */
133 ODB_PREPARE_FLUSH_CACHES = (1 << 0),
134 };
135
136 /*
137 * Prepare the object database for use. Calling this function is generally not
138 * needed, but can be useful in case the caller wants to pre-open individual
139 * sources.
140 */
141 void odb_prepare(struct object_database *o, enum odb_prepare_flags flags);
142
143 /* Equivalent to `odb_prepare(o, ODB_PREPARE_FLUSH_CACHES)`. */
144 void odb_reprepare(struct object_database *o);
145
146 /*
147 * Find source by its object directory path. Returns a `NULL` pointer in case
148 * the source could not be found.
149 */
150 struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir);
151
152 /* Same as `odb_find_source()`, but dies in case the source doesn't exist. */
153 struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir);
154
155 /*
156 * Replace the current writable object directory with the specified temporary
157 * object directory; returns the former primary source.
158 */
159 struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
160 const char *dir, int will_destroy);
161
162 /*
163 * Restore the primary source that was previously replaced by
164 * `odb_set_temporary_primary_source()`.
165 */
166 void odb_restore_primary_source(struct object_database *odb,
167 struct odb_source *restore_source,
168 const char *old_path);
169
170 /*
171 * Call odb_add_submodule_source_by_path() to add the submodule at the given
172 * path to a list. The object stores of all submodules in that list will be
173 * added as additional sources in the object store when looking up objects.
174 */
175 void odb_add_submodule_source_by_path(struct object_database *odb,
176 const char *path);
177
178 /*
179 * Iterate through all alternates of the database and execute the provided
180 * callback function for each of them. Stop iterating once the callback
181 * function returns a non-zero value, in which case the value is bubbled up
182 * from the callback.
183 */
184 typedef int odb_for_each_alternate_fn(struct odb_source *, void *);
185 int odb_for_each_alternate(struct object_database *odb,
186 odb_for_each_alternate_fn cb, void *payload);
187
188 /*
189 * Iterate through all alternates of the database and yield their respective
190 * references.
191 */
192 typedef void odb_for_each_alternate_ref_fn(const struct object_id *oid, void *);
193 void odb_for_each_alternate_ref(struct object_database *odb,
194 odb_for_each_alternate_ref_fn cb, void *payload);
195
196 /*
197 * Create a temporary file rooted in the primary alternate's directory, or die
198 * on failure. The filename is taken from "pattern", which should have the
199 * usual "XXXXXX" trailer, and the resulting filename is written into the
200 * "template" buffer. Returns the open descriptor.
201 */
202 int odb_mkstemp(struct object_database *odb,
203 struct strbuf *temp_filename, const char *pattern);
204
205 /*
206 * Prepare alternate object sources for the given database by reading
207 * "objects/info/alternates" and opening the respective sources.
208 */
209 void odb_prepare_alternates(struct object_database *odb);
210
211 /*
212 * Check whether the object database has any alternates. The primary object
213 * source does not count as alternate.
214 */
215 int odb_has_alternates(struct object_database *odb);
216
217 /*
218 * Add the directory to the on-disk alternates file; the new entry will also
219 * take effect in the current process.
220 */
221 void odb_add_to_alternates_file(struct object_database *odb,
222 const char *dir);
223
224 /*
225 * Add the directory to the in-memory list of alternate sources (along with any
226 * recursive alternates it points to), but do not modify the on-disk alternates
227 * file.
228 */
229 struct odb_source *odb_add_to_alternates_memory(struct object_database *odb,
230 const char *dir);
231
232 /*
233 * Read an object from the database. Returns the object data and assigns object
234 * type and size to the `type` and `size` pointers, if these pointers are
235 * non-NULL. Returns a `NULL` pointer in case the object does not exist.
236 *
237 * This function dies on corrupt objects; the callers who want to deal with
238 * them should arrange to call odb_read_object_info_extended() and give error
239 * messages themselves.
240 */
241 void *odb_read_object(struct object_database *odb,
242 const struct object_id *oid,
243 enum object_type *type,
244 size_t *size);
245
246 void *odb_read_object_peeled(struct object_database *odb,
247 const struct object_id *oid,
248 enum object_type required_type,
249 size_t *size,
250 struct object_id *oid_ret);
251
252 /*
253 * Add an object file to the in-memory object store, without writing it
254 * to disk.
255 *
256 * Callers are responsible for calling write_object_file to record the
257 * object in persistent storage before writing any other new objects
258 * that reference it.
259 */
260 int odb_pretend_object(struct object_database *odb,
261 void *buf, size_t len, enum object_type type,
262 struct object_id *oid);
263
264 /*
265 * Object database source information that can be used to uniquely identify an
266 * object and learn more about how exactly it is stored.
267 */
268 struct odb_source_info {
269 /* The source that this object has been looked up from. */
270 struct odb_source *source;
271
272 /*
273 * Backend-specific information about the specific object. This can be
274 * used for example to uniquely identify a given object in case it
275 * exists multiple times.
276 */
277 union {
278 /*
279 * struct {
280 * ... Nothing to expose in this case
281 * } cached;
282 * struct {
283 * ... Nothing to expose in this case
284 * } loose;
285 */
286 struct {
287 struct packed_git *pack;
288 off_t offset;
289 enum packed_object_type {
290 PACKED_OBJECT_TYPE_UNKNOWN,
291 PACKED_OBJECT_TYPE_FULL,
292 PACKED_OBJECT_TYPE_OFS_DELTA,
293 PACKED_OBJECT_TYPE_REF_DELTA,
294 } type;
295 } packed;
296 } u;
297 };
298
299 /*
300 * The object info contains the query and response that is to be used for
301 * functions that end up reading object information. Callers are expected to
302 * populate pointers whose information they want to request.
303 */
304 struct object_info {
305 /* The object type. */
306 enum object_type *typep;
307
308 /* The inflated object size in bytes. */
309 size_t *sizep;
310
311 /* The object size as stored on disk. */
312 off_t *disk_sizep;
313
314 /*
315 * The base the object is deltified against, in case it is stored as a
316 * delta.
317 */
318 struct object_id *delta_base_oid;
319
320 /* The object contents. Ownership of memory goes over to the caller. */
321 void **contentp;
322
323 /*
324 * The time the given looked-up object has been last modified.
325 *
326 * Note: the mtime may be ambiguous in case the object exists multiple
327 * times in the object database. It is thus _not_ recommended to use
328 * this field outside of contexts where you would read every instance
329 * of the object, like for example with `odb_for_each_object()`. As it
330 * is impossible to say at the ODB level what the intent of the caller
331 * is (e.g. whether to find the oldest or newest object), it is the
332 * responsibility of the caller to disambiguate the mtimes.
333 */
334 time_t *mtimep;
335
336 /*
337 * Backend-specific information that tells the caller where exactly an
338 * object was looked up from. This information should help disambiguate
339 * object lookups in case the same object exists in multiple sources,
340 * or multiple times in the same source.
341 */
342 struct odb_source_info *source_infop;
343
344 /*
345 * object-info protocol specific. Set by the protocol when the remote
346 * does not recognize the requested object.
347 */
348 unsigned int unrecognized:1;
349 };
350
351 /*
352 * Initializer for a "struct object_info" that wants no items. You may
353 * also memset() the memory to all-zeroes.
354 */
355 #define OBJECT_INFO_INIT { 0 }
356
357 /* Flags that can be passed to `odb_read_object_info_extended()`. */
358 enum object_info_flags {
359 /* Invoke lookup_replace_object() on the given hash. */
360 OBJECT_INFO_LOOKUP_REPLACE = (1 << 0),
361
362 /* Do not reprepare object sources when the first lookup has failed. */
363 OBJECT_INFO_QUICK = (1 << 1),
364
365 /*
366 * Do not attempt to fetch the object if missing (even if fetch_is_missing is
367 * nonzero).
368 */
369 OBJECT_INFO_SKIP_FETCH_OBJECT = (1 << 2),
370
371 /* Die if object corruption (not just an object being missing) was detected. */
372 OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3),
373
374 /*
375 * We have already tried reading the object, but it couldn't be found
376 * via any of the attached sources, and are now doing a second read.
377 * This second read asks the individual sources to also evaluate
378 * whether any on-disk state may have changed that may have caused the
379 * object to appear.
380 *
381 * This flag is for internal use, only. The second read only occurs
382 * when `OBJECT_INFO_QUICK` was not passed.
383 */
384 OBJECT_INFO_SECOND_READ = (1 << 4),
385
386 /*
387 * This is meant for bulk prefetching of missing blobs in a partial
388 * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK.
389 */
390 OBJECT_INFO_FOR_PREFETCH = (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK),
391 };
392
393 /*
394 * Read object info from the object database and populate the `object_info`
395 * structure. Returns 0 on success, a negative error code otherwise.
396 */
397 int odb_read_object_info_extended(struct object_database *odb,
398 const struct object_id *oid,
399 struct object_info *oi,
400 enum object_info_flags flags);
401
402 /*
403 * Read a subset of object info for the given object ID. Returns an `enum
404 * object_type` on success, a negative error code otherwise. If successful and
405 * `sizep` is non-NULL, then the size of the object will be written to the
406 * pointer.
407 */
408 int odb_read_object_info(struct object_database *odb,
409 const struct object_id *oid,
410 size_t *sizep);
411
412 enum odb_has_object_flags {
413 /* Retry packed storage after checking packed and loose storage */
414 ODB_HAS_OBJECT_RECHECK_PACKED = (1 << 0),
415 /* Allow fetching the object in case the repository has a promisor remote. */
416 ODB_HAS_OBJECT_FETCH_PROMISOR = (1 << 1),
417 };
418
419 /*
420 * Returns 1 if the object exists. This function will not lazily fetch objects
421 * in a partial clone by default.
422 */
423 int odb_has_object(struct object_database *odb,
424 const struct object_id *oid,
425 enum odb_has_object_flags flags);
426
427 int odb_freshen_object(struct object_database *odb,
428 const struct object_id *oid);
429
430 void odb_assert_oid_type(struct object_database *odb,
431 const struct object_id *oid, enum object_type expect);
432
433 /*
434 * Enabling the object read lock allows multiple threads to safely call the
435 * following functions in parallel: odb_read_object(),
436 * odb_read_object_peeled(), odb_read_object_info() and odb().
437 *
438 * obj_read_lock() and obj_read_unlock() may also be used to protect other
439 * section which cannot execute in parallel with object reading. Since the used
440 * lock is a recursive mutex, these sections can even contain calls to object
441 * reading functions. However, beware that in these cases zlib inflation won't
442 * be performed in parallel, losing performance.
443 *
444 * TODO: odb_read_object_info_extended()'s call stack has a recursive behavior. If
445 * any of its callees end up calling it, this recursive call won't benefit from
446 * parallel inflation.
447 */
448 void enable_obj_read_lock(void);
449 void disable_obj_read_lock(void);
450
451 extern int obj_read_use_lock;
452 extern pthread_mutex_t obj_read_mutex;
453
454 static inline void obj_read_lock(void)
455 {
456 if(obj_read_use_lock)
457 pthread_mutex_lock(&obj_read_mutex);
458 }
459
460 static inline void obj_read_unlock(void)
461 {
462 if(obj_read_use_lock)
463 pthread_mutex_unlock(&obj_read_mutex);
464 }
465
466 /* Flags for for_each_*_object(). */
467 enum odb_for_each_object_flags {
468 /* Iterate only over local objects, not alternates. */
469 ODB_FOR_EACH_OBJECT_LOCAL_ONLY = (1<<0),
470
471 /* Only iterate over packs obtained from the promisor remote. */
472 ODB_FOR_EACH_OBJECT_PROMISOR_ONLY = (1<<1),
473
474 /*
475 * Visit objects within a pack in packfile order rather than .idx order
476 */
477 ODB_FOR_EACH_OBJECT_PACK_ORDER = (1<<2),
478
479 /* Only iterate over packs that are not marked as kept in-core. */
480 ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS = (1<<3),
481
482 /* Only iterate over packs that do not have .keep files. */
483 ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4),
484 };
485
486 /*
487 * A callback function that can be used to iterate through objects. If given,
488 * the optional `oi` parameter will be populated the same as if you would call
489 * `odb_read_object_info()`.
490 *
491 * Returning a non-zero error code will cause iteration to abort. The error
492 * code will be propagated.
493 */
494 typedef int (*odb_for_each_object_cb)(const struct object_id *oid,
495 struct object_info *oi,
496 void *cb_data);
497
498 /*
499 * Options that can be passed to `odb_for_each_object()` and its
500 * backend-specific implementations.
501 */
502 struct odb_for_each_object_options {
503 /* A bitfield of `odb_for_each_object_flags`. */
504 enum odb_for_each_object_flags flags;
505
506 /*
507 * If set, only iterate through objects whose first `prefix_hex_len`
508 * hex characters matches the given prefix.
509 */
510 const struct object_id *prefix;
511 size_t prefix_hex_len;
512
513 /*
514 * Optional object filter that allows backends to skip yielding
515 * objects that are excluded by the filter as an optimization. The
516 * filter is a best-effort hint: backends may use it to skip
517 * excluded objects (e.g. by consulting a reachability bitmap), but
518 * are also free to ignore it entirely and yield every object. As a
519 * consequence, callers must re-apply the filter on yielded objects
520 * if they require strict filtering semantics.
521 */
522 const struct list_objects_filter_options *filter;
523 };
524
525 /*
526 * Iterate through all objects contained in the object database. Note that
527 * objects may be iterated over multiple times in case they are either stored
528 * in different backends or in case they are stored in multiple sources.
529 * If an object info request is given, then the object info will be read and
530 * passed to the callback as if `odb_read_object_info()` was called for the
531 * object.
532 *
533 * Returning a non-zero error code from the callback function will cause
534 * iteration to abort. The error code will be propagated.
535 *
536 * Returns 0 on success, a negative error code in case a failure occurred, or
537 * an arbitrary non-zero error code returned by the callback itself.
538 */
539 int odb_for_each_object_ext(struct object_database *odb,
540 const struct object_info *request,
541 odb_for_each_object_cb cb,
542 void *cb_data,
543 const struct odb_for_each_object_options *opts);
544
545 /* Same as `odb_for_each_object_ext()` with `opts.flags` set to the given flags. */
546 int odb_for_each_object(struct object_database *odb,
547 const struct object_info *request,
548 odb_for_each_object_cb cb,
549 void *cb_data,
550 enum odb_for_each_object_flags flags);
551
552 enum odb_count_objects_flags {
553 /*
554 * Instead of providing an accurate count, allow the number of objects
555 * to be approximated. Details of how this approximation works are
556 * subject to the specific source's implementation.
557 */
558 ODB_COUNT_OBJECTS_APPROXIMATE = (1 << 0),
559 };
560
561 /*
562 * Count the number of objects in the given object database. This object count
563 * may double-count objects that are stored in multiple backends, or which are
564 * stored multiple times in a single backend.
565 *
566 * Returns 0 on success, a negative error code otherwise. The number of objects
567 * will be assigned to the `out` pointer on success.
568 */
569 int odb_count_objects(struct object_database *odb,
570 enum odb_count_objects_flags flags,
571 unsigned long *out);
572
573 /*
574 * Given an object ID, find the minimum required length required to make the
575 * object ID unique across the whole object database.
576 *
577 * The `min_len` determines the minimum abbreviated length that'll be returned
578 * by this function. If `min_len < 0`, then the function will set a sensible
579 * default minimum abbreviation length.
580 *
581 * Returns 0 on success, a negative error code otherwise. The computed length
582 * will be assigned to `*out`.
583 */
584 int odb_find_abbrev_len(struct object_database *odb,
585 const struct object_id *oid,
586 int min_len,
587 unsigned *out);
588
589 enum odb_write_object_flags {
590 /*
591 * By default, `odb_write_object()` does not actually write anything
592 * into the object store, but only computes the object ID. This flag
593 * changes that so that the object will be written as a loose object
594 * and persisted.
595 */
596 ODB_WRITE_OBJECT_PERSIST = (1 << 0),
597
598 /*
599 * Do not print an error in case something goes wrong.
600 */
601 ODB_WRITE_OBJECT_SILENT = (1 << 1),
602 };
603
604 /*
605 * Write an object into the object database. The object is being written into
606 * the local alternate of the repository. If provided, the object ID of the
607 * final object is written into `oid`.
608 *
609 * If the caller provides a `compat_oid`, then this compatibility object hash
610 * will be stored instead of computing the compatibility hash ad-hoc.
611 *
612 * Returns 0 on success, a negative error code otherwise.
613 */
614 int odb_write_object_ext(struct object_database *odb,
615 const void *buf, unsigned long len,
616 enum object_type type,
617 struct object_id *oid,
618 const struct object_id *compat_oid,
619 enum odb_write_object_flags flags);
620
621 static inline int odb_write_object(struct object_database *odb,
622 const void *buf, unsigned long len,
623 enum object_type type,
624 struct object_id *oid)
625 {
626 return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0);
627 }
628
629 struct odb_write_stream;
630
631 int odb_write_object_stream(struct object_database *odb,
632 struct odb_write_stream *stream, size_t len,
633 struct object_id *oid);
634
635 void parse_alternates(const char *string,
636 int sep,
637 const char *relative_base,
638 struct strvec *out);
639
640 /* Free pointers inside of object_info, but not object_info itself */
641 void free_object_info_contents(struct object_info *object_info);
642
643 #endif /* ODB_H */