@matheusbsbs / matheus-bagliano / commits / 0abe14f6a5

pack: move {,re}prepare_packed_git and approximate_object_count

Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jonathan Tan committed Aug 18, 2017 at 15:20 UTC 0abe14f6a5b7681b4ea02d1bd0a64332587ae053

11 files changed +238 -232

builtin/gc.c

@@ -19,6 +19,7 @@

19 #include "sigchain.h"

20 #include "argv-array.h"

21 #include "commit.h"

22 +#include "packfile.h"

24 #define FAILED_RUN "failed to run %s"

bulk-checkin.c

@@ -6,6 +6,7 @@

6 #include "csum-file.h"

7 #include "pack.h"

8 #include "strbuf.h"

9 +#include "packfile.h"

11 static struct bulk_checkin_state {

12 unsigned plugged:1;

cache.h

-15

@@ -1605,21 +1605,6 @@ struct pack_entry {

1605 struct packed_git *p;

1606 };

1607

1608 -/* A hook to report invalid files in pack directory */

1609 -#define PACKDIR_FILE_PACK 1

1610 -#define PACKDIR_FILE_IDX 2

1611 -#define PACKDIR_FILE_GARBAGE 4

1612 -extern void (*report_garbage)(unsigned seen_bits, const char *path);

1613 -

1614 -extern void prepare_packed_git(void);

1615 -extern void reprepare_packed_git(void);

1616 -

1617 -/*

1618 - * Give a rough count of objects in the repository. This sacrifices accuracy

1619 - * for speed.

1620 - */

1621 -unsigned long approximate_object_count(void);

1622 -

1608 extern struct packed_git *find_sha1_pack(const unsigned char *sha1,

1609 struct packed_git *packs);

1610

connected.c

+1 -1

@@ -3,7 +3,7 @@

3 #include "sigchain.h"

4 #include "connected.h"

5 #include "transport.h"

6 -#include "pack.h"

6 +#include "packfile.h"

8 /*

9 * If we feed all the commits we want to verify to this command

fetch-pack.c

@@ -17,6 +17,7 @@

17 #include "prio-queue.h"

18 #include "sha1-array.h"

19 #include "oidset.h"

20 +#include "packfile.h"

22 static int transfer_unpack_limit = -1;

23 static int fetch_unpack_limit = -1;

http-backend.c

@@ -9,6 +9,7 @@

9 #include "string-list.h"

10 #include "url.h"

11 #include "argv-array.h"

12 +#include "packfile.h"

14 static const char content_type[] = "Content-Type";

15 static const char content_length[] = "Content-Length";

packfile.c

+217

@@ -1,6 +1,9 @@

1 #include "cache.h"

2 #include "mru.h"

3 #include "pack.h"

4 +#include "dir.h"

5 +#include "mergesort.h"

6 +#include "packfile.h"

8 char *odb_pack_name(struct strbuf *buf,

9 const unsigned char *sha1,

@@ -667,3 +670,217 @@ void install_packed_git(struct packed_git *pack)

670 pack->next = packed_git;

671 packed_git = pack;

672 }

673 +

674 +void (*report_garbage)(unsigned seen_bits, const char *path);

675 +

676 +static void report_helper(const struct string_list *list,

677 + int seen_bits, int first, int last)

678 +{

679 + if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))

680 + return;

681 +

682 + for (; first < last; first++)

683 + report_garbage(seen_bits, list->items[first].string);

684 +}

685 +

686 +static void report_pack_garbage(struct string_list *list)

687 +{

688 + int i, baselen = -1, first = 0, seen_bits = 0;

689 +

690 + if (!report_garbage)

691 + return;

692 +

693 + string_list_sort(list);

694 +

695 + for (i = 0; i < list->nr; i++) {

696 + const char *path = list->items[i].string;

697 + if (baselen != -1 &&

698 + strncmp(path, list->items[first].string, baselen)) {

699 + report_helper(list, seen_bits, first, i);

700 + baselen = -1;

701 + seen_bits = 0;

702 + }

703 + if (baselen == -1) {

704 + const char *dot = strrchr(path, '.');

705 + if (!dot) {

706 + report_garbage(PACKDIR_FILE_GARBAGE, path);

707 + continue;

708 + }

709 + baselen = dot - path + 1;

710 + first = i;

711 + }

712 + if (!strcmp(path + baselen, "pack"))

713 + seen_bits |= 1;

714 + else if (!strcmp(path + baselen, "idx"))

715 + seen_bits |= 2;

716 + }

717 + report_helper(list, seen_bits, first, list->nr);

718 +}

719 +

720 +static void prepare_packed_git_one(char *objdir, int local)

721 +{

722 + struct strbuf path = STRBUF_INIT;

723 + size_t dirnamelen;

724 + DIR *dir;

725 + struct dirent *de;

726 + struct string_list garbage = STRING_LIST_INIT_DUP;

727 +

728 + strbuf_addstr(&path, objdir);

729 + strbuf_addstr(&path, "/pack");

730 + dir = opendir(path.buf);

731 + if (!dir) {

732 + if (errno != ENOENT)

733 + error_errno("unable to open object pack directory: %s",

734 + path.buf);

735 + strbuf_release(&path);

736 + return;

737 + }

738 + strbuf_addch(&path, '/');

739 + dirnamelen = path.len;

740 + while ((de = readdir(dir)) != NULL) {

741 + struct packed_git *p;

742 + size_t base_len;

743 +

744 + if (is_dot_or_dotdot(de->d_name))

745 + continue;

746 +

747 + strbuf_setlen(&path, dirnamelen);

748 + strbuf_addstr(&path, de->d_name);

749 +

750 + base_len = path.len;

751 + if (strip_suffix_mem(path.buf, &base_len, ".idx")) {

752 + /* Don't reopen a pack we already have. */

753 + for (p = packed_git; p; p = p->next) {

754 + size_t len;

755 + if (strip_suffix(p->pack_name, ".pack", &len) &&

756 + len == base_len &&

757 + !memcmp(p->pack_name, path.buf, len))

758 + break;

759 + }

760 + if (p == NULL &&

761 + /*

762 + * See if it really is a valid .idx file with

763 + * corresponding .pack file that we can map.

764 + */

765 + (p = add_packed_git(path.buf, path.len, local)) != NULL)

766 + install_packed_git(p);

767 + }

768 +

769 + if (!report_garbage)

770 + continue;

771 +

772 + if (ends_with(de->d_name, ".idx") ||

773 + ends_with(de->d_name, ".pack") ||

774 + ends_with(de->d_name, ".bitmap") ||

775 + ends_with(de->d_name, ".keep"))

776 + string_list_append(&garbage, path.buf);

777 + else

778 + report_garbage(PACKDIR_FILE_GARBAGE, path.buf);

779 + }

780 + closedir(dir);

781 + report_pack_garbage(&garbage);

782 + string_list_clear(&garbage, 0);

783 + strbuf_release(&path);

784 +}

785 +

786 +static int approximate_object_count_valid;

787 +

788 +/*

789 + * Give a fast, rough count of the number of objects in the repository. This

790 + * ignores loose objects completely. If you have a lot of them, then either

791 + * you should repack because your performance will be awful, or they are

792 + * all unreachable objects about to be pruned, in which case they're not really

793 + * interesting as a measure of repo size in the first place.

794 + */

795 +unsigned long approximate_object_count(void)

796 +{

797 + static unsigned long count;

798 + if (!approximate_object_count_valid) {

799 + struct packed_git *p;

800 +

801 + prepare_packed_git();

802 + count = 0;

803 + for (p = packed_git; p; p = p->next) {

804 + if (open_pack_index(p))

805 + continue;

806 + count += p->num_objects;

807 + }

808 + }

809 + return count;

810 +}

811 +

812 +static void *get_next_packed_git(const void *p)

813 +{

814 + return ((const struct packed_git *)p)->next;

815 +}

816 +

817 +static void set_next_packed_git(void *p, void *next)

818 +{

819 + ((struct packed_git *)p)->next = next;

820 +}

821 +

822 +static int sort_pack(const void *a_, const void *b_)

823 +{

824 + const struct packed_git *a = a_;

825 + const struct packed_git *b = b_;

826 + int st;

827 +

828 + /*

829 + * Local packs tend to contain objects specific to our

830 + * variant of the project than remote ones. In addition,

831 + * remote ones could be on a network mounted filesystem.

832 + * Favor local ones for these reasons.

833 + */

834 + st = a->pack_local - b->pack_local;

835 + if (st)

836 + return -st;

837 +

838 + /*

839 + * Younger packs tend to contain more recent objects,

840 + * and more recent objects tend to get accessed more

841 + * often.

842 + */

843 + if (a->mtime < b->mtime)

844 + return 1;

845 + else if (a->mtime == b->mtime)

846 + return 0;

847 + return -1;

848 +}

849 +

850 +static void rearrange_packed_git(void)

851 +{

852 + packed_git = llist_mergesort(packed_git, get_next_packed_git,

853 + set_next_packed_git, sort_pack);

854 +}

855 +

856 +static void prepare_packed_git_mru(void)

857 +{

858 + struct packed_git *p;

859 +

860 + mru_clear(packed_git_mru);

861 + for (p = packed_git; p; p = p->next)

862 + mru_append(packed_git_mru, p);

863 +}

864 +

865 +static int prepare_packed_git_run_once = 0;

866 +void prepare_packed_git(void)

867 +{

868 + struct alternate_object_database *alt;

869 +

870 + if (prepare_packed_git_run_once)

871 + return;

872 + prepare_packed_git_one(get_object_directory(), 1);

873 + prepare_alt_odb();

874 + for (alt = alt_odb_list; alt; alt = alt->next)

875 + prepare_packed_git_one(alt->path, 0);

876 + rearrange_packed_git();

877 + prepare_packed_git_mru();

878 + prepare_packed_git_run_once = 1;

879 +}

880 +

881 +void reprepare_packed_git(void)

882 +{

883 + approximate_object_count_valid = 0;

884 + prepare_packed_git_run_once = 0;

885 + prepare_packed_git();

886 +}

packfile.h

+14 -2

@@ -24,12 +24,24 @@ extern char *sha1_pack_name(const unsigned char *sha1);

24 */

25 extern char *sha1_pack_index_name(const unsigned char *sha1);

27 -extern unsigned int pack_open_fds;

28 -

27 extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);

29 +/* A hook to report invalid files in pack directory */

30 +#define PACKDIR_FILE_PACK 1

31 +#define PACKDIR_FILE_IDX 2

32 +#define PACKDIR_FILE_GARBAGE 4

33 +extern void (*report_garbage)(unsigned seen_bits, const char *path);

34 +

35 +extern void prepare_packed_git(void);

36 +extern void reprepare_packed_git(void);

37 extern void install_packed_git(struct packed_git *pack);

39 +/*

40 + * Give a rough count of objects in the repository. This sacrifices accuracy

41 + * for speed.

42 + */

43 +unsigned long approximate_object_count(void);

44 +

45 extern void pack_report(void);

47 /*

path.c

@@ -9,6 +9,7 @@

9 #include "worktree.h"

10 #include "submodule-config.h"

11 #include "path.h"

12 +#include "packfile.h"

14 static int get_st_mode_bits(const char *path, int *mode)

15 {

server-info.c

@@ -3,6 +3,7 @@

3 #include "object.h"

4 #include "commit.h"

5 #include "tag.h"

6 +#include "packfile.h"

8 /*

9 * Create the file "path" by writing to a temporary file and renaming

sha1_file.c

-214

@@ -719,220 +719,6 @@ void *xmmap(void *start, size_t length,

719 return ret;

720 }

721

722 -void (*report_garbage)(unsigned seen_bits, const char *path);

723 -

724 -static void report_helper(const struct string_list *list,

725 - int seen_bits, int first, int last)

726 -{

727 - if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))

728 - return;

729 -

730 - for (; first < last; first++)

731 - report_garbage(seen_bits, list->items[first].string);

732 -}

733 -

734 -static void report_pack_garbage(struct string_list *list)

735 -{

736 - int i, baselen = -1, first = 0, seen_bits = 0;

737 -

738 - if (!report_garbage)

739 - return;

740 -

741 - string_list_sort(list);

742 -

743 - for (i = 0; i < list->nr; i++) {

744 - const char *path = list->items[i].string;

745 - if (baselen != -1 &&

746 - strncmp(path, list->items[first].string, baselen)) {

747 - report_helper(list, seen_bits, first, i);

748 - baselen = -1;

749 - seen_bits = 0;

750 - }

751 - if (baselen == -1) {

752 - const char *dot = strrchr(path, '.');

753 - if (!dot) {

754 - report_garbage(PACKDIR_FILE_GARBAGE, path);

755 - continue;

756 - }

757 - baselen = dot - path + 1;

758 - first = i;

759 - }

760 - if (!strcmp(path + baselen, "pack"))

761 - seen_bits |= 1;

762 - else if (!strcmp(path + baselen, "idx"))

763 - seen_bits |= 2;

764 - }

765 - report_helper(list, seen_bits, first, list->nr);

766 -}

767 -

768 -static void prepare_packed_git_one(char *objdir, int local)

769 -{

770 - struct strbuf path = STRBUF_INIT;

771 - size_t dirnamelen;

772 - DIR *dir;

773 - struct dirent *de;

774 - struct string_list garbage = STRING_LIST_INIT_DUP;

775 -

776 - strbuf_addstr(&path, objdir);

777 - strbuf_addstr(&path, "/pack");

778 - dir = opendir(path.buf);

779 - if (!dir) {

780 - if (errno != ENOENT)

781 - error_errno("unable to open object pack directory: %s",

782 - path.buf);

783 - strbuf_release(&path);

784 - return;

785 - }

786 - strbuf_addch(&path, '/');

787 - dirnamelen = path.len;

788 - while ((de = readdir(dir)) != NULL) {

789 - struct packed_git *p;

790 - size_t base_len;

791 -

792 - if (is_dot_or_dotdot(de->d_name))

793 - continue;

794 -

795 - strbuf_setlen(&path, dirnamelen);

796 - strbuf_addstr(&path, de->d_name);

797 -

798 - base_len = path.len;

799 - if (strip_suffix_mem(path.buf, &base_len, ".idx")) {

800 - /* Don't reopen a pack we already have. */

801 - for (p = packed_git; p; p = p->next) {

802 - size_t len;

803 - if (strip_suffix(p->pack_name, ".pack", &len) &&

804 - len == base_len &&

805 - !memcmp(p->pack_name, path.buf, len))

806 - break;

807 - }

808 - if (p == NULL &&

809 - /*

810 - * See if it really is a valid .idx file with

811 - * corresponding .pack file that we can map.

812 - */

813 - (p = add_packed_git(path.buf, path.len, local)) != NULL)

814 - install_packed_git(p);

815 - }

816 -

817 - if (!report_garbage)

818 - continue;

819 -

820 - if (ends_with(de->d_name, ".idx") ||

821 - ends_with(de->d_name, ".pack") ||

822 - ends_with(de->d_name, ".bitmap") ||

823 - ends_with(de->d_name, ".keep"))

824 - string_list_append(&garbage, path.buf);

825 - else

826 - report_garbage(PACKDIR_FILE_GARBAGE, path.buf);

827 - }

828 - closedir(dir);

829 - report_pack_garbage(&garbage);

830 - string_list_clear(&garbage, 0);

831 - strbuf_release(&path);

832 -}

833 -

834 -static int approximate_object_count_valid;

835 -

836 -/*

837 - * Give a fast, rough count of the number of objects in the repository. This

838 - * ignores loose objects completely. If you have a lot of them, then either

839 - * you should repack because your performance will be awful, or they are

840 - * all unreachable objects about to be pruned, in which case they're not really

841 - * interesting as a measure of repo size in the first place.

842 - */

843 -unsigned long approximate_object_count(void)

844 -{

845 - static unsigned long count;

846 - if (!approximate_object_count_valid) {

847 - struct packed_git *p;

848 -

849 - prepare_packed_git();

850 - count = 0;

851 - for (p = packed_git; p; p = p->next) {

852 - if (open_pack_index(p))

853 - continue;

854 - count += p->num_objects;

855 - }

856 - }

857 - return count;

858 -}

859 -

860 -static void *get_next_packed_git(const void *p)

861 -{

862 - return ((const struct packed_git *)p)->next;

863 -}

864 -

865 -static void set_next_packed_git(void *p, void *next)

866 -{

867 - ((struct packed_git *)p)->next = next;

868 -}

869 -

870 -static int sort_pack(const void *a_, const void *b_)

871 -{

872 - const struct packed_git *a = a_;

873 - const struct packed_git *b = b_;

874 - int st;

875 -

876 - /*

877 - * Local packs tend to contain objects specific to our

878 - * variant of the project than remote ones. In addition,

879 - * remote ones could be on a network mounted filesystem.

880 - * Favor local ones for these reasons.

881 - */

882 - st = a->pack_local - b->pack_local;

883 - if (st)

884 - return -st;

885 -

886 - /*

887 - * Younger packs tend to contain more recent objects,

888 - * and more recent objects tend to get accessed more

889 - * often.

890 - */

891 - if (a->mtime < b->mtime)

892 - return 1;

893 - else if (a->mtime == b->mtime)

894 - return 0;

895 - return -1;

896 -}

897 -

898 -static void rearrange_packed_git(void)

899 -{

900 - packed_git = llist_mergesort(packed_git, get_next_packed_git,

901 - set_next_packed_git, sort_pack);

902 -}

903 -

904 -static void prepare_packed_git_mru(void)

905 -{

906 - struct packed_git *p;

907 -

908 - mru_clear(packed_git_mru);

909 - for (p = packed_git; p; p = p->next)

910 - mru_append(packed_git_mru, p);

911 -}

912 -

913 -static int prepare_packed_git_run_once = 0;

914 -void prepare_packed_git(void)

915 -{

916 - struct alternate_object_database *alt;

917 -

918 - if (prepare_packed_git_run_once)

919 - return;

920 - prepare_packed_git_one(get_object_directory(), 1);

921 - prepare_alt_odb();

922 - for (alt = alt_odb_list; alt; alt = alt->next)

923 - prepare_packed_git_one(alt->path, 0);

924 - rearrange_packed_git();

925 - prepare_packed_git_mru();

926 - prepare_packed_git_run_once = 1;

927 -}

928 -

929 -void reprepare_packed_git(void)

930 -{

931 - approximate_object_count_valid = 0;

932 - prepare_packed_git_run_once = 0;

933 - prepare_packed_git();

934 -}

935 -

722 static void mark_bad_packed_object(struct packed_git *p,

723 const unsigned char *sha1)

724 {