scalar: teach `diagnose` to gather loose objects information
When operating at the scale that Scalar wants to support, certain data shapes are more likely to cause undesirable performance issues, such as large numbers of loose objects. By including statistics about this, `scalar diagnose` now makes it easier to identify such scenarios. Signed-off-by: Matthew John Cheetham <mjcheetham@outlook.com> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Matthew John Cheetham committed
May 28, 2022 at 16:11 UTC
15d8adccab9a3146b760b089df59ce3e7ca2b451
2 files changed
+63
-1
contrib/scalar/scalar.c
+59
@@ -618,6 +618,60 @@ static int dir_file_stats(struct object_directory *object_dir, void *data)
618
return 0;
619
}
620
621
+static int count_files(char *path)
622
+{
623
+ DIR *dir = opendir(path);
624
+ struct dirent *e;
625
+ int count = 0;
626
+
627
+ if (!dir)
628
+ return 0;
629
+
630
+ while ((e = readdir(dir)) != NULL)
631
+ if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG)
632
+ count++;
633
+
634
+ closedir(dir);
635
+ return count;
636
+}
637
+
638
+static void loose_objs_stats(struct strbuf *buf, const char *path)
639
+{
640
+ DIR *dir = opendir(path);
641
+ struct dirent *e;
642
+ int count;
643
+ int total = 0;
644
+ unsigned char c;
645
+ struct strbuf count_path = STRBUF_INIT;
646
+ size_t base_path_len;
647
+
648
+ if (!dir)
649
+ return;
650
+
651
+ strbuf_addstr(buf, "Object directory stats for ");
652
+ strbuf_add_absolute_path(buf, path);
653
+ strbuf_addstr(buf, ":\n");
654
+
655
+ strbuf_add_absolute_path(&count_path, path);
656
+ strbuf_addch(&count_path, '/');
657
+ base_path_len = count_path.len;
658
+
659
+ while ((e = readdir(dir)) != NULL)
660
+ if (!is_dot_or_dotdot(e->d_name) &&
661
+ e->d_type == DT_DIR && strlen(e->d_name) == 2 &&
662
+ !hex_to_bytes(&c, e->d_name, 1)) {
663
+ strbuf_setlen(&count_path, base_path_len);
664
+ strbuf_addstr(&count_path, e->d_name);
665
+ total += (count = count_files(count_path.buf));
666
+ strbuf_addf(buf, "%s : %7d files\n", e->d_name, count);
667
+ }
668
+
669
+ strbuf_addf(buf, "Total: %d loose objects", total);
670
+
671
+ strbuf_release(&count_path);
672
+ closedir(dir);
673
+}
674
+
675
static int cmd_diagnose(int argc, const char **argv)
676
{
677
struct option options[] = {
@@ -686,6 +740,11 @@ static int cmd_diagnose(int argc, const char **argv)
740
foreach_alt_odb(dir_file_stats, &buf);
741
strvec_push(&archiver_args, buf.buf);
742
743
+ strbuf_reset(&buf);
744
+ strbuf_addstr(&buf, "--add-virtual-file=objects-local.txt:");
745
+ loose_objs_stats(&buf, ".git/objects");
746
+ strvec_push(&archiver_args, buf.buf);
747
+
748
if ((res = add_directory_to_archiver(&archiver_args, ".git", 0)) ||
749
(res = add_directory_to_archiver(&archiver_args, ".git/hooks", 0)) ||
750
(res = add_directory_to_archiver(&archiver_args, ".git/info", 0)) ||
contrib/scalar/t/t9099-scalar.sh
+4
-1
@@ -103,6 +103,7 @@ test_expect_success UNZIP 'scalar diagnose' '
103
scalar clone "file://$(pwd)" cloned --single-branch &&
104
git repack &&
105
echo "$(pwd)/.git/objects/" >>cloned/src/.git/objects/info/alternates &&
106
+ test_commit -C cloned/src loose &&
107
scalar diagnose cloned >out 2>err &&
108
grep "Available space" out &&
109
sed -n "s/.*$SQ\\(.*\\.zip\\)$SQ.*/\\1/p" <err >zip_path &&
@@ -114,7 +115,9 @@ test_expect_success UNZIP 'scalar diagnose' '
115
unzip -p "$zip_path" diagnostics.log >out &&
116
test_file_not_empty out &&
117
unzip -p "$zip_path" packs-local.txt >out &&
117
- grep "$(pwd)/.git/objects" out
118
+ grep "$(pwd)/.git/objects" out &&
119
+ unzip -p "$zip_path" objects-local.txt >out &&
120
+ grep "^Total: [1-9]" out
121
'
122
123
test_done