@matheusbsbs / matheus-bagliano / commits / 230f13225d
Fork 0

Create object subdirectories on demand

This makes it possible to have a "sparse" git object subdirectory structure, something that has become much more attractive now that people use pack-files all the time. As a result of pack-files, a git object directory doesn't necessarily have any individual objects lying around, and in that case it's just wasting space to keep the empty first-level object directories around: on many filesystems the 256 empty directories will be aboue 1MB of diskspace. Even more importantly, after you re-pack a project that _used_ to be unpacked, you could be left with huge directories that no longer contain anything, but that waste space and take time to look through. With this change, "git prune-packed" can just do an rmdir() on the directories, and they'll get removed if empty, and re-created on demand. This patch also tries to fix up "write_sha1_from_fd()" to use the new common infrastructure for creating the object files, closing a hole where we might otherwise leave half-written objects in the object database. [jc: I unoptimized the part that really removes the fan-out directories to ease transition. init-db still wastes 1MB of diskspace to hold 256 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories, but runs mkdir() immediately after that -- reducing the saving from 150KB to 146KB. These parts will be re-introduced when everybody has the on-demand capability.] Signed-off-by: Linus Torvalds <torvalds@osdl.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Linus Torvalds committed Oct 8, 2005 at 15:54 UTC 230f13225df8b7e7eb0acc91a8c630f9e84967c1
7 files changed +88 -44
daemon.c
+1 -1
@@ -142,7 +142,7 @@ static int upload(char *dir, int dirlen)
142 * is ok with us doing this.
143 */
144 if ((!export_all_trees && access("git-daemon-export-ok", F_OK)) ||
145 - access("objects/00", X_OK) ||
145 + access("objects/", X_OK) ||
146 access("HEAD", R_OK)) {
147 logerror("Not a valid git-daemon-enabled repository: '%s'", dir);
148 return -1;
fsck-objects.c
+2 -3
@@ -329,9 +329,8 @@ static int fsck_dir(int i, char *path)
329 DIR *dir = opendir(path);
330 struct dirent *de;
331
332 - if (!dir) {
333 - return error("missing sha1 directory '%s'", path);
334 - }
332 + if (!dir)
333 + return 0;
334
335 while ((de = readdir(dir)) != NULL) {
336 char name[100];
git-rename.perl
+1 -1
@@ -15,7 +15,7 @@ sub usage($);
15 my $GIT_DIR = $ENV{'GIT_DIR'} || ".git";
16
17 unless ( -d $GIT_DIR && -d $GIT_DIR . "/objects" &&
18 - -d $GIT_DIR . "/objects/00" && -d $GIT_DIR . "/refs") {
18 + -d $GIT_DIR . "/objects/" && -d $GIT_DIR . "/refs") {
19 usage("Git repository not found.");
20 }
21
git-sh-setup.sh
+1 -1
@@ -22,4 +22,4 @@ refs/*) : ;;
22 *) false ;;
23 esac &&
24 [ -d "$GIT_DIR/refs" ] &&
25 -[ -d "$GIT_OBJECT_DIRECTORY/00" ]
25 +[ -d "$GIT_OBJECT_DIRECTORY/" ]
prune-packed.c
+4 -1
@@ -26,6 +26,9 @@ static void prune_dir(int i, DIR *dir, char *pathname, int len)
26 else if (unlink(pathname) < 0)
27 error("unable to unlink %s", pathname);
28 }
29 + pathname[len] = 0;
30 + if (rmdir(pathname))
31 + mkdir(pathname, 0777);
32 }
33
34 static void prune_packed_objects(void)
@@ -46,7 +49,7 @@ static void prune_packed_objects(void)
49 sprintf(pathname + len, "%02x/", i);
50 d = opendir(pathname);
51 if (!d)
49 - die("unable to open %s", pathname);
52 + continue;
53 prune_dir(i, d, pathname, len + 3);
54 closedir(d);
55 }
sha1_file.c
+78 -36
@@ -1248,6 +1248,73 @@ char *write_sha1_file_prepare(void *buf,
1248 return sha1_file_name(sha1);
1249 }
1250
1251 +/*
1252 + * Link the tempfile to the final place, possibly creating the
1253 + * last directory level as you do so.
1254 + *
1255 + * Returns the errno on failure, 0 on success.
1256 + */
1257 +static int link_temp_to_file(const char *tmpfile, char *filename)
1258 +{
1259 + int ret;
1260 +
1261 + if (!link(tmpfile, filename))
1262 + return 0;
1263 +
1264 + /*
1265 + * Try to mkdir the last path component if that failed
1266 + * with an ENOENT.
1267 + *
1268 + * Re-try the "link()" regardless of whether the mkdir
1269 + * succeeds, since a race might mean that somebody
1270 + * else succeeded.
1271 + */
1272 + ret = errno;
1273 + if (ret == ENOENT) {
1274 + char *dir = strrchr(filename, '/');
1275 + if (dir) {
1276 + *dir = 0;
1277 + mkdir(filename, 0777);
1278 + *dir = '/';
1279 + if (!link(tmpfile, filename))
1280 + return 0;
1281 + ret = errno;
1282 + }
1283 + }
1284 + return ret;
1285 +}
1286 +
1287 +/*
1288 + * Move the just written object into its final resting place
1289 + */
1290 +static int move_temp_to_file(const char *tmpfile, char *filename)
1291 +{
1292 + int ret = link_temp_to_file(tmpfile, filename);
1293 + if (ret) {
1294 + /*
1295 + * Coda hack - coda doesn't like cross-directory links,
1296 + * so we fall back to a rename, which will mean that it
1297 + * won't be able to check collisions, but that's not a
1298 + * big deal.
1299 + *
1300 + * When this succeeds, we just return 0. We have nothing
1301 + * left to unlink.
1302 + */
1303 + if (ret == EXDEV && !rename(tmpfile, filename))
1304 + return 0;
1305 + }
1306 + unlink(tmpfile);
1307 + if (ret) {
1308 + if (ret != EEXIST) {
1309 + fprintf(stderr, "unable to write sha1 filename %s: %s", filename, strerror(ret));
1310 + return -1;
1311 + }
1312 + /* FIXME!!! Collision check here ? */
1313 + }
1314 +
1315 + return 0;
1316 +}
1317 +
1318 int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
1319 {
1320 int size;
@@ -1257,7 +1324,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
1324 char *filename;
1325 static char tmpfile[PATH_MAX];
1326 unsigned char hdr[50];
1260 - int fd, hdrlen, ret;
1327 + int fd, hdrlen;
1328
1329 /* Normally if we have it in the pack then we do not bother writing
1330 * it out into .git/objects/??/?{38} file.
@@ -1320,32 +1387,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
1387 close(fd);
1388 free(compressed);
1389
1323 - ret = link(tmpfile, filename);
1324 - if (ret < 0) {
1325 - ret = errno;
1326 -
1327 - /*
1328 - * Coda hack - coda doesn't like cross-directory links,
1329 - * so we fall back to a rename, which will mean that it
1330 - * won't be able to check collisions, but that's not a
1331 - * big deal.
1332 - *
1333 - * When this succeeds, we just return 0. We have nothing
1334 - * left to unlink.
1335 - */
1336 - if (ret == EXDEV && !rename(tmpfile, filename))
1337 - return 0;
1338 - }
1339 - unlink(tmpfile);
1340 - if (ret) {
1341 - if (ret != EEXIST) {
1342 - fprintf(stderr, "unable to write sha1 filename %s: %s", filename, strerror(ret));
1343 - return -1;
1344 - }
1345 - /* FIXME!!! Collision check here ? */
1346 - }
1347 -
1348 - return 0;
1390 + return move_temp_to_file(tmpfile, filename);
1391 }
1392
1393 int write_sha1_to_fd(int fd, const unsigned char *sha1)
@@ -1420,8 +1462,7 @@ int write_sha1_to_fd(int fd, const unsigned char *sha1)
1462 int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
1463 size_t bufsize, size_t *bufposn)
1464 {
1423 - char *filename = sha1_file_name(sha1);
1424 -
1465 + char tmpfile[PATH_MAX];
1466 int local;
1467 z_stream stream;
1468 unsigned char real_sha1[20];
@@ -1429,10 +1470,11 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
1470 int ret;
1471 SHA_CTX c;
1472
1432 - local = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0666);
1473 + snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX", get_object_directory());
1474
1475 + local = mkstemp(tmpfile);
1476 if (local < 0)
1435 - return error("Couldn't open %s\n", filename);
1477 + return error("Couldn't open %s for %s\n", tmpfile, sha1_to_hex(sha1));
1478
1479 memset(&stream, 0, sizeof(stream));
1480
@@ -1462,7 +1504,7 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
1504 size = read(fd, buffer + *bufposn, bufsize - *bufposn);
1505 if (size <= 0) {
1506 close(local);
1465 - unlink(filename);
1507 + unlink(tmpfile);
1508 if (!size)
1509 return error("Connection closed?");
1510 perror("Reading from connection");
@@ -1475,15 +1517,15 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
1517 close(local);
1518 SHA1_Final(real_sha1, &c);
1519 if (ret != Z_STREAM_END) {
1478 - unlink(filename);
1520 + unlink(tmpfile);
1521 return error("File %s corrupted", sha1_to_hex(sha1));
1522 }
1523 if (memcmp(sha1, real_sha1, 20)) {
1482 - unlink(filename);
1524 + unlink(tmpfile);
1525 return error("File %s has bad hash\n", sha1_to_hex(sha1));
1526 }
1485 -
1486 - return 0;
1527 +
1528 + return move_temp_to_file(tmpfile, sha1_file_name(sha1));
1529 }
1530
1531 int has_pack_index(const unsigned char *sha1)
t/t0000-basic.sh
+1 -1
@@ -28,7 +28,7 @@ test_expect_success \
28 '.git/objects should be empty after git-init-db in an empty repo.' \
29 'cmp -s /dev/null should-be-empty'
30
31 -# also it should have 258 subdirectories; 256 fan-out, pack, and info.
31 +# also it should have 258 subdirectories; 256 fan-out anymore, pack, and info.
32 # 259 is counting "objects" itself
33 find .git/objects -type d -print >full-of-directories
34 test_expect_success \