@matheusbsbs / matheus-bagliano / commits / 6c622f9f0b

commit-graph: write commit-graph chains

Extend write_commit_graph() to write a commit-graph chain when given the COMMIT_GRAPH_SPLIT flag. This implementation is purposefully simplistic in how it creates a new chain. The commits not already in the chain are added to a new tip commit-graph file. Much of the logic around writing a graph-{hash}.graph file and updating the commit-graph-chain file is the same as the commit-graph file case. However, there are several places where we need to do some extra logic in the split case. Track the list of graph filenames before and after the planned write. This will be more important when we start merging graph files, but it also allows us to upgrade our commit-graph file to the appropriate graph-{hash}.graph file when we upgrade to a chain of commit-graphs. Note that we use the eighth byte of the commit-graph header to store the number of base graph files. This determines the length of the base graphs chunk. A subtle change of behavior with the new logic is that we do not write a commit-graph if we our commit list is empty. This extends to the typical case, which is reflected in t5318-commit-graph.sh. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Jun 18, 2019 at 11:14 UTC 6c622f9f0bbb38a23341dc4294f56d0d909b3d50

3 files changed +278 -12

commit-graph.c

+275 -11

@@ -300,12 +300,18 @@ static struct commit_graph *load_commit_graph_one(const char *graph_file)

300

301 struct stat st;

302 int fd;

303 + struct commit_graph *g;

304 int open_ok = open_commit_graph(graph_file, &fd, &st);

305

306 if (!open_ok)

307 return NULL;

308

308 - return load_commit_graph_one_fd_st(fd, &st);

309 + g = load_commit_graph_one_fd_st(fd, &st);

310 +

311 + if (g)

312 + g->filename = xstrdup(graph_file);

313 +

314 + return g;

315 }

316

317 static struct commit_graph *load_commit_graph_v1(struct repository *r, const char *obj_dir)

@@ -730,8 +736,19 @@ struct write_commit_graph_context {

736 struct progress *progress;

737 int progress_done;

738 uint64_t progress_cnt;

739 +

740 + char *base_graph_name;

741 + int num_commit_graphs_before;

742 + int num_commit_graphs_after;

743 + char **commit_graph_filenames_before;

744 + char **commit_graph_filenames_after;

745 + char **commit_graph_hash_after;

746 + uint32_t new_num_commits_in_base;

747 + struct commit_graph *new_base_graph;

748 +

749 unsigned append:1,

734 - report_progress:1;

750 + report_progress:1,

751 + split:1;

752 };

753

754 static void write_graph_chunk_fanout(struct hashfile *f,

@@ -801,6 +818,16 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,

818 ctx->commits.nr,

819 commit_to_sha1);

820

821 + if (edge_value >= 0)

822 + edge_value += ctx->new_num_commits_in_base;

823 + else {

824 + uint32_t pos;

825 + if (find_commit_in_graph(parent->item,

826 + ctx->new_base_graph,

827 + &pos))

828 + edge_value = pos;

829 + }

830 +

831 if (edge_value < 0)

832 BUG("missing parent %s for commit %s",

833 oid_to_hex(&parent->item->object.oid),

@@ -821,6 +848,17 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,

848 ctx->commits.list,

849 ctx->commits.nr,

850 commit_to_sha1);

851 +

852 + if (edge_value >= 0)

853 + edge_value += ctx->new_num_commits_in_base;

854 + else {

855 + uint32_t pos;

856 + if (find_commit_in_graph(parent->item,

857 + ctx->new_base_graph,

858 + &pos))

859 + edge_value = pos;

860 + }

861 +

862 if (edge_value < 0)

863 BUG("missing parent %s for commit %s",

864 oid_to_hex(&parent->item->object.oid),

@@ -878,6 +916,16 @@ static void write_graph_chunk_extra_edges(struct hashfile *f,

916 ctx->commits.nr,

917 commit_to_sha1);

918

919 + if (edge_value >= 0)

920 + edge_value += ctx->new_num_commits_in_base;

921 + else {

922 + uint32_t pos;

923 + if (find_commit_in_graph(parent->item,

924 + ctx->new_base_graph,

925 + &pos))

926 + edge_value = pos;

927 + }

928 +

929 if (edge_value < 0)

930 BUG("missing parent %s for commit %s",

931 oid_to_hex(&parent->item->object.oid),

@@ -969,7 +1017,13 @@ static void close_reachable(struct write_commit_graph_context *ctx)

1017 display_progress(ctx->progress, i + 1);

1018 commit = lookup_commit(ctx->r, &ctx->oids.list[i]);

1019

972 - if (commit && !parse_commit_no_graph(commit))

1020 + if (!commit)

1021 + continue;

1022 + if (ctx->split) {

1023 + if (!parse_commit(commit) &&

1024 + commit->graph_pos == COMMIT_NOT_FROM_GRAPH)

1025 + add_missing_parents(ctx, commit);

1026 + } else if (!parse_commit_no_graph(commit))

1027 add_missing_parents(ctx, commit);

1028 }

1029 stop_progress(&ctx->progress);

@@ -1165,8 +1219,16 @@ static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx)

1219

1220 for (i = 1; i < ctx->oids.nr; i++) {

1221 display_progress(ctx->progress, i + 1);

1168 - if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i]))

1222 + if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) {

1223 + if (ctx->split) {

1224 + struct commit *c = lookup_commit(ctx->r, &ctx->oids.list[i]);

1225 +

1226 + if (!c || c->graph_pos != COMMIT_NOT_FROM_GRAPH)

1227 + continue;

1228 + }

1229 +

1230 count_distinct++;

1231 + }

1232 }

1233 stop_progress(&ctx->progress);

1234

@@ -1189,7 +1251,13 @@ static void copy_oids_to_commits(struct write_commit_graph_context *ctx)

1251 if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i]))

1252 continue;

1253

1254 + ALLOC_GROW(ctx->commits.list, ctx->commits.nr + 1, ctx->commits.alloc);

1255 ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]);

1256 +

1257 + if (ctx->split &&

1258 + ctx->commits.list[ctx->commits.nr]->graph_pos != COMMIT_NOT_FROM_GRAPH)

1259 + continue;

1260 +

1261 parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]);

1262

1263 for (parent = ctx->commits.list[ctx->commits.nr]->parents;

@@ -1204,18 +1272,86 @@ static void copy_oids_to_commits(struct write_commit_graph_context *ctx)

1272 stop_progress(&ctx->progress);

1273 }

1274

1275 +static int write_graph_chunk_base_1(struct hashfile *f,

1276 + struct commit_graph *g)

1277 +{

1278 + int num = 0;

1279 +

1280 + if (!g)

1281 + return 0;

1282 +

1283 + num = write_graph_chunk_base_1(f, g->base_graph);

1284 + hashwrite(f, g->oid.hash, the_hash_algo->rawsz);

1285 + return num + 1;

1286 +}

1287 +

1288 +static int write_graph_chunk_base(struct hashfile *f,

1289 + struct write_commit_graph_context *ctx)

1290 +{

1291 + int num = write_graph_chunk_base_1(f, ctx->new_base_graph);

1292 +

1293 + if (num != ctx->num_commit_graphs_after - 1) {

1294 + error(_("failed to write correct number of base graph ids"));

1295 + return -1;

1296 + }

1297 +

1298 + return 0;

1299 +}

1300 +

1301 +static void init_commit_graph_chain(struct write_commit_graph_context *ctx)

1302 +{

1303 + struct commit_graph *g = ctx->r->objects->commit_graph;

1304 + uint32_t i;

1305 +

1306 + ctx->new_base_graph = g;

1307 + ctx->base_graph_name = xstrdup(g->filename);

1308 + ctx->new_num_commits_in_base = g->num_commits + g->num_commits_in_base;

1309 +

1310 + ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;

1311 +

1312 + ALLOC_ARRAY(ctx->commit_graph_filenames_after, ctx->num_commit_graphs_after);

1313 + ALLOC_ARRAY(ctx->commit_graph_hash_after, ctx->num_commit_graphs_after);

1314 +

1315 + for (i = 0; i < ctx->num_commit_graphs_before - 1; i++)

1316 + ctx->commit_graph_filenames_after[i] = xstrdup(ctx->commit_graph_filenames_before[i]);

1317 +

1318 + if (ctx->num_commit_graphs_before)

1319 + ctx->commit_graph_filenames_after[ctx->num_commit_graphs_before - 1] =

1320 + get_split_graph_filename(ctx->obj_dir, oid_to_hex(&g->oid));

1321 +

1322 + i = ctx->num_commit_graphs_before - 1;

1323 +

1324 + while (g) {

1325 + ctx->commit_graph_hash_after[i] = xstrdup(oid_to_hex(&g->oid));

1326 + i--;

1327 + g = g->base_graph;

1328 + }

1329 +}

1330 +

1331 static int write_commit_graph_file(struct write_commit_graph_context *ctx)

1332 {

1333 uint32_t i;

1334 + int fd;

1335 struct hashfile *f;

1336 struct lock_file lk = LOCK_INIT;

1212 - uint32_t chunk_ids[5];

1213 - uint64_t chunk_offsets[5];

1337 + uint32_t chunk_ids[6];

1338 + uint64_t chunk_offsets[6];

1339 const unsigned hashsz = the_hash_algo->rawsz;

1340 struct strbuf progress_title = STRBUF_INIT;

1341 int num_chunks = 3;

1342 + struct object_id file_hash;

1343 +

1344 + if (ctx->split) {

1345 + struct strbuf tmp_file = STRBUF_INIT;

1346 +

1347 + strbuf_addf(&tmp_file,

1348 + "%s/info/commit-graphs/tmp_graph_XXXXXX",

1349 + ctx->obj_dir);

1350 + ctx->graph_name = strbuf_detach(&tmp_file, NULL);

1351 + } else {

1352 + ctx->graph_name = get_commit_graph_filename(ctx->obj_dir);

1353 + }

1354

1218 - ctx->graph_name = get_commit_graph_filename(ctx->obj_dir);

1355 if (safe_create_leading_directories(ctx->graph_name)) {

1356 UNLEAK(ctx->graph_name);

1357 error(_("unable to create leading directories of %s"),

@@ -1223,8 +1359,23 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)

1359 return -1;

1360 }

1361

1226 - hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR);

1227 - f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);

1362 + if (ctx->split) {

1363 + char *lock_name = get_chain_filename(ctx->obj_dir);

1364 +

1365 + hold_lock_file_for_update(&lk, lock_name, LOCK_DIE_ON_ERROR);

1366 +

1367 + fd = git_mkstemp_mode(ctx->graph_name, 0444);

1368 + if (fd < 0) {

1369 + error(_("unable to create '%s'"), ctx->graph_name);

1370 + return -1;

1371 + }

1372 +

1373 + f = hashfd(fd, ctx->graph_name);

1374 + } else {

1375 + hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR);

1376 + fd = lk.tempfile->fd;

1377 + f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);

1378 + }

1379

1380 chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT;

1381 chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;

@@ -1233,6 +1384,10 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)

1384 chunk_ids[num_chunks] = GRAPH_CHUNKID_EXTRAEDGES;

1385 num_chunks++;

1386 }

1387 + if (ctx->num_commit_graphs_after > 1) {

1388 + chunk_ids[num_chunks] = GRAPH_CHUNKID_BASE;

1389 + num_chunks++;

1390 + }

1391

1392 chunk_ids[num_chunks] = 0;

1393

@@ -1247,13 +1402,18 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)

1402 4 * ctx->num_extra_edges;

1403 num_chunks++;

1404 }

1405 + if (ctx->num_commit_graphs_after > 1) {

1406 + chunk_offsets[num_chunks + 1] = chunk_offsets[num_chunks] +

1407 + hashsz * (ctx->num_commit_graphs_after - 1);

1408 + num_chunks++;

1409 + }

1410

1411 hashwrite_be32(f, GRAPH_SIGNATURE);

1412

1413 hashwrite_u8(f, GRAPH_VERSION);

1414 hashwrite_u8(f, oid_version());

1415 hashwrite_u8(f, num_chunks);

1256 - hashwrite_u8(f, 0);

1416 + hashwrite_u8(f, ctx->num_commit_graphs_after - 1);

1417

1418 for (i = 0; i <= num_chunks; i++) {

1419 uint32_t chunk_write[3];

@@ -1279,11 +1439,67 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)

1439 write_graph_chunk_data(f, hashsz, ctx);

1440 if (ctx->num_extra_edges)

1441 write_graph_chunk_extra_edges(f, ctx);

1442 + if (ctx->num_commit_graphs_after > 1 &&

1443 + write_graph_chunk_base(f, ctx)) {

1444 + return -1;

1445 + }

1446 stop_progress(&ctx->progress);

1447 strbuf_release(&progress_title);

1448

1449 + if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) {

1450 + char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid));

1451 + char *new_base_name = get_split_graph_filename(ctx->obj_dir, new_base_hash);

1452 +

1453 + free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]);

1454 + free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]);

1455 + ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2] = new_base_name;

1456 + ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2] = new_base_hash;

1457 + }

1458 +

1459 close_commit_graph(ctx->r->objects);

1286 - finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);

1460 + finalize_hashfile(f, file_hash.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC);

1461 +

1462 + if (ctx->split) {

1463 + FILE *chainf = fdopen_lock_file(&lk, "w");

1464 + char *final_graph_name;

1465 + int result;

1466 +

1467 + close(fd);

1468 +

1469 + if (!chainf) {

1470 + error(_("unable to open commit-graph chain file"));

1471 + return -1;

1472 + }

1473 +

1474 + if (ctx->base_graph_name) {

1475 + result = rename(ctx->base_graph_name,

1476 + ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]);

1477 +

1478 + if (result) {

1479 + error(_("failed to rename base commit-graph file"));

1480 + return -1;

1481 + }

1482 + } else {

1483 + char *graph_name = get_commit_graph_filename(ctx->obj_dir);

1484 + unlink(graph_name);

1485 + }

1486 +

1487 + ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] = xstrdup(oid_to_hex(&file_hash));

1488 + final_graph_name = get_split_graph_filename(ctx->obj_dir,

1489 + ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]);

1490 + ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name;

1491 +

1492 + result = rename(ctx->graph_name, final_graph_name);

1493 +

1494 + for (i = 0; i < ctx->num_commit_graphs_after; i++)

1495 + fprintf(lk.tempfile->fp, "%s\n", ctx->commit_graph_hash_after[i]);

1496 +

1497 + if (result) {

1498 + error(_("failed to rename temporary commit-graph file"));

1499 + return -1;

1500 + }

1501 + }

1502 +

1503 commit_lock_file(&lk);

1504

1505 return 0;

@@ -1306,6 +1522,30 @@ int write_commit_graph(const char *obj_dir,

1522 ctx->obj_dir = obj_dir;

1523 ctx->append = flags & COMMIT_GRAPH_APPEND ? 1 : 0;

1524 ctx->report_progress = flags & COMMIT_GRAPH_PROGRESS ? 1 : 0;

1525 + ctx->split = flags & COMMIT_GRAPH_SPLIT ? 1 : 0;

1526 +

1527 + if (ctx->split) {

1528 + struct commit_graph *g;

1529 + prepare_commit_graph(ctx->r);

1530 +

1531 + g = ctx->r->objects->commit_graph;

1532 +

1533 + while (g) {

1534 + ctx->num_commit_graphs_before++;

1535 + g = g->base_graph;

1536 + }

1537 +

1538 + if (ctx->num_commit_graphs_before) {

1539 + ALLOC_ARRAY(ctx->commit_graph_filenames_before, ctx->num_commit_graphs_before);

1540 + i = ctx->num_commit_graphs_before;

1541 + g = ctx->r->objects->commit_graph;

1542 +

1543 + while (g) {

1544 + ctx->commit_graph_filenames_before[--i] = xstrdup(g->filename);

1545 + g = g->base_graph;

1546 + }

1547 + }

1548 + }

1549

1550 ctx->approx_nr_objects = approximate_object_count();

1551 ctx->oids.alloc = ctx->approx_nr_objects / 32;

@@ -1360,6 +1600,14 @@ int write_commit_graph(const char *obj_dir,

1600 goto cleanup;

1601 }

1602

1603 + if (!ctx->commits.nr)

1604 + goto cleanup;

1605 +

1606 + if (ctx->split)

1607 + init_commit_graph_chain(ctx);

1608 + else

1609 + ctx->num_commit_graphs_after = 1;

1610 +

1611 compute_generation_numbers(ctx);

1612

1613 res = write_commit_graph_file(ctx);

@@ -1368,6 +1616,21 @@ cleanup:

1616 free(ctx->graph_name);

1617 free(ctx->commits.list);

1618 free(ctx->oids.list);

1619 +

1620 + if (ctx->commit_graph_filenames_after) {

1621 + for (i = 0; i < ctx->num_commit_graphs_after; i++) {

1622 + free(ctx->commit_graph_filenames_after[i]);

1623 + free(ctx->commit_graph_hash_after[i]);

1624 + }

1625 +

1626 + for (i = 0; i < ctx->num_commit_graphs_before; i++)

1627 + free(ctx->commit_graph_filenames_before[i]);

1628 +

1629 + free(ctx->commit_graph_filenames_after);

1630 + free(ctx->commit_graph_filenames_before);

1631 + free(ctx->commit_graph_hash_after);

1632 + }

1633 +

1634 free(ctx);

1635

1636 return res;

@@ -1555,5 +1818,6 @@ void free_commit_graph(struct commit_graph *g)

1818 g->data = NULL;

1819 close(g->graph_fd);

1820 }

1821 + free(g->filename);

1822 free(g);

1823 }

commit-graph.h

@@ -47,6 +47,7 @@ struct commit_graph {

47 unsigned char num_chunks;

48 uint32_t num_commits;

49 struct object_id oid;

50 + char *filename;

52 uint32_t num_commits_in_base;

53 struct commit_graph *base_graph;

@@ -71,6 +72,7 @@ int generation_numbers_enabled(struct repository *r);

73 #define COMMIT_GRAPH_APPEND (1 << 0)

74 #define COMMIT_GRAPH_PROGRESS (1 << 1)

75 +#define COMMIT_GRAPH_SPLIT (1 << 2)

77 /*

78 * The write_commit_graph* methods return zero on success

t/t5318-commit-graph.sh

+1 -1

@@ -20,7 +20,7 @@ test_expect_success 'verify graph with no graph file' '

20 test_expect_success 'write graph with no packs' '

21 cd "$TRASH_DIRECTORY/full" &&

22 git commit-graph write --object-dir . &&

23 - test_path_is_file info/commit-graph

23 + test_path_is_missing info/commit-graph

24 '

26 test_expect_success 'close with correct error on bad input' '