| 1 | #define USE_THE_REPOSITORY_VARIABLE |
| 2 | |
| 3 | #include "test-tool.h" |
| 4 | #include "git-compat-util.h" |
| 5 | #include "git-zlib.h" |
| 6 | #include "hash.h" |
| 7 | #include "hex.h" |
| 8 | #include "object-file.h" |
| 9 | #include "object.h" |
| 10 | #include "pack.h" |
| 11 | #include "parse-options.h" |
| 12 | #include "parse.h" |
| 13 | #include "repository.h" |
| 14 | #include "setup.h" |
| 15 | #include "strbuf.h" |
| 16 | #include "write-or-die.h" |
| 17 | |
| 18 | #define BLOCK_SIZE 0xffff |
| 19 | static const unsigned char zeros[BLOCK_SIZE]; |
| 20 | |
| 21 | /* |
| 22 | * Write data as an uncompressed zlib stream. |
| 23 | * For data larger than 64KB, writes multiple uncompressed blocks. |
| 24 | * If data is NULL, writes zeros. |
| 25 | * Updates the pack checksum context. |
| 26 | */ |
| 27 | static void write_uncompressed_zlib(FILE *f, struct git_hash_ctx *pack_ctx, |
| 28 | const void *data, size_t len) |
| 29 | { |
| 30 | unsigned char zlib_header[2] = { 0x78, 0x01 }; /* CMF, FLG */ |
| 31 | unsigned char block_header[5]; |
| 32 | const unsigned char *p = data; |
| 33 | size_t remaining = len; |
| 34 | uint32_t adler = 1L; /* adler32 initial value */ |
| 35 | unsigned char adler_buf[4]; |
| 36 | |
| 37 | /* Write zlib header */ |
| 38 | fwrite_or_die(f, zlib_header, sizeof(zlib_header)); |
| 39 | git_hash_update(pack_ctx, zlib_header, 2); |
| 40 | |
| 41 | /* Write uncompressed blocks (max 64KB each) */ |
| 42 | do { |
| 43 | size_t block_len = remaining > BLOCK_SIZE ? BLOCK_SIZE : remaining; |
| 44 | int is_final = (block_len == remaining); |
| 45 | const unsigned char *block_data = data ? p : zeros; |
| 46 | |
| 47 | block_header[0] = is_final ? 0x01 : 0x00; |
| 48 | block_header[1] = block_len & 0xff; |
| 49 | block_header[2] = (block_len >> 8) & 0xff; |
| 50 | block_header[3] = block_header[1] ^ 0xff; |
| 51 | block_header[4] = block_header[2] ^ 0xff; |
| 52 | |
| 53 | fwrite_or_die(f, block_header, sizeof(block_header)); |
| 54 | git_hash_update(pack_ctx, block_header, 5); |
| 55 | |
| 56 | if (block_len) { |
| 57 | fwrite_or_die(f, block_data, block_len); |
| 58 | git_hash_update(pack_ctx, block_data, block_len); |
| 59 | adler = adler32(adler, block_data, block_len); |
| 60 | } |
| 61 | |
| 62 | if (data) |
| 63 | p += block_len; |
| 64 | remaining -= block_len; |
| 65 | } while (remaining > 0); |
| 66 | |
| 67 | /* Write adler32 checksum */ |
| 68 | put_be32(adler_buf, adler); |
| 69 | fwrite_or_die(f, adler_buf, sizeof(adler_buf)); |
| 70 | git_hash_update(pack_ctx, adler_buf, 4); |
| 71 | } |
| 72 | |
| 73 | /* |
| 74 | * Write an uncompressed object to the pack file. |
| 75 | * If `data == NULL`, it is treated like a buffer to NUL bytes. |
| 76 | * Updates the pack checksum context. |
| 77 | */ |
| 78 | static void write_pack_object(FILE *f, struct git_hash_ctx *pack_ctx, |
| 79 | enum object_type type, |
| 80 | const void *data, size_t len, |
| 81 | struct object_id *oid, |
| 82 | const struct git_hash_algo *algo) |
| 83 | { |
| 84 | unsigned char pack_header[MAX_PACK_OBJECT_HEADER]; |
| 85 | char object_header[32]; |
| 86 | int pack_header_len, object_header_len; |
| 87 | struct git_hash_ctx ctx; |
| 88 | |
| 89 | /* Write pack object header */ |
| 90 | pack_header_len = encode_in_pack_object_header(pack_header, |
| 91 | sizeof(pack_header), |
| 92 | type, len); |
| 93 | fwrite_or_die(f, pack_header, pack_header_len); |
| 94 | git_hash_update(pack_ctx, pack_header, pack_header_len); |
| 95 | |
| 96 | /* Write the data as uncompressed zlib */ |
| 97 | write_uncompressed_zlib(f, pack_ctx, data, len); |
| 98 | |
| 99 | git_hash_init(&ctx, algo); |
| 100 | object_header_len = format_object_header(object_header, |
| 101 | sizeof(object_header), |
| 102 | type, len); |
| 103 | git_hash_update(&ctx, object_header, object_header_len); |
| 104 | if (data) |
| 105 | git_hash_update(&ctx, data, len); |
| 106 | else { |
| 107 | for (size_t i = len / BLOCK_SIZE; i; i--) |
| 108 | git_hash_update(&ctx, zeros, BLOCK_SIZE); |
| 109 | git_hash_update(&ctx, zeros, len % BLOCK_SIZE); |
| 110 | } |
| 111 | git_hash_final_oid(oid, &ctx); |
| 112 | } |
| 113 | |
| 114 | /* |
| 115 | * Fast path: precomputed pack data for a 4 GiB + 1 all-NUL blob. |
| 116 | * |
| 117 | * The generated pack is almost entirely zeros with a small constant |
| 118 | * prefix, periodic deflate block headers, and a constant suffix |
| 119 | * containing the tree, two commits, and the pack checksum. Because |
| 120 | * every byte is deterministic for a given blob size and hash algorithm, |
| 121 | * we can write the pack without computing any hashes at all, reducing |
| 122 | * runtime from minutes of hash computation to seconds of pure I/O. |
| 123 | * |
| 124 | * The blob is stored as an uncompressed deflate stream: a two-byte |
| 125 | * zlib header, then 65538 blocks of up to 0xffff bytes each, followed |
| 126 | * by an adler32 checksum. The pack header and deflate framing are |
| 127 | * shared across hash algorithms; only the suffix (which contains OIDs |
| 128 | * and the pack checksum) differs. |
| 129 | * |
| 130 | * Constants were generated by running the generic path and extracting |
| 131 | * the non-zero bytes from the resulting pack file. |
| 132 | */ |
| 133 | |
| 134 | #define FAST_PACK_4G1_BLOB_SIZE ((size_t)4 * 1024 * 1024 * 1024 + 1) |
| 135 | #define FAST_PACK_4G1_N_FULL_BLOCKS 65537 |
| 136 | |
| 137 | /* |
| 138 | * Per-hash-algorithm constants for the fast path. The prefix and |
| 139 | * deflate block structure are identical across algorithms; only the |
| 140 | * suffix (tree, commits, pack checksum) and the commit OID differ. |
| 141 | */ |
| 142 | struct fast_pack { |
| 143 | uint32_t format_id; |
| 144 | const unsigned char *suffix; |
| 145 | size_t suffix_len; |
| 146 | const char *commit_oid; |
| 147 | }; |
| 148 | |
| 149 | /* Pack header + pack object header + zlib header + first block header */ |
| 150 | static const unsigned char fast_pack_prefix[] = { |
| 151 | /* PACK header: signature, version 2, 5 objects */ |
| 152 | 0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, |
| 153 | 0x00, 0x00, 0x00, 0x05, |
| 154 | /* pack object header: blob, size = 4294967297 */ |
| 155 | 0xb1, 0x80, 0x80, 0x80, 0x80, 0x01, |
| 156 | /* zlib header: CMF=0x78, FLG=0x01 */ |
| 157 | 0x78, 0x01, |
| 158 | /* first non-final block header: BFINAL=0, LEN=0xffff, NLEN=0x0000 */ |
| 159 | 0x00, 0xff, 0xff, 0x00, 0x00 |
| 160 | }; |
| 161 | |
| 162 | /* Every non-final deflate block header is identical */ |
| 163 | static const unsigned char fast_pack_block_header[] = { |
| 164 | 0x00, 0xff, 0xff, 0x00, 0x00 |
| 165 | }; |
| 166 | |
| 167 | /* Final block (2 data bytes) + adler32 of 4294967297 NUL bytes */ |
| 168 | static const unsigned char fast_pack_final_block[] = { |
| 169 | /* BFINAL=1, LEN=2, NLEN=0xfffd */ |
| 170 | 0x01, 0x02, 0x00, 0xfd, 0xff, |
| 171 | /* 2 NUL data bytes */ |
| 172 | 0x00, 0x00, |
| 173 | /* adler32 */ |
| 174 | 0x00, 0xe2, 0x00, 0x01 |
| 175 | }; |
| 176 | |
| 177 | /* |
| 178 | * SHA-1 suffix: tree, commit, empty tree, final commit, pack checksum. |
| 179 | */ |
| 180 | static const unsigned char fast_pack_sha1_suffix[] = { |
| 181 | 0xa0, 0x02, 0x78, 0x01, 0x01, 0x20, 0x00, 0xdf, |
| 182 | 0xff, 0x31, 0x30, 0x30, 0x36, 0x34, 0x34, 0x20, |
| 183 | 0x66, 0x69, 0x6c, 0x65, 0x00, 0x3e, 0xb7, 0xfe, |
| 184 | 0xb1, 0x41, 0x3c, 0x75, 0x7f, 0x0d, 0x81, 0x81, |
| 185 | 0xde, 0xb2, 0x8d, 0x1d, 0xab, 0x03, 0xd6, 0x48, |
| 186 | 0x46, 0xb4, 0xb4, 0x0c, 0x60, 0x95, 0x0b, 0x78, |
| 187 | 0x01, 0x01, 0xb5, 0x00, 0x4a, 0xff, 0x74, 0x72, |
| 188 | 0x65, 0x65, 0x20, 0x63, 0x36, 0x38, 0x33, 0x66, |
| 189 | 0x63, 0x63, 0x37, 0x64, 0x31, 0x64, 0x38, 0x33, |
| 190 | 0x65, 0x66, 0x32, 0x66, 0x65, 0x31, 0x61, 0x66, |
| 191 | 0x35, 0x35, 0x32, 0x31, 0x35, 0x64, 0x30, 0x31, |
| 192 | 0x36, 0x38, 0x64, 0x62, 0x35, 0x32, 0x61, 0x33, |
| 193 | 0x61, 0x33, 0x62, 0x0a, 0x61, 0x75, 0x74, 0x68, |
| 194 | 0x6f, 0x72, 0x20, 0x41, 0x20, 0x55, 0x20, 0x54, |
| 195 | 0x68, 0x6f, 0x72, 0x20, 0x3c, 0x61, 0x75, 0x74, |
| 196 | 0x68, 0x6f, 0x72, 0x40, 0x65, 0x78, 0x61, 0x6d, |
| 197 | 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x3e, |
| 198 | 0x20, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, |
| 199 | 0x38, 0x39, 0x30, 0x20, 0x2b, 0x30, 0x30, 0x30, |
| 200 | 0x30, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, |
| 201 | 0x74, 0x65, 0x72, 0x20, 0x43, 0x20, 0x4f, 0x20, |
| 202 | 0x4d, 0x69, 0x74, 0x74, 0x65, 0x72, 0x20, 0x3c, |
| 203 | 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, |
| 204 | 0x72, 0x40, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, |
| 205 | 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x3e, 0x20, 0x31, |
| 206 | 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, |
| 207 | 0x30, 0x20, 0x2b, 0x30, 0x30, 0x30, 0x30, 0x0a, |
| 208 | 0x0a, 0x4c, 0x61, 0x72, 0x67, 0x65, 0x20, 0x62, |
| 209 | 0x6c, 0x6f, 0x62, 0x20, 0x63, 0x6f, 0x6d, 0x6d, |
| 210 | 0x69, 0x74, 0x0a, 0xc6, 0x55, 0x37, 0x6b, 0x20, |
| 211 | 0x78, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, |
| 212 | 0x00, 0x00, 0x01, 0x95, 0x0e, 0x78, 0x01, 0x01, |
| 213 | 0xe5, 0x00, 0x1a, 0xff, 0x74, 0x72, 0x65, 0x65, |
| 214 | 0x20, 0x34, 0x62, 0x38, 0x32, 0x35, 0x64, 0x63, |
| 215 | 0x36, 0x34, 0x32, 0x63, 0x62, 0x36, 0x65, 0x62, |
| 216 | 0x39, 0x61, 0x30, 0x36, 0x30, 0x65, 0x35, 0x34, |
| 217 | 0x62, 0x66, 0x38, 0x64, 0x36, 0x39, 0x32, 0x38, |
| 218 | 0x38, 0x66, 0x62, 0x65, 0x65, 0x34, 0x39, 0x30, |
| 219 | 0x34, 0x0a, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, |
| 220 | 0x20, 0x63, 0x35, 0x62, 0x32, 0x31, 0x63, 0x36, |
| 221 | 0x31, 0x31, 0x61, 0x61, 0x35, 0x39, 0x34, 0x65, |
| 222 | 0x63, 0x39, 0x66, 0x64, 0x37, 0x65, 0x39, 0x32, |
| 223 | 0x63, 0x66, 0x39, 0x36, 0x34, 0x38, 0x39, 0x31, |
| 224 | 0x34, 0x63, 0x61, 0x34, 0x63, 0x32, 0x34, 0x31, |
| 225 | 0x32, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, |
| 226 | 0x20, 0x41, 0x20, 0x55, 0x20, 0x54, 0x68, 0x6f, |
| 227 | 0x72, 0x20, 0x3c, 0x61, 0x75, 0x74, 0x68, 0x6f, |
| 228 | 0x72, 0x40, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, |
| 229 | 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x3e, 0x20, 0x31, |
| 230 | 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, |
| 231 | 0x30, 0x20, 0x2b, 0x30, 0x30, 0x30, 0x30, 0x0a, |
| 232 | 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, |
| 233 | 0x72, 0x20, 0x43, 0x20, 0x4f, 0x20, 0x4d, 0x69, |
| 234 | 0x74, 0x74, 0x65, 0x72, 0x20, 0x3c, 0x63, 0x6f, |
| 235 | 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72, 0x40, |
| 236 | 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, |
| 237 | 0x63, 0x6f, 0x6d, 0x3e, 0x20, 0x31, 0x32, 0x33, |
| 238 | 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x20, |
| 239 | 0x2b, 0x30, 0x30, 0x30, 0x30, 0x0a, 0x0a, 0x45, |
| 240 | 0x6d, 0x70, 0x74, 0x79, 0x20, 0x74, 0x72, 0x65, |
| 241 | 0x65, 0x20, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, |
| 242 | 0x0a, 0xaa, 0xb8, 0x45, 0x01, 0x8e, 0xfc, 0xf0, |
| 243 | 0x2f, 0x9c, 0xc5, 0xcc, 0x4f, 0x6a, 0x1a, 0xc9, |
| 244 | 0x2b, 0x23, 0xa9, 0xff, 0x91, 0x06, 0xc2, 0x70, |
| 245 | 0xe3 |
| 246 | }; |
| 247 | |
| 248 | /* |
| 249 | * SHA-256 suffix: same structure, but with 32-byte OIDs and SHA-256 |
| 250 | * pack checksum (609 bytes vs 513 for SHA-1). |
| 251 | */ |
| 252 | static const unsigned char fast_pack_sha256_suffix[] = { |
| 253 | 0xac, 0x02, 0x78, 0x01, 0x01, 0x2c, 0x00, 0xd3, |
| 254 | 0xff, 0x31, 0x30, 0x30, 0x36, 0x34, 0x34, 0x20, |
| 255 | 0x66, 0x69, 0x6c, 0x65, 0x00, 0x42, 0x53, 0xc1, |
| 256 | 0x8a, 0x9f, 0x5e, 0xc3, 0xbb, 0x47, 0xb0, 0x83, |
| 257 | 0x8a, 0x19, 0xdb, 0x31, 0xbb, 0x7b, 0x0f, 0x3b, |
| 258 | 0x80, 0xa4, 0xbc, 0x2f, 0xaf, 0x72, 0x6b, 0xdb, |
| 259 | 0x62, 0xaa, 0xba, 0xdd, 0xde, 0x77, 0xc6, 0x13, |
| 260 | 0xeb, 0x9d, 0x0c, 0x78, 0x01, 0x01, 0xcd, 0x00, |
| 261 | 0x32, 0xff, 0x74, 0x72, 0x65, 0x65, 0x20, 0x62, |
| 262 | 0x36, 0x30, 0x39, 0x37, 0x37, 0x64, 0x37, 0x63, |
| 263 | 0x34, 0x63, 0x32, 0x64, 0x31, 0x65, 0x63, 0x63, |
| 264 | 0x33, 0x66, 0x62, 0x61, 0x31, 0x64, 0x39, 0x38, |
| 265 | 0x65, 0x65, 0x31, 0x32, 0x30, 0x61, 0x64, 0x63, |
| 266 | 0x32, 0x34, 0x38, 0x33, 0x34, 0x39, 0x35, 0x30, |
| 267 | 0x62, 0x65, 0x34, 0x31, 0x32, 0x64, 0x39, 0x34, |
| 268 | 0x63, 0x38, 0x30, 0x39, 0x34, 0x38, 0x30, 0x66, |
| 269 | 0x35, 0x38, 0x62, 0x61, 0x39, 0x64, 0x61, 0x0a, |
| 270 | 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x20, 0x41, |
| 271 | 0x20, 0x55, 0x20, 0x54, 0x68, 0x6f, 0x72, 0x20, |
| 272 | 0x3c, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x40, |
| 273 | 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, |
| 274 | 0x63, 0x6f, 0x6d, 0x3e, 0x20, 0x31, 0x32, 0x33, |
| 275 | 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x20, |
| 276 | 0x2b, 0x30, 0x30, 0x30, 0x30, 0x0a, 0x63, 0x6f, |
| 277 | 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72, 0x20, |
| 278 | 0x43, 0x20, 0x4f, 0x20, 0x4d, 0x69, 0x74, 0x74, |
| 279 | 0x65, 0x72, 0x20, 0x3c, 0x63, 0x6f, 0x6d, 0x6d, |
| 280 | 0x69, 0x74, 0x74, 0x65, 0x72, 0x40, 0x65, 0x78, |
| 281 | 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, |
| 282 | 0x6d, 0x3e, 0x20, 0x31, 0x32, 0x33, 0x34, 0x35, |
| 283 | 0x36, 0x37, 0x38, 0x39, 0x30, 0x20, 0x2b, 0x30, |
| 284 | 0x30, 0x30, 0x30, 0x0a, 0x0a, 0x4c, 0x61, 0x72, |
| 285 | 0x67, 0x65, 0x20, 0x62, 0x6c, 0x6f, 0x62, 0x20, |
| 286 | 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x0a, 0xb7, |
| 287 | 0x80, 0x3d, 0xd7, 0x20, 0x78, 0x01, 0x01, 0x00, |
| 288 | 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x95, |
| 289 | 0x11, 0x78, 0x01, 0x01, 0x15, 0x01, 0xea, 0xfe, |
| 290 | 0x74, 0x72, 0x65, 0x65, 0x20, 0x36, 0x65, 0x66, |
| 291 | 0x31, 0x39, 0x62, 0x34, 0x31, 0x32, 0x32, 0x35, |
| 292 | 0x63, 0x35, 0x33, 0x36, 0x39, 0x66, 0x31, 0x63, |
| 293 | 0x31, 0x30, 0x34, 0x64, 0x34, 0x35, 0x64, 0x38, |
| 294 | 0x64, 0x38, 0x35, 0x65, 0x66, 0x61, 0x39, 0x62, |
| 295 | 0x30, 0x35, 0x37, 0x62, 0x35, 0x33, 0x62, 0x31, |
| 296 | 0x34, 0x62, 0x34, 0x62, 0x39, 0x62, 0x39, 0x33, |
| 297 | 0x39, 0x64, 0x64, 0x37, 0x34, 0x64, 0x65, 0x63, |
| 298 | 0x63, 0x35, 0x33, 0x32, 0x31, 0x0a, 0x70, 0x61, |
| 299 | 0x72, 0x65, 0x6e, 0x74, 0x20, 0x37, 0x35, 0x62, |
| 300 | 0x66, 0x30, 0x63, 0x34, 0x37, 0x61, 0x65, 0x34, |
| 301 | 0x62, 0x62, 0x33, 0x30, 0x38, 0x65, 0x37, 0x63, |
| 302 | 0x63, 0x32, 0x34, 0x38, 0x32, 0x65, 0x32, 0x32, |
| 303 | 0x65, 0x66, 0x61, 0x65, 0x33, 0x37, 0x38, 0x37, |
| 304 | 0x61, 0x39, 0x36, 0x38, 0x34, 0x38, 0x62, 0x64, |
| 305 | 0x31, 0x37, 0x34, 0x39, 0x35, 0x36, 0x37, 0x31, |
| 306 | 0x34, 0x37, 0x31, 0x35, 0x32, 0x34, 0x36, 0x64, |
| 307 | 0x64, 0x62, 0x64, 0x35, 0x34, 0x0a, 0x61, 0x75, |
| 308 | 0x74, 0x68, 0x6f, 0x72, 0x20, 0x41, 0x20, 0x55, |
| 309 | 0x20, 0x54, 0x68, 0x6f, 0x72, 0x20, 0x3c, 0x61, |
| 310 | 0x75, 0x74, 0x68, 0x6f, 0x72, 0x40, 0x65, 0x78, |
| 311 | 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, |
| 312 | 0x6d, 0x3e, 0x20, 0x31, 0x32, 0x33, 0x34, 0x35, |
| 313 | 0x36, 0x37, 0x38, 0x39, 0x30, 0x20, 0x2b, 0x30, |
| 314 | 0x30, 0x30, 0x30, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, |
| 315 | 0x69, 0x74, 0x74, 0x65, 0x72, 0x20, 0x43, 0x20, |
| 316 | 0x4f, 0x20, 0x4d, 0x69, 0x74, 0x74, 0x65, 0x72, |
| 317 | 0x20, 0x3c, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, |
| 318 | 0x74, 0x65, 0x72, 0x40, 0x65, 0x78, 0x61, 0x6d, |
| 319 | 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x3e, |
| 320 | 0x20, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, |
| 321 | 0x38, 0x39, 0x30, 0x20, 0x2b, 0x30, 0x30, 0x30, |
| 322 | 0x30, 0x0a, 0x0a, 0x45, 0x6d, 0x70, 0x74, 0x79, |
| 323 | 0x20, 0x74, 0x72, 0x65, 0x65, 0x20, 0x63, 0x6f, |
| 324 | 0x6d, 0x6d, 0x69, 0x74, 0x0a, 0x6d, 0x6d, 0x51, |
| 325 | 0x9a, 0xc9, 0x11, 0x76, 0x61, 0xa3, 0x89, 0x49, |
| 326 | 0xb7, 0xa1, 0x58, 0xc6, 0x1d, 0x8c, 0x33, 0x75, |
| 327 | 0x8d, 0x7e, 0x4d, 0x8e, 0x58, 0x91, 0xf8, 0x5c, |
| 328 | 0x57, 0xd9, 0x89, 0x9e, 0xb8, 0xd2, 0x9a, 0xd8, |
| 329 | 0xc9 |
| 330 | }; |
| 331 | |
| 332 | static const struct fast_pack fast_packs[] = { |
| 333 | { |
| 334 | .format_id = GIT_SHA1_FORMAT_ID, |
| 335 | .suffix = fast_pack_sha1_suffix, |
| 336 | .suffix_len = sizeof(fast_pack_sha1_suffix), |
| 337 | .commit_oid = "aac43daf40d0377af31aa9c798a4ae8a31b55c1d", |
| 338 | }, |
| 339 | { |
| 340 | .format_id = GIT_SHA256_FORMAT_ID, |
| 341 | .suffix = fast_pack_sha256_suffix, |
| 342 | .suffix_len = sizeof(fast_pack_sha256_suffix), |
| 343 | .commit_oid = "63c46ca51267b1d45be69a044bb84b4bf0559f09" |
| 344 | "d727f861d2ae94ddebdddbc9", |
| 345 | }, |
| 346 | }; |
| 347 | |
| 348 | /* |
| 349 | * Try the fast path for known blob sizes. Returns 1 if the pack was |
| 350 | * written from precomputed constants, 0 if the caller should fall |
| 351 | * through to the generic path. |
| 352 | */ |
| 353 | static int generate_fast_pack(const char *path, size_t blob_size, |
| 354 | const struct git_hash_algo *algo) |
| 355 | { |
| 356 | const struct fast_pack *fp = NULL; |
| 357 | FILE *f; |
| 358 | size_t i; |
| 359 | |
| 360 | if (blob_size != FAST_PACK_4G1_BLOB_SIZE) |
| 361 | return 0; |
| 362 | |
| 363 | for (i = 0; i < ARRAY_SIZE(fast_packs); i++) { |
| 364 | if (fast_packs[i].format_id == algo->format_id) { |
| 365 | fp = &fast_packs[i]; |
| 366 | break; |
| 367 | } |
| 368 | } |
| 369 | if (!fp) |
| 370 | return 0; |
| 371 | |
| 372 | f = xfopen(path, "wb"); |
| 373 | |
| 374 | fwrite_or_die(f, fast_pack_prefix, sizeof(fast_pack_prefix)); |
| 375 | |
| 376 | /* First full block: 0xffff zero bytes (header already in prefix) */ |
| 377 | fwrite_or_die(f, zeros, BLOCK_SIZE); |
| 378 | |
| 379 | /* Remaining non-final full blocks */ |
| 380 | for (i = 1; i < FAST_PACK_4G1_N_FULL_BLOCKS; i++) { |
| 381 | fwrite_or_die(f, fast_pack_block_header, |
| 382 | sizeof(fast_pack_block_header)); |
| 383 | fwrite_or_die(f, zeros, BLOCK_SIZE); |
| 384 | } |
| 385 | |
| 386 | /* Final block (2 data bytes) + adler32 */ |
| 387 | fwrite_or_die(f, fast_pack_final_block, |
| 388 | sizeof(fast_pack_final_block)); |
| 389 | |
| 390 | /* Tree, commits, and pack checksum */ |
| 391 | fwrite_or_die(f, fp->suffix, fp->suffix_len); |
| 392 | |
| 393 | if (fclose(f)) |
| 394 | die_errno(_("could not close '%s'"), path); |
| 395 | |
| 396 | printf("%s\n", fp->commit_oid); |
| 397 | return 1; |
| 398 | } |
| 399 | |
| 400 | /* |
| 401 | * Generate a pack file with a single large (>4GB) reachable object. |
| 402 | * |
| 403 | * Creates: |
| 404 | * 1. A large blob (all NUL bytes) |
| 405 | * 2. A tree containing that blob as "file" |
| 406 | * 3. A commit using that tree |
| 407 | * 4. The empty tree |
| 408 | * 5. A child commit using the empty tree |
| 409 | * |
| 410 | * This is useful for testing that Git can handle objects larger than 4GB. |
| 411 | */ |
| 412 | static int generate_pack_with_large_object(const char *path, size_t blob_size, |
| 413 | const struct git_hash_algo *algo) |
| 414 | { |
| 415 | FILE *f; |
| 416 | struct git_hash_ctx pack_ctx; |
| 417 | unsigned char pack_hash[GIT_MAX_RAWSZ]; |
| 418 | struct object_id blob_oid, tree_oid, commit_oid, empty_tree_oid, final_commit_oid; |
| 419 | struct strbuf buf = STRBUF_INIT; |
| 420 | const uint32_t object_count = 5; |
| 421 | struct pack_header pack_header = { |
| 422 | .hdr_signature = htonl(PACK_SIGNATURE), |
| 423 | .hdr_version = htonl(PACK_VERSION), |
| 424 | .hdr_entries = htonl(object_count), |
| 425 | }; |
| 426 | |
| 427 | if (generate_fast_pack(path, blob_size, algo)) |
| 428 | return 0; |
| 429 | |
| 430 | f = xfopen(path, "wb"); |
| 431 | |
| 432 | git_hash_init(&pack_ctx, algo); |
| 433 | |
| 434 | /* Write pack header */ |
| 435 | fwrite_or_die(f, &pack_header, sizeof(pack_header)); |
| 436 | git_hash_update(&pack_ctx, &pack_header, sizeof(pack_header)); |
| 437 | |
| 438 | /* 1. Write the large blob */ |
| 439 | write_pack_object(f, &pack_ctx, OBJ_BLOB, NULL, blob_size, &blob_oid, algo); |
| 440 | |
| 441 | /* 2. Write tree containing the blob as "file" */ |
| 442 | strbuf_addf(&buf, "100644 file%c", '\0'); |
| 443 | strbuf_add(&buf, blob_oid.hash, algo->rawsz); |
| 444 | write_pack_object(f, &pack_ctx, OBJ_TREE, buf.buf, buf.len, &tree_oid, algo); |
| 445 | |
| 446 | /* 3. Write commit using that tree */ |
| 447 | strbuf_reset(&buf); |
| 448 | strbuf_addf(&buf, |
| 449 | "tree %s\n" |
| 450 | "author A U Thor <author@example.com> 1234567890 +0000\n" |
| 451 | "committer C O Mitter <committer@example.com> 1234567890 +0000\n" |
| 452 | "\n" |
| 453 | "Large blob commit\n", |
| 454 | oid_to_hex(&tree_oid)); |
| 455 | write_pack_object(f, &pack_ctx, OBJ_COMMIT, buf.buf, buf.len, &commit_oid, algo); |
| 456 | |
| 457 | /* 4. Write the empty tree */ |
| 458 | write_pack_object(f, &pack_ctx, OBJ_TREE, "", 0, &empty_tree_oid, algo); |
| 459 | |
| 460 | /* 5. Write final commit using empty tree, with previous commit as parent */ |
| 461 | strbuf_reset(&buf); |
| 462 | strbuf_addf(&buf, |
| 463 | "tree %s\n" |
| 464 | "parent %s\n" |
| 465 | "author A U Thor <author@example.com> 1234567890 +0000\n" |
| 466 | "committer C O Mitter <committer@example.com> 1234567890 +0000\n" |
| 467 | "\n" |
| 468 | "Empty tree commit\n", |
| 469 | oid_to_hex(&empty_tree_oid), |
| 470 | oid_to_hex(&commit_oid)); |
| 471 | write_pack_object(f, &pack_ctx, OBJ_COMMIT, buf.buf, buf.len, &final_commit_oid, algo); |
| 472 | |
| 473 | /* Write pack trailer (checksum) */ |
| 474 | git_hash_final(pack_hash, &pack_ctx); |
| 475 | fwrite_or_die(f, pack_hash, algo->rawsz); |
| 476 | if (fclose(f)) |
| 477 | die_errno(_("could not close '%s'"), path); |
| 478 | |
| 479 | strbuf_release(&buf); |
| 480 | |
| 481 | /* Print the final commit OID so caller can set up refs */ |
| 482 | printf("%s\n", oid_to_hex(&final_commit_oid)); |
| 483 | |
| 484 | return 0; |
| 485 | } |
| 486 | |
| 487 | static int cmd__synthesize__pack(int argc, const char **argv, |
| 488 | const char *prefix UNUSED, |
| 489 | struct repository *repo) |
| 490 | { |
| 491 | int non_git; |
| 492 | int reachable_large = 0; |
| 493 | const struct git_hash_algo *algo; |
| 494 | size_t blob_size; |
| 495 | uintmax_t blob_size_u; |
| 496 | const char *path; |
| 497 | const char * const usage[] = { |
| 498 | "test-tool synthesize pack " |
| 499 | "--reachable-large <blob-size> <filename>", |
| 500 | NULL |
| 501 | }; |
| 502 | struct option options[] = { |
| 503 | OPT_BOOL(0, "reachable-large", &reachable_large, |
| 504 | N_("write a pack with a single reachable large blob")), |
| 505 | OPT_END() |
| 506 | }; |
| 507 | |
| 508 | setup_git_directory_gently(the_repository, &non_git); |
| 509 | repo = the_repository; |
| 510 | algo = unsafe_hash_algo(repo->hash_algo); |
| 511 | |
| 512 | argc = parse_options(argc, argv, NULL, options, usage, |
| 513 | PARSE_OPT_KEEP_ARGV0); |
| 514 | if (argc != 3 || !reachable_large) |
| 515 | usage_with_options(usage, options); |
| 516 | |
| 517 | if (!git_parse_unsigned(argv[1], &blob_size_u, |
| 518 | maximum_unsigned_value_of_type(size_t))) |
| 519 | die(_("'%s' is not a valid blob size"), argv[1]); |
| 520 | blob_size = blob_size_u; |
| 521 | path = argv[2]; |
| 522 | |
| 523 | return !!generate_pack_with_large_object(path, blob_size, algo); |
| 524 | } |
| 525 | |
| 526 | int cmd__synthesize(int argc, const char **argv) |
| 527 | { |
| 528 | const char *prefix = NULL; |
| 529 | char const * const synthesize_usage[] = { |
| 530 | "test-tool synthesize pack <options>", |
| 531 | NULL, |
| 532 | }; |
| 533 | parse_opt_subcommand_fn *fn = NULL; |
| 534 | struct option options[] = { |
| 535 | OPT_SUBCOMMAND("pack", &fn, cmd__synthesize__pack), |
| 536 | OPT_END() |
| 537 | }; |
| 538 | argc = parse_options(argc, argv, prefix, options, synthesize_usage, 0); |
| 539 | return !!fn(argc, argv, prefix, NULL); |
| 540 | } |