| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | ft "github.com/ipfs/boxo/ipld/unixfs" |
| 11 | "github.com/ipfs/kubo/test/cli/harness" |
| 12 | "github.com/ipfs/kubo/test/cli/testutils" |
| 13 | "github.com/stretchr/testify/assert" |
| 14 | "github.com/stretchr/testify/require" |
| 15 | ) |
| 16 | |
| 17 | // cidProfileExpectations defines expected behaviors for a UnixFS import profile. |
| 18 | // This allows DRY testing of multiple profiles with the same test logic. |
| 19 | // |
| 20 | // Each profile is tested against threshold boundaries to verify: |
| 21 | // - CID format (version, hash function, raw leaves vs dag-pb wrapped) |
| 22 | // - File chunking (UnixFSChunker size threshold) |
| 23 | // - DAG structure (UnixFSFileMaxLinks rebalancing threshold) |
| 24 | // - Directory sharding (HAMTThreshold for flat vs HAMT directories) |
| 25 | type cidProfileExpectations struct { |
| 26 | // Profile identification |
| 27 | Name string // canonical profile name from IPIP-499 |
| 28 | ProfileArgs []string // args to pass to ipfs init (empty for default behavior) |
| 29 | |
| 30 | // CID format expectations |
| 31 | CIDVersion int // 0 or 1 |
| 32 | HashFunc string // e.g., "sha2-256" |
| 33 | RawLeaves bool // true = raw codec for small files, false = dag-pb wrapped |
| 34 | |
| 35 | // File chunking expectations (UnixFSChunker config) |
| 36 | ChunkSize int // chunk size in bytes (e.g., 262144 for 256KiB, 1048576 for 1MiB) |
| 37 | ChunkSizeHuman string // human-readable chunk size (e.g., "256KiB", "1MiB") |
| 38 | FileMaxLinks int // max links before DAG rebalancing (UnixFSFileMaxLinks config) |
| 39 | |
| 40 | // HAMT directory sharding expectations (UnixFSHAMTDirectory* config). |
| 41 | // Threshold behavior: boxo converts to HAMT when size > HAMTThreshold (not >=). |
| 42 | // This means a directory exactly at the threshold stays as a basic (flat) directory. |
| 43 | HAMTFanout int // max links per HAMT shard bucket (256) |
| 44 | HAMTThreshold int // sharding threshold in bytes (262144 = 256 KiB) |
| 45 | HAMTSizeEstimation string // "block" (protobuf size) or "links" (legacy name+cid) |
| 46 | |
| 47 | // Test vector parameters for threshold boundary tests. |
| 48 | // - DirBasic: size == threshold (stays basic) |
| 49 | // - DirHAMT: size > threshold (converts to HAMT) |
| 50 | // For block estimation, last filename length is adjusted to hit exact thresholds. |
| 51 | DirBasicNameLen int // filename length for basic directory (files 0 to N-2) |
| 52 | DirBasicLastNameLen int // filename length for last file (0 = same as DirBasicNameLen) |
| 53 | DirBasicFiles int // file count for basic directory (at exact threshold) |
| 54 | DirHAMTNameLen int // filename length for HAMT directory (files 0 to N-2) |
| 55 | DirHAMTLastNameLen int // filename length for last file (0 = same as DirHAMTNameLen) |
| 56 | DirHAMTFiles int // total file count for HAMT directory (over threshold) |
| 57 | |
| 58 | // Expected deterministic CIDs for test vectors. |
| 59 | // These serve as regression tests to detect unintended changes in CID generation. |
| 60 | |
| 61 | // SmallFileCID is the deterministic CID for "hello world" string. |
| 62 | // Tests basic CID format (version, codec, hash). |
| 63 | SmallFileCID string |
| 64 | |
| 65 | // FileAtChunkSizeCID is the deterministic CID for a file exactly at chunk size. |
| 66 | // This file fits in a single block with no links: |
| 67 | // - v0-2015: dag-pb wrapped TFile node (CIDv0) |
| 68 | // - v1-2025: raw leaf block (CIDv1) |
| 69 | FileAtChunkSizeCID string |
| 70 | |
| 71 | // FileOverChunkSizeCID is the deterministic CID for a file 1 byte over chunk size. |
| 72 | // This file requires 2 chunks, producing a root dag-pb node with 2 links: |
| 73 | // - v0-2015: links point to dag-pb wrapped TFile leaf nodes |
| 74 | // - v1-2025: links point to raw leaf blocks |
| 75 | FileOverChunkSizeCID string |
| 76 | |
| 77 | // FileAtMaxLinksCID is the deterministic CID for a file at UnixFSFileMaxLinks threshold. |
| 78 | // File size = maxLinks * chunkSize, producing a single-layer DAG with exactly maxLinks children. |
| 79 | FileAtMaxLinksCID string |
| 80 | |
| 81 | // FileOverMaxLinksCID is the deterministic CID for a file 1 byte over max links threshold. |
| 82 | // The +1 byte requires an additional chunk, forcing DAG rebalancing to 2 layers. |
| 83 | FileOverMaxLinksCID string |
| 84 | |
| 85 | // DirBasicCID is the deterministic CID for a directory exactly at HAMTThreshold. |
| 86 | // With > comparison (not >=), directory at exact threshold stays as basic (flat) directory. |
| 87 | DirBasicCID string |
| 88 | |
| 89 | // DirHAMTCID is the deterministic CID for a directory 1 byte over HAMTThreshold. |
| 90 | // Crossing the threshold converts the directory to a HAMT sharded structure. |
| 91 | DirHAMTCID string |
| 92 | } |
| 93 | |
| 94 | // unixfsV02015 is the legacy profile for backward-compatible CID generation. |
| 95 | // Alias: legacy-cid-v0 |
| 96 | var unixfsV02015 = cidProfileExpectations{ |
| 97 | Name: "unixfs-v0-2015", |
| 98 | ProfileArgs: []string{"--profile=unixfs-v0-2015"}, |
| 99 | |
| 100 | CIDVersion: 0, |
| 101 | HashFunc: "sha2-256", |
| 102 | RawLeaves: false, |
| 103 | |
| 104 | ChunkSize: 262144, // 256 KiB |
| 105 | ChunkSizeHuman: "256KiB", |
| 106 | FileMaxLinks: 174, |
| 107 | |
| 108 | HAMTFanout: 256, |
| 109 | HAMTThreshold: 262144, // 256 KiB |
| 110 | HAMTSizeEstimation: "links", |
| 111 | DirBasicNameLen: 30, // 4096 * (30 + 34) = 262144 exactly at threshold |
| 112 | DirBasicFiles: 4096, // 4096 * 64 = 262144 (stays basic with >) |
| 113 | DirHAMTNameLen: 31, // 4033 * (31 + 34) = 262145 exactly +1 over threshold |
| 114 | DirHAMTLastNameLen: 0, // 0 = same as DirHAMTNameLen (uniform filenames) |
| 115 | DirHAMTFiles: 4033, // 4033 * 65 = 262145 (becomes HAMT) |
| 116 | |
| 117 | SmallFileCID: "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", // "hello world" dag-pb wrapped |
| 118 | FileAtChunkSizeCID: "QmWmRj3dFDZdb6ABvbmKhEL6TmPbAfBZ1t5BxsEyJrcZhE", // 262144 bytes with seed "chunk-v0-seed" |
| 119 | FileOverChunkSizeCID: "QmYyLxtzZyW22zpoVAtKANLRHpDjZtNeDjQdJrcQNWoRkJ", // 262145 bytes with seed "chunk-v0-seed" |
| 120 | FileAtMaxLinksCID: "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", // 174*256KiB bytes with seed "v0-seed" |
| 121 | FileOverMaxLinksCID: "QmV81WL765sC8DXsRhE5fJv2rwhS4icHRaf3J9Zk5FdRnW", // 174*256KiB+1 bytes with seed "v0-seed" |
| 122 | DirBasicCID: "QmX5GtRk3TSSEHtdrykgqm4eqMEn3n2XhfkFAis5fjyZmN", // 4096 files at threshold |
| 123 | DirHAMTCID: "QmeMiJzmhpJAUgynAcxTQYek5PPKgdv3qEvFsdV3XpVnvP", // 4033 files +1 over threshold |
| 124 | } |
| 125 | |
| 126 | // unixfsV12025 is the recommended profile for cross-implementation CID determinism. |
| 127 | var unixfsV12025 = cidProfileExpectations{ |
| 128 | Name: "unixfs-v1-2025", |
| 129 | ProfileArgs: []string{"--profile=unixfs-v1-2025"}, |
| 130 | |
| 131 | CIDVersion: 1, |
| 132 | HashFunc: "sha2-256", |
| 133 | RawLeaves: true, |
| 134 | |
| 135 | ChunkSize: 1048576, // 1 MiB |
| 136 | ChunkSizeHuman: "1MiB", |
| 137 | FileMaxLinks: 1024, |
| 138 | |
| 139 | HAMTFanout: 256, |
| 140 | HAMTThreshold: 262144, // 256 KiB |
| 141 | HAMTSizeEstimation: "block", |
| 142 | // Block size = numFiles * linkSize + 4 bytes overhead |
| 143 | // LinkSerializedSize(11, 36, 1) = 55, LinkSerializedSize(21, 36, 1) = 65, LinkSerializedSize(22, 36, 1) = 66 |
| 144 | DirBasicNameLen: 11, // 4765 files * 55 bytes |
| 145 | DirBasicLastNameLen: 21, // last file: 65 bytes; total: 4765*55 + 65 + 4 = 262144 (at threshold) |
| 146 | DirBasicFiles: 4766, // stays basic with > comparison |
| 147 | DirHAMTNameLen: 11, // 4765 files * 55 bytes |
| 148 | DirHAMTLastNameLen: 22, // last file: 66 bytes; total: 4765*55 + 66 + 4 = 262145 (+1 over threshold) |
| 149 | DirHAMTFiles: 4766, // becomes HAMT |
| 150 | |
| 151 | SmallFileCID: "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", // "hello world" raw leaf |
| 152 | FileAtChunkSizeCID: "bafkreiacndfy443ter6qr2tmbbdhadvxxheowwf75s6zehscklu6ezxmta", // 1048576 bytes with seed "chunk-v1-seed" |
| 153 | FileOverChunkSizeCID: "bafybeigmix7t42i6jacydtquhet7srwvgpizfg7gjbq7627d35mjomtu64", // 1048577 bytes with seed "chunk-v1-seed" |
| 154 | FileAtMaxLinksCID: "bafybeihmf37wcuvtx4hpu7he5zl5qaf2ineo2lqlfrapokkm5zzw7zyhvm", // 1024*1MiB bytes with seed "v1-2025-seed" |
| 155 | FileOverMaxLinksCID: "bafybeibdsi225ugbkmpbdohnxioyab6jsqrmkts3twhpvfnzp77xtzpyhe", // 1024*1MiB+1 bytes with seed "v1-2025-seed" |
| 156 | DirBasicCID: "bafybeic3h7rwruealwxkacabdy45jivq2crwz6bufb5ljwupn36gicplx4", // 4766 files at 262144 bytes (threshold) |
| 157 | DirHAMTCID: "bafybeiegvuterwurhdtkikfhbxcldohmxp566vpjdofhzmnhv6o4freidu", // 4766 files at 262145 bytes (+1 over) |
| 158 | } |
| 159 | |
| 160 | // defaultProfile points to the profile that matches Kubo's implicit default behavior. |
| 161 | // Today this is unixfs-v0-2015. When Kubo changes defaults, update this pointer. |
| 162 | var defaultProfile = unixfsV02015 |
| 163 | |
| 164 | const ( |
| 165 | cidV0Length = 34 // CIDv0 sha2-256 |
| 166 | cidV1Length = 36 // CIDv1 sha2-256 |
| 167 | ) |
| 168 | |
| 169 | // TestCIDProfiles generates deterministic test vectors for CID profile verification. |
| 170 | // Set CID_PROFILES_CAR_OUTPUT environment variable to export CAR files. |
| 171 | // Example: CID_PROFILES_CAR_OUTPUT=/tmp/cid-profiles go test -run TestCIDProfiles -v |
| 172 | func TestCIDProfiles(t *testing.T) { |
| 173 | t.Parallel() |
| 174 | |
| 175 | carOutputDir := os.Getenv("CID_PROFILES_CAR_OUTPUT") |
| 176 | exportCARs := carOutputDir != "" |
| 177 | if exportCARs { |
| 178 | if err := os.MkdirAll(carOutputDir, 0o755); err != nil { |
| 179 | t.Fatalf("failed to create CAR output directory: %v", err) |
| 180 | } |
| 181 | t.Logf("CAR export enabled, writing to: %s", carOutputDir) |
| 182 | } |
| 183 | |
| 184 | // Test both IPIP-499 profiles |
| 185 | for _, profile := range []cidProfileExpectations{unixfsV02015, unixfsV12025} { |
| 186 | t.Run(profile.Name, func(t *testing.T) { |
| 187 | t.Parallel() |
| 188 | runProfileTests(t, profile, carOutputDir, exportCARs) |
| 189 | }) |
| 190 | } |
| 191 | |
| 192 | // Test default behavior (no profile specified) |
| 193 | t.Run("default", func(t *testing.T) { |
| 194 | t.Parallel() |
| 195 | // Default behavior should match defaultProfile (currently unixfs-v0-2015) |
| 196 | defaultExp := defaultProfile |
| 197 | defaultExp.Name = "default" |
| 198 | defaultExp.ProfileArgs = nil // no profile args = default behavior |
| 199 | runProfileTests(t, defaultExp, carOutputDir, exportCARs) |
| 200 | }) |
| 201 | } |
| 202 | |
| 203 | // runProfileTests runs all test vectors for a given profile. |
| 204 | // Tests verify threshold behaviors for: |
| 205 | // - Small files (CID format verification) |
| 206 | // - UnixFSChunker threshold (single block vs multi-block) |
| 207 | // - UnixFSFileMaxLinks threshold (single-layer vs rebalanced DAG) |
| 208 | // - HAMTThreshold (basic flat directory vs HAMT sharded) |
| 209 | func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir string, exportCARs bool) { |
| 210 | cidLen := cidV0Length |
| 211 | if exp.CIDVersion == 1 { |
| 212 | cidLen = cidV1Length |
| 213 | } |
| 214 | |
| 215 | // Test: small file produces correct CID format |
| 216 | // Verifies the profile sets the expected CID version, hash function, and leaf encoding. |
| 217 | t.Run("small file produces correct CID format", func(t *testing.T) { |
| 218 | t.Parallel() |
| 219 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 220 | node.StartDaemon() |
| 221 | defer node.StopDaemon() |
| 222 | |
| 223 | // Use "hello world" for determinism |
| 224 | cidStr := node.IPFSAddStr("hello world") |
| 225 | |
| 226 | // Verify CID version (v0 starts with "Qm", v1 with "b") |
| 227 | verifyCIDVersion(t, node, cidStr, exp.CIDVersion) |
| 228 | |
| 229 | // Verify hash function (sha2-256 for both profiles) |
| 230 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 231 | |
| 232 | // Verify raw leaves vs dag-pb wrapped |
| 233 | // - v0-2015: dag-pb codec (wrapped) |
| 234 | // - v1-2025: raw codec (raw leaves) |
| 235 | verifyRawLeaves(t, node, cidStr, exp.RawLeaves) |
| 236 | |
| 237 | // Verify deterministic CID matches expected value |
| 238 | if exp.SmallFileCID != "" { |
| 239 | require.Equal(t, exp.SmallFileCID, cidStr, "expected deterministic CID for small file") |
| 240 | } |
| 241 | |
| 242 | if exportCARs { |
| 243 | carPath := filepath.Join(carOutputDir, exp.Name+"_small-file.car") |
| 244 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 245 | t.Logf("exported: %s -> %s", cidStr, carPath) |
| 246 | } |
| 247 | }) |
| 248 | |
| 249 | // Test: file at UnixFSChunker threshold (single block) |
| 250 | // A file exactly at chunk size fits in one block with no links. |
| 251 | // - v0-2015 (256KiB): produces dag-pb wrapped TFile node |
| 252 | // - v1-2025 (1MiB): produces raw leaf block |
| 253 | t.Run("file at UnixFSChunker threshold (single block)", func(t *testing.T) { |
| 254 | t.Parallel() |
| 255 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 256 | node.StartDaemon() |
| 257 | defer node.StopDaemon() |
| 258 | |
| 259 | // File exactly at chunk size = single block (no links) |
| 260 | seed := chunkSeedForProfile(exp) |
| 261 | cidStr := node.IPFSAddDeterministicBytes(int64(exp.ChunkSize), seed) |
| 262 | |
| 263 | // Verify block structure based on raw leaves setting |
| 264 | if exp.RawLeaves { |
| 265 | // v1-2025: single block is a raw leaf (no dag-pb structure) |
| 266 | codec := node.IPFS("cid", "format", "-f", "%c", cidStr).Stdout.Trimmed() |
| 267 | require.Equal(t, "raw", codec, "single block file is raw leaf") |
| 268 | } else { |
| 269 | // v0-2015: single block is a dag-pb node with no links (TFile type) |
| 270 | root, err := node.InspectPBNode(cidStr) |
| 271 | assert.NoError(t, err) |
| 272 | require.Equal(t, 0, len(root.Links), "single block file has no links") |
| 273 | fsType, err := node.UnixFSDataType(cidStr) |
| 274 | require.NoError(t, err) |
| 275 | require.Equal(t, ft.TFile, fsType, "single block file is dag-pb wrapped (TFile)") |
| 276 | } |
| 277 | |
| 278 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 279 | |
| 280 | if exp.FileAtChunkSizeCID != "" { |
| 281 | require.Equal(t, exp.FileAtChunkSizeCID, cidStr, "expected deterministic CID for file at chunk size") |
| 282 | } |
| 283 | |
| 284 | if exportCARs { |
| 285 | carPath := filepath.Join(carOutputDir, exp.Name+"_file-at-chunk-size.car") |
| 286 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 287 | t.Logf("exported: %s -> %s", cidStr, carPath) |
| 288 | } |
| 289 | }) |
| 290 | |
| 291 | // Test: file 1 byte over UnixFSChunker threshold (2 blocks) |
| 292 | // A file 1 byte over chunk size requires 2 chunks. |
| 293 | // Root is a dag-pb node with 2 links. Leaf encoding depends on profile: |
| 294 | // - v0-2015: leaf blocks are dag-pb wrapped TFile nodes |
| 295 | // - v1-2025: leaf blocks are raw codec blocks |
| 296 | t.Run("file 1 byte over UnixFSChunker threshold (2 blocks)", func(t *testing.T) { |
| 297 | t.Parallel() |
| 298 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 299 | node.StartDaemon() |
| 300 | defer node.StopDaemon() |
| 301 | |
| 302 | // File +1 byte over chunk size = 2 blocks |
| 303 | seed := chunkSeedForProfile(exp) |
| 304 | cidStr := node.IPFSAddDeterministicBytes(int64(exp.ChunkSize)+1, seed) |
| 305 | |
| 306 | root, err := node.InspectPBNode(cidStr) |
| 307 | assert.NoError(t, err) |
| 308 | require.Equal(t, 2, len(root.Links), "file over chunk size has 2 links") |
| 309 | |
| 310 | // Verify leaf block encoding |
| 311 | for _, link := range root.Links { |
| 312 | if exp.RawLeaves { |
| 313 | // v1-2025: leaves are raw blocks |
| 314 | leafCodec := node.IPFS("cid", "format", "-f", "%c", link.Hash.Slash).Stdout.Trimmed() |
| 315 | require.Equal(t, "raw", leafCodec, "leaf blocks are raw, not dag-pb") |
| 316 | } else { |
| 317 | // v0-2015: leaves are dag-pb wrapped (TFile type) |
| 318 | leafType, err := node.UnixFSDataType(link.Hash.Slash) |
| 319 | require.NoError(t, err) |
| 320 | require.Equal(t, ft.TFile, leafType, "leaf blocks are dag-pb wrapped (TFile)") |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 325 | |
| 326 | if exp.FileOverChunkSizeCID != "" { |
| 327 | require.Equal(t, exp.FileOverChunkSizeCID, cidStr, "expected deterministic CID for file over chunk size") |
| 328 | } |
| 329 | |
| 330 | if exportCARs { |
| 331 | carPath := filepath.Join(carOutputDir, exp.Name+"_file-over-chunk-size.car") |
| 332 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 333 | t.Logf("exported: %s -> %s", cidStr, carPath) |
| 334 | } |
| 335 | }) |
| 336 | |
| 337 | // Test: file at UnixFSFileMaxLinks threshold (single layer) |
| 338 | // A file of exactly maxLinks * chunkSize bytes fits in a single DAG layer. |
| 339 | // - v0-2015: 174 links (174 * 256KiB = ~44.6MiB) |
| 340 | // - v1-2025: 1024 links (1024 * 1MiB = 1GiB) |
| 341 | t.Run("file at UnixFSFileMaxLinks threshold (single layer)", func(t *testing.T) { |
| 342 | t.Parallel() |
| 343 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 344 | node.StartDaemon() |
| 345 | defer node.StopDaemon() |
| 346 | |
| 347 | // File size = maxLinks * chunkSize (exactly at threshold) |
| 348 | fileSize := fileAtMaxLinksBytes(exp) |
| 349 | seed := seedForProfile(exp) |
| 350 | cidStr := node.IPFSAddDeterministicBytes(fileSize, seed) |
| 351 | |
| 352 | root, err := node.InspectPBNode(cidStr) |
| 353 | assert.NoError(t, err) |
| 354 | require.Equal(t, exp.FileMaxLinks, len(root.Links), |
| 355 | "expected exactly %d links at max", exp.FileMaxLinks) |
| 356 | |
| 357 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 358 | |
| 359 | if exp.FileAtMaxLinksCID != "" { |
| 360 | require.Equal(t, exp.FileAtMaxLinksCID, cidStr, "expected deterministic CID for file at max links") |
| 361 | } |
| 362 | |
| 363 | if exportCARs { |
| 364 | carPath := filepath.Join(carOutputDir, exp.Name+"_file-at-max-links.car") |
| 365 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 366 | t.Logf("exported: %s -> %s", cidStr, carPath) |
| 367 | } |
| 368 | }) |
| 369 | |
| 370 | // Test: file 1 byte over UnixFSFileMaxLinks threshold (rebalanced DAG) |
| 371 | // Adding 1 byte requires an additional chunk, exceeding maxLinks. |
| 372 | // This triggers DAG rebalancing: chunks are grouped into intermediate nodes, |
| 373 | // producing a 2-layer DAG with 2 links at the root. |
| 374 | t.Run("file 1 byte over UnixFSFileMaxLinks threshold (rebalanced DAG)", func(t *testing.T) { |
| 375 | t.Parallel() |
| 376 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 377 | node.StartDaemon() |
| 378 | defer node.StopDaemon() |
| 379 | |
| 380 | // +1 byte over max links threshold triggers DAG rebalancing |
| 381 | fileSize := fileOverMaxLinksBytes(exp) |
| 382 | seed := seedForProfile(exp) |
| 383 | cidStr := node.IPFSAddDeterministicBytes(fileSize, seed) |
| 384 | |
| 385 | root, err := node.InspectPBNode(cidStr) |
| 386 | assert.NoError(t, err) |
| 387 | require.Equal(t, 2, len(root.Links), "expected 2 links after DAG rebalancing") |
| 388 | |
| 389 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 390 | |
| 391 | if exp.FileOverMaxLinksCID != "" { |
| 392 | require.Equal(t, exp.FileOverMaxLinksCID, cidStr, "expected deterministic CID for rebalanced file") |
| 393 | } |
| 394 | |
| 395 | if exportCARs { |
| 396 | carPath := filepath.Join(carOutputDir, exp.Name+"_file-over-max-links.car") |
| 397 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 398 | t.Logf("exported: %s -> %s", cidStr, carPath) |
| 399 | } |
| 400 | }) |
| 401 | |
| 402 | // Test: directory at HAMTThreshold (basic flat dir) |
| 403 | // A directory exactly at HAMTThreshold stays as a basic (flat) UnixFS directory. |
| 404 | // Threshold uses > comparison (not >=), so size == threshold stays basic. |
| 405 | // Size estimation method depends on profile: |
| 406 | // - v0-2015 "links": size = sum(nameLen + cidLen) |
| 407 | // - v1-2025 "block": size = serialized protobuf block size |
| 408 | t.Run("directory at HAMTThreshold (basic flat dir)", func(t *testing.T) { |
| 409 | t.Parallel() |
| 410 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 411 | node.StartDaemon() |
| 412 | defer node.StopDaemon() |
| 413 | |
| 414 | // Use consistent seed for deterministic CIDs |
| 415 | seed := hamtSeedForProfile(exp) |
| 416 | randDir, err := os.MkdirTemp(node.Dir, seed) |
| 417 | require.NoError(t, err) |
| 418 | |
| 419 | // Create basic (flat) directory exactly at threshold |
| 420 | basicLastNameLen := exp.DirBasicLastNameLen |
| 421 | if basicLastNameLen == 0 { |
| 422 | basicLastNameLen = exp.DirBasicNameLen |
| 423 | } |
| 424 | if exp.HAMTSizeEstimation == "block" { |
| 425 | err = createDirectoryForHAMTBlockEstimation(randDir, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) |
| 426 | } else { |
| 427 | err = createDirectoryForHAMTLinksEstimation(randDir, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) |
| 428 | } |
| 429 | require.NoError(t, err) |
| 430 | |
| 431 | cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() |
| 432 | |
| 433 | // Verify UnixFS type is TDirectory (1), not THAMTShard (5) |
| 434 | fsType, err := node.UnixFSDataType(cidStr) |
| 435 | require.NoError(t, err) |
| 436 | require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") |
| 437 | |
| 438 | root, err := node.InspectPBNode(cidStr) |
| 439 | assert.NoError(t, err) |
| 440 | require.Equal(t, exp.DirBasicFiles, len(root.Links), |
| 441 | "expected basic directory with %d links", exp.DirBasicFiles) |
| 442 | |
| 443 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 444 | |
| 445 | // Verify size is exactly at threshold |
| 446 | if exp.HAMTSizeEstimation == "block" { |
| 447 | blockSize := getBlockSize(t, node, cidStr) |
| 448 | require.Equal(t, exp.HAMTThreshold, blockSize, |
| 449 | "expected basic directory block size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, blockSize) |
| 450 | } |
| 451 | if exp.HAMTSizeEstimation == "links" { |
| 452 | linksSize := 0 |
| 453 | for _, link := range root.Links { |
| 454 | linksSize += len(link.Name) + cidLen |
| 455 | } |
| 456 | require.Equal(t, exp.HAMTThreshold, linksSize, |
| 457 | "expected basic directory links size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, linksSize) |
| 458 | } |
| 459 | |
| 460 | if exp.DirBasicCID != "" { |
| 461 | require.Equal(t, exp.DirBasicCID, cidStr, "expected deterministic CID for basic directory") |
| 462 | } |
| 463 | |
| 464 | if exportCARs { |
| 465 | carPath := filepath.Join(carOutputDir, exp.Name+"_dir-basic.car") |
| 466 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 467 | t.Logf("exported: %s (%d files) -> %s", cidStr, exp.DirBasicFiles, carPath) |
| 468 | } |
| 469 | }) |
| 470 | |
| 471 | // Test: directory 1 byte over HAMTThreshold (HAMT sharded) |
| 472 | // A directory 1 byte over HAMTThreshold is converted to a HAMT sharded structure. |
| 473 | // HAMT distributes entries across buckets using consistent hashing. |
| 474 | // Root has at most HAMTFanout links (256), with entries distributed across buckets. |
| 475 | t.Run("directory 1 byte over HAMTThreshold (HAMT sharded)", func(t *testing.T) { |
| 476 | t.Parallel() |
| 477 | node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) |
| 478 | node.StartDaemon() |
| 479 | defer node.StopDaemon() |
| 480 | |
| 481 | // Use consistent seed for deterministic CIDs |
| 482 | seed := hamtSeedForProfile(exp) |
| 483 | randDir, err := os.MkdirTemp(node.Dir, seed) |
| 484 | require.NoError(t, err) |
| 485 | |
| 486 | // Create HAMT (sharded) directory exactly +1 byte over threshold |
| 487 | lastNameLen := exp.DirHAMTLastNameLen |
| 488 | if lastNameLen == 0 { |
| 489 | lastNameLen = exp.DirHAMTNameLen |
| 490 | } |
| 491 | if exp.HAMTSizeEstimation == "block" { |
| 492 | err = createDirectoryForHAMTBlockEstimation(randDir, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) |
| 493 | } else { |
| 494 | err = createDirectoryForHAMTLinksEstimation(randDir, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) |
| 495 | } |
| 496 | require.NoError(t, err) |
| 497 | |
| 498 | cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() |
| 499 | |
| 500 | // Verify UnixFS type is THAMTShard (5), not TDirectory (1) |
| 501 | fsType, err := node.UnixFSDataType(cidStr) |
| 502 | require.NoError(t, err) |
| 503 | require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when over threshold") |
| 504 | |
| 505 | // HAMT root has at most fanout links (actual count depends on hash distribution) |
| 506 | root, err := node.InspectPBNode(cidStr) |
| 507 | assert.NoError(t, err) |
| 508 | require.LessOrEqual(t, len(root.Links), exp.HAMTFanout, |
| 509 | "expected HAMT directory root to have <= %d links", exp.HAMTFanout) |
| 510 | |
| 511 | verifyHashFunction(t, node, cidStr, exp.HashFunc) |
| 512 | |
| 513 | if exp.DirHAMTCID != "" { |
| 514 | require.Equal(t, exp.DirHAMTCID, cidStr, "expected deterministic CID for HAMT directory") |
| 515 | } |
| 516 | |
| 517 | if exportCARs { |
| 518 | carPath := filepath.Join(carOutputDir, exp.Name+"_dir-hamt.car") |
| 519 | require.NoError(t, node.IPFSDagExport(cidStr, carPath)) |
| 520 | t.Logf("exported: %s (%d files, HAMT root links: %d) -> %s", |
| 521 | cidStr, exp.DirHAMTFiles, len(root.Links), carPath) |
| 522 | } |
| 523 | }) |
| 524 | } |
| 525 | |
| 526 | // verifyCIDVersion checks that the CID has the expected version. |
| 527 | func verifyCIDVersion(t *testing.T, _ *harness.Node, cidStr string, expectedVersion int) { |
| 528 | t.Helper() |
| 529 | if expectedVersion == 0 { |
| 530 | require.True(t, strings.HasPrefix(cidStr, "Qm"), |
| 531 | "expected CIDv0 (starts with Qm), got: %s", cidStr) |
| 532 | } else { |
| 533 | require.True(t, strings.HasPrefix(cidStr, "b"), |
| 534 | "expected CIDv1 (base32, starts with b), got: %s", cidStr) |
| 535 | } |
| 536 | } |
| 537 | |
| 538 | // verifyHashFunction checks that the CID uses the expected hash function. |
| 539 | func verifyHashFunction(t *testing.T, node *harness.Node, cidStr, expectedHash string) { |
| 540 | t.Helper() |
| 541 | // Use ipfs cid format to get hash function info |
| 542 | // Format string %h gives the hash function name |
| 543 | res := node.IPFS("cid", "format", "-f", "%h", cidStr) |
| 544 | hashFunc := strings.TrimSpace(res.Stdout.String()) |
| 545 | require.Equal(t, expectedHash, hashFunc, |
| 546 | "expected hash function %s, got %s for CID %s", expectedHash, hashFunc, cidStr) |
| 547 | } |
| 548 | |
| 549 | // verifyRawLeaves checks whether the CID represents a raw leaf or dag-pb wrapped block. |
| 550 | // For CIDv1: raw leaves have codec 0x55 (raw), wrapped have codec 0x70 (dag-pb). |
| 551 | // For CIDv0: always dag-pb (no raw leaves possible). |
| 552 | func verifyRawLeaves(t *testing.T, node *harness.Node, cidStr string, expectRaw bool) { |
| 553 | t.Helper() |
| 554 | // Use ipfs cid format to get codec info |
| 555 | // Format string %c gives the codec name |
| 556 | res := node.IPFS("cid", "format", "-f", "%c", cidStr) |
| 557 | codec := strings.TrimSpace(res.Stdout.String()) |
| 558 | |
| 559 | if expectRaw { |
| 560 | require.Equal(t, "raw", codec, |
| 561 | "expected raw codec for raw leaves, got %s for CID %s", codec, cidStr) |
| 562 | } else { |
| 563 | require.Equal(t, "dag-pb", codec, |
| 564 | "expected dag-pb codec for wrapped leaves, got %s for CID %s", codec, cidStr) |
| 565 | } |
| 566 | } |
| 567 | |
| 568 | // getBlockSize returns the size of a block in bytes using ipfs block stat. |
| 569 | func getBlockSize(t *testing.T, node *harness.Node, cidStr string) int { |
| 570 | t.Helper() |
| 571 | res := node.IPFS("block", "stat", "--enc=json", cidStr) |
| 572 | var stat struct { |
| 573 | Size int `json:"Size"` |
| 574 | } |
| 575 | require.NoError(t, json.Unmarshal(res.Stdout.Bytes(), &stat)) |
| 576 | return stat.Size |
| 577 | } |
| 578 | |
| 579 | // fileAtMaxLinksBytes returns the file size in bytes that produces exactly FileMaxLinks chunks. |
| 580 | func fileAtMaxLinksBytes(exp cidProfileExpectations) int64 { |
| 581 | return int64(exp.FileMaxLinks) * int64(exp.ChunkSize) |
| 582 | } |
| 583 | |
| 584 | // fileOverMaxLinksBytes returns the file size in bytes that triggers DAG rebalancing (+1 byte over max links threshold). |
| 585 | func fileOverMaxLinksBytes(exp cidProfileExpectations) int64 { |
| 586 | return int64(exp.FileMaxLinks)*int64(exp.ChunkSize) + 1 |
| 587 | } |
| 588 | |
| 589 | // seedForProfile returns the deterministic seed used in add_test.go for file max links tests. |
| 590 | func seedForProfile(exp cidProfileExpectations) string { |
| 591 | switch exp.Name { |
| 592 | case "unixfs-v0-2015", "default": |
| 593 | return "v0-seed" |
| 594 | case "unixfs-v1-2025": |
| 595 | return "v1-2025-seed" |
| 596 | default: |
| 597 | return exp.Name + "-seed" |
| 598 | } |
| 599 | } |
| 600 | |
| 601 | // chunkSeedForProfile returns the deterministic seed for chunk threshold tests. |
| 602 | func chunkSeedForProfile(exp cidProfileExpectations) string { |
| 603 | switch exp.Name { |
| 604 | case "unixfs-v0-2015", "default": |
| 605 | return "chunk-v0-seed" |
| 606 | case "unixfs-v1-2025": |
| 607 | return "chunk-v1-seed" |
| 608 | default: |
| 609 | return "chunk-" + exp.Name + "-seed" |
| 610 | } |
| 611 | } |
| 612 | |
| 613 | // hamtSeedForProfile returns the deterministic seed for HAMT directory tests. |
| 614 | // Uses the same seed for both under/at threshold tests to ensure consistency. |
| 615 | func hamtSeedForProfile(exp cidProfileExpectations) string { |
| 616 | switch exp.Name { |
| 617 | case "unixfs-v0-2015", "default": |
| 618 | return "hamt-unixfs-v0-2015" |
| 619 | case "unixfs-v1-2025": |
| 620 | return "hamt-unixfs-v1-2025" |
| 621 | default: |
| 622 | return "hamt-" + exp.Name |
| 623 | } |
| 624 | } |
| 625 | |
| 626 | // TestDefaultMatchesExpectedProfile verifies that default ipfs add behavior |
| 627 | // matches the expected profile (currently unixfs-v0-2015). |
| 628 | func TestDefaultMatchesExpectedProfile(t *testing.T) { |
| 629 | t.Parallel() |
| 630 | |
| 631 | node := harness.NewT(t).NewNode().Init() |
| 632 | node.StartDaemon() |
| 633 | defer node.StopDaemon() |
| 634 | |
| 635 | // Small file test |
| 636 | cidDefault := node.IPFSAddStr("x") |
| 637 | |
| 638 | // Same file with explicit profile |
| 639 | nodeWithProfile := harness.NewT(t).NewNode().Init(defaultProfile.ProfileArgs...) |
| 640 | nodeWithProfile.StartDaemon() |
| 641 | defer nodeWithProfile.StopDaemon() |
| 642 | |
| 643 | cidWithProfile := nodeWithProfile.IPFSAddStr("x") |
| 644 | |
| 645 | require.Equal(t, cidWithProfile, cidDefault, |
| 646 | "default behavior should match %s profile", defaultProfile.Name) |
| 647 | } |
| 648 | |
| 649 | // TestProtobufHelpers verifies the protobuf size calculation helpers. |
| 650 | func TestProtobufHelpers(t *testing.T) { |
| 651 | t.Parallel() |
| 652 | |
| 653 | t.Run("VarintLen", func(t *testing.T) { |
| 654 | // Varint encoding: 7 bits per byte, MSB indicates continuation |
| 655 | cases := []struct { |
| 656 | value uint64 |
| 657 | expected int |
| 658 | }{ |
| 659 | {0, 1}, |
| 660 | {127, 1}, // 0x7F - max 1-byte varint |
| 661 | {128, 2}, // 0x80 - min 2-byte varint |
| 662 | {16383, 2}, // 0x3FFF - max 2-byte varint |
| 663 | {16384, 3}, // 0x4000 - min 3-byte varint |
| 664 | {2097151, 3}, // 0x1FFFFF - max 3-byte varint |
| 665 | {2097152, 4}, // 0x200000 - min 4-byte varint |
| 666 | {268435455, 4}, // 0xFFFFFFF - max 4-byte varint |
| 667 | {268435456, 5}, // 0x10000000 - min 5-byte varint |
| 668 | {34359738367, 5}, // 0x7FFFFFFFF - max 5-byte varint |
| 669 | } |
| 670 | |
| 671 | for _, tc := range cases { |
| 672 | got := testutils.VarintLen(tc.value) |
| 673 | require.Equal(t, tc.expected, got, "VarintLen(%d)", tc.value) |
| 674 | } |
| 675 | }) |
| 676 | |
| 677 | t.Run("LinkSerializedSize", func(t *testing.T) { |
| 678 | // Test typical cases for directory links |
| 679 | cases := []struct { |
| 680 | nameLen int |
| 681 | cidLen int |
| 682 | tsize uint64 |
| 683 | expected int |
| 684 | }{ |
| 685 | // 255-char name, CIDv0 (34 bytes), tsize=0 |
| 686 | // Inner: 1+1+34 + 1+2+255 + 1+1 = 296 |
| 687 | // Outer: 1 + 2 + 296 = 299 |
| 688 | {255, 34, 0, 299}, |
| 689 | // 255-char name, CIDv1 (36 bytes), tsize=0 |
| 690 | // Inner: 1+1+36 + 1+2+255 + 1+1 = 298 |
| 691 | // Outer: 1 + 2 + 298 = 301 |
| 692 | {255, 36, 0, 301}, |
| 693 | // Short name (10 chars), CIDv1, tsize=0 |
| 694 | // Inner: 1+1+36 + 1+1+10 + 1+1 = 52 |
| 695 | // Outer: 1 + 1 + 52 = 54 |
| 696 | {10, 36, 0, 54}, |
| 697 | // 255-char name, CIDv1, large tsize |
| 698 | // Inner: 1+1+36 + 1+2+255 + 1+5 = 302 (tsize uses 5-byte varint) |
| 699 | // Outer: 1 + 2 + 302 = 305 |
| 700 | {255, 36, 34359738367, 305}, |
| 701 | } |
| 702 | |
| 703 | for _, tc := range cases { |
| 704 | got := testutils.LinkSerializedSize(tc.nameLen, tc.cidLen, tc.tsize) |
| 705 | require.Equal(t, tc.expected, got, "LinkSerializedSize(%d, %d, %d)", tc.nameLen, tc.cidLen, tc.tsize) |
| 706 | } |
| 707 | }) |
| 708 | |
| 709 | t.Run("EstimateFilesForBlockThreshold", func(t *testing.T) { |
| 710 | threshold := 262144 |
| 711 | nameLen := 255 |
| 712 | cidLen := 36 |
| 713 | var tsize uint64 = 0 |
| 714 | |
| 715 | numFiles := testutils.EstimateFilesForBlockThreshold(threshold, nameLen, cidLen, tsize) |
| 716 | require.Equal(t, 870, numFiles, "expected 870 files for threshold 262144") |
| 717 | |
| 718 | numFilesUnder := testutils.EstimateFilesForBlockThreshold(threshold-1, nameLen, cidLen, tsize) |
| 719 | require.Equal(t, 870, numFilesUnder, "expected 870 files for threshold 262143") |
| 720 | |
| 721 | numFilesOver := testutils.EstimateFilesForBlockThreshold(262185, nameLen, cidLen, tsize) |
| 722 | require.Equal(t, 871, numFilesOver, "expected 871 files for threshold 262185") |
| 723 | }) |
| 724 | } |