@cryptotaxi247 / kubo / commits / 431d6a5a1

added random-files tool for testing

License: MIT Signed-off-by: Juan Batiz-Benet <juan@benet.ai>

Juan Batiz-Benet committed Jul 29, 2015 at 03:18 UTC 431d6a5a1407bfbcfb28a798657fa90e68a833a5
13 files changed +527 -1
Godeps/Godeps.json
+4
@@ -186,6 +186,10 @@
186 "ImportPath": "github.com/jbenet/go-random",
187 "Rev": "cd535bd25356746b9b1e824871dda7da932460e2"
188 },
189 + {
190 + "ImportPath": "github.com/jbenet/go-random-files",
191 + "Rev": "737479700b40b4b50e914e963ce8d9d44603e3c8"
192 + },
193 {
194 "ImportPath": "github.com/jbenet/go-reuseport",
195 "Rev": "48959f1fad204b6cf2c0e8d086ef69f03f2de961"
Godeps/_workspace/src/github.com/jbenet/go-random-files/LICENSE new
+21
@@ -0,0 +1,21 @@
1 +The MIT License (MIT)
2 +
3 +Copyright (c) 2014 Juan Batiz-Benet
4 +
5 +Permission is hereby granted, free of charge, to any person obtaining a copy
6 +of this software and associated documentation files (the "Software"), to deal
7 +in the Software without restriction, including without limitation the rights
8 +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 +copies of the Software, and to permit persons to whom the Software is
10 +furnished to do so, subject to the following conditions:
11 +
12 +The above copyright notice and this permission notice shall be included in
13 +all copies or substantial portions of the Software.
14 +
15 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 +THE SOFTWARE.
Godeps/_workspace/src/github.com/jbenet/go-random-files/Makefile new
+5
@@ -0,0 +1,5 @@
1 +build:
2 + cd random-files && go build
3 +
4 +install:
5 + cd random-files && go install
Godeps/_workspace/src/github.com/jbenet/go-random-files/README.md new
+108
@@ -0,0 +1,108 @@
1 +# go-random-files - create random fs hierarchies
2 +
3 +Useful for testing filesystems.
4 +
5 +
6 +```
7 +# library
8 +go get -u github.com/jbenet/go-random-files
9 +
10 +# binary
11 +go get -u github.com/jbenet/go-random-files/random-files
12 +```
13 +
14 +### Library godoc: https://godoc.org/github.com/jbenet/go-random-files
15 +
16 +# random-files - create random fs hierarchies
17 +
18 +See more about the binary at: github.com/jbenet/go-random-files/tree/master/random-files
19 +
20 +Useful for testing filesystems.
21 +
22 +## Install
23 +
24 +```
25 +go get -u github.com/jbenet/go-random-files/random-files
26 +```
27 +
28 +## Usage
29 +
30 +```sh
31 +> random-files --help
32 +usage: random-files [options] <path>...
33 +Write a random filesystem hierarchy to each <path>
34 +
35 +Options:
36 + -alphabet="easy": alphabet for filenames {easy, hard}
37 + -depth=2: fanout depth - how deep the hierarchy goes
38 + -dirs=5: fanout dirs - number of dirs per dir (or max)
39 + -files=10: fanout files - number of files per dir (or max
40 + -filesize=4096: filesize - how big to make each file (or max)
41 + -q=false: quiet output
42 + -random-crypto=false: use cryptographic randomness for files
43 + -random-fanout=false: randomize fanout numbers
44 + -random-size=true: randomize filesize
45 + -seed=0: random seed - 0 for current time
46 +```
47 +
48 +### Examples
49 +
50 +```sh
51 +> random-files --depth=2 --files=3 foo
52 +foo/h20uo3jrpihb
53 +foo/x6tef1
54 +foo/jh0c2vdci
55 +foo/fden012m368
56 +foo/fden012m368/p6n0chy4kg
57 +foo/fden012m368/h92_
58 +foo/fden012m368/kvjiya98p3
59 +foo/e_i6hwav1tb
60 +foo/e_i6hwav1tb/oj0-a
61 +foo/e_i6hwav1tb/1-pfgvim
62 +foo/e_i6hwav1tb/s_unf
63 +foo/bgvy8x-_hsm
64 +foo/bgvy8x-_hsm/98zcoz-9ng
65 +foo/bgvy8x-_hsm/j0see3qv
66 +foo/bgvy8x-_hsm/qntuf0r
67 +foo/6zjkw3ejm2awwt
68 +foo/6zjkw3ejm2awwt/iba52dh1lhnewh
69 +foo/6zjkw3ejm2awwt/n1bwcv5zpe
70 +foo/6zjkw3ejm2awwt/o8k89cc
71 +foo/efp_6
72 +foo/efp_6/qfap2
73 +foo/efp_6/v_kl_wlefsaa
74 +foo/efp_6/r7sdbph
75 +```
76 +
77 +It made:
78 +
79 +```
80 +> tree foo
81 +foo
82 +├── 6zjkw3ejm2awwt
83 +│   ├── iba52dh1lhnewh
84 +│   ├── n1bwcv5zpe
85 +│   └── o8k89cc
86 +├── bgvy8x-_hsm
87 +│   ├── 98zcoz-9ng
88 +│   ├── j0see3qv
89 +│   └── qntuf0r
90 +├── e_i6hwav1tb
91 +│   ├── 1-pfgvim
92 +│   ├── oj0-a
93 +│   └── s_unf
94 +├── efp_6
95 +│   ├── qfap2
96 +│   ├── r7sdbph
97 +│   └── v_kl_wlefsaa
98 +├── fden012m368
99 +│   ├── h92_
100 +│   ├── kvjiya98p3
101 +│   └── p6n0chy4kg
102 +├── h20uo3jrpihb
103 +├── jh0c2vdci
104 +└── x6tef1
105 +
106 +5 directories, 18 files
107 +```
108 +
Godeps/_workspace/src/github.com/jbenet/go-random-files/lib.go new
+110
@@ -0,0 +1,110 @@
1 +package randomfiles
2 +
3 +import (
4 + "fmt"
5 + "io"
6 + "math/rand"
7 + "os"
8 + "path"
9 +)
10 +
11 +type Options struct {
12 + Out io.Writer // output progress
13 + Source io.Reader // randomness source
14 +
15 + FileSize int // the size per file.
16 + Alphabet []rune // for filenames
17 +
18 + FanoutDepth int // how deep the hierarchy goes
19 + FanoutFiles int // how many files per dir
20 + FanoutDirs int // how many dirs per dir
21 +
22 + RandomSeed int64 // use a random seed. if 0, use a random seed
23 + RandomSize bool // randomize file sizes
24 + RandomFanout bool // randomize fanout numbers
25 +}
26 +
27 +func WriteRandomFiles(root string, depth int, opts *Options) error {
28 +
29 + numfiles := opts.FanoutFiles
30 + if opts.RandomFanout {
31 + numfiles = rand.Intn(numfiles) + 1
32 + }
33 +
34 + for i := 0; i < numfiles; i++ {
35 + if err := WriteRandomFile(root, opts); err != nil {
36 + return err
37 + }
38 + }
39 +
40 + if depth+1 <= opts.FanoutDepth {
41 + numdirs := opts.FanoutDirs
42 + if opts.RandomFanout {
43 + numdirs = rand.Intn(numdirs) + 1
44 + }
45 +
46 + for i := 0; i < numdirs; i++ {
47 + if err := WriteRandomDir(root, depth+1, opts); err != nil {
48 + return err
49 + }
50 + }
51 + }
52 +
53 + return nil
54 +}
55 +
56 +var FilenameSize = 16
57 +var RunesEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_")
58 +var RunesHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ")
59 +
60 +func RandomFilename(length int, alphabet []rune) string {
61 + b := make([]rune, length)
62 + for i := range b {
63 + b[i] = alphabet[rand.Intn(len(alphabet))]
64 + }
65 + return string(b)
66 +}
67 +
68 +func WriteRandomFile(root string, opts *Options) error {
69 + filesize := int64(opts.FileSize)
70 + if opts.RandomSize {
71 + filesize = rand.Int63n(filesize) + 1
72 + }
73 +
74 + n := rand.Intn(FilenameSize-4) + 4
75 + name := RandomFilename(n, opts.Alphabet)
76 + filepath := path.Join(root, name)
77 + f, err := os.Create(filepath)
78 + if err != nil {
79 + return err
80 + }
81 +
82 + if _, err := io.CopyN(f, opts.Source, filesize); err != nil {
83 + return err
84 + }
85 +
86 + if opts.Out != nil {
87 + fmt.Fprintln(opts.Out, filepath)
88 + }
89 +
90 + return f.Close()
91 +}
92 +
93 +func WriteRandomDir(root string, depth int, opts *Options) error {
94 + if depth > opts.FanoutDepth {
95 + return nil
96 + }
97 +
98 + n := rand.Intn(FilenameSize-4) + 4
99 + name := RandomFilename(n, opts.Alphabet)
100 + root = path.Join(root, name)
101 + if err := os.MkdirAll(root, 0755); err != nil {
102 + return err
103 + }
104 +
105 + if opts.Out != nil {
106 + fmt.Fprintln(opts.Out, root)
107 + }
108 +
109 + return WriteRandomFiles(root, depth, opts)
110 +}
Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/.gitignore new
+1
@@ -0,0 +1 @@
1 +random-files
Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/README.md new
+90
@@ -0,0 +1,90 @@
1 +# random-files - create random fs hierarchies
2 +
3 +random-files creates random fs hierarchies. Useful for testing filesystems.
4 +
5 +## Install
6 +
7 +```
8 +go get -u github.com/jbenet/go-random-files/random-files
9 +```
10 +
11 +## Usage
12 +
13 +```sh
14 +> random-files --help
15 +usage: random-files [options] <path>...
16 +Write a random filesystem hierarchy to each <path>
17 +
18 +Options:
19 + -alphabet="easy": alphabet for filenames {easy, hard}
20 + -depth=2: fanout depth - how deep the hierarchy goes
21 + -dirs=5: fanout dirs - number of dirs per dir (or max)
22 + -files=10: fanout files - number of files per dir (or max
23 + -filesize=4096: filesize - how big to make each file (or max)
24 + -q=false: quiet output
25 + -random-crypto=false: use cryptographic randomness for files
26 + -random-fanout=false: randomize fanout numbers
27 + -random-size=true: randomize filesize
28 + -seed=0: random seed - 0 for current time
29 +```
30 +
31 +## Examples
32 +
33 +```sh
34 +> random-files --depth=2 --files=3 foo
35 +foo/h20uo3jrpihb
36 +foo/x6tef1
37 +foo/jh0c2vdci
38 +foo/fden012m368
39 +foo/fden012m368/p6n0chy4kg
40 +foo/fden012m368/h92_
41 +foo/fden012m368/kvjiya98p3
42 +foo/e_i6hwav1tb
43 +foo/e_i6hwav1tb/oj0-a
44 +foo/e_i6hwav1tb/1-pfgvim
45 +foo/e_i6hwav1tb/s_unf
46 +foo/bgvy8x-_hsm
47 +foo/bgvy8x-_hsm/98zcoz-9ng
48 +foo/bgvy8x-_hsm/j0see3qv
49 +foo/bgvy8x-_hsm/qntuf0r
50 +foo/6zjkw3ejm2awwt
51 +foo/6zjkw3ejm2awwt/iba52dh1lhnewh
52 +foo/6zjkw3ejm2awwt/n1bwcv5zpe
53 +foo/6zjkw3ejm2awwt/o8k89cc
54 +foo/efp_6
55 +foo/efp_6/qfap2
56 +foo/efp_6/v_kl_wlefsaa
57 +foo/efp_6/r7sdbph
58 +```
59 +
60 +It made:
61 +
62 +```
63 +> tree foo
64 +foo
65 +├── 6zjkw3ejm2awwt
66 +│   ├── iba52dh1lhnewh
67 +│   ├── n1bwcv5zpe
68 +│   └── o8k89cc
69 +├── bgvy8x-_hsm
70 +│   ├── 98zcoz-9ng
71 +│   ├── j0see3qv
72 +│   └── qntuf0r
73 +├── e_i6hwav1tb
74 +│   ├── 1-pfgvim
75 +│   ├── oj0-a
76 +│   └── s_unf
77 +├── efp_6
78 +│   ├── qfap2
79 +│   ├── r7sdbph
80 +│   └── v_kl_wlefsaa
81 +├── fden012m368
82 +│   ├── h92_
83 +│   ├── kvjiya98p3
84 +│   └── p6n0chy4kg
85 +├── h20uo3jrpihb
86 +├── jh0c2vdci
87 +└── x6tef1
88 +
89 +5 directories, 18 files
90 +```
Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go new
+119
@@ -0,0 +1,119 @@
1 +package main
2 +
3 +import (
4 + crand "crypto/rand"
5 + "errors"
6 + "flag"
7 + "fmt"
8 + "math/rand"
9 + "os"
10 + "time"
11 +
12 + randomfiles "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files"
13 + ringreader "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader"
14 +)
15 +
16 +var usage = `usage: %s [options] <path>...
17 +Write a random filesystem hierarchy to each <path>
18 +
19 +Options:
20 +`
21 +
22 +// flags
23 +var opts randomfiles.Options
24 +var quiet bool
25 +var alphabet string
26 +var paths []string
27 +var cryptorand bool
28 +
29 +func init() {
30 + flag.Usage = func() {
31 + fmt.Fprintf(os.Stderr, usage, os.Args[0])
32 + flag.PrintDefaults()
33 + }
34 +
35 + flag.BoolVar(&quiet, "q", false, "quiet output")
36 + flag.BoolVar(&cryptorand, "random-crypto", false, "use cryptographic randomness for files")
37 + flag.StringVar(&alphabet, "alphabet", "easy", "alphabet for filenames {easy, hard}")
38 + flag.IntVar(&opts.FileSize, "filesize", 4096, "filesize - how big to make each file (or max)")
39 +
40 + flag.IntVar(&opts.FanoutDepth, "depth", 2, "fanout depth - how deep the hierarchy goes")
41 + flag.IntVar(&opts.FanoutDirs, "dirs", 5, "fanout dirs - number of dirs per dir (or max)")
42 + flag.IntVar(&opts.FanoutFiles, "files", 10, "fanout files - number of files per dir (or max")
43 +
44 + flag.Int64Var(&opts.RandomSeed, "seed", 0, "random seed - 0 for current time")
45 + flag.BoolVar(&opts.RandomFanout, "random-fanout", false, "randomize fanout numbers")
46 + flag.BoolVar(&opts.RandomSize, "random-size", true, "randomize filesize")
47 +}
48 +
49 +func parseArgs() error {
50 + flag.Parse()
51 +
52 + switch alphabet {
53 + case "easy":
54 + opts.Alphabet = randomfiles.RunesEasy
55 + case "hard":
56 + opts.Alphabet = randomfiles.RunesHard
57 + default:
58 + return errors.New("alphabet must be one of: easy, hard")
59 + }
60 +
61 + paths = flag.Args()
62 + if len(paths) < 1 {
63 + flag.Usage()
64 + os.Exit(0)
65 + }
66 +
67 + if !quiet {
68 + opts.Out = os.Stdout
69 + }
70 +
71 + switch opts.RandomSeed {
72 + case 0:
73 + rand.Seed(time.Now().UnixNano())
74 + default:
75 + rand.Seed(opts.RandomSeed)
76 + }
77 +
78 + // prepare randomn source.
79 + if cryptorand {
80 + opts.Source = crand.Reader
81 + } else {
82 + // if not crypto, we don't need a lot of random
83 + // data. we just need to sample from a sequence.
84 + s := 16777216 // 16MB
85 + r, err := ringreader.NewReader(s)
86 + if err != nil {
87 + return err
88 + }
89 + opts.Source = r
90 + }
91 +
92 + return nil
93 +}
94 +
95 +func run() error {
96 + if err := parseArgs(); err != nil {
97 + return err
98 + }
99 +
100 + for _, root := range paths {
101 + if err := os.MkdirAll(root, 0755); err != nil {
102 + return err
103 + }
104 +
105 + err := randomfiles.WriteRandomFiles(root, 1, &opts)
106 + if err != nil {
107 + return err
108 + }
109 + }
110 +
111 + return nil
112 +}
113 +
114 +func main() {
115 + if err := run(); err != nil {
116 + fmt.Fprintln(os.Stderr, "error:", err)
117 + os.Exit(1)
118 + }
119 +}
Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go new
+40
@@ -0,0 +1,40 @@
1 +package ringreader
2 +
3 +import (
4 + "bytes"
5 + "fmt"
6 + "math/rand"
7 +
8 + random "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random"
9 +)
10 +
11 +type Reader struct {
12 + Buf []byte
13 +}
14 +
15 +func NewReader(bufsize int) (*Reader, error) {
16 + buf := bytes.NewBuffer(nil)
17 + err := random.WritePseudoRandomBytes(int64(bufsize), buf, rand.Int63())
18 + return &Reader{Buf: buf.Bytes()}, err
19 +}
20 +
21 +func (r *Reader) Read(buf []byte) (n int, err error) {
22 + ibufl := len(r.Buf)
23 + left := len(buf)
24 + copied := 0
25 +
26 + for copied < left {
27 + pos1 := rand.Intn(len(r.Buf))
28 + pos2 := pos1 + left
29 + if pos2 > ibufl {
30 + pos2 = ibufl
31 + }
32 + copied += copy(buf[copied:], r.Buf[pos1:pos2])
33 + }
34 +
35 + if copied != left {
36 + err := fmt.Errorf("copied a different ammount: %d != %d", copied, left)
37 + panic(err.Error())
38 + }
39 + return copied, nil
40 +}
Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader_test.go new
+22
@@ -0,0 +1,22 @@
1 +package ringreader
2 +
3 +import (
4 + "testing"
5 +)
6 +
7 +func TestRingReader(t *testing.T) {
8 + r, _ := NewReader(256)
9 + t.Log("buffer:", r.Buf)
10 +
11 + for i := 1; i < 1048576; i = i * 2 {
12 + buf := make([]byte, i)
13 + n, err := r.Read(buf)
14 + if err != nil {
15 + t.Error(err)
16 + }
17 + if n != len(buf) {
18 + t.Error("did not read %d bytes", n)
19 + }
20 + t.Log("read:", buf)
21 + }
22 +}
test/Makefile
+5
@@ -3,6 +3,7 @@ BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint bin/iptb bin/go-sleep
3 IPFS_ROOT = ../
4 IPFS_CMD = ../cmd/ipfs
5 RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random
6 +RANDOM_FILES_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random-files
7 MULTIHASH_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-multihash
8 IPTB_SRC = ../Godeps/_workspace/src/github.com/whyrusleeping/iptb
9 POLLENDPOINT_SRC= ../thirdparty/pollEndpoint
@@ -26,6 +27,10 @@ bin/random: $(call find_go_files, $(RANDOM_SRC)) IPFS-BUILD-OPTIONS
27 @echo "*** installing $@ ***"
28 go build $(GOFLAGS) -o bin/random $(RANDOM_SRC)/random
29
30 +bin/random-files:
31 + @echo "*** installing $@ ***"
32 + go build $(GOFLAGS) -o bin/random-files $(RANDOM_FILES_SRC)/random-files
33 +
34 bin/multihash: $(call find_go_files, $(MULTIHASH_SRC)) IPFS-BUILD-OPTIONS
35 @echo "*** installing $@ ***"
36 go build $(GOFLAGS) -o bin/multihash $(MULTIHASH_SRC)/multihash
test/sharness/Makefile
+1 -1
@@ -8,7 +8,7 @@
8
9 T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
10 BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \
11 - bin/iptb bin/go-sleep
11 + bin/iptb bin/go-sleep bin/random-files
12 SHARNESS = lib/sharness/sharness.sh
13 IPFS_ROOT = ../..
14
test/sharness/lib/random-dep.go
+1
@@ -5,4 +5,5 @@ package randomdep
5
6 import (
7 _ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random"
8 + _ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files"
9 )