core/commands/unixfs/ls: Hash-map for Objects
Discussion with Juan on IRC ([1] through [2]) lead to this adjusted JSON output. Benefits over the old output include: * deduplication (we only check the children of a given Merkle node once, even if multiple arguments resolve to that hash) * alphabetized output (like POSIX's ls). As a side-effect of this change, I'm also matching GNU Coreutils' ls output (maybe in POSIX?) by printing an alphabetized list of non-directories (one per line) first, with alphabetized directory lists afterwards. [1]: https://botbot.me/freenode/ipfs/2015-06-12/?msg=41725570&page=5 [2]: https://botbot.me/freenode/ipfs/2015-06-12/?msg=41726547&page=5 License: MIT Signed-off-by: W. Trevor King <wking@tremily.us>
W. Trevor King committed
Jun 13, 2015 at 13:35 UTC
7fc2410d9504a029ccbb7f28b87a0525ea36aadc
2 files changed
+144
-35
core/commands/unixfs/ls.go
+68
-25
@@ -4,6 +4,7 @@ import (
4
"bytes"
5
"fmt"
6
"io"
7
+ "sort"
8
"text/tabwriter"
9
"time"
10
@@ -23,12 +24,12 @@ type LsLink struct {
24
}
25
26
type LsObject struct {
26
- Argument string
27
- Links []LsLink
27
+ Links []LsLink
28
}
29
30
type LsOutput struct {
31
- Objects []*LsObject
31
+ Arguments map[string]string
32
+ Objects map[string]*LsObject
33
}
34
35
var LsCmd = &cmds.Command{
@@ -57,8 +58,12 @@ directories, the child size is the IPFS link size.
58
59
paths := req.Arguments()
60
60
- output := make([]*LsObject, len(paths))
61
- for i, fpath := range paths {
61
+ output := LsOutput{
62
+ Arguments: map[string]string{},
63
+ Objects: map[string]*LsObject{},
64
+ }
65
+
66
+ for _, fpath := range paths {
67
ctx := req.Context().Context
68
merkleNode, err := core.Resolve(ctx, node, path.Path(fpath))
69
if err != nil {
@@ -66,13 +71,27 @@ directories, the child size is the IPFS link size.
71
return
72
}
73
69
- unixFSNode, err := unixfs.FromBytes(merkleNode.Data)
74
+ key, err := merkleNode.Key()
75
if err != nil {
76
res.SetError(err, cmds.ErrNormal)
77
return
78
}
79
75
- output[i] = &LsObject{Argument: fpath}
80
+ hash := key.B58String()
81
+ output.Arguments[fpath] = hash
82
+
83
+ if _, ok := output.Objects[hash]; ok {
84
+ // duplicate argument for an already-listed node
85
+ continue
86
+ }
87
+
88
+ output.Objects[hash] = &LsObject{}
89
+
90
+ unixFSNode, err := unixfs.FromBytes(merkleNode.Data)
91
+ if err != nil {
92
+ res.SetError(err, cmds.ErrNormal)
93
+ return
94
+ }
95
96
t := unixFSNode.GetType()
97
switch t {
@@ -85,15 +104,16 @@ directories, the child size is the IPFS link size.
104
res.SetError(err, cmds.ErrNormal)
105
return
106
}
88
- output[i].Links = []LsLink{LsLink{
107
+ output.Objects[hash].Links = []LsLink{LsLink{
108
Name: fpath,
109
Hash: key.String(),
110
Type: t.String(),
111
Size: unixFSNode.GetFilesize(),
112
}}
113
case unixfspb.Data_Directory:
95
- output[i].Links = make([]LsLink, len(merkleNode.Links))
96
- for j, link := range merkleNode.Links {
114
+ links := make([]LsLink, len(merkleNode.Links))
115
+ output.Objects[hash].Links = links
116
+ for i, link := range merkleNode.Links {
117
getCtx, cancel := context.WithTimeout(ctx, time.Minute)
118
defer cancel()
119
link.Node, err = link.GetNode(getCtx, node.DAG)
@@ -117,12 +137,12 @@ directories, the child size is the IPFS link size.
137
} else {
138
lsLink.Size = link.Size
139
}
120
- output[i].Links[j] = lsLink
140
+ links[i] = lsLink
141
}
142
}
143
}
144
125
- res.SetOutput(&LsOutput{Objects: output})
145
+ res.SetOutput(&output)
146
},
147
Marshalers: cmds.MarshalerMap{
148
cmds.Text: func(res cmds.Response) (io.Reader, error) {
@@ -130,21 +150,44 @@ directories, the child size is the IPFS link size.
150
output := res.Output().(*LsOutput)
151
buf := new(bytes.Buffer)
152
w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0)
133
- lastObjectDirHeader := false
134
- for i, object := range output.Objects {
135
- singleObject := (len(object.Links) == 1 &&
136
- object.Links[0].Name == object.Argument)
137
- if len(output.Objects) > 1 && !singleObject {
138
- if i > 0 {
139
- fmt.Fprintln(w)
140
- }
141
- fmt.Fprintf(w, "%s:\n", object.Argument)
142
- lastObjectDirHeader = true
153
+
154
+ nonDirectories := []string{}
155
+ directories := []string{}
156
+ for argument, hash := range output.Arguments {
157
+ object, ok := output.Objects[hash]
158
+ if !ok {
159
+ return nil, fmt.Errorf("unresolved hash: %s", hash)
160
+ }
161
+
162
+ if len(object.Links) == 1 && object.Links[0].Hash == hash {
163
+ nonDirectories = append(nonDirectories, argument)
164
} else {
144
- if lastObjectDirHeader {
145
- fmt.Fprintln(w)
165
+ directories = append(directories, argument)
166
+ }
167
+ }
168
+ sort.Strings(nonDirectories)
169
+ sort.Strings(directories)
170
+
171
+ for _, argument := range nonDirectories {
172
+ fmt.Fprintf(w, "%s\n", argument)
173
+ }
174
+
175
+ seen := map[string]bool{}
176
+ for i, argument := range directories {
177
+ hash := output.Arguments[argument]
178
+ if _, ok := seen[hash]; ok {
179
+ continue
180
+ }
181
+ seen[hash] = true
182
+
183
+ object := output.Objects[hash]
184
+ if i > 0 || len(nonDirectories) > 0 {
185
+ fmt.Fprintln(w)
186
+ }
187
+ for _, arg := range directories[i:] {
188
+ if output.Arguments[arg] == hash {
189
+ fmt.Fprintf(w, "%s:\n", arg)
190
}
147
- lastObjectDirHeader = false
191
}
192
for _, link := range object.Links {
193
fmt.Fprintf(w, "%s\n", link.Name)
test/sharness/t0200-unixfs-ls.sh
+76
-10
@@ -44,12 +44,6 @@ test_ls_cmd() {
44
45
test_expect_success "'ipfs file ls <three dir hashes>' output looks good" '
46
cat <<-\EOF >expected_ls &&
47
- QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
48
- d1
49
- d2
50
- f1
51
- f2
52
-
47
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
48
1024
49
a
@@ -57,6 +51,12 @@ test_ls_cmd() {
51
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
52
128
53
a
54
+
55
+ QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
56
+ d1
57
+ d2
58
+ f1
59
+ f2
60
EOF
61
test_cmp expected_ls actual_ls
62
'
@@ -73,6 +73,23 @@ test_ls_cmd() {
73
test_cmp expected_ls_file actual_ls_file
74
'
75
76
+ test_expect_success "'ipfs file ls <duplicates>' succeeds" '
77
+ ipfs file ls /ipfs/QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj/d1 /ipfs/QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss /ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024 /ipfs/QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd >actual_ls_duplicates_file
78
+ '
79
+
80
+ test_expect_success "'ipfs file ls <duplicates>' output looks good" '
81
+ cat <<-\EOF >expected_ls_duplicates_file &&
82
+ /ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024
83
+ /ipfs/QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd
84
+
85
+ /ipfs/QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
86
+ /ipfs/QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj/d1:
87
+ 128
88
+ a
89
+ EOF
90
+ test_cmp expected_ls_duplicates_file actual_ls_duplicates_file
91
+ '
92
+
93
test_expect_success "'ipfs --encoding=json file ls <file hashes>' succeeds" '
94
ipfs --encoding=json file ls /ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024 >actual_json_ls_file
95
'
@@ -80,9 +97,11 @@ test_ls_cmd() {
97
test_expect_success "'ipfs --encoding=json file ls <file hashes>' output looks good" '
98
cat <<-\EOF >expected_json_ls_file_trailing_newline &&
99
{
83
- "Objects": [
84
- {
85
- "Argument": "/ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024",
100
+ "Arguments": {
101
+ "/ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024": "QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd"
102
+ },
103
+ "Objects": {
104
+ "QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd": {
105
"Links": [
106
{
107
"Name": "/ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024",
@@ -92,12 +111,59 @@ test_ls_cmd() {
111
}
112
]
113
}
95
- ]
114
+ }
115
}
116
EOF
117
printf %s "$(cat expected_json_ls_file_trailing_newline)" >expected_json_ls_file &&
118
test_cmp expected_json_ls_file actual_json_ls_file
119
'
120
+
121
+ test_expect_success "'ipfs --encoding=json file ls <duplicates>' succeeds" '
122
+ ipfs --encoding=json file ls /ipfs/QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj/d1 /ipfs/QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss /ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024 /ipfs/QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd >actual_json_ls_duplicates_file
123
+ '
124
+
125
+ test_expect_success "'ipfs --encoding=json file ls <duplicates>' output looks good" '
126
+ cat <<-\EOF >expected_json_ls_duplicates_file_trailing_newline &&
127
+ {
128
+ "Arguments": {
129
+ "/ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024": "QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd",
130
+ "/ipfs/QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss": "QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss",
131
+ "/ipfs/QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd": "QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd",
132
+ "/ipfs/QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj/d1": "QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss"
133
+ },
134
+ "Objects": {
135
+ "QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss": {
136
+ "Links": [
137
+ {
138
+ "Name": "128",
139
+ "Hash": "QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe",
140
+ "Size": 128,
141
+ "Type": "File"
142
+ },
143
+ {
144
+ "Name": "a",
145
+ "Hash": "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN",
146
+ "Size": 6,
147
+ "Type": "File"
148
+ }
149
+ ]
150
+ },
151
+ "QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd": {
152
+ "Links": [
153
+ {
154
+ "Name": "/ipfs/QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy/1024",
155
+ "Hash": "QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd",
156
+ "Size": 1024,
157
+ "Type": "File"
158
+ }
159
+ ]
160
+ }
161
+ }
162
+ }
163
+ EOF
164
+ printf %s "$(cat expected_json_ls_duplicates_file_trailing_newline)" >expected_json_ls_duplicates_file &&
165
+ test_cmp expected_json_ls_duplicates_file actual_json_ls_duplicates_file
166
+ '
167
}
168
169