| 1 | package commands |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "io" |
| 7 | "os" |
| 8 | |
| 9 | filestore "github.com/ipfs/boxo/filestore" |
| 10 | cmds "github.com/ipfs/go-ipfs-cmds" |
| 11 | core "github.com/ipfs/kubo/core" |
| 12 | cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" |
| 13 | e "github.com/ipfs/kubo/core/commands/e" |
| 14 | |
| 15 | "github.com/ipfs/go-cid" |
| 16 | ) |
| 17 | |
| 18 | var FileStoreCmd = &cmds.Command{ |
| 19 | Helptext: cmds.HelpText{ |
| 20 | Tagline: "Interact with filestore objects.", |
| 21 | }, |
| 22 | Subcommands: map[string]*cmds.Command{ |
| 23 | "ls": lsFileStore, |
| 24 | "verify": verifyFileStore, |
| 25 | "dups": dupsFileStore, |
| 26 | }, |
| 27 | } |
| 28 | |
| 29 | const ( |
| 30 | fileOrderOptionName = "file-order" |
| 31 | removeBadBlocksOptionName = "remove-bad-blocks" |
| 32 | ) |
| 33 | |
| 34 | var lsFileStore = &cmds.Command{ |
| 35 | Helptext: cmds.HelpText{ |
| 36 | Tagline: "List objects in filestore.", |
| 37 | LongDescription: ` |
| 38 | List objects in the filestore. |
| 39 | |
| 40 | If one or more <obj> is specified only list those specific objects, |
| 41 | otherwise list all objects. |
| 42 | |
| 43 | The output is: |
| 44 | |
| 45 | <hash> <size> <path> <offset> |
| 46 | `, |
| 47 | }, |
| 48 | Arguments: []cmds.Argument{ |
| 49 | cmds.StringArg("obj", false, true, "Cid of objects to list."), |
| 50 | }, |
| 51 | Options: []cmds.Option{ |
| 52 | cmds.BoolOption(fileOrderOptionName, "sort the results based on the path of the backing file"), |
| 53 | }, |
| 54 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 55 | _, fs, err := getFilestore(env) |
| 56 | if err != nil { |
| 57 | return err |
| 58 | } |
| 59 | args := req.Arguments |
| 60 | if len(args) > 0 { |
| 61 | return listByArgs(req.Context, res, fs, args, false) |
| 62 | } |
| 63 | |
| 64 | fileOrder, _ := req.Options[fileOrderOptionName].(bool) |
| 65 | next, err := filestore.ListAll(req.Context, fs, fileOrder) |
| 66 | if err != nil { |
| 67 | return err |
| 68 | } |
| 69 | |
| 70 | for { |
| 71 | r := next(req.Context) |
| 72 | if r == nil { |
| 73 | break |
| 74 | } |
| 75 | if err := res.Emit(r); err != nil { |
| 76 | return err |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | return nil |
| 81 | }, |
| 82 | PostRun: cmds.PostRunMap{ |
| 83 | cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { |
| 84 | enc, err := cmdenv.GetCidEncoder(res.Request()) |
| 85 | if err != nil { |
| 86 | return err |
| 87 | } |
| 88 | return streamResult(func(v any, out io.Writer) nonFatalError { |
| 89 | r := v.(*filestore.ListRes) |
| 90 | if r.ErrorMsg != "" { |
| 91 | return nonFatalError(r.ErrorMsg) |
| 92 | } |
| 93 | fmt.Fprintf(out, "%s\n", r.FormatLong(enc.Encode)) |
| 94 | return "" |
| 95 | })(res, re) |
| 96 | }, |
| 97 | }, |
| 98 | Type: filestore.ListRes{}, |
| 99 | } |
| 100 | |
| 101 | var verifyFileStore = &cmds.Command{ |
| 102 | Helptext: cmds.HelpText{ |
| 103 | Tagline: "Verify objects in filestore.", |
| 104 | LongDescription: ` |
| 105 | Verify objects in the filestore. |
| 106 | |
| 107 | If one or more <obj> is specified only verify those specific objects, |
| 108 | otherwise verify all objects. |
| 109 | |
| 110 | The output is: |
| 111 | |
| 112 | <status> <hash> <size> <path> <offset> [<action>] |
| 113 | |
| 114 | Where <status> is one of: |
| 115 | ok: the block can be reconstructed |
| 116 | changed: the contents of the backing file have changed |
| 117 | no-file: the backing file could not be found |
| 118 | error: there was some other problem reading the file |
| 119 | missing: <obj> could not be found in the filestore |
| 120 | ERROR: internal error, most likely due to a corrupt database |
| 121 | |
| 122 | Where <action> is present only when removing bad blocks and is one of: |
| 123 | remove: link to the block will be removed from datastore |
| 124 | keep: keep link, nothing to do |
| 125 | |
| 126 | For ERROR entries the error will also be printed to stderr. |
| 127 | `, |
| 128 | }, |
| 129 | Arguments: []cmds.Argument{ |
| 130 | cmds.StringArg("obj", false, true, "Cid of objects to verify."), |
| 131 | }, |
| 132 | Options: []cmds.Option{ |
| 133 | cmds.BoolOption(fileOrderOptionName, "verify the objects based on the order of the backing file"), |
| 134 | cmds.BoolOption(removeBadBlocksOptionName, "remove bad blocks. WARNING: This may remove pinned data. You should run 'ipfs pin verify' after running this command and correct any issues."), |
| 135 | }, |
| 136 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 137 | _, fs, err := getFilestore(env) |
| 138 | if err != nil { |
| 139 | return err |
| 140 | } |
| 141 | |
| 142 | removeBadBlocks, _ := req.Options[removeBadBlocksOptionName].(bool) |
| 143 | args := req.Arguments |
| 144 | if len(args) > 0 { |
| 145 | return listByArgs(req.Context, res, fs, args, removeBadBlocks) |
| 146 | } |
| 147 | |
| 148 | fileOrder, _ := req.Options[fileOrderOptionName].(bool) |
| 149 | next, err := filestore.VerifyAll(req.Context, fs, fileOrder) |
| 150 | if err != nil { |
| 151 | return err |
| 152 | } |
| 153 | |
| 154 | for { |
| 155 | r := next(req.Context) |
| 156 | if r == nil { |
| 157 | break |
| 158 | } |
| 159 | |
| 160 | if removeBadBlocks && (r.Status != filestore.StatusOk) && (r.Status != filestore.StatusOtherError) { |
| 161 | if err = fs.FileManager().DeleteBlock(req.Context, r.Key); err != nil { |
| 162 | return err |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | if err = res.Emit(r); err != nil { |
| 167 | return err |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | return nil |
| 172 | }, |
| 173 | PostRun: cmds.PostRunMap{ |
| 174 | cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { |
| 175 | enc, err := cmdenv.GetCidEncoder(res.Request()) |
| 176 | if err != nil { |
| 177 | return err |
| 178 | } |
| 179 | |
| 180 | req := res.Request() |
| 181 | removeBadBlocks, _ := req.Options[removeBadBlocksOptionName].(bool) |
| 182 | for { |
| 183 | v, err := res.Next() |
| 184 | if err != nil { |
| 185 | if err == io.EOF { |
| 186 | return nil |
| 187 | } |
| 188 | return err |
| 189 | } |
| 190 | |
| 191 | list, ok := v.(*filestore.ListRes) |
| 192 | if !ok { |
| 193 | return e.TypeErr(list, v) |
| 194 | } |
| 195 | |
| 196 | if list.Status == filestore.StatusOtherError { |
| 197 | fmt.Fprintf(os.Stderr, "%s\n", list.ErrorMsg) |
| 198 | } |
| 199 | |
| 200 | if removeBadBlocks { |
| 201 | action := "keep" |
| 202 | if removeBadBlocks && (list.Status != filestore.StatusOk) && (list.Status != filestore.StatusOtherError) { |
| 203 | action = "remove" |
| 204 | } |
| 205 | fmt.Fprintf(os.Stdout, "%s %s %s\n", list.Status.Format(), list.FormatLong(enc.Encode), action) |
| 206 | } else { |
| 207 | fmt.Fprintf(os.Stdout, "%s %s\n", list.Status.Format(), list.FormatLong(enc.Encode)) |
| 208 | } |
| 209 | } |
| 210 | }, |
| 211 | }, |
| 212 | Type: filestore.ListRes{}, |
| 213 | } |
| 214 | |
| 215 | var dupsFileStore = &cmds.Command{ |
| 216 | Helptext: cmds.HelpText{ |
| 217 | Tagline: "List blocks that are both in the filestore and standard block storage.", |
| 218 | }, |
| 219 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 220 | _, fs, err := getFilestore(env) |
| 221 | if err != nil { |
| 222 | return err |
| 223 | } |
| 224 | |
| 225 | enc, err := cmdenv.GetCidEncoder(req) |
| 226 | if err != nil { |
| 227 | return err |
| 228 | } |
| 229 | |
| 230 | ch, err := fs.FileManager().AllKeysChan(req.Context) |
| 231 | if err != nil { |
| 232 | return err |
| 233 | } |
| 234 | |
| 235 | for cid := range ch { |
| 236 | have, err := fs.MainBlockstore().Has(req.Context, cid) |
| 237 | if err != nil { |
| 238 | return res.Emit(&RefWrapper{Err: err.Error()}) |
| 239 | } |
| 240 | if have { |
| 241 | if err := res.Emit(&RefWrapper{Ref: enc.Encode(cid)}); err != nil { |
| 242 | return err |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | return nil |
| 248 | }, |
| 249 | Encoders: refsEncoderMap, |
| 250 | Type: RefWrapper{}, |
| 251 | } |
| 252 | |
| 253 | func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, error) { |
| 254 | n, err := cmdenv.GetNode(env) |
| 255 | if err != nil { |
| 256 | return nil, nil, err |
| 257 | } |
| 258 | fs := n.Filestore |
| 259 | if fs == nil { |
| 260 | return n, nil, filestore.ErrFilestoreNotEnabled |
| 261 | } |
| 262 | return n, fs, err |
| 263 | } |
| 264 | |
| 265 | func listByArgs(ctx context.Context, res cmds.ResponseEmitter, fs *filestore.Filestore, args []string, removeBadBlocks bool) error { |
| 266 | for _, arg := range args { |
| 267 | c, err := cid.Decode(arg) |
| 268 | if err != nil { |
| 269 | ret := &filestore.ListRes{ |
| 270 | Status: filestore.StatusOtherError, |
| 271 | ErrorMsg: fmt.Sprintf("%s: %v", arg, err), |
| 272 | } |
| 273 | if err := res.Emit(ret); err != nil { |
| 274 | return err |
| 275 | } |
| 276 | continue |
| 277 | } |
| 278 | r := filestore.Verify(ctx, fs, c) |
| 279 | |
| 280 | if removeBadBlocks && (r.Status != filestore.StatusOk) && (r.Status != filestore.StatusOtherError) { |
| 281 | if err = fs.FileManager().DeleteBlock(ctx, r.Key); err != nil { |
| 282 | return err |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | if err = res.Emit(r); err != nil { |
| 287 | return err |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | return nil |
| 292 | } |