fix: Issue #9364 JSON config validation (#10679)
* Fix JSON config validation * updated ReflectToMap --------- Co-authored-by: galargh <piotr.galar@gmail.com> Co-authored-by: Guillaume Michel <guillaumemichel@users.noreply.github.com> Co-authored-by: guillaumemichel <guillaume@michel.id>
Sergey Gorbunov committed
Feb 4, 2025 at 19:26 UTC
032ceaf5d11d841b369e94d9f8a2d820f29903b1
7 files changed
+289
-57
config/config.go
+101
@@ -7,6 +7,7 @@ import (
7
"fmt"
8
"os"
9
"path/filepath"
10
+ "reflect"
11
"strings"
12
13
"github.com/ipfs/kubo/misc/fsutil"
@@ -137,6 +138,71 @@ func ToMap(conf *Config) (map[string]interface{}, error) {
138
return m, nil
139
}
140
141
+// Convert config to a map, without using encoding/json, since
142
+// zero/empty/'omitempty' fields are excluded by encoding/json during
143
+// marshaling.
144
+func ReflectToMap(conf interface{}) interface{} {
145
+ v := reflect.ValueOf(conf)
146
+ if !v.IsValid() {
147
+ return nil
148
+ }
149
+
150
+ // Handle pointer type
151
+ if v.Kind() == reflect.Ptr {
152
+ if v.IsNil() {
153
+ // Create a zero value of the pointer's element type
154
+ elemType := v.Type().Elem()
155
+ zero := reflect.Zero(elemType)
156
+ return ReflectToMap(zero.Interface())
157
+ }
158
+ v = v.Elem()
159
+ }
160
+
161
+ switch v.Kind() {
162
+ case reflect.Struct:
163
+ result := make(map[string]interface{})
164
+ t := v.Type()
165
+ for i := 0; i < v.NumField(); i++ {
166
+ field := v.Field(i)
167
+ // Only include exported fields
168
+ if field.CanInterface() {
169
+ result[t.Field(i).Name] = ReflectToMap(field.Interface())
170
+ }
171
+ }
172
+ return result
173
+
174
+ case reflect.Map:
175
+ result := make(map[string]interface{})
176
+ iter := v.MapRange()
177
+ for iter.Next() {
178
+ key := iter.Key()
179
+ // Convert map keys to strings for consistency
180
+ keyStr := fmt.Sprint(ReflectToMap(key.Interface()))
181
+ result[keyStr] = ReflectToMap(iter.Value().Interface())
182
+ }
183
+ // Add a sample to differentiate between a map and a struct on validation.
184
+ sample := reflect.Zero(v.Type().Elem())
185
+ if sample.CanInterface() {
186
+ result["*"] = ReflectToMap(sample.Interface())
187
+ }
188
+ return result
189
+
190
+ case reflect.Slice, reflect.Array:
191
+ result := make([]interface{}, v.Len())
192
+ for i := 0; i < v.Len(); i++ {
193
+ result[i] = ReflectToMap(v.Index(i).Interface())
194
+ }
195
+ return result
196
+
197
+ default:
198
+ // For basic types (int, string, etc.), just return the value
199
+ if v.CanInterface() {
200
+ return v.Interface()
201
+ }
202
+ return nil
203
+ }
204
+}
205
+
206
// Clone copies the config. Use when updating.
207
func (c *Config) Clone() (*Config, error) {
208
var newConfig Config
@@ -152,3 +218,38 @@ func (c *Config) Clone() (*Config, error) {
218
219
return &newConfig, nil
220
}
221
+
222
+// Check if the provided key is present in the structure.
223
+func CheckKey(key string) error {
224
+ conf := Config{}
225
+
226
+ // Convert an empty config to a map without JSON.
227
+ cursor := ReflectToMap(&conf)
228
+
229
+ // Parse the key and verify it's presence in the map.
230
+ var ok bool
231
+ var mapCursor map[string]interface{}
232
+
233
+ parts := strings.Split(key, ".")
234
+ for i, part := range parts {
235
+ mapCursor, ok = cursor.(map[string]interface{})
236
+ if !ok {
237
+ if cursor == nil {
238
+ return nil
239
+ }
240
+ path := strings.Join(parts[:i], ".")
241
+ return fmt.Errorf("%s key is not a map", path)
242
+ }
243
+
244
+ cursor, ok = mapCursor[part]
245
+ if !ok {
246
+ // If the config sections is a map, validate against the default entry.
247
+ if cursor, ok = mapCursor["*"]; ok {
248
+ continue
249
+ }
250
+ path := strings.Join(parts[:i+1], ".")
251
+ return fmt.Errorf("%s not found", path)
252
+ }
253
+ }
254
+ return nil
255
+}
config/config_test.go
+132
@@ -27,3 +27,135 @@ func TestClone(t *testing.T) {
27
t.Fatal("HTTP headers not preserved")
28
}
29
}
30
+
31
+func TestReflectToMap(t *testing.T) {
32
+ // Helper function to create a test config with various field types
33
+ reflectedConfig := ReflectToMap(new(Config))
34
+
35
+ mapConfig, ok := reflectedConfig.(map[string]interface{})
36
+ if !ok {
37
+ t.Fatal("Config didn't convert to map")
38
+ }
39
+
40
+ reflectedIdentity, ok := mapConfig["Identity"]
41
+ if !ok {
42
+ t.Fatal("Identity field not found")
43
+ }
44
+
45
+ mapIdentity, ok := reflectedIdentity.(map[string]interface{})
46
+ if !ok {
47
+ t.Fatal("Identity field didn't convert to map")
48
+ }
49
+
50
+ // Test string field reflection
51
+ reflectedPeerID, ok := mapIdentity["PeerID"]
52
+ if !ok {
53
+ t.Fatal("PeerID field not found in Identity")
54
+ }
55
+ if _, ok := reflectedPeerID.(string); !ok {
56
+ t.Fatal("PeerID field didn't convert to string")
57
+ }
58
+
59
+ // Test omitempty json string field
60
+ reflectedPrivKey, ok := mapIdentity["PrivKey"]
61
+ if !ok {
62
+ t.Fatal("PrivKey omitempty field not found in Identity")
63
+ }
64
+ if _, ok := reflectedPrivKey.(string); !ok {
65
+ t.Fatal("PrivKey omitempty field didn't convert to string")
66
+ }
67
+
68
+ // Test slices field
69
+ reflectedBootstrap, ok := mapConfig["Bootstrap"]
70
+ if !ok {
71
+ t.Fatal("Bootstrap field not found in config")
72
+ }
73
+ bootstrap, ok := reflectedBootstrap.([]interface{})
74
+ if !ok {
75
+ t.Fatal("Bootstrap field didn't convert to []string")
76
+ }
77
+ if len(bootstrap) != 0 {
78
+ t.Fatal("Bootstrap len is incorrect")
79
+ }
80
+
81
+ reflectedDatastore, ok := mapConfig["Datastore"]
82
+ if !ok {
83
+ t.Fatal("Datastore field not found in config")
84
+ }
85
+ datastore, ok := reflectedDatastore.(map[string]interface{})
86
+ if !ok {
87
+ t.Fatal("Datastore field didn't convert to map")
88
+ }
89
+ storageGCWatermark, ok := datastore["StorageGCWatermark"]
90
+ if !ok {
91
+ t.Fatal("StorageGCWatermark field not found in Datastore")
92
+ }
93
+ // Test int field
94
+ if _, ok := storageGCWatermark.(int64); !ok {
95
+ t.Fatal("StorageGCWatermark field didn't convert to int64")
96
+ }
97
+ noSync, ok := datastore["NoSync"]
98
+ if !ok {
99
+ t.Fatal("NoSync field not found in Datastore")
100
+ }
101
+ // Test bool field
102
+ if _, ok := noSync.(bool); !ok {
103
+ t.Fatal("NoSync field didn't convert to bool")
104
+ }
105
+
106
+ reflectedDNS, ok := mapConfig["DNS"]
107
+ if !ok {
108
+ t.Fatal("DNS field not found in config")
109
+ }
110
+ DNS, ok := reflectedDNS.(map[string]interface{})
111
+ if !ok {
112
+ t.Fatal("DNS field didn't convert to map")
113
+ }
114
+ reflectedResolvers, ok := DNS["Resolvers"]
115
+ if !ok {
116
+ t.Fatal("Resolvers field not found in DNS")
117
+ }
118
+ // Test map field
119
+ if _, ok := reflectedResolvers.(map[string]interface{}); !ok {
120
+ t.Fatal("Resolvers field didn't convert to map")
121
+ }
122
+
123
+ // Test pointer field
124
+ if _, ok := DNS["MaxCacheTTL"].(map[string]interface{}); !ok {
125
+ // Since OptionalDuration only field is private, we cannot test it
126
+ t.Fatal("MaxCacheTTL field didn't convert to map")
127
+ }
128
+}
129
+
130
+// Test validation of options set through "ipfs config"
131
+func TestCheckKey(t *testing.T) {
132
+ err := CheckKey("Foo.Bar")
133
+ if err == nil {
134
+ t.Fatal("Foo.Bar isn't a valid key in the config")
135
+ }
136
+
137
+ err = CheckKey("Provider.Strategy")
138
+ if err != nil {
139
+ t.Fatalf("%s: %s", err, "Provider.Strategy is a valid key in the config")
140
+ }
141
+
142
+ err = CheckKey("Provider.Foo")
143
+ if err == nil {
144
+ t.Fatal("Provider.Foo isn't a valid key in the config")
145
+ }
146
+
147
+ err = CheckKey("Gateway.PublicGateways.Foo.Paths")
148
+ if err != nil {
149
+ t.Fatalf("%s: %s", err, "Gateway.PublicGateways.Foo.Paths is a valid key in the config")
150
+ }
151
+
152
+ err = CheckKey("Gateway.PublicGateways.Foo.Bar")
153
+ if err == nil {
154
+ t.Fatal("Gateway.PublicGateways.Foo.Bar isn't a valid key in the config")
155
+ }
156
+
157
+ err = CheckKey("Plugins.Plugins.peerlog.Config.Enabled")
158
+ if err != nil {
159
+ t.Fatalf("%s: %s", err, "Plugins.Plugins.peerlog.Config.Enabled is a valid key in the config")
160
+ }
161
+}
docs/changelogs/v0.34.md
+6
-2
@@ -1,11 +1,12 @@
1
# Kubo changelog v0.34
2
3
-- [v0.34.0](#v0310)
3
+- [v0.34.0](#v0340)
4
5
## v0.34.0
6
7
- [Overview](#overview)
8
- [🔦 Highlights](#-highlights)
9
+ - [JSON config validation](#json-config-validation)
10
- [Reprovide command moved to routing](#reprovide-command-moved-to-routing)
11
- [Additional stats for Accelerated DHT Reprovides](#additional-stats-for-accelerated-dht-reprovides)
12
- [📝 Changelog](#-changelog)
@@ -15,6 +16,10 @@
16
17
### 🔦 Highlights
18
19
+#### JSON config validation
20
+
21
+`ipfs config` is now validating json fields ([#10679](https://github.com/ipfs/kubo/pull/10679)).
22
+
23
#### Reprovide command moved to routing
24
25
Moved the `bitswap reprovide` command to `routing reprovide`. ([#10677](https://github.com/ipfs/kubo/pull/10677))
@@ -26,4 +31,3 @@ The `stats reprovide` command now shows additional stats for the DHT Accelerated
31
### 📝 Changelog
32
33
### 👨👩👧👦 Contributors
29
-
repo/fsrepo/fsrepo.go
+6
@@ -676,6 +676,12 @@ func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
676
return errors.New("repo is closed")
677
}
678
679
+ // Validate the key's presence in the config structure.
680
+ err := config.CheckKey(key)
681
+ if err != nil {
682
+ return err
683
+ }
684
+
685
// Load into a map so we don't end up writing any additional defaults to the config file.
686
var mapconf map[string]interface{}
687
if err := serialize.ReadConfigFile(r.configFilePath, &mapconf); err != nil {
test/sharness/t0002-docker-image.sh
+4
-4
@@ -36,8 +36,8 @@ test_expect_success "docker image build succeeds" '
36
'
37
38
test_expect_success "write init scripts" '
39
- echo "ipfs config Foo Bar" > 001.sh &&
40
- echo "ipfs config Baz Qux" > 002.sh &&
39
+ echo "ipfs config Provider.Strategy Bar" > 001.sh &&
40
+ echo "ipfs config Pubsub.Router Qux" > 002.sh &&
41
chmod +x 002.sh
42
'
43
@@ -65,10 +65,10 @@ test_expect_success "check that init scripts were run correctly and in the corre
65
66
test_expect_success "check that init script configs were applied" '
67
echo Bar > expected &&
68
- docker exec "$DOC_ID" ipfs config Foo > actual &&
68
+ docker exec "$DOC_ID" ipfs config Provider.Strategy > actual &&
69
test_cmp actual expected &&
70
echo Qux > expected &&
71
- docker exec "$DOC_ID" ipfs config Baz > actual &&
71
+ docker exec "$DOC_ID" ipfs config Pubsub.Router > actual &&
72
test_cmp actual expected
73
'
74
test/sharness/t0021-config.sh
+35
-48
@@ -13,41 +13,23 @@ test_config_cmd_set() {
13
14
cfg_key=$1
15
cfg_val=$2
16
- test_expect_success "ipfs config succeeds" '
17
- ipfs config $cfg_flags "$cfg_key" "$cfg_val"
18
- '
19
-
20
- test_expect_success "ipfs config output looks good" '
21
- echo "$cfg_val" >expected &&
22
- ipfs config "$cfg_key" >actual &&
23
- test_cmp expected actual
24
- '
25
-
26
- # also test our lib function. it should work too.
27
- cfg_key="Lib.$cfg_key"
28
- test_expect_success "test_config_set succeeds" '
29
- test_config_set $cfg_flags "$cfg_key" "$cfg_val"
30
- '
31
-
32
- test_expect_success "test_config_set value looks good" '
33
- echo "$cfg_val" >expected &&
34
- ipfs config "$cfg_key" >actual &&
35
- test_cmp expected actual
36
- '
16
+ test_expect_success "ipfs config succeeds" "
17
+ ipfs config $cfg_flags \"$cfg_key\" \"$cfg_val\"
18
+ "
19
+
20
+ test_expect_success "ipfs config output looks good" "
21
+ echo \"$cfg_val\" >expected &&
22
+ if [$cfg_flags != \"--json\"]; then
23
+ ipfs config \"$cfg_key\" >actual &&
24
+ test_cmp expected actual
25
+ else
26
+ ipfs config \"$cfg_key\" | tr -d \"\\n\\t \" >actual &&
27
+ echo >>actual &&
28
+ test_cmp expected actual
29
+ fi
30
+ "
31
}
32
39
-# this is a bit brittle. the problem is we need to test
40
-# with something that will be forced to unmarshal as a struct.
41
-# (i.e. just setting 'ipfs config --json foo "[1, 2, 3]"') may
42
-# set it as astring instead of proper json. We leverage the
43
-# unmarshalling that has to happen.
44
-CONFIG_SET_JSON_TEST='{
45
- "MDNS": {
46
- "Enabled": true,
47
- "Interval": 10
48
- }
49
-}'
50
-
33
test_profile_apply_revert() {
34
profile=$1
35
inverse_profile=$2
@@ -87,27 +69,32 @@ test_profile_apply_dry_run_not_alter() {
69
}
70
71
test_config_cmd() {
90
- test_config_cmd_set "beep" "boop"
91
- test_config_cmd_set "beep1" "boop2"
92
- test_config_cmd_set "beep1" "boop2"
93
- test_config_cmd_set "--bool" "beep2" "true"
94
- test_config_cmd_set "--bool" "beep2" "false"
95
- test_config_cmd_set "--json" "beep3" "true"
96
- test_config_cmd_set "--json" "beep3" "false"
97
- test_config_cmd_set "--json" "Discovery" "$CONFIG_SET_JSON_TEST"
98
- test_config_cmd_set "--json" "deep-not-defined.prop" "true"
99
- test_config_cmd_set "--json" "deep-null" "null"
100
- test_config_cmd_set "--json" "deep-null.prop" "true"
72
+ test_config_cmd_set "Addresses.API" "foo"
73
+ test_config_cmd_set "Addresses.Gateway" "bar"
74
+ test_config_cmd_set "Datastore.GCPeriod" "baz"
75
+ test_config_cmd_set "AutoNAT.ServiceMode" "enabled"
76
+ test_config_cmd_set "--bool" "Discovery.MDNS.Enabled" "true"
77
+ test_config_cmd_set "--bool" "Discovery.MDNS.Enabled" "false"
78
+ test_config_cmd_set "--json" "Datastore.HashOnRead" "true"
79
+ test_config_cmd_set "--json" "Datastore.HashOnRead" "false"
80
+ test_config_cmd_set "--json" "Experimental.FilestoreEnabled" "true"
81
+ test_config_cmd_set "--json" "Import.BatchMaxSize" "null"
82
+ test_config_cmd_set "--json" "Import.UnixFSRawLeaves" "true"
83
+ test_config_cmd_set "--json" "Routing.Routers.Test" "{\\\"Parameters\\\":\\\"Test\\\",\\\"Type\\\":\\\"Test\\\"}"
84
+ test_config_cmd_set "--json" "Experimental.OptimisticProvideJobsPoolSize" "1337"
85
+ test_config_cmd_set "--json" "Addresses.Swarm" "[\\\"test\\\",\\\"test\\\",\\\"test\\\"]"
86
+ test_config_cmd_set "--json" "Gateway.PublicGateways.Foo" "{\\\"DeserializedResponses\\\":true,\\\"InlineDNSLink\\\":false,\\\"NoDNSLink\\\":false,\\\"Paths\\\":[\\\"Bar\\\",\\\"Baz\\\"],\\\"UseSubdomains\\\":true}"
87
+ test_config_cmd_set "--bool" "Gateway.PublicGateways.Foo.UseSubdomains" "false"
88
89
test_expect_success "'ipfs config show' works" '
90
ipfs config show >actual
91
'
92
93
test_expect_success "'ipfs config show' output looks good" '
107
- grep "\"beep\": \"boop\"," actual &&
108
- grep "\"beep1\": \"boop2\"," actual &&
109
- grep "\"beep2\": false," actual &&
110
- grep "\"beep3\": false," actual
94
+ grep "\"API\": \"foo\"," actual &&
95
+ grep "\"Gateway\": \"bar\"" actual &&
96
+ grep "\"Enabled\": false" actual &&
97
+ grep "\"HashOnRead\": false" actual
98
'
99
100
test_expect_success "'ipfs config show --config-file' works" '
test/sharness/t0070-user-config.sh
+5
-3
@@ -11,10 +11,12 @@ test_description="Test user-provided config values"
11
test_init_ipfs
12
13
test_expect_success "bootstrap doesn't overwrite user-provided config keys (top-level)" '
14
- ipfs config Foo.Bar baz &&
14
+ ipfs config Provider.Strategy >previous &&
15
+ ipfs config Provider.Strategy foo &&
16
ipfs bootstrap rm --all &&
16
- echo "baz" >expected &&
17
- ipfs config Foo.Bar >actual &&
17
+ echo "foo" >expected &&
18
+ ipfs config Provider.Strategy >actual &&
19
+ ipfs config Provider.Strategy $(cat previous) &&
20
test_cmp expected actual
21
'
22