| 1 | import argparse |
| 2 | import json |
| 3 | import logging |
| 4 | import posixpath |
| 5 | import random |
| 6 | import re |
| 7 | import requests |
| 8 | import string |
| 9 | import sys |
| 10 | import urllib.parse |
| 11 | |
| 12 | ####################################################################################################################### |
| 13 | # Utilities |
| 14 | |
| 15 | |
| 16 | def some(s): |
| 17 | return random.choice(sorted(s)) |
| 18 | |
| 19 | |
| 20 | def not_some(s): |
| 21 | test_set = random.choice([string.ascii_uppercase + string.ascii_lowercase, |
| 22 | string.digits, |
| 23 | string.digits + ".E-", |
| 24 | '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJK' |
| 25 | 'LMNOPQRSTUVWXYZ!"#$%\'()*+,-./:;<=>?@[\\]^_`{|}~ ']) |
| 26 | test_len = random.choice([1, 2, 3, 37, 61, 121]) |
| 27 | while True: |
| 28 | x = ''.join([random.choice(test_set) for _ in range(test_len)]) |
| 29 | if x not in s: |
| 30 | return x |
| 31 | |
| 32 | |
| 33 | def build_url(host_maybe_scheme, base_path): |
| 34 | try: |
| 35 | if '//' not in host_maybe_scheme: |
| 36 | host_maybe_scheme = '//' + host_maybe_scheme |
| 37 | url_tuple = urllib.parse.urlparse(host_maybe_scheme) |
| 38 | if base_path[0] == '/': |
| 39 | base_path = base_path[1:] |
| 40 | return url_tuple.netloc, posixpath.join(url_tuple.path, base_path) |
| 41 | except Exception as e: |
| 42 | L.error(f"Critical failure decoding arguments -> {e}") |
| 43 | sys.exit(-1) |
| 44 | |
| 45 | |
| 46 | ####################################################################################################################### |
| 47 | # Data-model and processing |
| 48 | |
| 49 | |
| 50 | class Param(object): |
| 51 | def __init__(self, name, location, kind): |
| 52 | self.location = location |
| 53 | self.kind = kind |
| 54 | self.name = name |
| 55 | self.values = set() |
| 56 | |
| 57 | def dump(self): |
| 58 | print(f"{self.name} in {self.location} is {self.kind} : {{{self.values}}}") |
| 59 | |
| 60 | |
| 61 | def does_response_fit_schema(schema_path, schema, resp): |
| 62 | '''The schema_path argument tells us where we are (globally) in the schema. The schema argument is the |
| 63 | sub-tree within the schema json that we are validating against. The resp is the json subtree from the |
| 64 | target host's response. |
| 65 | |
| 66 | The basic idea is this: swagger defines a model of valid json trees. In this sense it is a formal |
| 67 | language and we can validate a given server response by checking if the language accepts a particular |
| 68 | server response. This is basically a parser, but instead of strings we are operating on languages |
| 69 | of trees. |
| 70 | |
| 71 | This could probably be extended to arbitrary swagger definitions - but the amount of work increases |
| 72 | rapidly as we attempt to cover the full semantics of languages of trees defined in swagger. Instead |
| 73 | we have some special cases that describe the parts of the semantics that we've used to describe the |
| 74 | netdata API. |
| 75 | |
| 76 | If we hit an error (in the schema) that prevents further checks then we return early, otherwise we |
| 77 | try to collect as many errors as possible. |
| 78 | ''' |
| 79 | success = True |
| 80 | if "type" not in schema: |
| 81 | L.error(f"Cannot progress past {schema_path} -> no type specified in dictionary") |
| 82 | print(json.dumps(schema, indent=2)) |
| 83 | return False |
| 84 | if schema["type"] == "object": |
| 85 | if isinstance(resp, dict) and "properties" in schema and isinstance(schema["properties"], dict): |
| 86 | L.debug(f"Validate properties against dictionary at {schema_path}") |
| 87 | for k, v in schema["properties"].items(): |
| 88 | L.debug(f"Validate {k} received with {v}") |
| 89 | if v.get("required", False) and k not in resp: |
| 90 | L.error(f"Missing {k} in response at {schema_path}") |
| 91 | print(json.dumps(resp, indent=2)) |
| 92 | return False |
| 93 | if k in resp: |
| 94 | if not does_response_fit_schema(posixpath.join(schema_path, k), v, resp[k]): |
| 95 | success = False |
| 96 | elif isinstance(resp, dict) and "additionalProperties" in schema \ |
| 97 | and isinstance(schema["additionalProperties"], dict): |
| 98 | kv_schema = schema["additionalProperties"] |
| 99 | L.debug(f"Validate additionalProperties against every value in dictionary at {schema_path}") |
| 100 | if "type" in kv_schema and kv_schema["type"] == "object": |
| 101 | for k, v in resp.items(): |
| 102 | if not does_response_fit_schema(posixpath.join(schema_path, k), kv_schema, v): |
| 103 | success = False |
| 104 | else: |
| 105 | L.error("Don't understand what the additionalProperties means (it has no type?)") |
| 106 | return False |
| 107 | else: |
| 108 | L.error(f"Can't understand schema at {schema_path}") |
| 109 | print(json.dumps(schema, indent=2)) |
| 110 | return False |
| 111 | elif schema["type"] == "string": |
| 112 | if isinstance(resp, str): |
| 113 | L.debug(f"{repr(resp)} matches {repr(schema)} at {schema_path}") |
| 114 | return True |
| 115 | L.error(f"{repr(resp)} does not match schema {repr(schema)} at {schema_path}") |
| 116 | return False |
| 117 | elif schema["type"] == "boolean": |
| 118 | if isinstance(resp, bool): |
| 119 | L.debug(f"{repr(resp)} matches {repr(schema)} at {schema_path}") |
| 120 | return True |
| 121 | L.error(f"{repr(resp)} does not match schema {repr(schema)} at {schema_path}") |
| 122 | return False |
| 123 | elif schema["type"] == "number": |
| 124 | if 'nullable' in schema and resp is None: |
| 125 | L.debug(f"{repr(resp)} matches {repr(schema)} at {schema_path} (because nullable)") |
| 126 | return True |
| 127 | if isinstance(resp, int) or isinstance(resp, float): |
| 128 | L.debug(f"{repr(resp)} matches {repr(schema)} at {schema_path}") |
| 129 | return True |
| 130 | L.error(f"{repr(resp)} does not match schema {repr(schema)} at {schema_path}") |
| 131 | return False |
| 132 | elif schema["type"] == "integer": |
| 133 | if 'nullable' in schema and resp is None: |
| 134 | L.debug(f"{repr(resp)} matches {repr(schema)} at {schema_path} (because nullable)") |
| 135 | return True |
| 136 | if isinstance(resp, int): |
| 137 | L.debug(f"{repr(resp)} matches {repr(schema)} at {schema_path}") |
| 138 | return True |
| 139 | L.error(f"{repr(resp)} does not match schema {repr(schema)} at {schema_path}") |
| 140 | return False |
| 141 | elif schema["type"] == "array": |
| 142 | if "items" not in schema: |
| 143 | L.error(f"Schema for array at {schema_path} does not specify items!") |
| 144 | return False |
| 145 | item_schema = schema["items"] |
| 146 | if not isinstance(resp, list): |
| 147 | L.error(f"Server did not return a list for {schema_path} (typed as array in schema)") |
| 148 | return False |
| 149 | for i, item in enumerate(resp): |
| 150 | if not does_response_fit_schema(posixpath.join(schema_path, str(i)), item_schema, item): |
| 151 | success = False |
| 152 | else: |
| 153 | L.error(f"Invalid swagger type {schema['type']} for {type(resp)} at {schema_path}") |
| 154 | print(json.dumps(schema, indent=2)) |
| 155 | return False |
| 156 | return success |
| 157 | |
| 158 | |
| 159 | class GetPath(object): |
| 160 | def __init__(self, url, spec): |
| 161 | self.url = url |
| 162 | self.req_params = {} |
| 163 | self.opt_params = {} |
| 164 | self.success = None |
| 165 | self.failures = {} |
| 166 | if 'parameters' in spec.keys(): |
| 167 | for p in spec['parameters']: |
| 168 | name = p['name'] |
| 169 | req = p.get('required', False) |
| 170 | target = self.req_params if req else self.opt_params |
| 171 | target[name] = Param(name, p['in'], p['type']) |
| 172 | if 'default' in p: |
| 173 | defs = p['default'] |
| 174 | if isinstance(defs, list): |
| 175 | for d in defs: |
| 176 | target[name].values.add(d) |
| 177 | else: |
| 178 | target[name].values.add(defs) |
| 179 | if 'enum' in p: |
| 180 | for v in p['enum']: |
| 181 | target[name].values.add(v) |
| 182 | if req and len(target[name].values) == 0: |
| 183 | print(f"FAIL: No default values in swagger for required parameter {name} in {self.url}") |
| 184 | for code, schema in spec['responses'].items(): |
| 185 | if code[0] == "2" and 'schema' in schema: |
| 186 | self.success = schema['schema'] |
| 187 | elif code[0] == "2": |
| 188 | L.error(f"2xx response with no schema in {self.url}") |
| 189 | else: |
| 190 | self.failures[code] = schema |
| 191 | |
| 192 | def generate_success(self, host): |
| 193 | url_args = "&".join([f"{p.name}={some(p.values)}" for p in self.req_params.values()]) |
| 194 | base_url = urllib.parse.urljoin(host, self.url) |
| 195 | test_url = f"{base_url}?{url_args}" |
| 196 | if url_filter.match(test_url): |
| 197 | try: |
| 198 | resp = requests.get(url=test_url, verify=(not args.tls_no_verify)) |
| 199 | self.validate(test_url, resp, True) |
| 200 | except Exception as e: |
| 201 | L.error(f"Network failure in test {e}") |
| 202 | else: |
| 203 | L.debug(f"url_filter skips {test_url}") |
| 204 | |
| 205 | def generate_failure(self, host): |
| 206 | all_params = list(self.req_params.values()) + list(self.opt_params.values()) |
| 207 | bad_param = ''.join([random.choice(string.ascii_lowercase) for _ in range(5)]) |
| 208 | while bad_param in all_params: |
| 209 | bad_param = ''.join([random.choice(string.ascii_lowercase) for _ in range(5)]) |
| 210 | all_params.append(Param(bad_param, "query", "string")) |
| 211 | url_args = "&".join([f"{p.name}={not_some(p.values)}" for p in all_params]) |
| 212 | base_url = urllib.parse.urljoin(host, self.url) |
| 213 | test_url = f"{base_url}?{url_args}" |
| 214 | if url_filter.match(test_url): |
| 215 | try: |
| 216 | resp = requests.get(url=test_url, verify=(not args.tls_no_verify)) |
| 217 | self.validate(test_url, resp, False) |
| 218 | except Exception as e: |
| 219 | L.error(f"Network failure in test {e}") |
| 220 | |
| 221 | def validate(self, test_url, resp, expect_success): |
| 222 | try: |
| 223 | resp_json = json.loads(resp.text) |
| 224 | except json.decoder.JSONDecodeError as e: |
| 225 | L.error(f"Non-json response from {test_url}") |
| 226 | return |
| 227 | success_code = resp.status_code >= 200 and resp.status_code < 300 |
| 228 | if success_code and expect_success: |
| 229 | if self.success is not None: |
| 230 | if does_response_fit_schema(posixpath.join(self.url, str(resp.status_code)), self.success, resp_json): |
| 231 | L.info(f"tested {test_url}") |
| 232 | else: |
| 233 | L.error(f"tested {test_url}") |
| 234 | else: |
| 235 | L.error(f"Missing schema {test_url}") |
| 236 | elif not success_code and not expect_success: |
| 237 | schema = self.failures.get(str(resp.status_code), None) |
| 238 | if schema is not None: |
| 239 | if does_response_fit_schema(posixpath.join(self.url, str(resp.status_code)), schema, resp_json): |
| 240 | L.info(f"tested {test_url}") |
| 241 | else: |
| 242 | L.error(f"tested {test_url}") |
| 243 | else: |
| 244 | L.error("Missing schema for {resp.status_code} from {test_url}") |
| 245 | else: |
| 246 | L.error(f"Received incorrect status code {resp.status_code} against {test_url}") |
| 247 | |
| 248 | |
| 249 | def get_the_spec(url): |
| 250 | if url[:7] == "file://": |
| 251 | with open(url[7:]) as f: |
| 252 | return f.read() |
| 253 | return requests.get(url=url).text |
| 254 | |
| 255 | |
| 256 | # Swagger paths look absolute but they are relative to the base. |
| 257 | def not_absolute(path): |
| 258 | return path[1:] if path[0] == '/' else path |
| 259 | |
| 260 | |
| 261 | def find_ref(spec, path): |
| 262 | if len(path) > 0 and path[0] == '#': |
| 263 | return find_ref(spec, path[1:]) |
| 264 | if len(path) == 1: |
| 265 | return spec[path[0]] |
| 266 | return find_ref(spec[path[0]], path[1:]) |
| 267 | |
| 268 | |
| 269 | def resolve_refs(spec, spec_root=None): |
| 270 | '''Find all "$ref" keys in the swagger spec and inline their target schemas. |
| 271 | |
| 272 | As with all inliners this will break if a definition recursively links to itself, but this should not |
| 273 | happen in swagger as embedding a structure inside itself would produce a record of infinite size.''' |
| 274 | if spec_root is None: |
| 275 | spec_root = spec |
| 276 | newspec = {} |
| 277 | for k, v in spec.items(): |
| 278 | if k == "$ref": |
| 279 | path = v.split('/') |
| 280 | target = find_ref(spec_root, path) |
| 281 | # Unfold one level of the tree and erase the $ref if possible. |
| 282 | if isinstance(target, dict): |
| 283 | for kk, vv in resolve_refs(target, spec_root).items(): |
| 284 | newspec[kk] = vv |
| 285 | else: |
| 286 | newspec[k] = target |
| 287 | elif isinstance(v, dict): |
| 288 | newspec[k] = resolve_refs(v, spec_root) |
| 289 | else: |
| 290 | newspec[k] = v |
| 291 | # This is an artifact of inline the $refs when they are inside a properties key as their children should be |
| 292 | # pushed up into the parent dictionary. They must be merged (union) rather than replace as we use this to |
| 293 | # implement polymorphism in the data-model. |
| 294 | if 'properties' in newspec and isinstance(newspec['properties'], dict) and \ |
| 295 | 'properties' in newspec['properties']: |
| 296 | sub = newspec['properties']['properties'] |
| 297 | del newspec['properties']['properties'] |
| 298 | if 'type' in newspec['properties']: |
| 299 | del newspec['properties']['type'] |
| 300 | for k, v in sub.items(): |
| 301 | newspec['properties'][k] = v |
| 302 | return newspec |
| 303 | |
| 304 | |
| 305 | ####################################################################################################################### |
| 306 | # Initialization |
| 307 | |
| 308 | random.seed(7) # Default is reproducible sequences |
| 309 | |
| 310 | parser = argparse.ArgumentParser() |
| 311 | parser.add_argument('--url', type=str, |
| 312 | default='https://raw.githubusercontent.com/netdata/netdata/master/src/web/api/netdata-swagger.json', |
| 313 | help='The URL of the API definition in swagger. The default will pull the latest version ' |
| 314 | 'from the main branch.') |
| 315 | parser.add_argument('--host', type=str, |
| 316 | help='The URL of the target host to fuzz. The default will read the host from the swagger ' |
| 317 | 'definition.') |
| 318 | parser.add_argument('--reseed', action='store_true', |
| 319 | help="Pick a random seed for the PRNG. The default uses a constant seed for reproducibility.") |
| 320 | parser.add_argument('--passes', action='store_true', |
| 321 | help="Log information about tests that pass") |
| 322 | parser.add_argument('--detail', action='store_true', |
| 323 | help="Log information about the response/schema comparisons during each test") |
| 324 | parser.add_argument('--filter', type=str, |
| 325 | default=".*", |
| 326 | help="Supply a regex used to filter the testing URLs generated") |
| 327 | parser.add_argument('--tls-no-verify', action='store_true', |
| 328 | help="Disable TLS certification verification to allow connection to hosts that use" |
| 329 | "self-signed certificates") |
| 330 | parser.add_argument('--dump-inlined', action='store_true', |
| 331 | help='Dump the inlined swagger spec instead of fuzzing. For "reasons".') |
| 332 | |
| 333 | args = parser.parse_args() |
| 334 | if args.reseed: |
| 335 | random.seed() |
| 336 | |
| 337 | spec = json.loads(get_the_spec(args.url)) |
| 338 | inlined_spec = resolve_refs(spec) |
| 339 | if args.dump_inlined: |
| 340 | print(json.dumps(inlined_spec, indent=2)) |
| 341 | sys.exit(-1) |
| 342 | |
| 343 | logging.addLevelName(40, "FAIL") |
| 344 | logging.addLevelName(20, "PASS") |
| 345 | logging.addLevelName(10, "DETAIL") |
| 346 | L = logging.getLogger() |
| 347 | handler = logging.StreamHandler(sys.stdout) |
| 348 | if not args.passes and not args.detail: |
| 349 | L.setLevel(logging.ERROR) |
| 350 | elif args.passes and not args.detail: |
| 351 | L.setLevel(logging.INFO) |
| 352 | elif args.detail: |
| 353 | L.setLevel(logging.DEBUG) |
| 354 | handler.setFormatter(logging.Formatter(fmt="%(levelname)s %(message)s")) |
| 355 | L.addHandler(handler) |
| 356 | |
| 357 | url_filter = re.compile(args.filter) |
| 358 | |
| 359 | if spec['swagger'] != '2.0': |
| 360 | L.error(f"Unexpected swagger version") |
| 361 | sys.exit(-1) |
| 362 | L.info(f"Fuzzing {spec['info']['title']} / {spec['info']['version']}") |
| 363 | |
| 364 | host, base_url = build_url(args.host or spec['host'], inlined_spec['basePath']) |
| 365 | |
| 366 | L.info(f"Target host is {base_url}") |
| 367 | paths = [] |
| 368 | for name, p in inlined_spec['paths'].items(): |
| 369 | if 'get' in p: |
| 370 | name = not_absolute(name) |
| 371 | paths.append(GetPath(posixpath.join(base_url, name), p['get'])) |
| 372 | elif 'put' in p: |
| 373 | L.error(f"Generation of PUT methods (for {name} is unimplemented") |
| 374 | |
| 375 | for s in inlined_spec['schemes']: |
| 376 | for p in paths: |
| 377 | resp = p.generate_success(s + "://" + host) |
| 378 | resp = p.generate_failure(s+"://"+host) |