| 1 | # SPDX-License-Identifier: MIT |
| 2 | from __future__ import absolute_import |
| 3 | from collections import namedtuple |
| 4 | |
| 5 | from ..exceptions import LocationParseError |
| 6 | |
| 7 | |
| 8 | url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] |
| 9 | |
| 10 | # We only want to normalize urls with an HTTP(S) scheme. |
| 11 | # urllib3 infers URLs without a scheme (None) to be http. |
| 12 | NORMALIZABLE_SCHEMES = ('http', 'https', None) |
| 13 | |
| 14 | |
| 15 | class Url(namedtuple('Url', url_attrs)): |
| 16 | """ |
| 17 | Datastructure for representing an HTTP URL. Used as a return value for |
| 18 | :func:`parse_url`. Both the scheme and host are normalized as they are |
| 19 | both case-insensitive according to RFC 3986. |
| 20 | """ |
| 21 | __slots__ = () |
| 22 | |
| 23 | def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, |
| 24 | query=None, fragment=None): |
| 25 | if path and not path.startswith('/'): |
| 26 | path = '/' + path |
| 27 | if scheme: |
| 28 | scheme = scheme.lower() |
| 29 | if host and scheme in NORMALIZABLE_SCHEMES: |
| 30 | host = host.lower() |
| 31 | return super(Url, cls).__new__(cls, scheme, auth, host, port, path, |
| 32 | query, fragment) |
| 33 | |
| 34 | @property |
| 35 | def hostname(self): |
| 36 | """For backwards-compatibility with urlparse. We're nice like that.""" |
| 37 | return self.host |
| 38 | |
| 39 | @property |
| 40 | def request_uri(self): |
| 41 | """Absolute path including the query string.""" |
| 42 | uri = self.path or '/' |
| 43 | |
| 44 | if self.query is not None: |
| 45 | uri += '?' + self.query |
| 46 | |
| 47 | return uri |
| 48 | |
| 49 | @property |
| 50 | def netloc(self): |
| 51 | """Network location including host and port""" |
| 52 | if self.port: |
| 53 | return '%s:%d' % (self.host, self.port) |
| 54 | return self.host |
| 55 | |
| 56 | @property |
| 57 | def url(self): |
| 58 | """ |
| 59 | Convert self into a url |
| 60 | |
| 61 | This function should more or less round-trip with :func:`.parse_url`. The |
| 62 | returned url may not be exactly the same as the url inputted to |
| 63 | :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls |
| 64 | with a blank port will have : removed). |
| 65 | |
| 66 | Example: :: |
| 67 | |
| 68 | >>> U = parse_url('http://google.com/mail/') |
| 69 | >>> U.url |
| 70 | 'http://google.com/mail/' |
| 71 | >>> Url('http', 'username:password', 'host.com', 80, |
| 72 | ... '/path', 'query', 'fragment').url |
| 73 | 'http://username:password@host.com:80/path?query#fragment' |
| 74 | """ |
| 75 | scheme, auth, host, port, path, query, fragment = self |
| 76 | url = '' |
| 77 | |
| 78 | # We use "is not None" we want things to happen with empty strings (or 0 port) |
| 79 | if scheme is not None: |
| 80 | url += scheme + '://' |
| 81 | if auth is not None: |
| 82 | url += auth + '@' |
| 83 | if host is not None: |
| 84 | url += host |
| 85 | if port is not None: |
| 86 | url += ':' + str(port) |
| 87 | if path is not None: |
| 88 | url += path |
| 89 | if query is not None: |
| 90 | url += '?' + query |
| 91 | if fragment is not None: |
| 92 | url += '#' + fragment |
| 93 | |
| 94 | return url |
| 95 | |
| 96 | def __str__(self): |
| 97 | return self.url |
| 98 | |
| 99 | |
| 100 | def split_first(s, delims): |
| 101 | """ |
| 102 | Given a string and an iterable of delimiters, split on the first found |
| 103 | delimiter. Return two split parts and the matched delimiter. |
| 104 | |
| 105 | If not found, then the first part is the full input string. |
| 106 | |
| 107 | Example:: |
| 108 | |
| 109 | >>> split_first('foo/bar?baz', '?/=') |
| 110 | ('foo', 'bar?baz', '/') |
| 111 | >>> split_first('foo/bar?baz', '123') |
| 112 | ('foo/bar?baz', '', None) |
| 113 | |
| 114 | Scales linearly with number of delims. Not ideal for large number of delims. |
| 115 | """ |
| 116 | min_idx = None |
| 117 | min_delim = None |
| 118 | for d in delims: |
| 119 | idx = s.find(d) |
| 120 | if idx < 0: |
| 121 | continue |
| 122 | |
| 123 | if min_idx is None or idx < min_idx: |
| 124 | min_idx = idx |
| 125 | min_delim = d |
| 126 | |
| 127 | if min_idx is None or min_idx < 0: |
| 128 | return s, '', None |
| 129 | |
| 130 | return s[:min_idx], s[min_idx + 1:], min_delim |
| 131 | |
| 132 | |
| 133 | def parse_url(url): |
| 134 | """ |
| 135 | Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is |
| 136 | performed to parse incomplete urls. Fields not provided will be None. |
| 137 | |
| 138 | Partly backwards-compatible with :mod:`urlparse`. |
| 139 | |
| 140 | Example:: |
| 141 | |
| 142 | >>> parse_url('http://google.com/mail/') |
| 143 | Url(scheme='http', host='google.com', port=None, path='/mail/', ...) |
| 144 | >>> parse_url('google.com:80') |
| 145 | Url(scheme=None, host='google.com', port=80, path=None, ...) |
| 146 | >>> parse_url('/foo?bar') |
| 147 | Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) |
| 148 | """ |
| 149 | |
| 150 | # While this code has overlap with stdlib's urlparse, it is much |
| 151 | # simplified for our needs and less annoying. |
| 152 | # Additionally, this implementations does silly things to be optimal |
| 153 | # on CPython. |
| 154 | |
| 155 | if not url: |
| 156 | # Empty |
| 157 | return Url() |
| 158 | |
| 159 | scheme = None |
| 160 | auth = None |
| 161 | host = None |
| 162 | port = None |
| 163 | path = None |
| 164 | fragment = None |
| 165 | query = None |
| 166 | |
| 167 | # Scheme |
| 168 | if '://' in url: |
| 169 | scheme, url = url.split('://', 1) |
| 170 | |
| 171 | # Find the earliest Authority Terminator |
| 172 | # (http://tools.ietf.org/html/rfc3986#section-3.2) |
| 173 | url, path_, delim = split_first(url, ['/', '?', '#']) |
| 174 | |
| 175 | if delim: |
| 176 | # Reassemble the path |
| 177 | path = delim + path_ |
| 178 | |
| 179 | # Auth |
| 180 | if '@' in url: |
| 181 | # Last '@' denotes end of auth part |
| 182 | auth, url = url.rsplit('@', 1) |
| 183 | |
| 184 | # IPv6 |
| 185 | if url and url[0] == '[': |
| 186 | host, url = url.split(']', 1) |
| 187 | host += ']' |
| 188 | |
| 189 | # Port |
| 190 | if ':' in url: |
| 191 | _host, port = url.split(':', 1) |
| 192 | |
| 193 | if not host: |
| 194 | host = _host |
| 195 | |
| 196 | if port: |
| 197 | # If given, ports must be integers. No whitespace, no plus or |
| 198 | # minus prefixes, no non-integer digits such as ^2 (superscript). |
| 199 | if not port.isdigit(): |
| 200 | raise LocationParseError(url) |
| 201 | try: |
| 202 | port = int(port) |
| 203 | except ValueError: |
| 204 | raise LocationParseError(url) |
| 205 | else: |
| 206 | # Blank ports are cool, too. (rfc3986#section-3.2.3) |
| 207 | port = None |
| 208 | |
| 209 | elif not host and url: |
| 210 | host = url |
| 211 | |
| 212 | if not path: |
| 213 | return Url(scheme, auth, host, port, path, query, fragment) |
| 214 | |
| 215 | # Fragment |
| 216 | if '#' in path: |
| 217 | path, fragment = path.split('#', 1) |
| 218 | |
| 219 | # Query |
| 220 | if '?' in path: |
| 221 | path, query = path.split('?', 1) |
| 222 | |
| 223 | return Url(scheme, auth, host, port, path, query, fragment) |
| 224 | |
| 225 | |
| 226 | def get_host(url): |
| 227 | """ |
| 228 | Deprecated. Use :func:`parse_url` instead. |
| 229 | """ |
| 230 | p = parse_url(url) |
| 231 | return p.scheme or 'http', p.hostname, p.port |