| 1 | # SPDX-License-Identifier: MIT |
| 2 | """ |
| 3 | This module provides a pool manager that uses Google App Engine's |
| 4 | `URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. |
| 5 | |
| 6 | Example usage:: |
| 7 | |
| 8 | from urllib3 import PoolManager |
| 9 | from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox |
| 10 | |
| 11 | if is_appengine_sandbox(): |
| 12 | # AppEngineManager uses AppEngine's URLFetch API behind the scenes |
| 13 | http = AppEngineManager() |
| 14 | else: |
| 15 | # PoolManager uses a socket-level API behind the scenes |
| 16 | http = PoolManager() |
| 17 | |
| 18 | r = http.request('GET', 'https://google.com/') |
| 19 | |
| 20 | There are `limitations <https://cloud.google.com/appengine/docs/python/\ |
| 21 | urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be |
| 22 | the best choice for your application. There are three options for using |
| 23 | urllib3 on Google App Engine: |
| 24 | |
| 25 | 1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is |
| 26 | cost-effective in many circumstances as long as your usage is within the |
| 27 | limitations. |
| 28 | 2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. |
| 29 | Sockets also have `limitations and restrictions |
| 30 | <https://cloud.google.com/appengine/docs/python/sockets/\ |
| 31 | #limitations-and-restrictions>`_ and have a lower free quota than URLFetch. |
| 32 | To use sockets, be sure to specify the following in your ``app.yaml``:: |
| 33 | |
| 34 | env_variables: |
| 35 | GAE_USE_SOCKETS_HTTPLIB : 'true' |
| 36 | |
| 37 | 3. If you are using `App Engine Flexible |
| 38 | <https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard |
| 39 | :class:`PoolManager` without any configuration or special environment variables. |
| 40 | """ |
| 41 | |
| 42 | from __future__ import absolute_import |
| 43 | import logging |
| 44 | import os |
| 45 | import warnings |
| 46 | from ..packages.six.moves.urllib.parse import urljoin |
| 47 | |
| 48 | from ..exceptions import ( |
| 49 | HTTPError, |
| 50 | HTTPWarning, |
| 51 | MaxRetryError, |
| 52 | ProtocolError, |
| 53 | TimeoutError, |
| 54 | SSLError |
| 55 | ) |
| 56 | |
| 57 | from ..packages.six import BytesIO |
| 58 | from ..request import RequestMethods |
| 59 | from ..response import HTTPResponse |
| 60 | from ..util.timeout import Timeout |
| 61 | from ..util.retry import Retry |
| 62 | |
| 63 | try: |
| 64 | from google.appengine.api import urlfetch |
| 65 | except ImportError: |
| 66 | urlfetch = None |
| 67 | |
| 68 | |
| 69 | log = logging.getLogger(__name__) |
| 70 | |
| 71 | |
| 72 | class AppEnginePlatformWarning(HTTPWarning): |
| 73 | pass |
| 74 | |
| 75 | |
| 76 | class AppEnginePlatformError(HTTPError): |
| 77 | pass |
| 78 | |
| 79 | |
| 80 | class AppEngineManager(RequestMethods): |
| 81 | """ |
| 82 | Connection manager for Google App Engine sandbox applications. |
| 83 | |
| 84 | This manager uses the URLFetch service directly instead of using the |
| 85 | emulated httplib, and is subject to URLFetch limitations as described in |
| 86 | the App Engine documentation `here |
| 87 | <https://cloud.google.com/appengine/docs/python/urlfetch>`_. |
| 88 | |
| 89 | Notably it will raise an :class:`AppEnginePlatformError` if: |
| 90 | * URLFetch is not available. |
| 91 | * If you attempt to use this on App Engine Flexible, as full socket |
| 92 | support is available. |
| 93 | * If a request size is more than 10 megabytes. |
| 94 | * If a response size is more than 32 megabtyes. |
| 95 | * If you use an unsupported request method such as OPTIONS. |
| 96 | |
| 97 | Beyond those cases, it will raise normal urllib3 errors. |
| 98 | """ |
| 99 | |
| 100 | def __init__(self, headers=None, retries=None, validate_certificate=True, |
| 101 | urlfetch_retries=True): |
| 102 | if not urlfetch: |
| 103 | raise AppEnginePlatformError( |
| 104 | "URLFetch is not available in this environment.") |
| 105 | |
| 106 | if is_prod_appengine_mvms(): |
| 107 | raise AppEnginePlatformError( |
| 108 | "Use normal urllib3.PoolManager instead of AppEngineManager" |
| 109 | "on Managed VMs, as using URLFetch is not necessary in " |
| 110 | "this environment.") |
| 111 | |
| 112 | warnings.warn( |
| 113 | "urllib3 is using URLFetch on Google App Engine sandbox instead " |
| 114 | "of sockets. To use sockets directly instead of URLFetch see " |
| 115 | "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", |
| 116 | AppEnginePlatformWarning) |
| 117 | |
| 118 | RequestMethods.__init__(self, headers) |
| 119 | self.validate_certificate = validate_certificate |
| 120 | self.urlfetch_retries = urlfetch_retries |
| 121 | |
| 122 | self.retries = retries or Retry.DEFAULT |
| 123 | |
| 124 | def __enter__(self): |
| 125 | return self |
| 126 | |
| 127 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 128 | # Return False to re-raise any potential exceptions |
| 129 | return False |
| 130 | |
| 131 | def urlopen(self, method, url, body=None, headers=None, |
| 132 | retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT, |
| 133 | **response_kw): |
| 134 | |
| 135 | retries = self._get_retries(retries, redirect) |
| 136 | |
| 137 | try: |
| 138 | follow_redirects = ( |
| 139 | redirect and |
| 140 | retries.redirect != 0 and |
| 141 | retries.total) |
| 142 | response = urlfetch.fetch( |
| 143 | url, |
| 144 | payload=body, |
| 145 | method=method, |
| 146 | headers=headers or {}, |
| 147 | allow_truncated=False, |
| 148 | follow_redirects=self.urlfetch_retries and follow_redirects, |
| 149 | deadline=self._get_absolute_timeout(timeout), |
| 150 | validate_certificate=self.validate_certificate, |
| 151 | ) |
| 152 | except urlfetch.DeadlineExceededError as e: |
| 153 | raise TimeoutError(self, e) |
| 154 | |
| 155 | except urlfetch.InvalidURLError as e: |
| 156 | if 'too large' in str(e): |
| 157 | raise AppEnginePlatformError( |
| 158 | "URLFetch request too large, URLFetch only " |
| 159 | "supports requests up to 10mb in size.", e) |
| 160 | raise ProtocolError(e) |
| 161 | |
| 162 | except urlfetch.DownloadError as e: |
| 163 | if 'Too many redirects' in str(e): |
| 164 | raise MaxRetryError(self, url, reason=e) |
| 165 | raise ProtocolError(e) |
| 166 | |
| 167 | except urlfetch.ResponseTooLargeError as e: |
| 168 | raise AppEnginePlatformError( |
| 169 | "URLFetch response too large, URLFetch only supports" |
| 170 | "responses up to 32mb in size.", e) |
| 171 | |
| 172 | except urlfetch.SSLCertificateError as e: |
| 173 | raise SSLError(e) |
| 174 | |
| 175 | except urlfetch.InvalidMethodError as e: |
| 176 | raise AppEnginePlatformError( |
| 177 | "URLFetch does not support method: %s" % method, e) |
| 178 | |
| 179 | http_response = self._urlfetch_response_to_http_response( |
| 180 | response, retries=retries, **response_kw) |
| 181 | |
| 182 | # Handle redirect? |
| 183 | redirect_location = redirect and http_response.get_redirect_location() |
| 184 | if redirect_location: |
| 185 | # Check for redirect response |
| 186 | if (self.urlfetch_retries and retries.raise_on_redirect): |
| 187 | raise MaxRetryError(self, url, "too many redirects") |
| 188 | else: |
| 189 | if http_response.status == 303: |
| 190 | method = 'GET' |
| 191 | |
| 192 | try: |
| 193 | retries = retries.increment(method, url, response=http_response, _pool=self) |
| 194 | except MaxRetryError: |
| 195 | if retries.raise_on_redirect: |
| 196 | raise MaxRetryError(self, url, "too many redirects") |
| 197 | return http_response |
| 198 | |
| 199 | retries.sleep_for_retry(http_response) |
| 200 | log.debug("Redirecting %s -> %s", url, redirect_location) |
| 201 | redirect_url = urljoin(url, redirect_location) |
| 202 | return self.urlopen( |
| 203 | method, redirect_url, body, headers, |
| 204 | retries=retries, redirect=redirect, |
| 205 | timeout=timeout, **response_kw) |
| 206 | |
| 207 | # Check if we should retry the HTTP response. |
| 208 | has_retry_after = bool(http_response.getheader('Retry-After')) |
| 209 | if retries.is_retry(method, http_response.status, has_retry_after): |
| 210 | retries = retries.increment( |
| 211 | method, url, response=http_response, _pool=self) |
| 212 | log.debug("Retry: %s", url) |
| 213 | retries.sleep(http_response) |
| 214 | return self.urlopen( |
| 215 | method, url, |
| 216 | body=body, headers=headers, |
| 217 | retries=retries, redirect=redirect, |
| 218 | timeout=timeout, **response_kw) |
| 219 | |
| 220 | return http_response |
| 221 | |
| 222 | def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): |
| 223 | |
| 224 | if is_prod_appengine(): |
| 225 | # Production GAE handles deflate encoding automatically, but does |
| 226 | # not remove the encoding header. |
| 227 | content_encoding = urlfetch_resp.headers.get('content-encoding') |
| 228 | |
| 229 | if content_encoding == 'deflate': |
| 230 | del urlfetch_resp.headers['content-encoding'] |
| 231 | |
| 232 | transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') |
| 233 | # We have a full response's content, |
| 234 | # so let's make sure we don't report ourselves as chunked data. |
| 235 | if transfer_encoding == 'chunked': |
| 236 | encodings = transfer_encoding.split(",") |
| 237 | encodings.remove('chunked') |
| 238 | urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) |
| 239 | |
| 240 | return HTTPResponse( |
| 241 | # In order for decoding to work, we must present the content as |
| 242 | # a file-like object. |
| 243 | body=BytesIO(urlfetch_resp.content), |
| 244 | headers=urlfetch_resp.headers, |
| 245 | status=urlfetch_resp.status_code, |
| 246 | **response_kw |
| 247 | ) |
| 248 | |
| 249 | def _get_absolute_timeout(self, timeout): |
| 250 | if timeout is Timeout.DEFAULT_TIMEOUT: |
| 251 | return None # Defer to URLFetch's default. |
| 252 | if isinstance(timeout, Timeout): |
| 253 | if timeout._read is not None or timeout._connect is not None: |
| 254 | warnings.warn( |
| 255 | "URLFetch does not support granular timeout settings, " |
| 256 | "reverting to total or default URLFetch timeout.", |
| 257 | AppEnginePlatformWarning) |
| 258 | return timeout.total |
| 259 | return timeout |
| 260 | |
| 261 | def _get_retries(self, retries, redirect): |
| 262 | if not isinstance(retries, Retry): |
| 263 | retries = Retry.from_int( |
| 264 | retries, redirect=redirect, default=self.retries) |
| 265 | |
| 266 | if retries.connect or retries.read or retries.redirect: |
| 267 | warnings.warn( |
| 268 | "URLFetch only supports total retries and does not " |
| 269 | "recognize connect, read, or redirect retry parameters.", |
| 270 | AppEnginePlatformWarning) |
| 271 | |
| 272 | return retries |
| 273 | |
| 274 | |
| 275 | def is_appengine(): |
| 276 | return (is_local_appengine() or |
| 277 | is_prod_appengine() or |
| 278 | is_prod_appengine_mvms()) |
| 279 | |
| 280 | |
| 281 | def is_appengine_sandbox(): |
| 282 | return is_appengine() and not is_prod_appengine_mvms() |
| 283 | |
| 284 | |
| 285 | def is_local_appengine(): |
| 286 | return ('APPENGINE_RUNTIME' in os.environ and |
| 287 | 'Development/' in os.environ['SERVER_SOFTWARE']) |
| 288 | |
| 289 | |
| 290 | def is_prod_appengine(): |
| 291 | return ('APPENGINE_RUNTIME' in os.environ and |
| 292 | 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and |
| 293 | not is_prod_appengine_mvms()) |
| 294 | |
| 295 | |
| 296 | def is_prod_appengine_mvms(): |
| 297 | return os.environ.get('GAE_VM', False) == 'true' |