One Hat Cyber Team

Edit File: _browser.py

# This file is part of lazr.restfulclient.
#
# lazr.restfulclient is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# lazr.restfulclient is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with lazr.restfulclient.  If not, see
# <http://www.gnu.org/licenses/>.

"""Browser object to make requests of lazr.restful web services.

The `Browser` class does some massage of HTTP requests and responses,
and handles custom caches. It is not part of the public
lazr.restfulclient API. (But maybe it should be?)
"""

__metaclass__ = type
__all__ = [
    'Browser',
    'RestfulHttp',
    'ssl_certificate_validation_disabled',
    ]

import atexit
import errno
from hashlib import md5
import os
import re
import shutil
import sys
import tempfile
# Import sleep directly into the module so we can monkey-patch it
# during a test.
from time import sleep
from httplib2 import (
    Http,
    urlnorm,
    )
try:
    from httplib2 import proxy_info_from_environment
except ImportError:
    from httplib2 import ProxyInfo
    proxy_info_from_environment = ProxyInfo.from_environment
try:
    # Python 3 and sufficiently modern Python 2 versions.
    from json import dumps
except ImportError:
    from simplejson import dumps

try:
    # Python 3.
    from io import BytesIO
except ImportError:
    from cStringIO import StringIO as BytesIO

try:
    # Python 3.
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode

from wadllib.application import Application
from lazr.uri import URI
from lazr.restfulclient.errors import error_for, HTTPError
from lazr.restfulclient._json import DatetimeJSONEncoder

if bytes is str:
    # Python 2
    unicode_type = unicode
    str_types = basestring
else:
    unicode_type = str
    str_types = str

# A drop-in replacement for httplib2's safename.  Substantially borrowed
# from httplib2, but its cache name format changed in 0.12.0 and we want to
# stick with the previous version.

re_url_scheme = re.compile(br'^\w+://')
re_url_scheme_s = re.compile(r'^\w+://')
re_slash = re.compile(br'[?/:|]+')

def safename(filename):
    """Return a filename suitable for the cache.

Strips dangerous and common characters to create a filename we
    can use to store the cache in.
    """
    try:
        if isinstance(filename, bytes):
            filename_match = filename.decode('utf-8')
        else:
            filename_match = filename

if re_url_scheme_s.match(filename_match):
            if isinstance(filename, bytes):
                filename = filename.decode('utf-8')
                filename = filename.encode('idna')
            else:
                filename = filename.encode('idna')
    except UnicodeError:
        pass
    if isinstance(filename, unicode_type):
        filename = filename.encode('utf-8')
    filemd5 = md5(filename).hexdigest()
    filename = re_url_scheme.sub(b"", filename)
    filename = re_slash.sub(b",", filename)

# This is the part that we changed. In stock httplib2, the
    # filename is trimmed if it's longer than 200 characters, and then
    # a comma and a 32-character md5 sum are appended. This causes
    # problems on eCryptfs filesystems, where the maximum safe
    # filename length is closer to 143 characters.
    #
    # We take a (user-hackable) maximum filename length from
    # RestfulHttp and subtract 33 characters to make room for the comma
    # and the md5 sum.
    #
    # See:
    #  http://code.google.com/p/httplib2/issues/detail?id=92
    #  https://bugs.launchpad.net/bugs/344878
    #  https://bugs.launchpad.net/bugs/545197
    maximum_filename_length = RestfulHttp.maximum_cache_filename_length
    maximum_length_before_md5_sum = maximum_filename_length - 32 - 1
    if len(filename) > maximum_length_before_md5_sum:
        filename = filename[:maximum_length_before_md5_sum]
    return ",".join((filename.decode('utf-8'), filemd5))

def ssl_certificate_validation_disabled():
    """Whether the user has disabled SSL certificate connection.

Some testing servers have broken certificates.  Rather than raising an
    error, we allow an environment variable,
    ``LP_DISABLE_SSL_CERTIFICATE_VALIDATION`` to disable the check.
    """
    return bool(
        os.environ.get('LP_DISABLE_SSL_CERTIFICATE_VALIDATION', False))

if os.path.exists('/etc/ssl/certs/ca-certificates.crt'):
    SYSTEM_CA_CERTS = '/etc/ssl/certs/ca-certificates.crt'
else:
    from httplib2 import CA_CERTS as SYSTEM_CA_CERTS

class RestfulHttp(Http):
    """An Http subclass with some custom behavior.

This Http client uses the TE header instead of the Accept-Encoding
    header to ask for compressed representations. It also knows how to
    react when its cache is a MultipleRepresentationCache.
    """

maximum_cache_filename_length = 143

def __init__(self, authorizer=None, cache=None, timeout=None,
                 proxy_info=proxy_info_from_environment):
        cert_disabled = ssl_certificate_validation_disabled()
        super(RestfulHttp, self).__init__(
            cache, timeout, proxy_info,
            disable_ssl_certificate_validation=cert_disabled,
            ca_certs=SYSTEM_CA_CERTS)
        self.authorizer = authorizer
        if self.authorizer is not None:
            self.authorizer.authorizeSession(self)

def _request(self, conn, host, absolute_uri, request_uri, method, body,
                 headers, redirections, cachekey):
        """Use the authorizer to authorize an outgoing request."""
        if 'authorization' in headers:
            # There's an authorization header left over from a
            # previous request that resulted in a redirect. Resources
            # protected by OAuth or HTTP Digest must send a distinct
            # Authorization header with each request, to prevent
            # playback attacks. Remove the Authorization header and
            # start again.
            del headers['authorization']
        if self.authorizer is not None:
            self.authorizer.authorizeRequest(
                absolute_uri, method, body, headers)
        return super(RestfulHttp, self)._request(
            conn, host, absolute_uri, request_uri, method, body, headers,
            redirections, cachekey)

def _getCachedHeader(self, uri, header):
        """Retrieve a cached value for an HTTP header."""
        if isinstance(self.cache, MultipleRepresentationCache):
            return self.cache._getCachedHeader(uri, header)
        return None

class AtomicFileCache(object):
    """A FileCache that can be shared by multiple processes.

Based on a patch found at
    <http://code.google.com/p/httplib2/issues/detail?id=125>.
    """

TEMPFILE_PREFIX = ".temp"

def __init__(self, cache, safe=safename):
        """Construct an ``AtomicFileCache``.

:param cache: The directory to use as a cache.
        :param safe: A function that takes a key and returns a name that's
            safe to use as a filename.  The key must never return a string
            that begins with ``TEMPFILE_PREFIX``.  By default uses
            ``safename``.
        """
        self._cache_dir = os.path.normpath(cache)
        self._get_safe_name = safe
        try:
             os.makedirs(self._cache_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

def _get_key_path(self, key):
        """Return the path on disk where ``key`` is stored."""
        safe_key = self._get_safe_name(key)
        if safe_key.startswith(self.TEMPFILE_PREFIX):
            # If the cache key starts with the tempfile prefix, then it's
            # possible that it will clash with a temporary file that we
            # create.
            raise ValueError(
                "Cache key cannot start with '%s'" % self.TEMPFILE_PREFIX)
        return os.path.join(self._cache_dir, safe_key)

def get(self, key):
        """Get the value of ``key`` if set.

This behaves slightly differently to ``FileCache`` in that if
        ``set()`` fails to store a key, this ``get()`` will behave as if that
        key were never set whereas ``FileCache`` returns the empty string.

:param key: The key to retrieve.  Must be either bytes or unicode
            text.
        :return: The value of ``key`` if set, None otherwise.
        """
        cache_full_path = self._get_key_path(key)
        try:
            f = open(cache_full_path, 'rb')
            try:
                return f.read()
            finally:
                f.close()
        except (IOError, OSError) as e:
            if e.errno != errno.ENOENT:
                raise

def set(self, key, value):
        """Set ``key`` to ``value``.

:param key: The key to set.  Must be either bytes or unicode text.
        :param value: The value to set ``key`` to.  Must be bytes.
        """
        # Open a temporary file
        handle, path_name = tempfile.mkstemp(
            prefix=self.TEMPFILE_PREFIX, dir=self._cache_dir)
        f = os.fdopen(handle, 'wb')
        f.write(value)
        f.close()
        cache_full_path = self._get_key_path(key)
        # And rename atomically (on POSIX at least)
        if sys.platform == 'win32' and os.path.exists(cache_full_path):
            os.unlink(cache_full_path)
        os.rename(path_name, cache_full_path)

def delete(self, key):
        """Delete ``key`` from the cache.

If ``key`` has not already been set then has no effect.

:param key: The key to delete.  Must be either bytes or unicode text.
        """
        cache_full_path = self._get_key_path(key)
        try:
            os.remove(cache_full_path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise

class MultipleRepresentationCache(AtomicFileCache):
    """A cache that can hold different representations of the same resource.

If a resource has two representations with two media types,
    FileCache will only store the most recently fetched
    representation. This cache can keep track of multiple
    representations of the same resource.

This class works on the assumption that outside calling code sets
    an instance's request_media_type attribute to the value of the
    'Accept' header before initiating the request.

This class is very much not thread-safe, but FileCache isn't
    thread-safe anyway.
    """
    def __init__(self, cache):
        """Tell FileCache to call append_media_type when generating keys."""
        super(MultipleRepresentationCache, self).__init__(
            cache, self.append_media_type)
        self.request_media_type = None

def append_media_type(self, key):
        """Append the request media type to the cache key.

This ensures that representations of the same resource will be
        cached separately, so long as they're served as different
        media types.
        """
        if self.request_media_type is not None:
            key = key + '-' + self.request_media_type
        return safename(key)

def _getCachedHeader(self, uri, header):
        """Retrieve a cached value for an HTTP header."""
        (scheme, authority, request_uri, cachekey) = urlnorm(uri)
        cached_value = self.get(cachekey)
        header_start = header + ':'
        if not isinstance(header_start, bytes):
            header_start = header_start.encode('utf-8')
        if cached_value is not None:
            for line in BytesIO(cached_value):
                if line.startswith(header_start):
                    return line[len(header_start):].strip()
        return None

class Browser:
    """A class for making calls to lazr.restful web services."""

NOT_MODIFIED = object()
    MAX_RETRIES = 6

def __init__(self, service_root, credentials, cache=None, timeout=None,
                 proxy_info=None, user_agent=None, max_retries=MAX_RETRIES):
        """Initialize, possibly creating a cache.

If no cache is provided, a temporary directory will be used as
        a cache. The temporary directory will be automatically removed
        when the Python process exits.
        """
        if cache is None:
            cache = tempfile.mkdtemp()
            atexit.register(shutil.rmtree, cache)
        if isinstance(cache, str_types):
            cache = MultipleRepresentationCache(cache)
        self._connection = service_root.httpFactory(
            credentials, cache, timeout, proxy_info)
        self.user_agent = user_agent
        self.max_retries = max_retries

def _request_and_retry(self, url, method, body, headers):
        for retry_count in range(0, self.max_retries+1):
            response, content = self._connection.request(
                url, method=method, body=body, headers=headers)
            if (response.status in [502, 503]
                and retry_count < self.max_retries):
                # The server returned a 502 or 503. Sleep for 0, 1, 2,
                # 4, 8, 16, ... seconds and try again.
                sleep_for = int(2**(retry_count-1))
                sleep(sleep_for)
            else:
                break
        # Either the request succeeded or we gave up.
        return response, content

def _request(self, url, data=None, method='GET',
                 media_type='application/json', extra_headers=None):
        """Create an authenticated request object."""
        # If the user is trying to get data that has been redacted,
        # give a helpful message.
        if url == "tag:launchpad.net:2008:redacted":
            raise ValueError("You tried to access a resource that you "
                             "don't have the server-side permission to see.")

# Add extra headers for the request.
        headers = {'Accept': media_type}
        if self.user_agent is not None:
            headers['User-Agent'] = self.user_agent
        if isinstance(self._connection.cache, MultipleRepresentationCache):
            self._connection.cache.request_media_type = media_type
        if extra_headers is not None:
            headers.update(extra_headers)
        response, content = self._request_and_retry(
            str(url), method=method, body=data, headers=headers)
        if response.status == 304:
            # The resource didn't change.
            if content == b'':
                if ('If-None-Match' in headers
                    or 'If-Modified-Since' in headers):
                    # The caller made a conditional request, and the
                    # condition failed. Rather than send an empty
                    # representation, which might be misinterpreted,
                    # send a special object that will let the calling code know
                    # that the resource was not modified.
                    return response, self.NOT_MODIFIED
                else:
                    # The caller didn't make a conditional request,
                    # but the response code is 304 and there's no
                    # content. The only way to handle this is to raise
                    # an error.
                    #
                    # We don't use error_for() here because 304 is not
                    # normally considered an error condition.
                    raise HTTPError(response, content)
            else:
                # XXX leonardr 2010/04/12 bug=httplib2#97
                #
                # Why is this check here? Why would there ever be any
                # content when the response code is 304? It's because of
                # an httplib2 bug that sometimes sets a 304 response
                # code when caching retrieved documents. When the
                # cached document is retrieved, we get a 304 response
                # code and a full representation.
                #
                # Since the cache lookup succeeded, the 'real'
                # response code is 200. This code undoes the bad
                # behavior in httplib2.
                response.status = 200
            return response, content
        # Turn non-2xx responses into appropriate HTTPError subclasses.
        error = error_for(response, content)
        if error is not None:
            raise error
        return response, content

def get(self, resource_or_uri, headers=None, return_response=False):
        """GET a representation of the given resource or URI."""
        if isinstance(resource_or_uri, (str_types, URI)):
            url = resource_or_uri
        else:
            method = resource_or_uri.get_method('get')
            url = method.build_request_url()
        response, content = self._request(url, extra_headers=headers)
        if return_response:
            return (response, content)
        return content

def get_wadl_application(self, url):
        """GET a WADL representation of the resource at the requested url."""
        wadl_type = 'application/vnd.sun.wadl+xml'
        response, content = self._request(url, media_type=wadl_type)
        url = str(url)
        if not isinstance(content, bytes):
            content = content.encode('utf-8')
        return Application(url, content)

def post(self, url, method_name, **kws):
        """POST a request to the web service."""
        kws['ws.op'] = method_name
        data = urlencode(kws)
        return self._request(url, data, 'POST')

def put(self, url, representation, media_type, headers=None):
        """PUT the given representation to the URL."""
        extra_headers = {'Content-Type': media_type}
        if headers is not None:
            extra_headers.update(headers)
        return self._request(
            url, representation, 'PUT', extra_headers=extra_headers)

def delete(self, url):
        """DELETE the resource at the given URL."""
        self._request(url, method='DELETE')
        return None

def patch(self, url, representation, headers=None):
        """PATCH the object at url with the updated representation."""
        extra_headers = {'Content-Type': 'application/json'}
        if headers is not None:
            extra_headers.update(headers)
        # httplib2 doesn't know about the PATCH method, so we need to
        # do some work ourselves. Pull any cached value of "ETag" out
        # and use it as the value for "If-Match".
        cached_etag = self._connection._getCachedHeader(str(url), 'etag')
        if cached_etag is not None and not self._connection.ignore_etag:
            # http://www.w3.org/1999/04/Editing/
            headers['If-Match'] = cached_etag

return self._request(
            url, dumps(representation, cls=DatetimeJSONEncoder),
            'PATCH', extra_headers=extra_headers)