Admin Control Panel v2.0

✎ Editing: ConnMan.py ← Back

# -*- coding: utf-8 -*-

## --------------------------------------------------------------------
## Amazon S3 manager
##
## Authors   : Michal Ludvig <michal@logix.cz> (https://www.logix.cz/michal)
##             Florent Viard <florent@sodria.com> (https://www.sodria.com)
## Copyright : TGRMN Software, Sodria SAS and contributors
## License   : GPL Version 2
## Website   : https://s3tools.org
## --------------------------------------------------------------------

from __future__ import absolute_import

import sys
if sys.version_info >= (3, 0):
    from .Custom_httplib3x import httplib
else:
    from .Custom_httplib27 import httplib
import ssl

from logging import debug
from threading import Semaphore
from time import time
try:
    # python 3 support
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

from .Config import Config
from .Exceptions import ParameterError, S3SSLCertificateError
from .Utils import getBucketFromHostname

__all__ = ["ConnMan"]

class http_connection(object):
    context = None
    context_set = False

@staticmethod
    def _ssl_verified_context(cafile):
        cfg = Config()
        context = None
        try:
            context = ssl.create_default_context(cafile=cafile)
        except AttributeError: # no ssl.create_default_context
            pass
        if context and not cfg.check_ssl_hostname:
            context.check_hostname = False
            debug(u'Disabling SSL certificate hostname checking')

return context

@staticmethod
    def _ssl_unverified_context(cafile):
        debug(u'Disabling SSL certificate checking')
        context = None
        try:
            context = ssl._create_unverified_context(cafile=cafile,
                                                     cert_reqs=ssl.CERT_NONE)
        except AttributeError: # no ssl._create_unverified_context
            pass
        return context

@staticmethod
    def _ssl_client_auth_context(certfile, keyfile, check_server_cert, cafile):
        context = None
        try:
            cert_reqs = ssl.CERT_REQUIRED if check_server_cert else ssl.CERT_NONE
            context = ssl._create_unverified_context(cafile=cafile,
                                                     keyfile=keyfile,
                                                     certfile=certfile,
                                                     cert_reqs=cert_reqs)
        except AttributeError: # no ssl._create_unverified_context
            pass
        return context

@staticmethod
    def _ssl_context():
        if http_connection.context_set:
            return http_connection.context

cfg = Config()
        cafile = cfg.ca_certs_file
        if cafile == "":
            cafile = None
        certfile = cfg.ssl_client_cert_file or None
        keyfile = cfg.ssl_client_key_file or None # the key may be embedded into cert file

debug(u"Using ca_certs_file %s", cafile)
        debug(u"Using ssl_client_cert_file %s", certfile)
        debug(u"Using ssl_client_key_file %s", keyfile)

if certfile is not None:
            context = http_connection._ssl_client_auth_context(certfile, keyfile, cfg.check_ssl_certificate, cafile)
        elif cfg.check_ssl_certificate:
            context = http_connection._ssl_verified_context(cafile)
        else:
            context = http_connection._ssl_unverified_context(cafile)

http_connection.context = context
        http_connection.context_set = True
        return context

def forgive_wildcard_cert(self, cert, hostname):
        """
        Wildcard matching for *.s3.amazonaws.com and similar per region.

Per http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html:
        "We recommend that all bucket names comply with DNS naming conventions."

Per http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html:
        "When using virtual hosted-style buckets with SSL, the SSL
        wild card certificate only matches buckets that do not contain
        periods. To work around this, use HTTP or write your own
        certificate verification logic."

Therefore, we need a custom validation routine that allows
        mybucket.example.com.s3.amazonaws.com to be considered a valid
        hostname for the *.s3.amazonaws.com wildcard cert, and for the
        region-specific *.s3-[region].amazonaws.com wildcard cert.

We also forgive non-S3 wildcard certificates should the
        hostname match, to allow compatibility with other S3
        API-compatible storage providers.
        """
        debug(u'checking SSL subjectAltName as forgiving wildcard cert')
        san = cert.get('subjectAltName', ())
        hostname = hostname.lower()
        cleaned_host_bucket_config = urlparse('https://' + Config.host_bucket).hostname
        for key, value in san:
            if key == 'DNS':
                value = value.lower()
                if value.startswith('*.s3') and \
                   (value.endswith('.amazonaws.com') and hostname.endswith('.amazonaws.com')) or \
                   (value.endswith('.amazonaws.com.cn') and hostname.endswith('.amazonaws.com.cn')):
                    return True
                elif value == cleaned_host_bucket_config % \
                               {'bucket': '*', 'location': Config.bucket_location.lower()} and \
                     hostname.endswith(cleaned_host_bucket_config % \
                                       {'bucket': '', 'location': Config.bucket_location.lower()}):
                    return True
        return False

def match_hostname(self):
        cert = self.c.sock.getpeercert()
        try:
            ssl.match_hostname(cert, self.hostname)
        except AttributeError:
            # old ssl module doesn't have this function
            return
        except ValueError:
            # empty SSL cert means underlying SSL library didn't validate it, we don't either.
            return
        except S3CertificateError as e:
            if not self.forgive_wildcard_cert(cert, self.hostname):
                raise e

@staticmethod
    def _https_connection(hostname, port=None):
        try:
            context = http_connection._ssl_context()
            # Wildcard certificates do not work with DNS-style named buckets.
            bucket_name, success = getBucketFromHostname(hostname)
            if success and '.' in bucket_name:
                # this merely delays running the hostname check until
                # after the connection is made and we get control
                # back.  We then run the same check, relaxed for S3's
                # wildcard certificates.
                debug(u'Bucket name contains "." character, disabling initial SSL hostname check')
                check_hostname = False
                if context:
                    context.check_hostname = False
            else:
                if context:
                    check_hostname = context.check_hostname
                else:
                    # Earliest version of python that don't have context,
                    # don't check hostnames anyway
                    check_hostname = True
            # Note, we are probably needed to try to set check_hostname because of that bug:
            # http://bugs.python.org/issue22959
            conn = httplib.HTTPSConnection(hostname, port, context=context, check_hostname=check_hostname)
            debug(u'httplib.HTTPSConnection() has both context and check_hostname')
        except TypeError:
            try:
                # in case check_hostname parameter is not present try again
                conn = httplib.HTTPSConnection(hostname, port, context=context)
                debug(u'httplib.HTTPSConnection() has only context')
            except TypeError:
                # in case even context parameter is not present try one last time
                conn = httplib.HTTPSConnection(hostname, port)
                debug(u'httplib.HTTPSConnection() has neither context nor check_hostname')
        return conn

def __init__(self, id, hostname, ssl, cfg):
        self.ssl = ssl
        self.id = id
        self.counter = 0
        # Whatever is the input, ensure to have clean hostname and port
        parsed_hostname = urlparse('https://' + hostname)
        self.hostname = parsed_hostname.hostname
        self.port = parsed_hostname.port
        if parsed_hostname.path and parsed_hostname.path != '/':
            self.path = parsed_hostname.path.rstrip('/')
            debug(u'endpoint path set to %s', self.path)
        else:
            self.path = None

"""
        History note:
        In a perfect world, or in the future:
        - All http proxies would support CONNECT/tunnel, and so there would be no need
        for using "absolute URIs" in format_uri.
        - All s3-like servers would work well whether using relative or ABSOLUTE URIs.
        But currently, what is currently common:
        - Proxies without support for CONNECT for http, and so "absolute URIs" have to
        be used.
        - Proxies with support for CONNECT for httpS but s3-like servers having issues
        with "absolute URIs", so relative one still have to be used as the requests will
        pass as-is, through the proxy because of the CONNECT mode.
        """

if not cfg.proxy_host:
            if ssl:
                self.c = http_connection._https_connection(self.hostname, self.port)
                debug(u'non-proxied HTTPSConnection(%s, %s)', self.hostname, self.port)
            else:
                self.c = httplib.HTTPConnection(self.hostname, self.port)
                debug(u'non-proxied HTTPConnection(%s, %s)', self.hostname, self.port)
        else:
            if ssl:
                self.c = http_connection._https_connection(cfg.proxy_host, cfg.proxy_port)
                debug(u'proxied HTTPSConnection(%s, %s)', cfg.proxy_host, cfg.proxy_port)
                port = self.port and self.port or 443
                self.c.set_tunnel(self.hostname, port)
                debug(u'tunnel to %s, %s', self.hostname, port)
            else:
                self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port)
                debug(u'proxied HTTPConnection(%s, %s)', cfg.proxy_host, cfg.proxy_port)
                # No tunnel here for the moment

self.last_used_time = time()

class ConnMan(object):
    _CS_REQ_SENT = httplib._CS_REQ_SENT
    CONTINUE = httplib.CONTINUE
    conn_pool_sem = Semaphore()
    conn_pool = {}
    conn_max_counter = 800  ## AWS closes connection after some ~90 requests

@staticmethod
    def get(hostname, ssl=None):
        cfg = Config()
        if ssl is None:
            ssl = cfg.use_https
        conn = None
        if cfg.proxy_host != "":
            if ssl and sys.hexversion < 0x02070000:
                raise ParameterError("use_https=True can't be used with proxy on Python <2.7")
            conn_id = "proxy://%s:%s" % (cfg.proxy_host, cfg.proxy_port)
        else:
            conn_id = "http%s://%s" % (ssl and "s" or "", hostname)
        ConnMan.conn_pool_sem.acquire()
        if conn_id not in ConnMan.conn_pool:
            ConnMan.conn_pool[conn_id] = []
        while ConnMan.conn_pool[conn_id]:
            conn = ConnMan.conn_pool[conn_id].pop()
            cur_time = time()
            if cur_time < conn.last_used_time + cfg.connection_max_age \
               and cur_time >= conn.last_used_time:
                debug("ConnMan.get(): re-using connection: %s#%d"
                      % (conn.id, conn.counter))
                break
            # Conn is too old or wall clock went back in the past
            debug("ConnMan.get(): closing expired connection")
            ConnMan.close(conn)
            conn = None

ConnMan.conn_pool_sem.release()
        if not conn:
            debug("ConnMan.get(): creating new connection: %s" % conn_id)
            conn = http_connection(conn_id, hostname, ssl, cfg)
            conn.c.connect()
            if conn.ssl and cfg.check_ssl_certificate and cfg.check_ssl_hostname:
                conn.match_hostname()
        conn.counter += 1
        return conn

@staticmethod
    def put(conn):
        if conn.id.startswith("proxy://"):
            ConnMan.close(conn)
            debug("ConnMan.put(): closing proxy connection (keep-alive not yet"
                  " supported)")
            return

if conn.counter >= ConnMan.conn_max_counter:
            ConnMan.close(conn)
            debug("ConnMan.put(): closing over-used connection")
            return

cfg = Config()
        if not cfg.connection_pooling:
            ConnMan.close(conn)
            debug("ConnMan.put(): closing connection (connection pooling disabled)")
            return

# Update timestamp of conn to record when was its last use
        conn.last_used_time = time()

ConnMan.conn_pool_sem.acquire()
        ConnMan.conn_pool[conn.id].append(conn)
        ConnMan.conn_pool_sem.release()
        debug("ConnMan.put(): connection put back to pool (%s#%d)"
              % (conn.id, conn.counter))

@staticmethod
    def close(conn):
        if conn:
            conn.c.close()

Cancel