python连接elasticsearch_urllib3.exceptions.protocolerror: ('connection abo

问题一：urllib3.exceptions.ProtocolError: (‘Connection aborted.’, RemoteDisconnected(‘Remote end closed connection without response’))

协议写错了，是https

问题一：SSLError([SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain

import ssl
from elasticsearch import Elasticsearch

# 不验证ssl_context=context，6.1.3版本有用
# 6.1.3版本设置verify_certs=False没用
# 因为` verify_certs=False`在后续创建连接时还是会使用默认的ssl加载，需设置context才有用
context = ssl._create_unverified_context()
client = Elasticsearch(hosts="https://1.1.1.1:9200/", http_auth=('用户名', '密码'), ssl_context=context)

# 不验证  verify_certs=False ，8版本有用
# 8版本设置ssl_context没用
# 因为8版本直接使用verify_certs这个变量进行控制，即使传参时传了不需要验证的上下文，他自己还会根据verify_certs参数，
# 如果它是true就会更新对应的数据，False时，直接创建一个不需要验证的上下文，连接的时候把这个上下文传递过去，
# 8版本的在一开始transport传参时就进行了判断创建
client = Elasticsearch(hosts="https://1.1.1.1:9200/", http_auth=('用户名', '密码'), verify_certs=False)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

结论

1、elasticsearch Python 6.1.3 使用verify_certs=False时，不会创建ssl上下文，等到连接时发现没有上下文会自动创建一个默认的上下文，因此指定了上下文会起作用。

C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\connection\http_urllib3.py

class Urllib3HttpConnection(Connection):
    """
    Default connection class using the `urllib3` library and the http protocol.

    :arg host: hostname of the node (default: localhost)
    :arg port: port to use (integer, default: 9200)
    :arg url_prefix: optional url prefix for elasticsearch
    :arg timeout: default timeout in seconds (float, default: 10)
    :arg http_auth: optional http auth information as either ':' separated
        string or a tuple
    :arg use_ssl: use ssl for the connection if `True`
    :arg verify_certs: whether to verify SSL certificates
    :arg ca_certs: optional path to CA bundle.
        See https://urllib3.readthedocs.io/en/latest/security.html#using-certifi-with-urllib3
        for instructions how to get default set
    :arg client_cert: path to the file containing the private key and the
        certificate, or cert only if using client_key
    :arg client_key: path to the file containing the private key if using
        separate cert and key files (client_cert will contain only the cert)
    :arg ssl_version: version of the SSL protocol to use. Choices are:
        SSLv23 (default) SSLv2 SSLv3 TLSv1 (see ``PROTOCOL_*`` constants in the
        ``ssl`` module for exact options for your environment).
    :arg ssl_assert_hostname: use hostname verification if not `False`
    :arg ssl_assert_fingerprint: verify the supplied certificate fingerprint if not `None`
    :arg maxsize: the number of connections which will be kept open to this
        host. See https://urllib3.readthedocs.io/en/1.4/pools.html#api for more
        information.
    :arg headers: any custom http headers to be add to requests
    :arg http_compress: Use gzip compression
    """
    def __init__(self, host='localhost', port=9200, http_auth=None,
            use_ssl=False, verify_certs=VERIFY_CERTS_DEFAULT, ca_certs=None, client_cert=None,
            client_key=None, ssl_version=None, ssl_assert_hostname=None,
            ssl_assert_fingerprint=None, maxsize=10, headers=None, ssl_context=None, http_compress=False, **kwargs):

        '''省略'''
        kw = {}

        # if providing an SSL context, raise error if any other SSL related flag is used
        if ssl_context and ( (verify_certs is not VERIFY_CERTS_DEFAULT) or ca_certs
                             or client_cert or client_key or ssl_version):
            warnings.warn("When using `ssl_context`, all other SSL related kwargs are ignored")

        # if ssl_context provided use SSL by default
        if ssl_context and self.use_ssl:
            pool_class = urllib3.HTTPSConnectionPool
            kw.update({
                'assert_fingerprint': ssl_assert_fingerprint,
                'ssl_context': ssl_context,
            })

        elif self.use_ssl:
            pool_class = urllib3.HTTPSConnectionPool
            kw.update({
                'ssl_version': ssl_version,
                'assert_hostname': ssl_assert_hostname,
                'assert_fingerprint': ssl_assert_fingerprint,
            })

            # If `verify_certs` is sentinal value, default `verify_certs` to `True`
            if verify_certs is VERIFY_CERTS_DEFAULT:
                verify_certs = True

            ca_certs = CA_CERTS if ca_certs is None else ca_certs
            # verify_certs为false则不会更新任何ssl的信息，下边又因为没有cert_reqs和上下文变量，
            # 导致创建了默认的上下文变量即VerifyMode.CERT_REQUIRED
            if verify_certs:
                if not ca_certs:
                    raise ImproperlyConfigured("Root certificates are missing for certificate "
                        "validation. Either pass them in using the ca_certs parameter or "
                        "install certifi to use it automatically.")

                kw.update({
                    'cert_reqs': 'CERT_REQUIRED',
                    'ca_certs': ca_certs,
                    'cert_file': client_cert,
                    'key_file': client_key,
                })
            else:
                warnings.warn(
                    'Connecting to %s using SSL with verify_certs=False is insecure.' % host)

		# 该处会调用下边的HTTPSConnectionPool
        self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize, **kw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py

class HTTPSConnectionPool(HTTPConnectionPool):
    """
    Same as :class:`.HTTPConnectionPool`, but HTTPS.

    :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
    ``assert_hostname`` and ``host`` in this order to verify connections.
    If ``assert_hostname`` is False, no verification is done.

    The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
    ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
    is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
    the connection socket into an SSL socket.
    """

    scheme = "https"
    ConnectionCls = HTTPSConnection

    def __init__(
        self,
        host,
        port=None,
        strict=False,
        timeout=Timeout.DEFAULT_TIMEOUT,
        maxsize=1,
        block=False,
        headers=None,
        retries=None,
        _proxy=None,
        _proxy_headers=None,
        key_file=None,
        cert_file=None,
        cert_reqs=None,
        key_password=None,
        ca_certs=None,
        ssl_version=None,
        assert_hostname=None,
        assert_fingerprint=None,
        ca_cert_dir=None,
        **conn_kw
    ):

        HTTPConnectionPool.__init__(
            self,
            host,
            port,
            strict,
            timeout,
            maxsize,
            block,
            headers,
            retries,
            _proxy,
            _proxy_headers,
            **conn_kw
        )

        self.key_file = key_file
        self.cert_file = cert_file
        # 更新cert_reqs
        self.cert_reqs = cert_reqs
        self.key_password = key_password
        self.ca_certs = ca_certs
        self.ca_cert_dir = ca_cert_dir
        self.ssl_version = ssl_version
        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint

    def _prepare_conn(self, conn):
        """
        Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
        and establish the tunnel if proxy is used.
        """

        if isinstance(conn, VerifiedHTTPSConnection):
        	# 调用创建ssl上下文变量
            conn.set_cert(
                key_file=self.key_file,
                key_password=self.key_password,
                cert_file=self.cert_file,
                cert_reqs=self.cert_reqs,
                ca_certs=self.ca_certs,
                ca_cert_dir=self.ca_cert_dir,
                assert_hostname=self.assert_hostname,
                assert_fingerprint=self.assert_fingerprint,
            )
            conn.ssl_version = self.ssl_version
        return conn

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connection.py

    def set_cert(
        self,
        key_file=None,
        cert_file=None,
        cert_reqs=None,
        key_password=None,
        ca_certs=None,
        assert_hostname=None,
        assert_fingerprint=None,
        ca_cert_dir=None,
        ca_cert_data=None,
    ):
        """
        This method should only be called once, before the connection is used.
        """
        # 如果没有传cert_reqs 会根据上下文获取，因为没有创建上下文，所以会用默认的VerifyMode.CERT_REQUIRED
        if cert_reqs is None:
            if self.ssl_context is not None:
                cert_reqs = self.ssl_context.verify_mode
            else:
                cert_reqs = resolve_cert_reqs(None)

        self.key_file = key_file
        self.cert_file = cert_file
        # 赋值，下边调用connect时会使用
        self.cert_reqs = cert_reqs

    def connect(self):
        # Add certificate verification
        conn = self._new_conn()
        hostname = self.host
        tls_in_tls = False

        if self._is_using_tunnel():
            if self.tls_in_tls_required:
                conn = self._connect_tls_proxy(hostname, conn)
                tls_in_tls = True

            self.sock = conn

            # Calls self._set_hostport(), so self.host is
            # self._tunnel_host below.
            self._tunnel()
            # Mark this connection as not reusable
            self.auto_open = 0

            # Override the host with the one we're requesting data from.
            hostname = self._tunnel_host

        server_hostname = hostname
        if self.server_hostname is not None:
            server_hostname = self.server_hostname

        is_time_off = datetime.date.today() < RECENT_DATE
        if is_time_off:
            warnings.warn(
                (
                    "System time is way off (before {0}). This will probably "
                    "lead to SSL verification errors"
                ).format(RECENT_DATE),
                SystemTimeWarning,
            )

        # Wrap socket using verification with the root certs in
        # trusted_root_certs
        default_ssl_context = False
        # 如果没有上下文变量，则会根据创建默认的上下文变量，导致出现验证
        if self.ssl_context is None:
            default_ssl_context = True
            self.ssl_context = create_urllib3_context(
                ssl_version=resolve_ssl_version(self.ssl_version),
                cert_reqs=resolve_cert_reqs(self.cert_reqs),
            )

        context = self.ssl_context
        context.verify_mode = resolve_cert_reqs(self.cert_reqs)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

2、elasticsearch Python 8 直接使用verify_certs这个变量进行控制，即使传参时传了不需要验证的上下文，他自己还会根据verify_certs参数，如果它是true就会更新对应的数据，False时，直接创建一个不需要验证的上下文，连接的时候把这个上下文传递过去，8版本的在一开始transport传参时就进行了判断创建。

C:\Users\admin\Desktop\second\venv\Lib\site-packages\elastic_transport_node_http_urllib3.py

C:\Users\admin\Desktop\second\venv\Lib\site-packages\elastic_transport_node_http_urllib3.py

class Urllib3HttpNode(BaseNode):
    def __init__(self, config: NodeConfig):
		'''省略'''
        if config.scheme == "https":
            pool_class = HTTPSConnectionPool
            # 根据你的配置创建上下文，如果你传了上下文直接返回你自己的上下文，但是下边会根据verify_certs的值给你更新掉
            ssl_context = ssl_context_from_node_config(config) 
            kw["ssl_context"] = ssl_context  

            if config.ssl_assert_hostname and config.ssl_assert_fingerprint:
                raise ValueError(
                    "Can't specify both 'ssl_assert_hostname' and 'ssl_assert_fingerprint'"
                )
            elif config.ssl_assert_fingerprint:
                ssl_context.check_hostname = False
                ssl_context.verify_mode = ssl.CERT_NONE

                kw.update(
                    {
                        "assert_fingerprint": config.ssl_assert_fingerprint,
                        "assert_hostname": False,
                        "cert_reqs": "CERT_NONE",
                    }
                )

            else:
                kw["assert_hostname"] = config.ssl_assert_hostname

                # Convert all sentinel values to their actual default
                # values if not using an SSLContext.
                ca_certs = (
                    DEFAULT_CA_CERTS if config.ca_certs is None else config.ca_certs
                )
                # 就在这里根据verify_certs把你的上下文更新掉，cert_reqs为CERT_NONE即不需要验证
                if config.verify_certs:
                    if not ca_certs:
                        raise ValueError(
                            "Root certificates are missing for certificate "
                            "validation. Either pass them in using the ca_certs parameter or "
                            "install certifi to use it automatically."
                        )

                    kw.update(
                        {
                            "cert_reqs": "CERT_REQUIRED",
                            "ca_certs": ca_certs,
                            "cert_file": config.client_cert,
                            "key_file": config.client_key,
                        }
                    )
                else:
                    kw["cert_reqs"] = "CERT_NONE"
         '''省略'''
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

下边的代码是6.1.3版本的

大致看了一眼代码：

1、

client = Elasticsearch(hosts="https://ip:9200/", http_auth=('用户名', '密码'), ssl=True, verify_certs=False)
1

1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\client_init_.py

1.1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\transport.py


1.1.1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\connection\http_urllib3.py

1.1.1.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\util\request.py

设置的用户名密码就是在这里被翻译的

1.1.1.2、

        和上边1.1.1.1函数在同一个函数中
        if ssl_context and self.use_ssl:
            pool_class = urllib3.HTTPSConnectionPool
            kw.update({
                'assert_fingerprint': ssl_assert_fingerprint,
                'ssl_context': ssl_context,
            })

        elif self.use_ssl:
            pool_class = urllib3.HTTPSConnectionPool
            kw.update({
                'ssl_version': ssl_version,
                'assert_hostname': ssl_assert_hostname,
                'assert_fingerprint': ssl_assert_fingerprint,
            })

            # If `verify_certs` is sentinal value, default `verify_certs` to `True`
            if verify_certs is VERIFY_CERTS_DEFAULT:  # 不验证的话直接跳到else，不处理
                verify_certs = True

            ca_certs = CA_CERTS if ca_certs is None else ca_certs
            if verify_certs:
                if not ca_certs:
                    raise ImproperlyConfigured("Root certificates are missing for certificate "
                        "validation. Either pass them in using the ca_certs parameter or "
                        "install certifi to use it automatically.")

                kw.update({
                    'cert_reqs': 'CERT_REQUIRED',
                    'ca_certs': ca_certs,
                    'cert_file': client_cert,
                    'key_file': client_key,
                })
            else:
                warnings.warn(
                    'Connecting to %s using SSL with verify_certs=False is insecure.' % host)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

1.2、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\transport.py


1.2.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\connection\http_urllib3.py


1.2.1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py


1.2.1.1.1

C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py

1.2.1.1.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connection.py

1.2.1.2、

1.2.1.2.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py

1.2.1.2.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connection.py

1.2.1.2.1.1.1