mirror of
https://github.com/moraroy/NonSteamLaunchers-On-Steam-Deck.git
synced 2024-12-20 23:01:53 +01:00
chore: remove modules
Use directly in main python script
This commit is contained in:
parent
34f8921273
commit
f6c291dd3d
@ -1,180 +0,0 @@
|
||||
# __
|
||||
# /__) _ _ _ _ _/ _
|
||||
# / ( (- (/ (/ (- _) / _)
|
||||
# /
|
||||
|
||||
"""
|
||||
Requests HTTP Library
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Requests is an HTTP library, written in Python, for human beings.
|
||||
Basic GET usage:
|
||||
|
||||
>>> import requests
|
||||
>>> r = requests.get('https://www.python.org')
|
||||
>>> r.status_code
|
||||
200
|
||||
>>> b'Python is a programming language' in r.content
|
||||
True
|
||||
|
||||
... or POST:
|
||||
|
||||
>>> payload = dict(key1='value1', key2='value2')
|
||||
>>> r = requests.post('https://httpbin.org/post', data=payload)
|
||||
>>> print(r.text)
|
||||
{
|
||||
...
|
||||
"form": {
|
||||
"key1": "value1",
|
||||
"key2": "value2"
|
||||
},
|
||||
...
|
||||
}
|
||||
|
||||
The other HTTP methods are supported - see `requests.api`. Full documentation
|
||||
is at <https://requests.readthedocs.io>.
|
||||
|
||||
:copyright: (c) 2017 by Kenneth Reitz.
|
||||
:license: Apache 2.0, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
import urllib3
|
||||
|
||||
from .exceptions import RequestsDependencyWarning
|
||||
|
||||
try:
|
||||
from charset_normalizer import __version__ as charset_normalizer_version
|
||||
except ImportError:
|
||||
charset_normalizer_version = None
|
||||
|
||||
try:
|
||||
from chardet import __version__ as chardet_version
|
||||
except ImportError:
|
||||
chardet_version = None
|
||||
|
||||
|
||||
def check_compatibility(urllib3_version, chardet_version, charset_normalizer_version):
|
||||
urllib3_version = urllib3_version.split(".")
|
||||
assert urllib3_version != ["dev"] # Verify urllib3 isn't installed from git.
|
||||
|
||||
# Sometimes, urllib3 only reports its version as 16.1.
|
||||
if len(urllib3_version) == 2:
|
||||
urllib3_version.append("0")
|
||||
|
||||
# Check urllib3 for compatibility.
|
||||
major, minor, patch = urllib3_version # noqa: F811
|
||||
major, minor, patch = int(major), int(minor), int(patch)
|
||||
# urllib3 >= 1.21.1
|
||||
assert major >= 1
|
||||
if major == 1:
|
||||
assert minor >= 21
|
||||
|
||||
# Check charset_normalizer for compatibility.
|
||||
if chardet_version:
|
||||
major, minor, patch = chardet_version.split(".")[:3]
|
||||
major, minor, patch = int(major), int(minor), int(patch)
|
||||
# chardet_version >= 3.0.2, < 6.0.0
|
||||
assert (3, 0, 2) <= (major, minor, patch) < (6, 0, 0)
|
||||
elif charset_normalizer_version:
|
||||
major, minor, patch = charset_normalizer_version.split(".")[:3]
|
||||
major, minor, patch = int(major), int(minor), int(patch)
|
||||
# charset_normalizer >= 2.0.0 < 4.0.0
|
||||
assert (2, 0, 0) <= (major, minor, patch) < (4, 0, 0)
|
||||
else:
|
||||
raise Exception("You need either charset_normalizer or chardet installed")
|
||||
|
||||
|
||||
def _check_cryptography(cryptography_version):
|
||||
# cryptography < 1.3.4
|
||||
try:
|
||||
cryptography_version = list(map(int, cryptography_version.split(".")))
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
if cryptography_version < [1, 3, 4]:
|
||||
warning = "Old version of cryptography ({}) may cause slowdown.".format(
|
||||
cryptography_version
|
||||
)
|
||||
warnings.warn(warning, RequestsDependencyWarning)
|
||||
|
||||
|
||||
# Check imported dependencies for compatibility.
|
||||
try:
|
||||
check_compatibility(
|
||||
urllib3.__version__, chardet_version, charset_normalizer_version
|
||||
)
|
||||
except (AssertionError, ValueError):
|
||||
warnings.warn(
|
||||
"urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported "
|
||||
"version!".format(
|
||||
urllib3.__version__, chardet_version, charset_normalizer_version
|
||||
),
|
||||
RequestsDependencyWarning,
|
||||
)
|
||||
|
||||
# Attempt to enable urllib3's fallback for SNI support
|
||||
# if the standard library doesn't support SNI or the
|
||||
# 'ssl' library isn't available.
|
||||
try:
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
ssl = None
|
||||
|
||||
if not getattr(ssl, "HAS_SNI", False):
|
||||
from urllib3.contrib import pyopenssl
|
||||
|
||||
pyopenssl.inject_into_urllib3()
|
||||
|
||||
# Check cryptography version
|
||||
from cryptography import __version__ as cryptography_version
|
||||
|
||||
_check_cryptography(cryptography_version)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# urllib3's DependencyWarnings should be silenced.
|
||||
from urllib3.exceptions import DependencyWarning
|
||||
|
||||
warnings.simplefilter("ignore", DependencyWarning)
|
||||
|
||||
# Set default logging handler to avoid "No handler found" warnings.
|
||||
import logging
|
||||
from logging import NullHandler
|
||||
|
||||
from . import packages, utils
|
||||
from .__version__ import (
|
||||
__author__,
|
||||
__author_email__,
|
||||
__build__,
|
||||
__cake__,
|
||||
__copyright__,
|
||||
__description__,
|
||||
__license__,
|
||||
__title__,
|
||||
__url__,
|
||||
__version__,
|
||||
)
|
||||
from .api import delete, get, head, options, patch, post, put, request
|
||||
from .exceptions import (
|
||||
ConnectionError,
|
||||
ConnectTimeout,
|
||||
FileModeWarning,
|
||||
HTTPError,
|
||||
JSONDecodeError,
|
||||
ReadTimeout,
|
||||
RequestException,
|
||||
Timeout,
|
||||
TooManyRedirects,
|
||||
URLRequired,
|
||||
)
|
||||
from .models import PreparedRequest, Request, Response
|
||||
from .sessions import Session, session
|
||||
from .status_codes import codes
|
||||
|
||||
logging.getLogger(__name__).addHandler(NullHandler())
|
||||
|
||||
# FileModeWarnings go off per the default.
|
||||
warnings.simplefilter("default", FileModeWarning, append=True)
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,14 +0,0 @@
|
||||
# .-. .-. .-. . . .-. .-. .-. .-.
|
||||
# |( |- |.| | | |- `-. | `-.
|
||||
# ' ' `-' `-`.`-' `-' `-' ' `-'
|
||||
|
||||
__title__ = "requests"
|
||||
__description__ = "Python HTTP for Humans."
|
||||
__url__ = "https://requests.readthedocs.io"
|
||||
__version__ = "2.31.0"
|
||||
__build__ = 0x023100
|
||||
__author__ = "Kenneth Reitz"
|
||||
__author_email__ = "me@kennethreitz.org"
|
||||
__license__ = "Apache 2.0"
|
||||
__copyright__ = "Copyright Kenneth Reitz"
|
||||
__cake__ = "\u2728 \U0001f370 \u2728"
|
@ -1,50 +0,0 @@
|
||||
"""
|
||||
requests._internal_utils
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
Provides utility functions that are consumed internally by Requests
|
||||
which depend on extremely few external helpers (such as compat)
|
||||
"""
|
||||
import re
|
||||
|
||||
from .compat import builtin_str
|
||||
|
||||
_VALID_HEADER_NAME_RE_BYTE = re.compile(rb"^[^:\s][^:\r\n]*$")
|
||||
_VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*$")
|
||||
_VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*$|^$")
|
||||
_VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*$|^$")
|
||||
|
||||
_HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR)
|
||||
_HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE)
|
||||
HEADER_VALIDATORS = {
|
||||
bytes: _HEADER_VALIDATORS_BYTE,
|
||||
str: _HEADER_VALIDATORS_STR,
|
||||
}
|
||||
|
||||
|
||||
def to_native_string(string, encoding="ascii"):
|
||||
"""Given a string object, regardless of type, returns a representation of
|
||||
that string in the native string type, encoding and decoding where
|
||||
necessary. This assumes ASCII unless told otherwise.
|
||||
"""
|
||||
if isinstance(string, builtin_str):
|
||||
out = string
|
||||
else:
|
||||
out = string.decode(encoding)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def unicode_is_ascii(u_string):
|
||||
"""Determine if unicode string only contains ASCII characters.
|
||||
|
||||
:param str u_string: unicode string to check. Must be unicode
|
||||
and not Python 2 `str`.
|
||||
:rtype: bool
|
||||
"""
|
||||
assert isinstance(u_string, str)
|
||||
try:
|
||||
u_string.encode("ascii")
|
||||
return True
|
||||
except UnicodeEncodeError:
|
||||
return False
|
@ -1,538 +0,0 @@
|
||||
"""
|
||||
requests.adapters
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
This module contains the transport adapters that Requests uses to define
|
||||
and maintain connections.
|
||||
"""
|
||||
|
||||
import os.path
|
||||
import socket # noqa: F401
|
||||
|
||||
from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
|
||||
from urllib3.exceptions import HTTPError as _HTTPError
|
||||
from urllib3.exceptions import InvalidHeader as _InvalidHeader
|
||||
from urllib3.exceptions import (
|
||||
LocationValueError,
|
||||
MaxRetryError,
|
||||
NewConnectionError,
|
||||
ProtocolError,
|
||||
)
|
||||
from urllib3.exceptions import ProxyError as _ProxyError
|
||||
from urllib3.exceptions import ReadTimeoutError, ResponseError
|
||||
from urllib3.exceptions import SSLError as _SSLError
|
||||
from urllib3.poolmanager import PoolManager, proxy_from_url
|
||||
from urllib3.util import Timeout as TimeoutSauce
|
||||
from urllib3.util import parse_url
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from .auth import _basic_auth_str
|
||||
from .compat import basestring, urlparse
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .exceptions import (
|
||||
ConnectionError,
|
||||
ConnectTimeout,
|
||||
InvalidHeader,
|
||||
InvalidProxyURL,
|
||||
InvalidSchema,
|
||||
InvalidURL,
|
||||
ProxyError,
|
||||
ReadTimeout,
|
||||
RetryError,
|
||||
SSLError,
|
||||
)
|
||||
from .models import Response
|
||||
from .structures import CaseInsensitiveDict
|
||||
from .utils import (
|
||||
DEFAULT_CA_BUNDLE_PATH,
|
||||
extract_zipped_paths,
|
||||
get_auth_from_url,
|
||||
get_encoding_from_headers,
|
||||
prepend_scheme_if_needed,
|
||||
select_proxy,
|
||||
urldefragauth,
|
||||
)
|
||||
|
||||
try:
|
||||
from urllib3.contrib.socks import SOCKSProxyManager
|
||||
except ImportError:
|
||||
|
||||
def SOCKSProxyManager(*args, **kwargs):
|
||||
raise InvalidSchema("Missing dependencies for SOCKS support.")
|
||||
|
||||
|
||||
DEFAULT_POOLBLOCK = False
|
||||
DEFAULT_POOLSIZE = 10
|
||||
DEFAULT_RETRIES = 0
|
||||
DEFAULT_POOL_TIMEOUT = None
|
||||
|
||||
|
||||
class BaseAdapter:
|
||||
"""The Base Transport Adapter"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def send(
|
||||
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
|
||||
):
|
||||
"""Sends PreparedRequest object. Returns Response object.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||
:param stream: (optional) Whether to stream the request content.
|
||||
:param timeout: (optional) How long to wait for the server to send
|
||||
data before giving up, as a float, or a :ref:`(connect timeout,
|
||||
read timeout) <timeouts>` tuple.
|
||||
:type timeout: float or tuple
|
||||
:param verify: (optional) Either a boolean, in which case it controls whether we verify
|
||||
the server's TLS certificate, or a string, in which case it must be a path
|
||||
to a CA bundle to use
|
||||
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||||
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def close(self):
|
||||
"""Cleans up adapter specific items."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class HTTPAdapter(BaseAdapter):
|
||||
"""The built-in HTTP Adapter for urllib3.
|
||||
|
||||
Provides a general-case interface for Requests sessions to contact HTTP and
|
||||
HTTPS urls by implementing the Transport Adapter interface. This class will
|
||||
usually be created by the :class:`Session <Session>` class under the
|
||||
covers.
|
||||
|
||||
:param pool_connections: The number of urllib3 connection pools to cache.
|
||||
:param pool_maxsize: The maximum number of connections to save in the pool.
|
||||
:param max_retries: The maximum number of retries each connection
|
||||
should attempt. Note, this applies only to failed DNS lookups, socket
|
||||
connections and connection timeouts, never to requests where data has
|
||||
made it to the server. By default, Requests does not retry failed
|
||||
connections. If you need granular control over the conditions under
|
||||
which we retry a request, import urllib3's ``Retry`` class and pass
|
||||
that instead.
|
||||
:param pool_block: Whether the connection pool should block for connections.
|
||||
|
||||
Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> s = requests.Session()
|
||||
>>> a = requests.adapters.HTTPAdapter(max_retries=3)
|
||||
>>> s.mount('http://', a)
|
||||
"""
|
||||
|
||||
__attrs__ = [
|
||||
"max_retries",
|
||||
"config",
|
||||
"_pool_connections",
|
||||
"_pool_maxsize",
|
||||
"_pool_block",
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pool_connections=DEFAULT_POOLSIZE,
|
||||
pool_maxsize=DEFAULT_POOLSIZE,
|
||||
max_retries=DEFAULT_RETRIES,
|
||||
pool_block=DEFAULT_POOLBLOCK,
|
||||
):
|
||||
if max_retries == DEFAULT_RETRIES:
|
||||
self.max_retries = Retry(0, read=False)
|
||||
else:
|
||||
self.max_retries = Retry.from_int(max_retries)
|
||||
self.config = {}
|
||||
self.proxy_manager = {}
|
||||
|
||||
super().__init__()
|
||||
|
||||
self._pool_connections = pool_connections
|
||||
self._pool_maxsize = pool_maxsize
|
||||
self._pool_block = pool_block
|
||||
|
||||
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
|
||||
|
||||
def __getstate__(self):
|
||||
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
|
||||
|
||||
def __setstate__(self, state):
|
||||
# Can't handle by adding 'proxy_manager' to self.__attrs__ because
|
||||
# self.poolmanager uses a lambda function, which isn't pickleable.
|
||||
self.proxy_manager = {}
|
||||
self.config = {}
|
||||
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.init_poolmanager(
|
||||
self._pool_connections, self._pool_maxsize, block=self._pool_block
|
||||
)
|
||||
|
||||
def init_poolmanager(
|
||||
self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
|
||||
):
|
||||
"""Initializes a urllib3 PoolManager.
|
||||
|
||||
This method should not be called from user code, and is only
|
||||
exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param connections: The number of urllib3 connection pools to cache.
|
||||
:param maxsize: The maximum number of connections to save in the pool.
|
||||
:param block: Block when no free connections are available.
|
||||
:param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
|
||||
"""
|
||||
# save these values for pickling
|
||||
self._pool_connections = connections
|
||||
self._pool_maxsize = maxsize
|
||||
self._pool_block = block
|
||||
|
||||
self.poolmanager = PoolManager(
|
||||
num_pools=connections,
|
||||
maxsize=maxsize,
|
||||
block=block,
|
||||
**pool_kwargs,
|
||||
)
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
"""Return urllib3 ProxyManager for the given proxy.
|
||||
|
||||
This method should not be called from user code, and is only
|
||||
exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param proxy: The proxy to return a urllib3 ProxyManager for.
|
||||
:param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
|
||||
:returns: ProxyManager
|
||||
:rtype: urllib3.ProxyManager
|
||||
"""
|
||||
if proxy in self.proxy_manager:
|
||||
manager = self.proxy_manager[proxy]
|
||||
elif proxy.lower().startswith("socks"):
|
||||
username, password = get_auth_from_url(proxy)
|
||||
manager = self.proxy_manager[proxy] = SOCKSProxyManager(
|
||||
proxy,
|
||||
username=username,
|
||||
password=password,
|
||||
num_pools=self._pool_connections,
|
||||
maxsize=self._pool_maxsize,
|
||||
block=self._pool_block,
|
||||
**proxy_kwargs,
|
||||
)
|
||||
else:
|
||||
proxy_headers = self.proxy_headers(proxy)
|
||||
manager = self.proxy_manager[proxy] = proxy_from_url(
|
||||
proxy,
|
||||
proxy_headers=proxy_headers,
|
||||
num_pools=self._pool_connections,
|
||||
maxsize=self._pool_maxsize,
|
||||
block=self._pool_block,
|
||||
**proxy_kwargs,
|
||||
)
|
||||
|
||||
return manager
|
||||
|
||||
def cert_verify(self, conn, url, verify, cert):
|
||||
"""Verify a SSL certificate. This method should not be called from user
|
||||
code, and is only exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param conn: The urllib3 connection object associated with the cert.
|
||||
:param url: The requested URL.
|
||||
:param verify: Either a boolean, in which case it controls whether we verify
|
||||
the server's TLS certificate, or a string, in which case it must be a path
|
||||
to a CA bundle to use
|
||||
:param cert: The SSL certificate to verify.
|
||||
"""
|
||||
if url.lower().startswith("https") and verify:
|
||||
|
||||
cert_loc = None
|
||||
|
||||
# Allow self-specified cert location.
|
||||
if verify is not True:
|
||||
cert_loc = verify
|
||||
|
||||
if not cert_loc:
|
||||
cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
|
||||
|
||||
if not cert_loc or not os.path.exists(cert_loc):
|
||||
raise OSError(
|
||||
f"Could not find a suitable TLS CA certificate bundle, "
|
||||
f"invalid path: {cert_loc}"
|
||||
)
|
||||
|
||||
conn.cert_reqs = "CERT_REQUIRED"
|
||||
|
||||
if not os.path.isdir(cert_loc):
|
||||
conn.ca_certs = cert_loc
|
||||
else:
|
||||
conn.ca_cert_dir = cert_loc
|
||||
else:
|
||||
conn.cert_reqs = "CERT_NONE"
|
||||
conn.ca_certs = None
|
||||
conn.ca_cert_dir = None
|
||||
|
||||
if cert:
|
||||
if not isinstance(cert, basestring):
|
||||
conn.cert_file = cert[0]
|
||||
conn.key_file = cert[1]
|
||||
else:
|
||||
conn.cert_file = cert
|
||||
conn.key_file = None
|
||||
if conn.cert_file and not os.path.exists(conn.cert_file):
|
||||
raise OSError(
|
||||
f"Could not find the TLS certificate file, "
|
||||
f"invalid path: {conn.cert_file}"
|
||||
)
|
||||
if conn.key_file and not os.path.exists(conn.key_file):
|
||||
raise OSError(
|
||||
f"Could not find the TLS key file, invalid path: {conn.key_file}"
|
||||
)
|
||||
|
||||
def build_response(self, req, resp):
|
||||
"""Builds a :class:`Response <requests.Response>` object from a urllib3
|
||||
response. This should not be called from user code, and is only exposed
|
||||
for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
|
||||
|
||||
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
|
||||
:param resp: The urllib3 response object.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
response = Response()
|
||||
|
||||
# Fallback to None if there's no status_code, for whatever reason.
|
||||
response.status_code = getattr(resp, "status", None)
|
||||
|
||||
# Make headers case-insensitive.
|
||||
response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
|
||||
|
||||
# Set encoding.
|
||||
response.encoding = get_encoding_from_headers(response.headers)
|
||||
response.raw = resp
|
||||
response.reason = response.raw.reason
|
||||
|
||||
if isinstance(req.url, bytes):
|
||||
response.url = req.url.decode("utf-8")
|
||||
else:
|
||||
response.url = req.url
|
||||
|
||||
# Add new cookies from the server.
|
||||
extract_cookies_to_jar(response.cookies, req, resp)
|
||||
|
||||
# Give the Response some context.
|
||||
response.request = req
|
||||
response.connection = self
|
||||
|
||||
return response
|
||||
|
||||
def get_connection(self, url, proxies=None):
|
||||
"""Returns a urllib3 connection for the given URL. This should not be
|
||||
called from user code, and is only exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param url: The URL to connect to.
|
||||
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
|
||||
:rtype: urllib3.ConnectionPool
|
||||
"""
|
||||
proxy = select_proxy(url, proxies)
|
||||
|
||||
if proxy:
|
||||
proxy = prepend_scheme_if_needed(proxy, "http")
|
||||
proxy_url = parse_url(proxy)
|
||||
if not proxy_url.host:
|
||||
raise InvalidProxyURL(
|
||||
"Please check proxy URL. It is malformed "
|
||||
"and could be missing the host."
|
||||
)
|
||||
proxy_manager = self.proxy_manager_for(proxy)
|
||||
conn = proxy_manager.connection_from_url(url)
|
||||
else:
|
||||
# Only scheme should be lower case
|
||||
parsed = urlparse(url)
|
||||
url = parsed.geturl()
|
||||
conn = self.poolmanager.connection_from_url(url)
|
||||
|
||||
return conn
|
||||
|
||||
def close(self):
|
||||
"""Disposes of any internal state.
|
||||
|
||||
Currently, this closes the PoolManager and any active ProxyManager,
|
||||
which closes any pooled connections.
|
||||
"""
|
||||
self.poolmanager.clear()
|
||||
for proxy in self.proxy_manager.values():
|
||||
proxy.clear()
|
||||
|
||||
def request_url(self, request, proxies):
|
||||
"""Obtain the url to use when making the final request.
|
||||
|
||||
If the message is being sent through a HTTP proxy, the full URL has to
|
||||
be used. Otherwise, we should only use the path portion of the URL.
|
||||
|
||||
This should not be called from user code, and is only exposed for use
|
||||
when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||
:param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
|
||||
:rtype: str
|
||||
"""
|
||||
proxy = select_proxy(request.url, proxies)
|
||||
scheme = urlparse(request.url).scheme
|
||||
|
||||
is_proxied_http_request = proxy and scheme != "https"
|
||||
using_socks_proxy = False
|
||||
if proxy:
|
||||
proxy_scheme = urlparse(proxy).scheme.lower()
|
||||
using_socks_proxy = proxy_scheme.startswith("socks")
|
||||
|
||||
url = request.path_url
|
||||
if is_proxied_http_request and not using_socks_proxy:
|
||||
url = urldefragauth(request.url)
|
||||
|
||||
return url
|
||||
|
||||
def add_headers(self, request, **kwargs):
|
||||
"""Add any headers needed by the connection. As of v2.0 this does
|
||||
nothing by default, but is left for overriding by users that subclass
|
||||
the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
This should not be called from user code, and is only exposed for use
|
||||
when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
|
||||
:param kwargs: The keyword arguments from the call to send().
|
||||
"""
|
||||
pass
|
||||
|
||||
def proxy_headers(self, proxy):
|
||||
"""Returns a dictionary of the headers to add to any request sent
|
||||
through a proxy. This works with urllib3 magic to ensure that they are
|
||||
correctly sent to the proxy, rather than in a tunnelled request if
|
||||
CONNECT is being used.
|
||||
|
||||
This should not be called from user code, and is only exposed for use
|
||||
when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param proxy: The url of the proxy being used for this request.
|
||||
:rtype: dict
|
||||
"""
|
||||
headers = {}
|
||||
username, password = get_auth_from_url(proxy)
|
||||
|
||||
if username:
|
||||
headers["Proxy-Authorization"] = _basic_auth_str(username, password)
|
||||
|
||||
return headers
|
||||
|
||||
def send(
|
||||
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
|
||||
):
|
||||
"""Sends PreparedRequest object. Returns Response object.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||
:param stream: (optional) Whether to stream the request content.
|
||||
:param timeout: (optional) How long to wait for the server to send
|
||||
data before giving up, as a float, or a :ref:`(connect timeout,
|
||||
read timeout) <timeouts>` tuple.
|
||||
:type timeout: float or tuple or urllib3 Timeout object
|
||||
:param verify: (optional) Either a boolean, in which case it controls whether
|
||||
we verify the server's TLS certificate, or a string, in which case it
|
||||
must be a path to a CA bundle to use
|
||||
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||||
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
try:
|
||||
conn = self.get_connection(request.url, proxies)
|
||||
except LocationValueError as e:
|
||||
raise InvalidURL(e, request=request)
|
||||
|
||||
self.cert_verify(conn, request.url, verify, cert)
|
||||
url = self.request_url(request, proxies)
|
||||
self.add_headers(
|
||||
request,
|
||||
stream=stream,
|
||||
timeout=timeout,
|
||||
verify=verify,
|
||||
cert=cert,
|
||||
proxies=proxies,
|
||||
)
|
||||
|
||||
chunked = not (request.body is None or "Content-Length" in request.headers)
|
||||
|
||||
if isinstance(timeout, tuple):
|
||||
try:
|
||||
connect, read = timeout
|
||||
timeout = TimeoutSauce(connect=connect, read=read)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
|
||||
f"or a single float to set both timeouts to the same value."
|
||||
)
|
||||
elif isinstance(timeout, TimeoutSauce):
|
||||
pass
|
||||
else:
|
||||
timeout = TimeoutSauce(connect=timeout, read=timeout)
|
||||
|
||||
try:
|
||||
resp = conn.urlopen(
|
||||
method=request.method,
|
||||
url=url,
|
||||
body=request.body,
|
||||
headers=request.headers,
|
||||
redirect=False,
|
||||
assert_same_host=False,
|
||||
preload_content=False,
|
||||
decode_content=False,
|
||||
retries=self.max_retries,
|
||||
timeout=timeout,
|
||||
chunked=chunked,
|
||||
)
|
||||
|
||||
except (ProtocolError, OSError) as err:
|
||||
raise ConnectionError(err, request=request)
|
||||
|
||||
except MaxRetryError as e:
|
||||
if isinstance(e.reason, ConnectTimeoutError):
|
||||
# TODO: Remove this in 3.0.0: see #2811
|
||||
if not isinstance(e.reason, NewConnectionError):
|
||||
raise ConnectTimeout(e, request=request)
|
||||
|
||||
if isinstance(e.reason, ResponseError):
|
||||
raise RetryError(e, request=request)
|
||||
|
||||
if isinstance(e.reason, _ProxyError):
|
||||
raise ProxyError(e, request=request)
|
||||
|
||||
if isinstance(e.reason, _SSLError):
|
||||
# This branch is for urllib3 v1.22 and later.
|
||||
raise SSLError(e, request=request)
|
||||
|
||||
raise ConnectionError(e, request=request)
|
||||
|
||||
except ClosedPoolError as e:
|
||||
raise ConnectionError(e, request=request)
|
||||
|
||||
except _ProxyError as e:
|
||||
raise ProxyError(e)
|
||||
|
||||
except (_SSLError, _HTTPError) as e:
|
||||
if isinstance(e, _SSLError):
|
||||
# This branch is for urllib3 versions earlier than v1.22
|
||||
raise SSLError(e, request=request)
|
||||
elif isinstance(e, ReadTimeoutError):
|
||||
raise ReadTimeout(e, request=request)
|
||||
elif isinstance(e, _InvalidHeader):
|
||||
raise InvalidHeader(e, request=request)
|
||||
else:
|
||||
raise
|
||||
|
||||
return self.build_response(request, resp)
|
@ -1,157 +0,0 @@
|
||||
"""
|
||||
requests.api
|
||||
~~~~~~~~~~~~
|
||||
|
||||
This module implements the Requests API.
|
||||
|
||||
:copyright: (c) 2012 by Kenneth Reitz.
|
||||
:license: Apache2, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
from . import sessions
|
||||
|
||||
|
||||
def request(method, url, **kwargs):
|
||||
"""Constructs and sends a :class:`Request <Request>`.
|
||||
|
||||
:param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``.
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary, list of tuples or bytes to send
|
||||
in the query string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
|
||||
``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
|
||||
or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string
|
||||
defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
|
||||
to add for the file.
|
||||
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) How many seconds to wait for the server to send data
|
||||
before giving up, as a float, or a :ref:`(connect timeout, read
|
||||
timeout) <timeouts>` tuple.
|
||||
:type timeout: float or tuple
|
||||
:param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
|
||||
:type allow_redirects: bool
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param verify: (optional) Either a boolean, in which case it controls whether we verify
|
||||
the server's TLS certificate, or a string, in which case it must be a path
|
||||
to a CA bundle to use. Defaults to ``True``.
|
||||
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
|
||||
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
|
||||
Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> req = requests.request('GET', 'https://httpbin.org/get')
|
||||
>>> req
|
||||
<Response [200]>
|
||||
"""
|
||||
|
||||
# By using the 'with' statement we are sure the session is closed, thus we
|
||||
# avoid leaving sockets open which can trigger a ResourceWarning in some
|
||||
# cases, and look like a memory leak in others.
|
||||
with sessions.Session() as session:
|
||||
return session.request(method=method, url=url, **kwargs)
|
||||
|
||||
|
||||
def get(url, params=None, **kwargs):
|
||||
r"""Sends a GET request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary, list of tuples or bytes to send
|
||||
in the query string for the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return request("get", url, params=params, **kwargs)
|
||||
|
||||
|
||||
def options(url, **kwargs):
|
||||
r"""Sends an OPTIONS request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return request("options", url, **kwargs)
|
||||
|
||||
|
||||
def head(url, **kwargs):
|
||||
r"""Sends a HEAD request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes. If
|
||||
`allow_redirects` is not provided, it will be set to `False` (as
|
||||
opposed to the default :meth:`request` behavior).
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault("allow_redirects", False)
|
||||
return request("head", url, **kwargs)
|
||||
|
||||
|
||||
def post(url, data=None, json=None, **kwargs):
|
||||
r"""Sends a POST request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return request("post", url, data=data, json=json, **kwargs)
|
||||
|
||||
|
||||
def put(url, data=None, **kwargs):
|
||||
r"""Sends a PUT request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return request("put", url, data=data, **kwargs)
|
||||
|
||||
|
||||
def patch(url, data=None, **kwargs):
|
||||
r"""Sends a PATCH request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return request("patch", url, data=data, **kwargs)
|
||||
|
||||
|
||||
def delete(url, **kwargs):
|
||||
r"""Sends a DELETE request.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:return: :class:`Response <Response>` object
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return request("delete", url, **kwargs)
|
@ -1,315 +0,0 @@
|
||||
"""
|
||||
requests.auth
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
This module contains the authentication handlers for Requests.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import warnings
|
||||
from base64 import b64encode
|
||||
|
||||
from ._internal_utils import to_native_string
|
||||
from .compat import basestring, str, urlparse
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .utils import parse_dict_header
|
||||
|
||||
CONTENT_TYPE_FORM_URLENCODED = "application/x-www-form-urlencoded"
|
||||
CONTENT_TYPE_MULTI_PART = "multipart/form-data"
|
||||
|
||||
|
||||
def _basic_auth_str(username, password):
|
||||
"""Returns a Basic Auth string."""
|
||||
|
||||
# "I want us to put a big-ol' comment on top of it that
|
||||
# says that this behaviour is dumb but we need to preserve
|
||||
# it because people are relying on it."
|
||||
# - Lukasa
|
||||
#
|
||||
# These are here solely to maintain backwards compatibility
|
||||
# for things like ints. This will be removed in 3.0.0.
|
||||
if not isinstance(username, basestring):
|
||||
warnings.warn(
|
||||
"Non-string usernames will no longer be supported in Requests "
|
||||
"3.0.0. Please convert the object you've passed in ({!r}) to "
|
||||
"a string or bytes object in the near future to avoid "
|
||||
"problems.".format(username),
|
||||
category=DeprecationWarning,
|
||||
)
|
||||
username = str(username)
|
||||
|
||||
if not isinstance(password, basestring):
|
||||
warnings.warn(
|
||||
"Non-string passwords will no longer be supported in Requests "
|
||||
"3.0.0. Please convert the object you've passed in ({!r}) to "
|
||||
"a string or bytes object in the near future to avoid "
|
||||
"problems.".format(type(password)),
|
||||
category=DeprecationWarning,
|
||||
)
|
||||
password = str(password)
|
||||
# -- End Removal --
|
||||
|
||||
if isinstance(username, str):
|
||||
username = username.encode("latin1")
|
||||
|
||||
if isinstance(password, str):
|
||||
password = password.encode("latin1")
|
||||
|
||||
authstr = "Basic " + to_native_string(
|
||||
b64encode(b":".join((username, password))).strip()
|
||||
)
|
||||
|
||||
return authstr
|
||||
|
||||
|
||||
class AuthBase:
|
||||
"""Base class that all auth implementations derive from"""
|
||||
|
||||
def __call__(self, r):
|
||||
raise NotImplementedError("Auth hooks must be callable.")
|
||||
|
||||
|
||||
class HTTPBasicAuth(AuthBase):
|
||||
"""Attaches HTTP Basic Authentication to the given Request object."""
|
||||
|
||||
def __init__(self, username, password):
|
||||
self.username = username
|
||||
self.password = password
|
||||
|
||||
def __eq__(self, other):
|
||||
return all(
|
||||
[
|
||||
self.username == getattr(other, "username", None),
|
||||
self.password == getattr(other, "password", None),
|
||||
]
|
||||
)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self == other
|
||||
|
||||
def __call__(self, r):
|
||||
r.headers["Authorization"] = _basic_auth_str(self.username, self.password)
|
||||
return r
|
||||
|
||||
|
||||
class HTTPProxyAuth(HTTPBasicAuth):
|
||||
"""Attaches HTTP Proxy Authentication to a given Request object."""
|
||||
|
||||
def __call__(self, r):
|
||||
r.headers["Proxy-Authorization"] = _basic_auth_str(self.username, self.password)
|
||||
return r
|
||||
|
||||
|
||||
class HTTPDigestAuth(AuthBase):
|
||||
"""Attaches HTTP Digest Authentication to the given Request object."""
|
||||
|
||||
def __init__(self, username, password):
|
||||
self.username = username
|
||||
self.password = password
|
||||
# Keep state in per-thread local storage
|
||||
self._thread_local = threading.local()
|
||||
|
||||
def init_per_thread_state(self):
|
||||
# Ensure state is initialized just once per-thread
|
||||
if not hasattr(self._thread_local, "init"):
|
||||
self._thread_local.init = True
|
||||
self._thread_local.last_nonce = ""
|
||||
self._thread_local.nonce_count = 0
|
||||
self._thread_local.chal = {}
|
||||
self._thread_local.pos = None
|
||||
self._thread_local.num_401_calls = None
|
||||
|
||||
def build_digest_header(self, method, url):
|
||||
"""
|
||||
:rtype: str
|
||||
"""
|
||||
|
||||
realm = self._thread_local.chal["realm"]
|
||||
nonce = self._thread_local.chal["nonce"]
|
||||
qop = self._thread_local.chal.get("qop")
|
||||
algorithm = self._thread_local.chal.get("algorithm")
|
||||
opaque = self._thread_local.chal.get("opaque")
|
||||
hash_utf8 = None
|
||||
|
||||
if algorithm is None:
|
||||
_algorithm = "MD5"
|
||||
else:
|
||||
_algorithm = algorithm.upper()
|
||||
# lambdas assume digest modules are imported at the top level
|
||||
if _algorithm == "MD5" or _algorithm == "MD5-SESS":
|
||||
|
||||
def md5_utf8(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode("utf-8")
|
||||
return hashlib.md5(x).hexdigest()
|
||||
|
||||
hash_utf8 = md5_utf8
|
||||
elif _algorithm == "SHA":
|
||||
|
||||
def sha_utf8(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode("utf-8")
|
||||
return hashlib.sha1(x).hexdigest()
|
||||
|
||||
hash_utf8 = sha_utf8
|
||||
elif _algorithm == "SHA-256":
|
||||
|
||||
def sha256_utf8(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode("utf-8")
|
||||
return hashlib.sha256(x).hexdigest()
|
||||
|
||||
hash_utf8 = sha256_utf8
|
||||
elif _algorithm == "SHA-512":
|
||||
|
||||
def sha512_utf8(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode("utf-8")
|
||||
return hashlib.sha512(x).hexdigest()
|
||||
|
||||
hash_utf8 = sha512_utf8
|
||||
|
||||
KD = lambda s, d: hash_utf8(f"{s}:{d}") # noqa:E731
|
||||
|
||||
if hash_utf8 is None:
|
||||
return None
|
||||
|
||||
# XXX not implemented yet
|
||||
entdig = None
|
||||
p_parsed = urlparse(url)
|
||||
#: path is request-uri defined in RFC 2616 which should not be empty
|
||||
path = p_parsed.path or "/"
|
||||
if p_parsed.query:
|
||||
path += f"?{p_parsed.query}"
|
||||
|
||||
A1 = f"{self.username}:{realm}:{self.password}"
|
||||
A2 = f"{method}:{path}"
|
||||
|
||||
HA1 = hash_utf8(A1)
|
||||
HA2 = hash_utf8(A2)
|
||||
|
||||
if nonce == self._thread_local.last_nonce:
|
||||
self._thread_local.nonce_count += 1
|
||||
else:
|
||||
self._thread_local.nonce_count = 1
|
||||
ncvalue = f"{self._thread_local.nonce_count:08x}"
|
||||
s = str(self._thread_local.nonce_count).encode("utf-8")
|
||||
s += nonce.encode("utf-8")
|
||||
s += time.ctime().encode("utf-8")
|
||||
s += os.urandom(8)
|
||||
|
||||
cnonce = hashlib.sha1(s).hexdigest()[:16]
|
||||
if _algorithm == "MD5-SESS":
|
||||
HA1 = hash_utf8(f"{HA1}:{nonce}:{cnonce}")
|
||||
|
||||
if not qop:
|
||||
respdig = KD(HA1, f"{nonce}:{HA2}")
|
||||
elif qop == "auth" or "auth" in qop.split(","):
|
||||
noncebit = f"{nonce}:{ncvalue}:{cnonce}:auth:{HA2}"
|
||||
respdig = KD(HA1, noncebit)
|
||||
else:
|
||||
# XXX handle auth-int.
|
||||
return None
|
||||
|
||||
self._thread_local.last_nonce = nonce
|
||||
|
||||
# XXX should the partial digests be encoded too?
|
||||
base = (
|
||||
f'username="{self.username}", realm="{realm}", nonce="{nonce}", '
|
||||
f'uri="{path}", response="{respdig}"'
|
||||
)
|
||||
if opaque:
|
||||
base += f', opaque="{opaque}"'
|
||||
if algorithm:
|
||||
base += f', algorithm="{algorithm}"'
|
||||
if entdig:
|
||||
base += f', digest="{entdig}"'
|
||||
if qop:
|
||||
base += f', qop="auth", nc={ncvalue}, cnonce="{cnonce}"'
|
||||
|
||||
return f"Digest {base}"
|
||||
|
||||
def handle_redirect(self, r, **kwargs):
|
||||
"""Reset num_401_calls counter on redirects."""
|
||||
if r.is_redirect:
|
||||
self._thread_local.num_401_calls = 1
|
||||
|
||||
def handle_401(self, r, **kwargs):
|
||||
"""
|
||||
Takes the given response and tries digest-auth, if needed.
|
||||
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
# If response is not 4xx, do not auth
|
||||
# See https://github.com/psf/requests/issues/3772
|
||||
if not 400 <= r.status_code < 500:
|
||||
self._thread_local.num_401_calls = 1
|
||||
return r
|
||||
|
||||
if self._thread_local.pos is not None:
|
||||
# Rewind the file position indicator of the body to where
|
||||
# it was to resend the request.
|
||||
r.request.body.seek(self._thread_local.pos)
|
||||
s_auth = r.headers.get("www-authenticate", "")
|
||||
|
||||
if "digest" in s_auth.lower() and self._thread_local.num_401_calls < 2:
|
||||
|
||||
self._thread_local.num_401_calls += 1
|
||||
pat = re.compile(r"digest ", flags=re.IGNORECASE)
|
||||
self._thread_local.chal = parse_dict_header(pat.sub("", s_auth, count=1))
|
||||
|
||||
# Consume content and release the original connection
|
||||
# to allow our new request to reuse the same one.
|
||||
r.content
|
||||
r.close()
|
||||
prep = r.request.copy()
|
||||
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
|
||||
prep.prepare_cookies(prep._cookies)
|
||||
|
||||
prep.headers["Authorization"] = self.build_digest_header(
|
||||
prep.method, prep.url
|
||||
)
|
||||
_r = r.connection.send(prep, **kwargs)
|
||||
_r.history.append(r)
|
||||
_r.request = prep
|
||||
|
||||
return _r
|
||||
|
||||
self._thread_local.num_401_calls = 1
|
||||
return r
|
||||
|
||||
def __call__(self, r):
|
||||
# Initialize per-thread state, if needed
|
||||
self.init_per_thread_state()
|
||||
# If we have a saved nonce, skip the 401
|
||||
if self._thread_local.last_nonce:
|
||||
r.headers["Authorization"] = self.build_digest_header(r.method, r.url)
|
||||
try:
|
||||
self._thread_local.pos = r.body.tell()
|
||||
except AttributeError:
|
||||
# In the case of HTTPDigestAuth being reused and the body of
|
||||
# the previous request was a file-like object, pos has the
|
||||
# file position of the previous body. Ensure it's set to
|
||||
# None.
|
||||
self._thread_local.pos = None
|
||||
r.register_hook("response", self.handle_401)
|
||||
r.register_hook("response", self.handle_redirect)
|
||||
self._thread_local.num_401_calls = 1
|
||||
|
||||
return r
|
||||
|
||||
def __eq__(self, other):
|
||||
return all(
|
||||
[
|
||||
self.username == getattr(other, "username", None),
|
||||
self.password == getattr(other, "password", None),
|
||||
]
|
||||
)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self == other
|
@ -1,17 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
requests.certs
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
This module returns the preferred default CA certificate bundle. There is
|
||||
only one — the one from the certifi package.
|
||||
|
||||
If you are packaging Requests, e.g., for a Linux distribution or a managed
|
||||
environment, you can change the definition of where() to return a separately
|
||||
packaged CA bundle.
|
||||
"""
|
||||
from certifi import where
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(where())
|
@ -1,79 +0,0 @@
|
||||
"""
|
||||
requests.compat
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
This module previously handled import compatibility issues
|
||||
between Python 2 and Python 3. It remains for backwards
|
||||
compatibility until the next major version.
|
||||
"""
|
||||
|
||||
try:
|
||||
import chardet
|
||||
except ImportError:
|
||||
import charset_normalizer as chardet
|
||||
|
||||
import sys
|
||||
|
||||
# -------
|
||||
# Pythons
|
||||
# -------
|
||||
|
||||
# Syntax sugar.
|
||||
_ver = sys.version_info
|
||||
|
||||
#: Python 2.x?
|
||||
is_py2 = _ver[0] == 2
|
||||
|
||||
#: Python 3.x?
|
||||
is_py3 = _ver[0] == 3
|
||||
|
||||
# json/simplejson module import resolution
|
||||
has_simplejson = False
|
||||
try:
|
||||
import simplejson as json
|
||||
|
||||
has_simplejson = True
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
if has_simplejson:
|
||||
from simplejson import JSONDecodeError
|
||||
else:
|
||||
from json import JSONDecodeError
|
||||
|
||||
# Keep OrderedDict for backwards compatibility.
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable, Mapping, MutableMapping
|
||||
from http import cookiejar as cookielib
|
||||
from http.cookies import Morsel
|
||||
from io import StringIO
|
||||
|
||||
# --------------
|
||||
# Legacy Imports
|
||||
# --------------
|
||||
from urllib.parse import (
|
||||
quote,
|
||||
quote_plus,
|
||||
unquote,
|
||||
unquote_plus,
|
||||
urldefrag,
|
||||
urlencode,
|
||||
urljoin,
|
||||
urlparse,
|
||||
urlsplit,
|
||||
urlunparse,
|
||||
)
|
||||
from urllib.request import (
|
||||
getproxies,
|
||||
getproxies_environment,
|
||||
parse_http_list,
|
||||
proxy_bypass,
|
||||
proxy_bypass_environment,
|
||||
)
|
||||
|
||||
builtin_str = str
|
||||
str = str
|
||||
bytes = bytes
|
||||
basestring = (str, bytes)
|
||||
numeric_types = (int, float)
|
||||
integer_types = (int,)
|
@ -1,561 +0,0 @@
|
||||
"""
|
||||
requests.cookies
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Compatibility code to be able to use `cookielib.CookieJar` with requests.
|
||||
|
||||
requests.utils imports from here, so be careful with imports.
|
||||
"""
|
||||
|
||||
import calendar
|
||||
import copy
|
||||
import time
|
||||
|
||||
from ._internal_utils import to_native_string
|
||||
from .compat import Morsel, MutableMapping, cookielib, urlparse, urlunparse
|
||||
|
||||
try:
|
||||
import threading
|
||||
except ImportError:
|
||||
import dummy_threading as threading
|
||||
|
||||
|
||||
class MockRequest:
|
||||
"""Wraps a `requests.Request` to mimic a `urllib2.Request`.
|
||||
|
||||
The code in `cookielib.CookieJar` expects this interface in order to correctly
|
||||
manage cookie policies, i.e., determine whether a cookie can be set, given the
|
||||
domains of the request and the cookie.
|
||||
|
||||
The original request object is read-only. The client is responsible for collecting
|
||||
the new headers via `get_new_headers()` and interpreting them appropriately. You
|
||||
probably want `get_cookie_header`, defined below.
|
||||
"""
|
||||
|
||||
def __init__(self, request):
|
||||
self._r = request
|
||||
self._new_headers = {}
|
||||
self.type = urlparse(self._r.url).scheme
|
||||
|
||||
def get_type(self):
|
||||
return self.type
|
||||
|
||||
def get_host(self):
|
||||
return urlparse(self._r.url).netloc
|
||||
|
||||
def get_origin_req_host(self):
|
||||
return self.get_host()
|
||||
|
||||
def get_full_url(self):
|
||||
# Only return the response's URL if the user hadn't set the Host
|
||||
# header
|
||||
if not self._r.headers.get("Host"):
|
||||
return self._r.url
|
||||
# If they did set it, retrieve it and reconstruct the expected domain
|
||||
host = to_native_string(self._r.headers["Host"], encoding="utf-8")
|
||||
parsed = urlparse(self._r.url)
|
||||
# Reconstruct the URL as we expect it
|
||||
return urlunparse(
|
||||
[
|
||||
parsed.scheme,
|
||||
host,
|
||||
parsed.path,
|
||||
parsed.params,
|
||||
parsed.query,
|
||||
parsed.fragment,
|
||||
]
|
||||
)
|
||||
|
||||
def is_unverifiable(self):
|
||||
return True
|
||||
|
||||
def has_header(self, name):
|
||||
return name in self._r.headers or name in self._new_headers
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return self._r.headers.get(name, self._new_headers.get(name, default))
|
||||
|
||||
def add_header(self, key, val):
|
||||
"""cookielib has no legitimate use for this method; add it back if you find one."""
|
||||
raise NotImplementedError(
|
||||
"Cookie headers should be added with add_unredirected_header()"
|
||||
)
|
||||
|
||||
def add_unredirected_header(self, name, value):
|
||||
self._new_headers[name] = value
|
||||
|
||||
def get_new_headers(self):
|
||||
return self._new_headers
|
||||
|
||||
@property
|
||||
def unverifiable(self):
|
||||
return self.is_unverifiable()
|
||||
|
||||
@property
|
||||
def origin_req_host(self):
|
||||
return self.get_origin_req_host()
|
||||
|
||||
@property
|
||||
def host(self):
|
||||
return self.get_host()
|
||||
|
||||
|
||||
class MockResponse:
|
||||
"""Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.
|
||||
|
||||
...what? Basically, expose the parsed HTTP headers from the server response
|
||||
the way `cookielib` expects to see them.
|
||||
"""
|
||||
|
||||
def __init__(self, headers):
|
||||
"""Make a MockResponse for `cookielib` to read.
|
||||
|
||||
:param headers: a httplib.HTTPMessage or analogous carrying the headers
|
||||
"""
|
||||
self._headers = headers
|
||||
|
||||
def info(self):
|
||||
return self._headers
|
||||
|
||||
def getheaders(self, name):
|
||||
self._headers.getheaders(name)
|
||||
|
||||
|
||||
def extract_cookies_to_jar(jar, request, response):
|
||||
"""Extract the cookies from the response into a CookieJar.
|
||||
|
||||
:param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar)
|
||||
:param request: our own requests.Request object
|
||||
:param response: urllib3.HTTPResponse object
|
||||
"""
|
||||
if not (hasattr(response, "_original_response") and response._original_response):
|
||||
return
|
||||
# the _original_response field is the wrapped httplib.HTTPResponse object,
|
||||
req = MockRequest(request)
|
||||
# pull out the HTTPMessage with the headers and put it in the mock:
|
||||
res = MockResponse(response._original_response.msg)
|
||||
jar.extract_cookies(res, req)
|
||||
|
||||
|
||||
def get_cookie_header(jar, request):
|
||||
"""
|
||||
Produce an appropriate Cookie header string to be sent with `request`, or None.
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
r = MockRequest(request)
|
||||
jar.add_cookie_header(r)
|
||||
return r.get_new_headers().get("Cookie")
|
||||
|
||||
|
||||
def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
|
||||
"""Unsets a cookie by name, by default over all domains and paths.
|
||||
|
||||
Wraps CookieJar.clear(), is O(n).
|
||||
"""
|
||||
clearables = []
|
||||
for cookie in cookiejar:
|
||||
if cookie.name != name:
|
||||
continue
|
||||
if domain is not None and domain != cookie.domain:
|
||||
continue
|
||||
if path is not None and path != cookie.path:
|
||||
continue
|
||||
clearables.append((cookie.domain, cookie.path, cookie.name))
|
||||
|
||||
for domain, path, name in clearables:
|
||||
cookiejar.clear(domain, path, name)
|
||||
|
||||
|
||||
class CookieConflictError(RuntimeError):
|
||||
"""There are two cookies that meet the criteria specified in the cookie jar.
|
||||
Use .get and .set and include domain and path args in order to be more specific.
|
||||
"""
|
||||
|
||||
|
||||
class RequestsCookieJar(cookielib.CookieJar, MutableMapping):
|
||||
"""Compatibility class; is a cookielib.CookieJar, but exposes a dict
|
||||
interface.
|
||||
|
||||
This is the CookieJar we create by default for requests and sessions that
|
||||
don't specify one, since some clients may expect response.cookies and
|
||||
session.cookies to support dict operations.
|
||||
|
||||
Requests does not use the dict interface internally; it's just for
|
||||
compatibility with external client code. All requests code should work
|
||||
out of the box with externally provided instances of ``CookieJar``, e.g.
|
||||
``LWPCookieJar`` and ``FileCookieJar``.
|
||||
|
||||
Unlike a regular CookieJar, this class is pickleable.
|
||||
|
||||
.. warning:: dictionary operations that are normally O(1) may be O(n).
|
||||
"""
|
||||
|
||||
def get(self, name, default=None, domain=None, path=None):
|
||||
"""Dict-like get() that also supports optional domain and path args in
|
||||
order to resolve naming collisions from using one cookie jar over
|
||||
multiple domains.
|
||||
|
||||
.. warning:: operation is O(n), not O(1).
|
||||
"""
|
||||
try:
|
||||
return self._find_no_duplicates(name, domain, path)
|
||||
except KeyError:
|
||||
return default
|
||||
|
||||
def set(self, name, value, **kwargs):
|
||||
"""Dict-like set() that also supports optional domain and path args in
|
||||
order to resolve naming collisions from using one cookie jar over
|
||||
multiple domains.
|
||||
"""
|
||||
# support client code that unsets cookies by assignment of a None value:
|
||||
if value is None:
|
||||
remove_cookie_by_name(
|
||||
self, name, domain=kwargs.get("domain"), path=kwargs.get("path")
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(value, Morsel):
|
||||
c = morsel_to_cookie(value)
|
||||
else:
|
||||
c = create_cookie(name, value, **kwargs)
|
||||
self.set_cookie(c)
|
||||
return c
|
||||
|
||||
def iterkeys(self):
|
||||
"""Dict-like iterkeys() that returns an iterator of names of cookies
|
||||
from the jar.
|
||||
|
||||
.. seealso:: itervalues() and iteritems().
|
||||
"""
|
||||
for cookie in iter(self):
|
||||
yield cookie.name
|
||||
|
||||
def keys(self):
|
||||
"""Dict-like keys() that returns a list of names of cookies from the
|
||||
jar.
|
||||
|
||||
.. seealso:: values() and items().
|
||||
"""
|
||||
return list(self.iterkeys())
|
||||
|
||||
def itervalues(self):
|
||||
"""Dict-like itervalues() that returns an iterator of values of cookies
|
||||
from the jar.
|
||||
|
||||
.. seealso:: iterkeys() and iteritems().
|
||||
"""
|
||||
for cookie in iter(self):
|
||||
yield cookie.value
|
||||
|
||||
def values(self):
|
||||
"""Dict-like values() that returns a list of values of cookies from the
|
||||
jar.
|
||||
|
||||
.. seealso:: keys() and items().
|
||||
"""
|
||||
return list(self.itervalues())
|
||||
|
||||
def iteritems(self):
|
||||
"""Dict-like iteritems() that returns an iterator of name-value tuples
|
||||
from the jar.
|
||||
|
||||
.. seealso:: iterkeys() and itervalues().
|
||||
"""
|
||||
for cookie in iter(self):
|
||||
yield cookie.name, cookie.value
|
||||
|
||||
def items(self):
|
||||
"""Dict-like items() that returns a list of name-value tuples from the
|
||||
jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a
|
||||
vanilla python dict of key value pairs.
|
||||
|
||||
.. seealso:: keys() and values().
|
||||
"""
|
||||
return list(self.iteritems())
|
||||
|
||||
def list_domains(self):
|
||||
"""Utility method to list all the domains in the jar."""
|
||||
domains = []
|
||||
for cookie in iter(self):
|
||||
if cookie.domain not in domains:
|
||||
domains.append(cookie.domain)
|
||||
return domains
|
||||
|
||||
def list_paths(self):
|
||||
"""Utility method to list all the paths in the jar."""
|
||||
paths = []
|
||||
for cookie in iter(self):
|
||||
if cookie.path not in paths:
|
||||
paths.append(cookie.path)
|
||||
return paths
|
||||
|
||||
def multiple_domains(self):
|
||||
"""Returns True if there are multiple domains in the jar.
|
||||
Returns False otherwise.
|
||||
|
||||
:rtype: bool
|
||||
"""
|
||||
domains = []
|
||||
for cookie in iter(self):
|
||||
if cookie.domain is not None and cookie.domain in domains:
|
||||
return True
|
||||
domains.append(cookie.domain)
|
||||
return False # there is only one domain in jar
|
||||
|
||||
def get_dict(self, domain=None, path=None):
|
||||
"""Takes as an argument an optional domain and path and returns a plain
|
||||
old Python dict of name-value pairs of cookies that meet the
|
||||
requirements.
|
||||
|
||||
:rtype: dict
|
||||
"""
|
||||
dictionary = {}
|
||||
for cookie in iter(self):
|
||||
if (domain is None or cookie.domain == domain) and (
|
||||
path is None or cookie.path == path
|
||||
):
|
||||
dictionary[cookie.name] = cookie.value
|
||||
return dictionary
|
||||
|
||||
def __contains__(self, name):
|
||||
try:
|
||||
return super().__contains__(name)
|
||||
except CookieConflictError:
|
||||
return True
|
||||
|
||||
def __getitem__(self, name):
|
||||
"""Dict-like __getitem__() for compatibility with client code. Throws
|
||||
exception if there are more than one cookie with name. In that case,
|
||||
use the more explicit get() method instead.
|
||||
|
||||
.. warning:: operation is O(n), not O(1).
|
||||
"""
|
||||
return self._find_no_duplicates(name)
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
"""Dict-like __setitem__ for compatibility with client code. Throws
|
||||
exception if there is already a cookie of that name in the jar. In that
|
||||
case, use the more explicit set() method instead.
|
||||
"""
|
||||
self.set(name, value)
|
||||
|
||||
def __delitem__(self, name):
|
||||
"""Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s
|
||||
``remove_cookie_by_name()``.
|
||||
"""
|
||||
remove_cookie_by_name(self, name)
|
||||
|
||||
def set_cookie(self, cookie, *args, **kwargs):
|
||||
if (
|
||||
hasattr(cookie.value, "startswith")
|
||||
and cookie.value.startswith('"')
|
||||
and cookie.value.endswith('"')
|
||||
):
|
||||
cookie.value = cookie.value.replace('\\"', "")
|
||||
return super().set_cookie(cookie, *args, **kwargs)
|
||||
|
||||
def update(self, other):
|
||||
"""Updates this jar with cookies from another CookieJar or dict-like"""
|
||||
if isinstance(other, cookielib.CookieJar):
|
||||
for cookie in other:
|
||||
self.set_cookie(copy.copy(cookie))
|
||||
else:
|
||||
super().update(other)
|
||||
|
||||
def _find(self, name, domain=None, path=None):
|
||||
"""Requests uses this method internally to get cookie values.
|
||||
|
||||
If there are conflicting cookies, _find arbitrarily chooses one.
|
||||
See _find_no_duplicates if you want an exception thrown if there are
|
||||
conflicting cookies.
|
||||
|
||||
:param name: a string containing name of cookie
|
||||
:param domain: (optional) string containing domain of cookie
|
||||
:param path: (optional) string containing path of cookie
|
||||
:return: cookie.value
|
||||
"""
|
||||
for cookie in iter(self):
|
||||
if cookie.name == name:
|
||||
if domain is None or cookie.domain == domain:
|
||||
if path is None or cookie.path == path:
|
||||
return cookie.value
|
||||
|
||||
raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}")
|
||||
|
||||
def _find_no_duplicates(self, name, domain=None, path=None):
|
||||
"""Both ``__get_item__`` and ``get`` call this function: it's never
|
||||
used elsewhere in Requests.
|
||||
|
||||
:param name: a string containing name of cookie
|
||||
:param domain: (optional) string containing domain of cookie
|
||||
:param path: (optional) string containing path of cookie
|
||||
:raises KeyError: if cookie is not found
|
||||
:raises CookieConflictError: if there are multiple cookies
|
||||
that match name and optionally domain and path
|
||||
:return: cookie.value
|
||||
"""
|
||||
toReturn = None
|
||||
for cookie in iter(self):
|
||||
if cookie.name == name:
|
||||
if domain is None or cookie.domain == domain:
|
||||
if path is None or cookie.path == path:
|
||||
if toReturn is not None:
|
||||
# if there are multiple cookies that meet passed in criteria
|
||||
raise CookieConflictError(
|
||||
f"There are multiple cookies with name, {name!r}"
|
||||
)
|
||||
# we will eventually return this as long as no cookie conflict
|
||||
toReturn = cookie.value
|
||||
|
||||
if toReturn:
|
||||
return toReturn
|
||||
raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}")
|
||||
|
||||
def __getstate__(self):
|
||||
"""Unlike a normal CookieJar, this class is pickleable."""
|
||||
state = self.__dict__.copy()
|
||||
# remove the unpickleable RLock object
|
||||
state.pop("_cookies_lock")
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
"""Unlike a normal CookieJar, this class is pickleable."""
|
||||
self.__dict__.update(state)
|
||||
if "_cookies_lock" not in self.__dict__:
|
||||
self._cookies_lock = threading.RLock()
|
||||
|
||||
def copy(self):
|
||||
"""Return a copy of this RequestsCookieJar."""
|
||||
new_cj = RequestsCookieJar()
|
||||
new_cj.set_policy(self.get_policy())
|
||||
new_cj.update(self)
|
||||
return new_cj
|
||||
|
||||
def get_policy(self):
|
||||
"""Return the CookiePolicy instance used."""
|
||||
return self._policy
|
||||
|
||||
|
||||
def _copy_cookie_jar(jar):
|
||||
if jar is None:
|
||||
return None
|
||||
|
||||
if hasattr(jar, "copy"):
|
||||
# We're dealing with an instance of RequestsCookieJar
|
||||
return jar.copy()
|
||||
# We're dealing with a generic CookieJar instance
|
||||
new_jar = copy.copy(jar)
|
||||
new_jar.clear()
|
||||
for cookie in jar:
|
||||
new_jar.set_cookie(copy.copy(cookie))
|
||||
return new_jar
|
||||
|
||||
|
||||
def create_cookie(name, value, **kwargs):
|
||||
"""Make a cookie from underspecified parameters.
|
||||
|
||||
By default, the pair of `name` and `value` will be set for the domain ''
|
||||
and sent on every request (this is sometimes called a "supercookie").
|
||||
"""
|
||||
result = {
|
||||
"version": 0,
|
||||
"name": name,
|
||||
"value": value,
|
||||
"port": None,
|
||||
"domain": "",
|
||||
"path": "/",
|
||||
"secure": False,
|
||||
"expires": None,
|
||||
"discard": True,
|
||||
"comment": None,
|
||||
"comment_url": None,
|
||||
"rest": {"HttpOnly": None},
|
||||
"rfc2109": False,
|
||||
}
|
||||
|
||||
badargs = set(kwargs) - set(result)
|
||||
if badargs:
|
||||
raise TypeError(
|
||||
f"create_cookie() got unexpected keyword arguments: {list(badargs)}"
|
||||
)
|
||||
|
||||
result.update(kwargs)
|
||||
result["port_specified"] = bool(result["port"])
|
||||
result["domain_specified"] = bool(result["domain"])
|
||||
result["domain_initial_dot"] = result["domain"].startswith(".")
|
||||
result["path_specified"] = bool(result["path"])
|
||||
|
||||
return cookielib.Cookie(**result)
|
||||
|
||||
|
||||
def morsel_to_cookie(morsel):
|
||||
"""Convert a Morsel object into a Cookie containing the one k/v pair."""
|
||||
|
||||
expires = None
|
||||
if morsel["max-age"]:
|
||||
try:
|
||||
expires = int(time.time() + int(morsel["max-age"]))
|
||||
except ValueError:
|
||||
raise TypeError(f"max-age: {morsel['max-age']} must be integer")
|
||||
elif morsel["expires"]:
|
||||
time_template = "%a, %d-%b-%Y %H:%M:%S GMT"
|
||||
expires = calendar.timegm(time.strptime(morsel["expires"], time_template))
|
||||
return create_cookie(
|
||||
comment=morsel["comment"],
|
||||
comment_url=bool(morsel["comment"]),
|
||||
discard=False,
|
||||
domain=morsel["domain"],
|
||||
expires=expires,
|
||||
name=morsel.key,
|
||||
path=morsel["path"],
|
||||
port=None,
|
||||
rest={"HttpOnly": morsel["httponly"]},
|
||||
rfc2109=False,
|
||||
secure=bool(morsel["secure"]),
|
||||
value=morsel.value,
|
||||
version=morsel["version"] or 0,
|
||||
)
|
||||
|
||||
|
||||
def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True):
|
||||
"""Returns a CookieJar from a key/value dictionary.
|
||||
|
||||
:param cookie_dict: Dict of key/values to insert into CookieJar.
|
||||
:param cookiejar: (optional) A cookiejar to add the cookies to.
|
||||
:param overwrite: (optional) If False, will not replace cookies
|
||||
already in the jar with new ones.
|
||||
:rtype: CookieJar
|
||||
"""
|
||||
if cookiejar is None:
|
||||
cookiejar = RequestsCookieJar()
|
||||
|
||||
if cookie_dict is not None:
|
||||
names_from_jar = [cookie.name for cookie in cookiejar]
|
||||
for name in cookie_dict:
|
||||
if overwrite or (name not in names_from_jar):
|
||||
cookiejar.set_cookie(create_cookie(name, cookie_dict[name]))
|
||||
|
||||
return cookiejar
|
||||
|
||||
|
||||
def merge_cookies(cookiejar, cookies):
|
||||
"""Add cookies to cookiejar and returns a merged CookieJar.
|
||||
|
||||
:param cookiejar: CookieJar object to add the cookies to.
|
||||
:param cookies: Dictionary or CookieJar object to be added.
|
||||
:rtype: CookieJar
|
||||
"""
|
||||
if not isinstance(cookiejar, cookielib.CookieJar):
|
||||
raise ValueError("You can only merge into CookieJar")
|
||||
|
||||
if isinstance(cookies, dict):
|
||||
cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False)
|
||||
elif isinstance(cookies, cookielib.CookieJar):
|
||||
try:
|
||||
cookiejar.update(cookies)
|
||||
except AttributeError:
|
||||
for cookie_in_jar in cookies:
|
||||
cookiejar.set_cookie(cookie_in_jar)
|
||||
|
||||
return cookiejar
|
@ -1,141 +0,0 @@
|
||||
"""
|
||||
requests.exceptions
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
This module contains the set of Requests' exceptions.
|
||||
"""
|
||||
from urllib3.exceptions import HTTPError as BaseHTTPError
|
||||
|
||||
from .compat import JSONDecodeError as CompatJSONDecodeError
|
||||
|
||||
|
||||
class RequestException(IOError):
|
||||
"""There was an ambiguous exception that occurred while handling your
|
||||
request.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""Initialize RequestException with `request` and `response` objects."""
|
||||
response = kwargs.pop("response", None)
|
||||
self.response = response
|
||||
self.request = kwargs.pop("request", None)
|
||||
if response is not None and not self.request and hasattr(response, "request"):
|
||||
self.request = self.response.request
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class InvalidJSONError(RequestException):
|
||||
"""A JSON error occurred."""
|
||||
|
||||
|
||||
class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError):
|
||||
"""Couldn't decode the text into json"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
Construct the JSONDecodeError instance first with all
|
||||
args. Then use it's args to construct the IOError so that
|
||||
the json specific args aren't used as IOError specific args
|
||||
and the error message from JSONDecodeError is preserved.
|
||||
"""
|
||||
CompatJSONDecodeError.__init__(self, *args)
|
||||
InvalidJSONError.__init__(self, *self.args, **kwargs)
|
||||
|
||||
|
||||
class HTTPError(RequestException):
|
||||
"""An HTTP error occurred."""
|
||||
|
||||
|
||||
class ConnectionError(RequestException):
|
||||
"""A Connection error occurred."""
|
||||
|
||||
|
||||
class ProxyError(ConnectionError):
|
||||
"""A proxy error occurred."""
|
||||
|
||||
|
||||
class SSLError(ConnectionError):
|
||||
"""An SSL error occurred."""
|
||||
|
||||
|
||||
class Timeout(RequestException):
|
||||
"""The request timed out.
|
||||
|
||||
Catching this error will catch both
|
||||
:exc:`~requests.exceptions.ConnectTimeout` and
|
||||
:exc:`~requests.exceptions.ReadTimeout` errors.
|
||||
"""
|
||||
|
||||
|
||||
class ConnectTimeout(ConnectionError, Timeout):
|
||||
"""The request timed out while trying to connect to the remote server.
|
||||
|
||||
Requests that produced this error are safe to retry.
|
||||
"""
|
||||
|
||||
|
||||
class ReadTimeout(Timeout):
|
||||
"""The server did not send any data in the allotted amount of time."""
|
||||
|
||||
|
||||
class URLRequired(RequestException):
|
||||
"""A valid URL is required to make a request."""
|
||||
|
||||
|
||||
class TooManyRedirects(RequestException):
|
||||
"""Too many redirects."""
|
||||
|
||||
|
||||
class MissingSchema(RequestException, ValueError):
|
||||
"""The URL scheme (e.g. http or https) is missing."""
|
||||
|
||||
|
||||
class InvalidSchema(RequestException, ValueError):
|
||||
"""The URL scheme provided is either invalid or unsupported."""
|
||||
|
||||
|
||||
class InvalidURL(RequestException, ValueError):
|
||||
"""The URL provided was somehow invalid."""
|
||||
|
||||
|
||||
class InvalidHeader(RequestException, ValueError):
|
||||
"""The header value provided was somehow invalid."""
|
||||
|
||||
|
||||
class InvalidProxyURL(InvalidURL):
|
||||
"""The proxy URL provided is invalid."""
|
||||
|
||||
|
||||
class ChunkedEncodingError(RequestException):
|
||||
"""The server declared chunked encoding but sent an invalid chunk."""
|
||||
|
||||
|
||||
class ContentDecodingError(RequestException, BaseHTTPError):
|
||||
"""Failed to decode response content."""
|
||||
|
||||
|
||||
class StreamConsumedError(RequestException, TypeError):
|
||||
"""The content for this response was already consumed."""
|
||||
|
||||
|
||||
class RetryError(RequestException):
|
||||
"""Custom retries logic failed"""
|
||||
|
||||
|
||||
class UnrewindableBodyError(RequestException):
|
||||
"""Requests encountered an error when trying to rewind a body."""
|
||||
|
||||
|
||||
# Warnings
|
||||
|
||||
|
||||
class RequestsWarning(Warning):
|
||||
"""Base warning for Requests."""
|
||||
|
||||
|
||||
class FileModeWarning(RequestsWarning, DeprecationWarning):
|
||||
"""A file was opened in text mode, but Requests determined its binary length."""
|
||||
|
||||
|
||||
class RequestsDependencyWarning(RequestsWarning):
|
||||
"""An imported dependency doesn't match the expected version range."""
|
@ -1,134 +0,0 @@
|
||||
"""Module containing bug report helper(s)."""
|
||||
|
||||
import json
|
||||
import platform
|
||||
import ssl
|
||||
import sys
|
||||
|
||||
import idna
|
||||
import urllib3
|
||||
|
||||
from . import __version__ as requests_version
|
||||
|
||||
try:
|
||||
import charset_normalizer
|
||||
except ImportError:
|
||||
charset_normalizer = None
|
||||
|
||||
try:
|
||||
import chardet
|
||||
except ImportError:
|
||||
chardet = None
|
||||
|
||||
try:
|
||||
from urllib3.contrib import pyopenssl
|
||||
except ImportError:
|
||||
pyopenssl = None
|
||||
OpenSSL = None
|
||||
cryptography = None
|
||||
else:
|
||||
import cryptography
|
||||
import OpenSSL
|
||||
|
||||
|
||||
def _implementation():
|
||||
"""Return a dict with the Python implementation and version.
|
||||
|
||||
Provide both the name and the version of the Python implementation
|
||||
currently running. For example, on CPython 3.10.3 it will return
|
||||
{'name': 'CPython', 'version': '3.10.3'}.
|
||||
|
||||
This function works best on CPython and PyPy: in particular, it probably
|
||||
doesn't work for Jython or IronPython. Future investigation should be done
|
||||
to work out the correct shape of the code for those platforms.
|
||||
"""
|
||||
implementation = platform.python_implementation()
|
||||
|
||||
if implementation == "CPython":
|
||||
implementation_version = platform.python_version()
|
||||
elif implementation == "PyPy":
|
||||
implementation_version = "{}.{}.{}".format(
|
||||
sys.pypy_version_info.major,
|
||||
sys.pypy_version_info.minor,
|
||||
sys.pypy_version_info.micro,
|
||||
)
|
||||
if sys.pypy_version_info.releaselevel != "final":
|
||||
implementation_version = "".join(
|
||||
[implementation_version, sys.pypy_version_info.releaselevel]
|
||||
)
|
||||
elif implementation == "Jython":
|
||||
implementation_version = platform.python_version() # Complete Guess
|
||||
elif implementation == "IronPython":
|
||||
implementation_version = platform.python_version() # Complete Guess
|
||||
else:
|
||||
implementation_version = "Unknown"
|
||||
|
||||
return {"name": implementation, "version": implementation_version}
|
||||
|
||||
|
||||
def info():
|
||||
"""Generate information for a bug report."""
|
||||
try:
|
||||
platform_info = {
|
||||
"system": platform.system(),
|
||||
"release": platform.release(),
|
||||
}
|
||||
except OSError:
|
||||
platform_info = {
|
||||
"system": "Unknown",
|
||||
"release": "Unknown",
|
||||
}
|
||||
|
||||
implementation_info = _implementation()
|
||||
urllib3_info = {"version": urllib3.__version__}
|
||||
charset_normalizer_info = {"version": None}
|
||||
chardet_info = {"version": None}
|
||||
if charset_normalizer:
|
||||
charset_normalizer_info = {"version": charset_normalizer.__version__}
|
||||
if chardet:
|
||||
chardet_info = {"version": chardet.__version__}
|
||||
|
||||
pyopenssl_info = {
|
||||
"version": None,
|
||||
"openssl_version": "",
|
||||
}
|
||||
if OpenSSL:
|
||||
pyopenssl_info = {
|
||||
"version": OpenSSL.__version__,
|
||||
"openssl_version": f"{OpenSSL.SSL.OPENSSL_VERSION_NUMBER:x}",
|
||||
}
|
||||
cryptography_info = {
|
||||
"version": getattr(cryptography, "__version__", ""),
|
||||
}
|
||||
idna_info = {
|
||||
"version": getattr(idna, "__version__", ""),
|
||||
}
|
||||
|
||||
system_ssl = ssl.OPENSSL_VERSION_NUMBER
|
||||
system_ssl_info = {"version": f"{system_ssl:x}" if system_ssl is not None else ""}
|
||||
|
||||
return {
|
||||
"platform": platform_info,
|
||||
"implementation": implementation_info,
|
||||
"system_ssl": system_ssl_info,
|
||||
"using_pyopenssl": pyopenssl is not None,
|
||||
"using_charset_normalizer": chardet is None,
|
||||
"pyOpenSSL": pyopenssl_info,
|
||||
"urllib3": urllib3_info,
|
||||
"chardet": chardet_info,
|
||||
"charset_normalizer": charset_normalizer_info,
|
||||
"cryptography": cryptography_info,
|
||||
"idna": idna_info,
|
||||
"requests": {
|
||||
"version": requests_version,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Pretty-print the bug information as JSON."""
|
||||
print(json.dumps(info(), sort_keys=True, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
@ -1,33 +0,0 @@
|
||||
"""
|
||||
requests.hooks
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
This module provides the capabilities for the Requests hooks system.
|
||||
|
||||
Available hooks:
|
||||
|
||||
``response``:
|
||||
The response generated from a Request.
|
||||
"""
|
||||
HOOKS = ["response"]
|
||||
|
||||
|
||||
def default_hooks():
|
||||
return {event: [] for event in HOOKS}
|
||||
|
||||
|
||||
# TODO: response is the only one
|
||||
|
||||
|
||||
def dispatch_hook(key, hooks, hook_data, **kwargs):
|
||||
"""Dispatches a hook dictionary on a given piece of data."""
|
||||
hooks = hooks or {}
|
||||
hooks = hooks.get(key)
|
||||
if hooks:
|
||||
if hasattr(hooks, "__call__"):
|
||||
hooks = [hooks]
|
||||
for hook in hooks:
|
||||
_hook_data = hook(hook_data, **kwargs)
|
||||
if _hook_data is not None:
|
||||
hook_data = _hook_data
|
||||
return hook_data
|
File diff suppressed because it is too large
Load Diff
@ -1,28 +0,0 @@
|
||||
import sys
|
||||
|
||||
try:
|
||||
import chardet
|
||||
except ImportError:
|
||||
import warnings
|
||||
|
||||
import charset_normalizer as chardet
|
||||
|
||||
warnings.filterwarnings("ignore", "Trying to detect", module="charset_normalizer")
|
||||
|
||||
# This code exists for backwards compatibility reasons.
|
||||
# I don't like it either. Just look the other way. :)
|
||||
|
||||
for package in ("urllib3", "idna"):
|
||||
locals()[package] = __import__(package)
|
||||
# This traversal is apparently necessary such that the identities are
|
||||
# preserved (requests.packages.urllib3.* is urllib3.*)
|
||||
for mod in list(sys.modules):
|
||||
if mod == package or mod.startswith(f"{package}."):
|
||||
sys.modules[f"requests.packages.{mod}"] = sys.modules[mod]
|
||||
|
||||
target = chardet.__name__
|
||||
for mod in list(sys.modules):
|
||||
if mod == target or mod.startswith(f"{target}."):
|
||||
target = target.replace(target, "chardet")
|
||||
sys.modules[f"requests.packages.{target}"] = sys.modules[mod]
|
||||
# Kinda cool, though, right?
|
@ -1,833 +0,0 @@
|
||||
"""
|
||||
requests.sessions
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
This module provides a Session object to manage and persist settings across
|
||||
requests (cookies, auth, proxies).
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from datetime import timedelta
|
||||
|
||||
from ._internal_utils import to_native_string
|
||||
from .adapters import HTTPAdapter
|
||||
from .auth import _basic_auth_str
|
||||
from .compat import Mapping, cookielib, urljoin, urlparse
|
||||
from .cookies import (
|
||||
RequestsCookieJar,
|
||||
cookiejar_from_dict,
|
||||
extract_cookies_to_jar,
|
||||
merge_cookies,
|
||||
)
|
||||
from .exceptions import (
|
||||
ChunkedEncodingError,
|
||||
ContentDecodingError,
|
||||
InvalidSchema,
|
||||
TooManyRedirects,
|
||||
)
|
||||
from .hooks import default_hooks, dispatch_hook
|
||||
|
||||
# formerly defined here, reexposed here for backward compatibility
|
||||
from .models import ( # noqa: F401
|
||||
DEFAULT_REDIRECT_LIMIT,
|
||||
REDIRECT_STATI,
|
||||
PreparedRequest,
|
||||
Request,
|
||||
)
|
||||
from .status_codes import codes
|
||||
from .structures import CaseInsensitiveDict
|
||||
from .utils import ( # noqa: F401
|
||||
DEFAULT_PORTS,
|
||||
default_headers,
|
||||
get_auth_from_url,
|
||||
get_environ_proxies,
|
||||
get_netrc_auth,
|
||||
requote_uri,
|
||||
resolve_proxies,
|
||||
rewind_body,
|
||||
should_bypass_proxies,
|
||||
to_key_val_list,
|
||||
)
|
||||
|
||||
# Preferred clock, based on which one is more accurate on a given system.
|
||||
if sys.platform == "win32":
|
||||
preferred_clock = time.perf_counter
|
||||
else:
|
||||
preferred_clock = time.time
|
||||
|
||||
|
||||
def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
|
||||
"""Determines appropriate setting for a given request, taking into account
|
||||
the explicit setting on that request, and the setting in the session. If a
|
||||
setting is a dictionary, they will be merged together using `dict_class`
|
||||
"""
|
||||
|
||||
if session_setting is None:
|
||||
return request_setting
|
||||
|
||||
if request_setting is None:
|
||||
return session_setting
|
||||
|
||||
# Bypass if not a dictionary (e.g. verify)
|
||||
if not (
|
||||
isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping)
|
||||
):
|
||||
return request_setting
|
||||
|
||||
merged_setting = dict_class(to_key_val_list(session_setting))
|
||||
merged_setting.update(to_key_val_list(request_setting))
|
||||
|
||||
# Remove keys that are set to None. Extract keys first to avoid altering
|
||||
# the dictionary during iteration.
|
||||
none_keys = [k for (k, v) in merged_setting.items() if v is None]
|
||||
for key in none_keys:
|
||||
del merged_setting[key]
|
||||
|
||||
return merged_setting
|
||||
|
||||
|
||||
def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict):
|
||||
"""Properly merges both requests and session hooks.
|
||||
|
||||
This is necessary because when request_hooks == {'response': []}, the
|
||||
merge breaks Session hooks entirely.
|
||||
"""
|
||||
if session_hooks is None or session_hooks.get("response") == []:
|
||||
return request_hooks
|
||||
|
||||
if request_hooks is None or request_hooks.get("response") == []:
|
||||
return session_hooks
|
||||
|
||||
return merge_setting(request_hooks, session_hooks, dict_class)
|
||||
|
||||
|
||||
class SessionRedirectMixin:
|
||||
def get_redirect_target(self, resp):
|
||||
"""Receives a Response. Returns a redirect URI or ``None``"""
|
||||
# Due to the nature of how requests processes redirects this method will
|
||||
# be called at least once upon the original response and at least twice
|
||||
# on each subsequent redirect response (if any).
|
||||
# If a custom mixin is used to handle this logic, it may be advantageous
|
||||
# to cache the redirect location onto the response object as a private
|
||||
# attribute.
|
||||
if resp.is_redirect:
|
||||
location = resp.headers["location"]
|
||||
# Currently the underlying http module on py3 decode headers
|
||||
# in latin1, but empirical evidence suggests that latin1 is very
|
||||
# rarely used with non-ASCII characters in HTTP headers.
|
||||
# It is more likely to get UTF8 header rather than latin1.
|
||||
# This causes incorrect handling of UTF8 encoded location headers.
|
||||
# To solve this, we re-encode the location in latin1.
|
||||
location = location.encode("latin1")
|
||||
return to_native_string(location, "utf8")
|
||||
return None
|
||||
|
||||
def should_strip_auth(self, old_url, new_url):
|
||||
"""Decide whether Authorization header should be removed when redirecting"""
|
||||
old_parsed = urlparse(old_url)
|
||||
new_parsed = urlparse(new_url)
|
||||
if old_parsed.hostname != new_parsed.hostname:
|
||||
return True
|
||||
# Special case: allow http -> https redirect when using the standard
|
||||
# ports. This isn't specified by RFC 7235, but is kept to avoid
|
||||
# breaking backwards compatibility with older versions of requests
|
||||
# that allowed any redirects on the same host.
|
||||
if (
|
||||
old_parsed.scheme == "http"
|
||||
and old_parsed.port in (80, None)
|
||||
and new_parsed.scheme == "https"
|
||||
and new_parsed.port in (443, None)
|
||||
):
|
||||
return False
|
||||
|
||||
# Handle default port usage corresponding to scheme.
|
||||
changed_port = old_parsed.port != new_parsed.port
|
||||
changed_scheme = old_parsed.scheme != new_parsed.scheme
|
||||
default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None)
|
||||
if (
|
||||
not changed_scheme
|
||||
and old_parsed.port in default_port
|
||||
and new_parsed.port in default_port
|
||||
):
|
||||
return False
|
||||
|
||||
# Standard case: root URI must match
|
||||
return changed_port or changed_scheme
|
||||
|
||||
def resolve_redirects(
|
||||
self,
|
||||
resp,
|
||||
req,
|
||||
stream=False,
|
||||
timeout=None,
|
||||
verify=True,
|
||||
cert=None,
|
||||
proxies=None,
|
||||
yield_requests=False,
|
||||
**adapter_kwargs,
|
||||
):
|
||||
"""Receives a Response. Returns a generator of Responses or Requests."""
|
||||
|
||||
hist = [] # keep track of history
|
||||
|
||||
url = self.get_redirect_target(resp)
|
||||
previous_fragment = urlparse(req.url).fragment
|
||||
while url:
|
||||
prepared_request = req.copy()
|
||||
|
||||
# Update history and keep track of redirects.
|
||||
# resp.history must ignore the original request in this loop
|
||||
hist.append(resp)
|
||||
resp.history = hist[1:]
|
||||
|
||||
try:
|
||||
resp.content # Consume socket so it can be released
|
||||
except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
|
||||
resp.raw.read(decode_content=False)
|
||||
|
||||
if len(resp.history) >= self.max_redirects:
|
||||
raise TooManyRedirects(
|
||||
f"Exceeded {self.max_redirects} redirects.", response=resp
|
||||
)
|
||||
|
||||
# Release the connection back into the pool.
|
||||
resp.close()
|
||||
|
||||
# Handle redirection without scheme (see: RFC 1808 Section 4)
|
||||
if url.startswith("//"):
|
||||
parsed_rurl = urlparse(resp.url)
|
||||
url = ":".join([to_native_string(parsed_rurl.scheme), url])
|
||||
|
||||
# Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2)
|
||||
parsed = urlparse(url)
|
||||
if parsed.fragment == "" and previous_fragment:
|
||||
parsed = parsed._replace(fragment=previous_fragment)
|
||||
elif parsed.fragment:
|
||||
previous_fragment = parsed.fragment
|
||||
url = parsed.geturl()
|
||||
|
||||
# Facilitate relative 'location' headers, as allowed by RFC 7231.
|
||||
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
|
||||
# Compliant with RFC3986, we percent encode the url.
|
||||
if not parsed.netloc:
|
||||
url = urljoin(resp.url, requote_uri(url))
|
||||
else:
|
||||
url = requote_uri(url)
|
||||
|
||||
prepared_request.url = to_native_string(url)
|
||||
|
||||
self.rebuild_method(prepared_request, resp)
|
||||
|
||||
# https://github.com/psf/requests/issues/1084
|
||||
if resp.status_code not in (
|
||||
codes.temporary_redirect,
|
||||
codes.permanent_redirect,
|
||||
):
|
||||
# https://github.com/psf/requests/issues/3490
|
||||
purged_headers = ("Content-Length", "Content-Type", "Transfer-Encoding")
|
||||
for header in purged_headers:
|
||||
prepared_request.headers.pop(header, None)
|
||||
prepared_request.body = None
|
||||
|
||||
headers = prepared_request.headers
|
||||
headers.pop("Cookie", None)
|
||||
|
||||
# Extract any cookies sent on the response to the cookiejar
|
||||
# in the new request. Because we've mutated our copied prepared
|
||||
# request, use the old one that we haven't yet touched.
|
||||
extract_cookies_to_jar(prepared_request._cookies, req, resp.raw)
|
||||
merge_cookies(prepared_request._cookies, self.cookies)
|
||||
prepared_request.prepare_cookies(prepared_request._cookies)
|
||||
|
||||
# Rebuild auth and proxy information.
|
||||
proxies = self.rebuild_proxies(prepared_request, proxies)
|
||||
self.rebuild_auth(prepared_request, resp)
|
||||
|
||||
# A failed tell() sets `_body_position` to `object()`. This non-None
|
||||
# value ensures `rewindable` will be True, allowing us to raise an
|
||||
# UnrewindableBodyError, instead of hanging the connection.
|
||||
rewindable = prepared_request._body_position is not None and (
|
||||
"Content-Length" in headers or "Transfer-Encoding" in headers
|
||||
)
|
||||
|
||||
# Attempt to rewind consumed file-like object.
|
||||
if rewindable:
|
||||
rewind_body(prepared_request)
|
||||
|
||||
# Override the original request.
|
||||
req = prepared_request
|
||||
|
||||
if yield_requests:
|
||||
yield req
|
||||
else:
|
||||
|
||||
resp = self.send(
|
||||
req,
|
||||
stream=stream,
|
||||
timeout=timeout,
|
||||
verify=verify,
|
||||
cert=cert,
|
||||
proxies=proxies,
|
||||
allow_redirects=False,
|
||||
**adapter_kwargs,
|
||||
)
|
||||
|
||||
extract_cookies_to_jar(self.cookies, prepared_request, resp.raw)
|
||||
|
||||
# extract redirect url, if any, for the next loop
|
||||
url = self.get_redirect_target(resp)
|
||||
yield resp
|
||||
|
||||
def rebuild_auth(self, prepared_request, response):
|
||||
"""When being redirected we may want to strip authentication from the
|
||||
request to avoid leaking credentials. This method intelligently removes
|
||||
and reapplies authentication where possible to avoid credential loss.
|
||||
"""
|
||||
headers = prepared_request.headers
|
||||
url = prepared_request.url
|
||||
|
||||
if "Authorization" in headers and self.should_strip_auth(
|
||||
response.request.url, url
|
||||
):
|
||||
# If we get redirected to a new host, we should strip out any
|
||||
# authentication headers.
|
||||
del headers["Authorization"]
|
||||
|
||||
# .netrc might have more auth for us on our new host.
|
||||
new_auth = get_netrc_auth(url) if self.trust_env else None
|
||||
if new_auth is not None:
|
||||
prepared_request.prepare_auth(new_auth)
|
||||
|
||||
def rebuild_proxies(self, prepared_request, proxies):
|
||||
"""This method re-evaluates the proxy configuration by considering the
|
||||
environment variables. If we are redirected to a URL covered by
|
||||
NO_PROXY, we strip the proxy configuration. Otherwise, we set missing
|
||||
proxy keys for this URL (in case they were stripped by a previous
|
||||
redirect).
|
||||
|
||||
This method also replaces the Proxy-Authorization header where
|
||||
necessary.
|
||||
|
||||
:rtype: dict
|
||||
"""
|
||||
headers = prepared_request.headers
|
||||
scheme = urlparse(prepared_request.url).scheme
|
||||
new_proxies = resolve_proxies(prepared_request, proxies, self.trust_env)
|
||||
|
||||
if "Proxy-Authorization" in headers:
|
||||
del headers["Proxy-Authorization"]
|
||||
|
||||
try:
|
||||
username, password = get_auth_from_url(new_proxies[scheme])
|
||||
except KeyError:
|
||||
username, password = None, None
|
||||
|
||||
# urllib3 handles proxy authorization for us in the standard adapter.
|
||||
# Avoid appending this to TLS tunneled requests where it may be leaked.
|
||||
if not scheme.startswith('https') and username and password:
|
||||
headers["Proxy-Authorization"] = _basic_auth_str(username, password)
|
||||
|
||||
return new_proxies
|
||||
|
||||
def rebuild_method(self, prepared_request, response):
|
||||
"""When being redirected we may want to change the method of the request
|
||||
based on certain specs or browser behavior.
|
||||
"""
|
||||
method = prepared_request.method
|
||||
|
||||
# https://tools.ietf.org/html/rfc7231#section-6.4.4
|
||||
if response.status_code == codes.see_other and method != "HEAD":
|
||||
method = "GET"
|
||||
|
||||
# Do what the browsers do, despite standards...
|
||||
# First, turn 302s into GETs.
|
||||
if response.status_code == codes.found and method != "HEAD":
|
||||
method = "GET"
|
||||
|
||||
# Second, if a POST is responded to with a 301, turn it into a GET.
|
||||
# This bizarre behaviour is explained in Issue 1704.
|
||||
if response.status_code == codes.moved and method == "POST":
|
||||
method = "GET"
|
||||
|
||||
prepared_request.method = method
|
||||
|
||||
|
||||
class Session(SessionRedirectMixin):
|
||||
"""A Requests session.
|
||||
|
||||
Provides cookie persistence, connection-pooling, and configuration.
|
||||
|
||||
Basic Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> s = requests.Session()
|
||||
>>> s.get('https://httpbin.org/get')
|
||||
<Response [200]>
|
||||
|
||||
Or as a context manager::
|
||||
|
||||
>>> with requests.Session() as s:
|
||||
... s.get('https://httpbin.org/get')
|
||||
<Response [200]>
|
||||
"""
|
||||
|
||||
__attrs__ = [
|
||||
"headers",
|
||||
"cookies",
|
||||
"auth",
|
||||
"proxies",
|
||||
"hooks",
|
||||
"params",
|
||||
"verify",
|
||||
"cert",
|
||||
"adapters",
|
||||
"stream",
|
||||
"trust_env",
|
||||
"max_redirects",
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
|
||||
#: A case-insensitive dictionary of headers to be sent on each
|
||||
#: :class:`Request <Request>` sent from this
|
||||
#: :class:`Session <Session>`.
|
||||
self.headers = default_headers()
|
||||
|
||||
#: Default Authentication tuple or object to attach to
|
||||
#: :class:`Request <Request>`.
|
||||
self.auth = None
|
||||
|
||||
#: Dictionary mapping protocol or protocol and host to the URL of the proxy
|
||||
#: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to
|
||||
#: be used on each :class:`Request <Request>`.
|
||||
self.proxies = {}
|
||||
|
||||
#: Event-handling hooks.
|
||||
self.hooks = default_hooks()
|
||||
|
||||
#: Dictionary of querystring data to attach to each
|
||||
#: :class:`Request <Request>`. The dictionary values may be lists for
|
||||
#: representing multivalued query parameters.
|
||||
self.params = {}
|
||||
|
||||
#: Stream response content default.
|
||||
self.stream = False
|
||||
|
||||
#: SSL Verification default.
|
||||
#: Defaults to `True`, requiring requests to verify the TLS certificate at the
|
||||
#: remote end.
|
||||
#: If verify is set to `False`, requests will accept any TLS certificate
|
||||
#: presented by the server, and will ignore hostname mismatches and/or
|
||||
#: expired certificates, which will make your application vulnerable to
|
||||
#: man-in-the-middle (MitM) attacks.
|
||||
#: Only set this to `False` for testing.
|
||||
self.verify = True
|
||||
|
||||
#: SSL client certificate default, if String, path to ssl client
|
||||
#: cert file (.pem). If Tuple, ('cert', 'key') pair.
|
||||
self.cert = None
|
||||
|
||||
#: Maximum number of redirects allowed. If the request exceeds this
|
||||
#: limit, a :class:`TooManyRedirects` exception is raised.
|
||||
#: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is
|
||||
#: 30.
|
||||
self.max_redirects = DEFAULT_REDIRECT_LIMIT
|
||||
|
||||
#: Trust environment settings for proxy configuration, default
|
||||
#: authentication and similar.
|
||||
self.trust_env = True
|
||||
|
||||
#: A CookieJar containing all currently outstanding cookies set on this
|
||||
#: session. By default it is a
|
||||
#: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but
|
||||
#: may be any other ``cookielib.CookieJar`` compatible object.
|
||||
self.cookies = cookiejar_from_dict({})
|
||||
|
||||
# Default connection adapters.
|
||||
self.adapters = OrderedDict()
|
||||
self.mount("https://", HTTPAdapter())
|
||||
self.mount("http://", HTTPAdapter())
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.close()
|
||||
|
||||
def prepare_request(self, request):
|
||||
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for
|
||||
transmission and returns it. The :class:`PreparedRequest` has settings
|
||||
merged from the :class:`Request <Request>` instance and those of the
|
||||
:class:`Session`.
|
||||
|
||||
:param request: :class:`Request` instance to prepare with this
|
||||
session's settings.
|
||||
:rtype: requests.PreparedRequest
|
||||
"""
|
||||
cookies = request.cookies or {}
|
||||
|
||||
# Bootstrap CookieJar.
|
||||
if not isinstance(cookies, cookielib.CookieJar):
|
||||
cookies = cookiejar_from_dict(cookies)
|
||||
|
||||
# Merge with session cookies
|
||||
merged_cookies = merge_cookies(
|
||||
merge_cookies(RequestsCookieJar(), self.cookies), cookies
|
||||
)
|
||||
|
||||
# Set environment's basic authentication if not explicitly set.
|
||||
auth = request.auth
|
||||
if self.trust_env and not auth and not self.auth:
|
||||
auth = get_netrc_auth(request.url)
|
||||
|
||||
p = PreparedRequest()
|
||||
p.prepare(
|
||||
method=request.method.upper(),
|
||||
url=request.url,
|
||||
files=request.files,
|
||||
data=request.data,
|
||||
json=request.json,
|
||||
headers=merge_setting(
|
||||
request.headers, self.headers, dict_class=CaseInsensitiveDict
|
||||
),
|
||||
params=merge_setting(request.params, self.params),
|
||||
auth=merge_setting(auth, self.auth),
|
||||
cookies=merged_cookies,
|
||||
hooks=merge_hooks(request.hooks, self.hooks),
|
||||
)
|
||||
return p
|
||||
|
||||
def request(
|
||||
self,
|
||||
method,
|
||||
url,
|
||||
params=None,
|
||||
data=None,
|
||||
headers=None,
|
||||
cookies=None,
|
||||
files=None,
|
||||
auth=None,
|
||||
timeout=None,
|
||||
allow_redirects=True,
|
||||
proxies=None,
|
||||
hooks=None,
|
||||
stream=None,
|
||||
verify=None,
|
||||
cert=None,
|
||||
json=None,
|
||||
):
|
||||
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
|
||||
Returns :class:`Response <Response>` object.
|
||||
|
||||
:param method: method for the new :class:`Request` object.
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary or bytes to be sent in the query
|
||||
string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json to send in the body of the
|
||||
:class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the
|
||||
:class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the
|
||||
:class:`Request`.
|
||||
:param files: (optional) Dictionary of ``'filename': file-like-objects``
|
||||
for multipart encoding upload.
|
||||
:param auth: (optional) Auth tuple or callable to enable
|
||||
Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) How long to wait for the server to send
|
||||
data before giving up, as a float, or a :ref:`(connect timeout,
|
||||
read timeout) <timeouts>` tuple.
|
||||
:type timeout: float or tuple
|
||||
:param allow_redirects: (optional) Set to True by default.
|
||||
:type allow_redirects: bool
|
||||
:param proxies: (optional) Dictionary mapping protocol or protocol and
|
||||
hostname to the URL of the proxy.
|
||||
:param stream: (optional) whether to immediately download the response
|
||||
content. Defaults to ``False``.
|
||||
:param verify: (optional) Either a boolean, in which case it controls whether we verify
|
||||
the server's TLS certificate, or a string, in which case it must be a path
|
||||
to a CA bundle to use. Defaults to ``True``. When set to
|
||||
``False``, requests will accept any TLS certificate presented by
|
||||
the server, and will ignore hostname mismatches and/or expired
|
||||
certificates, which will make your application vulnerable to
|
||||
man-in-the-middle (MitM) attacks. Setting verify to ``False``
|
||||
may be useful during local development or testing.
|
||||
:param cert: (optional) if String, path to ssl client cert file (.pem).
|
||||
If Tuple, ('cert', 'key') pair.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
# Create the Request.
|
||||
req = Request(
|
||||
method=method.upper(),
|
||||
url=url,
|
||||
headers=headers,
|
||||
files=files,
|
||||
data=data or {},
|
||||
json=json,
|
||||
params=params or {},
|
||||
auth=auth,
|
||||
cookies=cookies,
|
||||
hooks=hooks,
|
||||
)
|
||||
prep = self.prepare_request(req)
|
||||
|
||||
proxies = proxies or {}
|
||||
|
||||
settings = self.merge_environment_settings(
|
||||
prep.url, proxies, stream, verify, cert
|
||||
)
|
||||
|
||||
# Send the request.
|
||||
send_kwargs = {
|
||||
"timeout": timeout,
|
||||
"allow_redirects": allow_redirects,
|
||||
}
|
||||
send_kwargs.update(settings)
|
||||
resp = self.send(prep, **send_kwargs)
|
||||
|
||||
return resp
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
r"""Sends a GET request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault("allow_redirects", True)
|
||||
return self.request("GET", url, **kwargs)
|
||||
|
||||
def options(self, url, **kwargs):
|
||||
r"""Sends a OPTIONS request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault("allow_redirects", True)
|
||||
return self.request("OPTIONS", url, **kwargs)
|
||||
|
||||
def head(self, url, **kwargs):
|
||||
r"""Sends a HEAD request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault("allow_redirects", False)
|
||||
return self.request("HEAD", url, **kwargs)
|
||||
|
||||
def post(self, url, data=None, json=None, **kwargs):
|
||||
r"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request("POST", url, data=data, json=json, **kwargs)
|
||||
|
||||
def put(self, url, data=None, **kwargs):
|
||||
r"""Sends a PUT request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request("PUT", url, data=data, **kwargs)
|
||||
|
||||
def patch(self, url, data=None, **kwargs):
|
||||
r"""Sends a PATCH request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request("PATCH", url, data=data, **kwargs)
|
||||
|
||||
def delete(self, url, **kwargs):
|
||||
r"""Sends a DELETE request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request("DELETE", url, **kwargs)
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
"""Send a given PreparedRequest.
|
||||
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
# Set defaults that the hooks can utilize to ensure they always have
|
||||
# the correct parameters to reproduce the previous request.
|
||||
kwargs.setdefault("stream", self.stream)
|
||||
kwargs.setdefault("verify", self.verify)
|
||||
kwargs.setdefault("cert", self.cert)
|
||||
if "proxies" not in kwargs:
|
||||
kwargs["proxies"] = resolve_proxies(request, self.proxies, self.trust_env)
|
||||
|
||||
# It's possible that users might accidentally send a Request object.
|
||||
# Guard against that specific failure case.
|
||||
if isinstance(request, Request):
|
||||
raise ValueError("You can only send PreparedRequests.")
|
||||
|
||||
# Set up variables needed for resolve_redirects and dispatching of hooks
|
||||
allow_redirects = kwargs.pop("allow_redirects", True)
|
||||
stream = kwargs.get("stream")
|
||||
hooks = request.hooks
|
||||
|
||||
# Get the appropriate adapter to use
|
||||
adapter = self.get_adapter(url=request.url)
|
||||
|
||||
# Start time (approximately) of the request
|
||||
start = preferred_clock()
|
||||
|
||||
# Send the request
|
||||
r = adapter.send(request, **kwargs)
|
||||
|
||||
# Total elapsed time of the request (approximately)
|
||||
elapsed = preferred_clock() - start
|
||||
r.elapsed = timedelta(seconds=elapsed)
|
||||
|
||||
# Response manipulation hooks
|
||||
r = dispatch_hook("response", hooks, r, **kwargs)
|
||||
|
||||
# Persist cookies
|
||||
if r.history:
|
||||
|
||||
# If the hooks create history then we want those cookies too
|
||||
for resp in r.history:
|
||||
extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
|
||||
|
||||
extract_cookies_to_jar(self.cookies, request, r.raw)
|
||||
|
||||
# Resolve redirects if allowed.
|
||||
if allow_redirects:
|
||||
# Redirect resolving generator.
|
||||
gen = self.resolve_redirects(r, request, **kwargs)
|
||||
history = [resp for resp in gen]
|
||||
else:
|
||||
history = []
|
||||
|
||||
# Shuffle things around if there's history.
|
||||
if history:
|
||||
# Insert the first (original) request at the start
|
||||
history.insert(0, r)
|
||||
# Get the last request made
|
||||
r = history.pop()
|
||||
r.history = history
|
||||
|
||||
# If redirects aren't being followed, store the response on the Request for Response.next().
|
||||
if not allow_redirects:
|
||||
try:
|
||||
r._next = next(
|
||||
self.resolve_redirects(r, request, yield_requests=True, **kwargs)
|
||||
)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
if not stream:
|
||||
r.content
|
||||
|
||||
return r
|
||||
|
||||
def merge_environment_settings(self, url, proxies, stream, verify, cert):
|
||||
"""
|
||||
Check the environment and merge it with some settings.
|
||||
|
||||
:rtype: dict
|
||||
"""
|
||||
# Gather clues from the surrounding environment.
|
||||
if self.trust_env:
|
||||
# Set environment's proxies.
|
||||
no_proxy = proxies.get("no_proxy") if proxies is not None else None
|
||||
env_proxies = get_environ_proxies(url, no_proxy=no_proxy)
|
||||
for (k, v) in env_proxies.items():
|
||||
proxies.setdefault(k, v)
|
||||
|
||||
# Look for requests environment configuration
|
||||
# and be compatible with cURL.
|
||||
if verify is True or verify is None:
|
||||
verify = (
|
||||
os.environ.get("REQUESTS_CA_BUNDLE")
|
||||
or os.environ.get("CURL_CA_BUNDLE")
|
||||
or verify
|
||||
)
|
||||
|
||||
# Merge all the kwargs.
|
||||
proxies = merge_setting(proxies, self.proxies)
|
||||
stream = merge_setting(stream, self.stream)
|
||||
verify = merge_setting(verify, self.verify)
|
||||
cert = merge_setting(cert, self.cert)
|
||||
|
||||
return {"proxies": proxies, "stream": stream, "verify": verify, "cert": cert}
|
||||
|
||||
def get_adapter(self, url):
|
||||
"""
|
||||
Returns the appropriate connection adapter for the given URL.
|
||||
|
||||
:rtype: requests.adapters.BaseAdapter
|
||||
"""
|
||||
for (prefix, adapter) in self.adapters.items():
|
||||
|
||||
if url.lower().startswith(prefix.lower()):
|
||||
return adapter
|
||||
|
||||
# Nothing matches :-/
|
||||
raise InvalidSchema(f"No connection adapters were found for {url!r}")
|
||||
|
||||
def close(self):
|
||||
"""Closes all adapters and as such the session"""
|
||||
for v in self.adapters.values():
|
||||
v.close()
|
||||
|
||||
def mount(self, prefix, adapter):
|
||||
"""Registers a connection adapter to a prefix.
|
||||
|
||||
Adapters are sorted in descending order by prefix length.
|
||||
"""
|
||||
self.adapters[prefix] = adapter
|
||||
keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]
|
||||
|
||||
for key in keys_to_move:
|
||||
self.adapters[key] = self.adapters.pop(key)
|
||||
|
||||
def __getstate__(self):
|
||||
state = {attr: getattr(self, attr, None) for attr in self.__attrs__}
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
|
||||
def session():
|
||||
"""
|
||||
Returns a :class:`Session` for context-management.
|
||||
|
||||
.. deprecated:: 1.0.0
|
||||
|
||||
This method has been deprecated since version 1.0.0 and is only kept for
|
||||
backwards compatibility. New code should use :class:`~requests.sessions.Session`
|
||||
to create a session. This may be removed at a future date.
|
||||
|
||||
:rtype: Session
|
||||
"""
|
||||
return Session()
|
@ -1,128 +0,0 @@
|
||||
r"""
|
||||
The ``codes`` object defines a mapping from common names for HTTP statuses
|
||||
to their numerical codes, accessible either as attributes or as dictionary
|
||||
items.
|
||||
|
||||
Example::
|
||||
|
||||
>>> import requests
|
||||
>>> requests.codes['temporary_redirect']
|
||||
307
|
||||
>>> requests.codes.teapot
|
||||
418
|
||||
>>> requests.codes['\o/']
|
||||
200
|
||||
|
||||
Some codes have multiple names, and both upper- and lower-case versions of
|
||||
the names are allowed. For example, ``codes.ok``, ``codes.OK``, and
|
||||
``codes.okay`` all correspond to the HTTP status code 200.
|
||||
"""
|
||||
|
||||
from .structures import LookupDict
|
||||
|
||||
_codes = {
|
||||
# Informational.
|
||||
100: ("continue",),
|
||||
101: ("switching_protocols",),
|
||||
102: ("processing",),
|
||||
103: ("checkpoint",),
|
||||
122: ("uri_too_long", "request_uri_too_long"),
|
||||
200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"),
|
||||
201: ("created",),
|
||||
202: ("accepted",),
|
||||
203: ("non_authoritative_info", "non_authoritative_information"),
|
||||
204: ("no_content",),
|
||||
205: ("reset_content", "reset"),
|
||||
206: ("partial_content", "partial"),
|
||||
207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"),
|
||||
208: ("already_reported",),
|
||||
226: ("im_used",),
|
||||
# Redirection.
|
||||
300: ("multiple_choices",),
|
||||
301: ("moved_permanently", "moved", "\\o-"),
|
||||
302: ("found",),
|
||||
303: ("see_other", "other"),
|
||||
304: ("not_modified",),
|
||||
305: ("use_proxy",),
|
||||
306: ("switch_proxy",),
|
||||
307: ("temporary_redirect", "temporary_moved", "temporary"),
|
||||
308: (
|
||||
"permanent_redirect",
|
||||
"resume_incomplete",
|
||||
"resume",
|
||||
), # "resume" and "resume_incomplete" to be removed in 3.0
|
||||
# Client Error.
|
||||
400: ("bad_request", "bad"),
|
||||
401: ("unauthorized",),
|
||||
402: ("payment_required", "payment"),
|
||||
403: ("forbidden",),
|
||||
404: ("not_found", "-o-"),
|
||||
405: ("method_not_allowed", "not_allowed"),
|
||||
406: ("not_acceptable",),
|
||||
407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"),
|
||||
408: ("request_timeout", "timeout"),
|
||||
409: ("conflict",),
|
||||
410: ("gone",),
|
||||
411: ("length_required",),
|
||||
412: ("precondition_failed", "precondition"),
|
||||
413: ("request_entity_too_large",),
|
||||
414: ("request_uri_too_large",),
|
||||
415: ("unsupported_media_type", "unsupported_media", "media_type"),
|
||||
416: (
|
||||
"requested_range_not_satisfiable",
|
||||
"requested_range",
|
||||
"range_not_satisfiable",
|
||||
),
|
||||
417: ("expectation_failed",),
|
||||
418: ("im_a_teapot", "teapot", "i_am_a_teapot"),
|
||||
421: ("misdirected_request",),
|
||||
422: ("unprocessable_entity", "unprocessable"),
|
||||
423: ("locked",),
|
||||
424: ("failed_dependency", "dependency"),
|
||||
425: ("unordered_collection", "unordered"),
|
||||
426: ("upgrade_required", "upgrade"),
|
||||
428: ("precondition_required", "precondition"),
|
||||
429: ("too_many_requests", "too_many"),
|
||||
431: ("header_fields_too_large", "fields_too_large"),
|
||||
444: ("no_response", "none"),
|
||||
449: ("retry_with", "retry"),
|
||||
450: ("blocked_by_windows_parental_controls", "parental_controls"),
|
||||
451: ("unavailable_for_legal_reasons", "legal_reasons"),
|
||||
499: ("client_closed_request",),
|
||||
# Server Error.
|
||||
500: ("internal_server_error", "server_error", "/o\\", "✗"),
|
||||
501: ("not_implemented",),
|
||||
502: ("bad_gateway",),
|
||||
503: ("service_unavailable", "unavailable"),
|
||||
504: ("gateway_timeout",),
|
||||
505: ("http_version_not_supported", "http_version"),
|
||||
506: ("variant_also_negotiates",),
|
||||
507: ("insufficient_storage",),
|
||||
509: ("bandwidth_limit_exceeded", "bandwidth"),
|
||||
510: ("not_extended",),
|
||||
511: ("network_authentication_required", "network_auth", "network_authentication"),
|
||||
}
|
||||
|
||||
codes = LookupDict(name="status_codes")
|
||||
|
||||
|
||||
def _init():
|
||||
for code, titles in _codes.items():
|
||||
for title in titles:
|
||||
setattr(codes, title, code)
|
||||
if not title.startswith(("\\", "/")):
|
||||
setattr(codes, title.upper(), code)
|
||||
|
||||
def doc(code):
|
||||
names = ", ".join(f"``{n}``" for n in _codes[code])
|
||||
return "* %d: %s" % (code, names)
|
||||
|
||||
global __doc__
|
||||
__doc__ = (
|
||||
__doc__ + "\n" + "\n".join(doc(code) for code in sorted(_codes))
|
||||
if __doc__ is not None
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
_init()
|
@ -1,99 +0,0 @@
|
||||
"""
|
||||
requests.structures
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Data structures that power Requests.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from .compat import Mapping, MutableMapping
|
||||
|
||||
|
||||
class CaseInsensitiveDict(MutableMapping):
|
||||
"""A case-insensitive ``dict``-like object.
|
||||
|
||||
Implements all methods and operations of
|
||||
``MutableMapping`` as well as dict's ``copy``. Also
|
||||
provides ``lower_items``.
|
||||
|
||||
All keys are expected to be strings. The structure remembers the
|
||||
case of the last key to be set, and ``iter(instance)``,
|
||||
``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
|
||||
will contain case-sensitive keys. However, querying and contains
|
||||
testing is case insensitive::
|
||||
|
||||
cid = CaseInsensitiveDict()
|
||||
cid['Accept'] = 'application/json'
|
||||
cid['aCCEPT'] == 'application/json' # True
|
||||
list(cid) == ['Accept'] # True
|
||||
|
||||
For example, ``headers['content-encoding']`` will return the
|
||||
value of a ``'Content-Encoding'`` response header, regardless
|
||||
of how the header name was originally stored.
|
||||
|
||||
If the constructor, ``.update``, or equality comparison
|
||||
operations are given keys that have equal ``.lower()``s, the
|
||||
behavior is undefined.
|
||||
"""
|
||||
|
||||
def __init__(self, data=None, **kwargs):
|
||||
self._store = OrderedDict()
|
||||
if data is None:
|
||||
data = {}
|
||||
self.update(data, **kwargs)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
# Use the lowercased key for lookups, but store the actual
|
||||
# key alongside the value.
|
||||
self._store[key.lower()] = (key, value)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._store[key.lower()][1]
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._store[key.lower()]
|
||||
|
||||
def __iter__(self):
|
||||
return (casedkey for casedkey, mappedvalue in self._store.values())
|
||||
|
||||
def __len__(self):
|
||||
return len(self._store)
|
||||
|
||||
def lower_items(self):
|
||||
"""Like iteritems(), but with all lowercase keys."""
|
||||
return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items())
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, Mapping):
|
||||
other = CaseInsensitiveDict(other)
|
||||
else:
|
||||
return NotImplemented
|
||||
# Compare insensitively
|
||||
return dict(self.lower_items()) == dict(other.lower_items())
|
||||
|
||||
# Copy is required
|
||||
def copy(self):
|
||||
return CaseInsensitiveDict(self._store.values())
|
||||
|
||||
def __repr__(self):
|
||||
return str(dict(self.items()))
|
||||
|
||||
|
||||
class LookupDict(dict):
|
||||
"""Dictionary lookup object."""
|
||||
|
||||
def __init__(self, name=None):
|
||||
self.name = name
|
||||
super().__init__()
|
||||
|
||||
def __repr__(self):
|
||||
return f"<lookup '{self.name}'>"
|
||||
|
||||
def __getitem__(self, key):
|
||||
# We allow fall-through here, so values default to None
|
||||
|
||||
return self.__dict__.get(key, None)
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.__dict__.get(key, default)
|
File diff suppressed because it is too large
Load Diff
@ -1,149 +0,0 @@
|
||||
"""
|
||||
Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Set default logging handler to avoid "No handler found" warnings.
|
||||
import logging
|
||||
import typing
|
||||
import warnings
|
||||
from logging import NullHandler
|
||||
|
||||
from . import exceptions
|
||||
from ._base_connection import _TYPE_BODY
|
||||
from ._collections import HTTPHeaderDict
|
||||
from ._version import __version__
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
|
||||
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
|
||||
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
|
||||
from .response import BaseHTTPResponse, HTTPResponse
|
||||
from .util.request import make_headers
|
||||
from .util.retry import Retry
|
||||
from .util.timeout import Timeout
|
||||
|
||||
# Ensure that Python is compiled with OpenSSL 1.1.1+
|
||||
# If the 'ssl' module isn't available at all that's
|
||||
# fine, we only care if the module is available.
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive:
|
||||
warnings.warn(
|
||||
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
|
||||
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
|
||||
"See: https://github.com/urllib3/urllib3/issues/3020",
|
||||
exceptions.NotOpenSSLWarning,
|
||||
)
|
||||
elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive:
|
||||
raise ImportError(
|
||||
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
|
||||
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
|
||||
"See: https://github.com/urllib3/urllib3/issues/2168"
|
||||
)
|
||||
|
||||
__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
|
||||
__license__ = "MIT"
|
||||
__version__ = __version__
|
||||
|
||||
__all__ = (
|
||||
"HTTPConnectionPool",
|
||||
"HTTPHeaderDict",
|
||||
"HTTPSConnectionPool",
|
||||
"PoolManager",
|
||||
"ProxyManager",
|
||||
"HTTPResponse",
|
||||
"Retry",
|
||||
"Timeout",
|
||||
"add_stderr_logger",
|
||||
"connection_from_url",
|
||||
"disable_warnings",
|
||||
"encode_multipart_formdata",
|
||||
"make_headers",
|
||||
"proxy_from_url",
|
||||
"request",
|
||||
"BaseHTTPResponse",
|
||||
)
|
||||
|
||||
logging.getLogger(__name__).addHandler(NullHandler())
|
||||
|
||||
|
||||
def add_stderr_logger(
|
||||
level: int = logging.DEBUG,
|
||||
) -> logging.StreamHandler[typing.TextIO]:
|
||||
"""
|
||||
Helper for quickly adding a StreamHandler to the logger. Useful for
|
||||
debugging.
|
||||
|
||||
Returns the handler after adding it.
|
||||
"""
|
||||
# This method needs to be in this __init__.py to get the __name__ correct
|
||||
# even if urllib3 is vendored within another package.
|
||||
logger = logging.getLogger(__name__)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(level)
|
||||
logger.debug("Added a stderr logging handler to logger: %s", __name__)
|
||||
return handler
|
||||
|
||||
|
||||
# ... Clean up.
|
||||
del NullHandler
|
||||
|
||||
|
||||
# All warning filters *must* be appended unless you're really certain that they
|
||||
# shouldn't be: otherwise, it's very hard for users to use most Python
|
||||
# mechanisms to silence them.
|
||||
# SecurityWarning's always go off by default.
|
||||
warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
|
||||
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
|
||||
warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
|
||||
|
||||
|
||||
def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None:
|
||||
"""
|
||||
Helper for quickly disabling all urllib3 warnings.
|
||||
"""
|
||||
warnings.simplefilter("ignore", category)
|
||||
|
||||
|
||||
_DEFAULT_POOL = PoolManager()
|
||||
|
||||
|
||||
def request(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
body: _TYPE_BODY | None = None,
|
||||
fields: _TYPE_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
preload_content: bool | None = True,
|
||||
decode_content: bool | None = True,
|
||||
redirect: bool | None = True,
|
||||
retries: Retry | bool | int | None = None,
|
||||
timeout: Timeout | float | int | None = 3,
|
||||
json: typing.Any | None = None,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
|
||||
Therefore, its side effects could be shared across dependencies relying on it.
|
||||
To avoid side effects create a new ``PoolManager`` instance and use it instead.
|
||||
The method does not accept low-level ``**urlopen_kw`` keyword arguments.
|
||||
"""
|
||||
|
||||
return _DEFAULT_POOL.request(
|
||||
method,
|
||||
url,
|
||||
body=body,
|
||||
fields=fields,
|
||||
headers=headers,
|
||||
preload_content=preload_content,
|
||||
decode_content=decode_content,
|
||||
redirect=redirect,
|
||||
retries=retries,
|
||||
timeout=timeout,
|
||||
json=json,
|
||||
)
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,172 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from .util.connection import _TYPE_SOCKET_OPTIONS
|
||||
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
||||
from .util.url import Url
|
||||
|
||||
_TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes], str]
|
||||
|
||||
|
||||
class ProxyConfig(typing.NamedTuple):
|
||||
ssl_context: ssl.SSLContext | None
|
||||
use_forwarding_for_https: bool
|
||||
assert_hostname: None | str | Literal[False]
|
||||
assert_fingerprint: str | None
|
||||
|
||||
|
||||
class _ResponseOptions(typing.NamedTuple):
|
||||
# TODO: Remove this in favor of a better
|
||||
# HTTP request/response lifecycle tracking.
|
||||
request_method: str
|
||||
request_url: str
|
||||
preload_content: bool
|
||||
decode_content: bool
|
||||
enforce_content_length: bool
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
import ssl
|
||||
from typing import Literal, Protocol
|
||||
|
||||
from .response import BaseHTTPResponse
|
||||
|
||||
class BaseHTTPConnection(Protocol):
|
||||
default_port: typing.ClassVar[int]
|
||||
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
||||
|
||||
host: str
|
||||
port: int
|
||||
timeout: None | (
|
||||
float
|
||||
) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved.
|
||||
blocksize: int
|
||||
source_address: tuple[str, int] | None
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None
|
||||
|
||||
proxy: Url | None
|
||||
proxy_config: ProxyConfig | None
|
||||
|
||||
is_verified: bool
|
||||
proxy_is_verified: bool | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 8192,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
) -> None:
|
||||
...
|
||||
|
||||
def set_tunnel(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
scheme: str = "http",
|
||||
) -> None:
|
||||
...
|
||||
|
||||
def connect(self) -> None:
|
||||
...
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
# We know *at least* botocore is depending on the order of the
|
||||
# first 3 parameters so to be safe we only mark the later ones
|
||||
# as keyword-only to ensure we have space to extend.
|
||||
*,
|
||||
chunked: bool = False,
|
||||
preload_content: bool = True,
|
||||
decode_content: bool = True,
|
||||
enforce_content_length: bool = True,
|
||||
) -> None:
|
||||
...
|
||||
|
||||
def getresponse(self) -> BaseHTTPResponse:
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
...
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
"""Whether the connection either is brand new or has been previously closed.
|
||||
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
|
||||
properties must be False.
|
||||
"""
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Whether the connection is actively connected to any origin (proxy or target)"""
|
||||
|
||||
@property
|
||||
def has_connected_to_proxy(self) -> bool:
|
||||
"""Whether the connection has successfully connected to its proxy.
|
||||
This returns False if no proxy is in use. Used to determine whether
|
||||
errors are coming from the proxy layer or from tunnelling to the target origin.
|
||||
"""
|
||||
|
||||
class BaseHTTPSConnection(BaseHTTPConnection, Protocol):
|
||||
default_port: typing.ClassVar[int]
|
||||
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
||||
|
||||
# Certificate verification methods
|
||||
cert_reqs: int | str | None
|
||||
assert_hostname: None | str | Literal[False]
|
||||
assert_fingerprint: str | None
|
||||
ssl_context: ssl.SSLContext | None
|
||||
|
||||
# Trusted CAs
|
||||
ca_certs: str | None
|
||||
ca_cert_dir: str | None
|
||||
ca_cert_data: None | str | bytes
|
||||
|
||||
# TLS version
|
||||
ssl_minimum_version: int | None
|
||||
ssl_maximum_version: int | None
|
||||
ssl_version: int | str | None # Deprecated
|
||||
|
||||
# Client certificates
|
||||
cert_file: str | None
|
||||
key_file: str | None
|
||||
key_password: str | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 16384,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
cert_reqs: int | str | None = None,
|
||||
assert_hostname: None | str | Literal[False] = None,
|
||||
assert_fingerprint: str | None = None,
|
||||
server_hostname: str | None = None,
|
||||
ssl_context: ssl.SSLContext | None = None,
|
||||
ca_certs: str | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
ssl_minimum_version: int | None = None,
|
||||
ssl_maximum_version: int | None = None,
|
||||
ssl_version: int | str | None = None, # Deprecated
|
||||
cert_file: str | None = None,
|
||||
key_file: str | None = None,
|
||||
key_password: str | None = None,
|
||||
) -> None:
|
||||
...
|
@ -1,483 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
from collections import OrderedDict
|
||||
from enum import Enum, auto
|
||||
from threading import RLock
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
# We can only import Protocol if TYPE_CHECKING because it's a development
|
||||
# dependency, and is not available at runtime.
|
||||
from typing import Protocol
|
||||
|
||||
from typing_extensions import Self
|
||||
|
||||
class HasGettableStringKeys(Protocol):
|
||||
def keys(self) -> typing.Iterator[str]:
|
||||
...
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
...
|
||||
|
||||
|
||||
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
|
||||
|
||||
|
||||
# Key type
|
||||
_KT = typing.TypeVar("_KT")
|
||||
# Value type
|
||||
_VT = typing.TypeVar("_VT")
|
||||
# Default type
|
||||
_DT = typing.TypeVar("_DT")
|
||||
|
||||
ValidHTTPHeaderSource = typing.Union[
|
||||
"HTTPHeaderDict",
|
||||
typing.Mapping[str, str],
|
||||
typing.Iterable[typing.Tuple[str, str]],
|
||||
"HasGettableStringKeys",
|
||||
]
|
||||
|
||||
|
||||
class _Sentinel(Enum):
|
||||
not_passed = auto()
|
||||
|
||||
|
||||
def ensure_can_construct_http_header_dict(
|
||||
potential: object,
|
||||
) -> ValidHTTPHeaderSource | None:
|
||||
if isinstance(potential, HTTPHeaderDict):
|
||||
return potential
|
||||
elif isinstance(potential, typing.Mapping):
|
||||
# Full runtime checking of the contents of a Mapping is expensive, so for the
|
||||
# purposes of typechecking, we assume that any Mapping is the right shape.
|
||||
return typing.cast(typing.Mapping[str, str], potential)
|
||||
elif isinstance(potential, typing.Iterable):
|
||||
# Similarly to Mapping, full runtime checking of the contents of an Iterable is
|
||||
# expensive, so for the purposes of typechecking, we assume that any Iterable
|
||||
# is the right shape.
|
||||
return typing.cast(typing.Iterable[typing.Tuple[str, str]], potential)
|
||||
elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"):
|
||||
return typing.cast("HasGettableStringKeys", potential)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]):
|
||||
"""
|
||||
Provides a thread-safe dict-like container which maintains up to
|
||||
``maxsize`` keys while throwing away the least-recently-used keys beyond
|
||||
``maxsize``.
|
||||
|
||||
:param maxsize:
|
||||
Maximum number of recent elements to retain.
|
||||
|
||||
:param dispose_func:
|
||||
Every time an item is evicted from the container,
|
||||
``dispose_func(value)`` is called. Callback which will get called
|
||||
"""
|
||||
|
||||
_container: typing.OrderedDict[_KT, _VT]
|
||||
_maxsize: int
|
||||
dispose_func: typing.Callable[[_VT], None] | None
|
||||
lock: RLock
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
maxsize: int = 10,
|
||||
dispose_func: typing.Callable[[_VT], None] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._maxsize = maxsize
|
||||
self.dispose_func = dispose_func
|
||||
self._container = OrderedDict()
|
||||
self.lock = RLock()
|
||||
|
||||
def __getitem__(self, key: _KT) -> _VT:
|
||||
# Re-insert the item, moving it to the end of the eviction line.
|
||||
with self.lock:
|
||||
item = self._container.pop(key)
|
||||
self._container[key] = item
|
||||
return item
|
||||
|
||||
def __setitem__(self, key: _KT, value: _VT) -> None:
|
||||
evicted_item = None
|
||||
with self.lock:
|
||||
# Possibly evict the existing value of 'key'
|
||||
try:
|
||||
# If the key exists, we'll overwrite it, which won't change the
|
||||
# size of the pool. Because accessing a key should move it to
|
||||
# the end of the eviction line, we pop it out first.
|
||||
evicted_item = key, self._container.pop(key)
|
||||
self._container[key] = value
|
||||
except KeyError:
|
||||
# When the key does not exist, we insert the value first so that
|
||||
# evicting works in all cases, including when self._maxsize is 0
|
||||
self._container[key] = value
|
||||
if len(self._container) > self._maxsize:
|
||||
# If we didn't evict an existing value, and we've hit our maximum
|
||||
# size, then we have to evict the least recently used item from
|
||||
# the beginning of the container.
|
||||
evicted_item = self._container.popitem(last=False)
|
||||
|
||||
# After releasing the lock on the pool, dispose of any evicted value.
|
||||
if evicted_item is not None and self.dispose_func:
|
||||
_, evicted_value = evicted_item
|
||||
self.dispose_func(evicted_value)
|
||||
|
||||
def __delitem__(self, key: _KT) -> None:
|
||||
with self.lock:
|
||||
value = self._container.pop(key)
|
||||
|
||||
if self.dispose_func:
|
||||
self.dispose_func(value)
|
||||
|
||||
def __len__(self) -> int:
|
||||
with self.lock:
|
||||
return len(self._container)
|
||||
|
||||
def __iter__(self) -> typing.NoReturn:
|
||||
raise NotImplementedError(
|
||||
"Iteration over this class is unlikely to be threadsafe."
|
||||
)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self.lock:
|
||||
# Copy pointers to all values, then wipe the mapping
|
||||
values = list(self._container.values())
|
||||
self._container.clear()
|
||||
|
||||
if self.dispose_func:
|
||||
for value in values:
|
||||
self.dispose_func(value)
|
||||
|
||||
def keys(self) -> set[_KT]: # type: ignore[override]
|
||||
with self.lock:
|
||||
return set(self._container.keys())
|
||||
|
||||
|
||||
class HTTPHeaderDictItemView(typing.Set[typing.Tuple[str, str]]):
|
||||
"""
|
||||
HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of
|
||||
address.
|
||||
|
||||
If we directly try to get an item with a particular name, we will get a string
|
||||
back that is the concatenated version of all the values:
|
||||
|
||||
>>> d['X-Header-Name']
|
||||
'Value1, Value2, Value3'
|
||||
|
||||
However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
|
||||
these values based on whether combine=True was called when building up the dictionary
|
||||
|
||||
>>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
|
||||
>>> d.add("A", "2", combine=True)
|
||||
>>> d.add("B", "bar")
|
||||
>>> list(d.items())
|
||||
[
|
||||
('A', '1, 2'),
|
||||
('B', 'foo'),
|
||||
('B', 'bar'),
|
||||
]
|
||||
|
||||
This class conforms to the interface required by the MutableMapping ABC while
|
||||
also giving us the nonstandard iteration behavior we want; items with duplicate
|
||||
keys, ordered by time of first insertion.
|
||||
"""
|
||||
|
||||
_headers: HTTPHeaderDict
|
||||
|
||||
def __init__(self, headers: HTTPHeaderDict) -> None:
|
||||
self._headers = headers
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(list(self._headers.iteritems()))
|
||||
|
||||
def __iter__(self) -> typing.Iterator[tuple[str, str]]:
|
||||
return self._headers.iteritems()
|
||||
|
||||
def __contains__(self, item: object) -> bool:
|
||||
if isinstance(item, tuple) and len(item) == 2:
|
||||
passed_key, passed_val = item
|
||||
if isinstance(passed_key, str) and isinstance(passed_val, str):
|
||||
return self._headers._has_value_for_header(passed_key, passed_val)
|
||||
return False
|
||||
|
||||
|
||||
class HTTPHeaderDict(typing.MutableMapping[str, str]):
|
||||
"""
|
||||
:param headers:
|
||||
An iterable of field-value pairs. Must not contain multiple field names
|
||||
when compared case-insensitively.
|
||||
|
||||
:param kwargs:
|
||||
Additional field-value pairs to pass in to ``dict.update``.
|
||||
|
||||
A ``dict`` like container for storing HTTP Headers.
|
||||
|
||||
Field names are stored and compared case-insensitively in compliance with
|
||||
RFC 7230. Iteration provides the first case-sensitive key seen for each
|
||||
case-insensitive pair.
|
||||
|
||||
Using ``__setitem__`` syntax overwrites fields that compare equal
|
||||
case-insensitively in order to maintain ``dict``'s api. For fields that
|
||||
compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
|
||||
in a loop.
|
||||
|
||||
If multiple fields that are equal case-insensitively are passed to the
|
||||
constructor or ``.update``, the behavior is undefined and some will be
|
||||
lost.
|
||||
|
||||
>>> headers = HTTPHeaderDict()
|
||||
>>> headers.add('Set-Cookie', 'foo=bar')
|
||||
>>> headers.add('set-cookie', 'baz=quxx')
|
||||
>>> headers['content-length'] = '7'
|
||||
>>> headers['SET-cookie']
|
||||
'foo=bar, baz=quxx'
|
||||
>>> headers['Content-Length']
|
||||
'7'
|
||||
"""
|
||||
|
||||
_container: typing.MutableMapping[str, list[str]]
|
||||
|
||||
def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str):
|
||||
super().__init__()
|
||||
self._container = {} # 'dict' is insert-ordered
|
||||
if headers is not None:
|
||||
if isinstance(headers, HTTPHeaderDict):
|
||||
self._copy_from(headers)
|
||||
else:
|
||||
self.extend(headers)
|
||||
if kwargs:
|
||||
self.extend(kwargs)
|
||||
|
||||
def __setitem__(self, key: str, val: str) -> None:
|
||||
# avoid a bytes/str comparison by decoding before httplib
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
self._container[key.lower()] = [key, val]
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
val = self._container[key.lower()]
|
||||
return ", ".join(val[1:])
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
del self._container[key.lower()]
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
if isinstance(key, str):
|
||||
return key.lower() in self._container
|
||||
return False
|
||||
|
||||
def setdefault(self, key: str, default: str = "") -> str:
|
||||
return super().setdefault(key, default)
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return False
|
||||
else:
|
||||
other_as_http_header_dict = type(self)(maybe_constructable)
|
||||
|
||||
return {k.lower(): v for k, v in self.itermerged()} == {
|
||||
k.lower(): v for k, v in other_as_http_header_dict.itermerged()
|
||||
}
|
||||
|
||||
def __ne__(self, other: object) -> bool:
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._container)
|
||||
|
||||
def __iter__(self) -> typing.Iterator[str]:
|
||||
# Only provide the originally cased names
|
||||
for vals in self._container.values():
|
||||
yield vals[0]
|
||||
|
||||
def discard(self, key: str) -> None:
|
||||
try:
|
||||
del self[key]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def add(self, key: str, val: str, *, combine: bool = False) -> None:
|
||||
"""Adds a (name, value) pair, doesn't overwrite the value if it already
|
||||
exists.
|
||||
|
||||
If this is called with combine=True, instead of adding a new header value
|
||||
as a distinct item during iteration, this will instead append the value to
|
||||
any existing header value with a comma. If no existing header value exists
|
||||
for the key, then the value will simply be added, ignoring the combine parameter.
|
||||
|
||||
>>> headers = HTTPHeaderDict(foo='bar')
|
||||
>>> headers.add('Foo', 'baz')
|
||||
>>> headers['foo']
|
||||
'bar, baz'
|
||||
>>> list(headers.items())
|
||||
[('foo', 'bar'), ('foo', 'baz')]
|
||||
>>> headers.add('foo', 'quz', combine=True)
|
||||
>>> list(headers.items())
|
||||
[('foo', 'bar, baz, quz')]
|
||||
"""
|
||||
# avoid a bytes/str comparison by decoding before httplib
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
key_lower = key.lower()
|
||||
new_vals = [key, val]
|
||||
# Keep the common case aka no item present as fast as possible
|
||||
vals = self._container.setdefault(key_lower, new_vals)
|
||||
if new_vals is not vals:
|
||||
# if there are values here, then there is at least the initial
|
||||
# key/value pair
|
||||
assert len(vals) >= 2
|
||||
if combine:
|
||||
vals[-1] = vals[-1] + ", " + val
|
||||
else:
|
||||
vals.append(val)
|
||||
|
||||
def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:
|
||||
"""Generic import function for any type of header-like object.
|
||||
Adapted version of MutableMapping.update in order to insert items
|
||||
with self.add instead of self.__setitem__
|
||||
"""
|
||||
if len(args) > 1:
|
||||
raise TypeError(
|
||||
f"extend() takes at most 1 positional arguments ({len(args)} given)"
|
||||
)
|
||||
other = args[0] if len(args) >= 1 else ()
|
||||
|
||||
if isinstance(other, HTTPHeaderDict):
|
||||
for key, val in other.iteritems():
|
||||
self.add(key, val)
|
||||
elif isinstance(other, typing.Mapping):
|
||||
for key, val in other.items():
|
||||
self.add(key, val)
|
||||
elif isinstance(other, typing.Iterable):
|
||||
other = typing.cast(typing.Iterable[typing.Tuple[str, str]], other)
|
||||
for key, value in other:
|
||||
self.add(key, value)
|
||||
elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
|
||||
# THIS IS NOT A TYPESAFE BRANCH
|
||||
# In this branch, the object has a `keys` attr but is not a Mapping or any of
|
||||
# the other types indicated in the method signature. We do some stuff with
|
||||
# it as though it partially implements the Mapping interface, but we're not
|
||||
# doing that stuff safely AT ALL.
|
||||
for key in other.keys():
|
||||
self.add(key, other[key])
|
||||
|
||||
for key, value in kwargs.items():
|
||||
self.add(key, value)
|
||||
|
||||
@typing.overload
|
||||
def getlist(self, key: str) -> list[str]:
|
||||
...
|
||||
|
||||
@typing.overload
|
||||
def getlist(self, key: str, default: _DT) -> list[str] | _DT:
|
||||
...
|
||||
|
||||
def getlist(
|
||||
self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed
|
||||
) -> list[str] | _DT:
|
||||
"""Returns a list of all the values for the named field. Returns an
|
||||
empty list if the key doesn't exist."""
|
||||
try:
|
||||
vals = self._container[key.lower()]
|
||||
except KeyError:
|
||||
if default is _Sentinel.not_passed:
|
||||
# _DT is unbound; empty list is instance of List[str]
|
||||
return []
|
||||
# _DT is bound; default is instance of _DT
|
||||
return default
|
||||
else:
|
||||
# _DT may or may not be bound; vals[1:] is instance of List[str], which
|
||||
# meets our external interface requirement of `Union[List[str], _DT]`.
|
||||
return vals[1:]
|
||||
|
||||
def _prepare_for_method_change(self) -> Self:
|
||||
"""
|
||||
Remove content-specific header fields before changing the request
|
||||
method to GET or HEAD according to RFC 9110, Section 15.4.
|
||||
"""
|
||||
content_specific_headers = [
|
||||
"Content-Encoding",
|
||||
"Content-Language",
|
||||
"Content-Location",
|
||||
"Content-Type",
|
||||
"Content-Length",
|
||||
"Digest",
|
||||
"Last-Modified",
|
||||
]
|
||||
for header in content_specific_headers:
|
||||
self.discard(header)
|
||||
return self
|
||||
|
||||
# Backwards compatibility for httplib
|
||||
getheaders = getlist
|
||||
getallmatchingheaders = getlist
|
||||
iget = getlist
|
||||
|
||||
# Backwards compatibility for http.cookiejar
|
||||
get_all = getlist
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{type(self).__name__}({dict(self.itermerged())})"
|
||||
|
||||
def _copy_from(self, other: HTTPHeaderDict) -> None:
|
||||
for key in other:
|
||||
val = other.getlist(key)
|
||||
self._container[key.lower()] = [key, *val]
|
||||
|
||||
def copy(self) -> HTTPHeaderDict:
|
||||
clone = type(self)()
|
||||
clone._copy_from(self)
|
||||
return clone
|
||||
|
||||
def iteritems(self) -> typing.Iterator[tuple[str, str]]:
|
||||
"""Iterate over all header lines, including duplicate ones."""
|
||||
for key in self:
|
||||
vals = self._container[key.lower()]
|
||||
for val in vals[1:]:
|
||||
yield vals[0], val
|
||||
|
||||
def itermerged(self) -> typing.Iterator[tuple[str, str]]:
|
||||
"""Iterate over all headers, merging duplicate ones together."""
|
||||
for key in self:
|
||||
val = self._container[key.lower()]
|
||||
yield val[0], ", ".join(val[1:])
|
||||
|
||||
def items(self) -> HTTPHeaderDictItemView: # type: ignore[override]
|
||||
return HTTPHeaderDictItemView(self)
|
||||
|
||||
def _has_value_for_header(self, header_name: str, potential_value: str) -> bool:
|
||||
if header_name in self:
|
||||
return potential_value in self._container[header_name.lower()][1:]
|
||||
return False
|
||||
|
||||
def __ior__(self, other: object) -> HTTPHeaderDict:
|
||||
# Supports extending a header dict in-place using operator |=
|
||||
# combining items with add instead of __setitem__
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return NotImplemented
|
||||
self.extend(maybe_constructable)
|
||||
return self
|
||||
|
||||
def __or__(self, other: object) -> HTTPHeaderDict:
|
||||
# Supports merging header dicts using operator |
|
||||
# combining items with add instead of __setitem__
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return NotImplemented
|
||||
result = self.copy()
|
||||
result.extend(maybe_constructable)
|
||||
return result
|
||||
|
||||
def __ror__(self, other: object) -> HTTPHeaderDict:
|
||||
# Supports merging header dicts using operator | when other is on left side
|
||||
# combining items with add instead of __setitem__
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return NotImplemented
|
||||
result = type(self)(maybe_constructable)
|
||||
result.extend(self)
|
||||
return result
|
@ -1,217 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json as _json
|
||||
import typing
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from ._base_connection import _TYPE_BODY
|
||||
from ._collections import HTTPHeaderDict
|
||||
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
|
||||
from .response import BaseHTTPResponse
|
||||
|
||||
__all__ = ["RequestMethods"]
|
||||
|
||||
_TYPE_ENCODE_URL_FIELDS = typing.Union[
|
||||
typing.Sequence[typing.Tuple[str, typing.Union[str, bytes]]],
|
||||
typing.Mapping[str, typing.Union[str, bytes]],
|
||||
]
|
||||
|
||||
|
||||
class RequestMethods:
|
||||
"""
|
||||
Convenience mixin for classes who implement a :meth:`urlopen` method, such
|
||||
as :class:`urllib3.HTTPConnectionPool` and
|
||||
:class:`urllib3.PoolManager`.
|
||||
|
||||
Provides behavior for making common types of HTTP request methods and
|
||||
decides which type of request field encoding to use.
|
||||
|
||||
Specifically,
|
||||
|
||||
:meth:`.request_encode_url` is for sending requests whose fields are
|
||||
encoded in the URL (such as GET, HEAD, DELETE).
|
||||
|
||||
:meth:`.request_encode_body` is for sending requests whose fields are
|
||||
encoded in the *body* of the request using multipart or www-form-urlencoded
|
||||
(such as for POST, PUT, PATCH).
|
||||
|
||||
:meth:`.request` is for making any kind of request, it will look up the
|
||||
appropriate encoding format and use one of the above two methods to make
|
||||
the request.
|
||||
|
||||
Initializer parameters:
|
||||
|
||||
:param headers:
|
||||
Headers to include with all requests, unless other headers are given
|
||||
explicitly.
|
||||
"""
|
||||
|
||||
_encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
|
||||
|
||||
def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None:
|
||||
self.headers = headers or {}
|
||||
|
||||
def urlopen(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
encode_multipart: bool = True,
|
||||
multipart_boundary: str | None = None,
|
||||
**kw: typing.Any,
|
||||
) -> BaseHTTPResponse: # Abstract
|
||||
raise NotImplementedError(
|
||||
"Classes extending RequestMethods must implement "
|
||||
"their own ``urlopen`` method."
|
||||
)
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
fields: _TYPE_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
json: typing.Any | None = None,
|
||||
**urlopen_kw: typing.Any,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Make a request using :meth:`urlopen` with the appropriate encoding of
|
||||
``fields`` based on the ``method`` used.
|
||||
|
||||
This is a convenience method that requires the least amount of manual
|
||||
effort. It can be used in most situations, while still having the
|
||||
option to drop down to more specific methods when necessary, such as
|
||||
:meth:`request_encode_url`, :meth:`request_encode_body`,
|
||||
or even the lowest level :meth:`urlopen`.
|
||||
"""
|
||||
method = method.upper()
|
||||
|
||||
if json is not None and body is not None:
|
||||
raise TypeError(
|
||||
"request got values for both 'body' and 'json' parameters which are mutually exclusive"
|
||||
)
|
||||
|
||||
if json is not None:
|
||||
if headers is None:
|
||||
headers = self.headers.copy() # type: ignore
|
||||
if not ("content-type" in map(str.lower, headers.keys())):
|
||||
headers["Content-Type"] = "application/json" # type: ignore
|
||||
|
||||
body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
if body is not None:
|
||||
urlopen_kw["body"] = body
|
||||
|
||||
if method in self._encode_url_methods:
|
||||
return self.request_encode_url(
|
||||
method,
|
||||
url,
|
||||
fields=fields, # type: ignore[arg-type]
|
||||
headers=headers,
|
||||
**urlopen_kw,
|
||||
)
|
||||
else:
|
||||
return self.request_encode_body(
|
||||
method, url, fields=fields, headers=headers, **urlopen_kw
|
||||
)
|
||||
|
||||
def request_encode_url(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
fields: _TYPE_ENCODE_URL_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**urlopen_kw: str,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Make a request using :meth:`urlopen` with the ``fields`` encoded in
|
||||
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
extra_kw: dict[str, typing.Any] = {"headers": headers}
|
||||
extra_kw.update(urlopen_kw)
|
||||
|
||||
if fields:
|
||||
url += "?" + urlencode(fields)
|
||||
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
|
||||
def request_encode_body(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
fields: _TYPE_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
encode_multipart: bool = True,
|
||||
multipart_boundary: str | None = None,
|
||||
**urlopen_kw: str,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Make a request using :meth:`urlopen` with the ``fields`` encoded in
|
||||
the body. This is useful for request methods like POST, PUT, PATCH, etc.
|
||||
|
||||
When ``encode_multipart=True`` (default), then
|
||||
:func:`urllib3.encode_multipart_formdata` is used to encode
|
||||
the payload with the appropriate content type. Otherwise
|
||||
:func:`urllib.parse.urlencode` is used with the
|
||||
'application/x-www-form-urlencoded' content type.
|
||||
|
||||
Multipart encoding must be used when posting files, and it's reasonably
|
||||
safe to use it in other times too. However, it may break request
|
||||
signing, such as with OAuth.
|
||||
|
||||
Supports an optional ``fields`` parameter of key/value strings AND
|
||||
key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
|
||||
the MIME type is optional. For example::
|
||||
|
||||
fields = {
|
||||
'foo': 'bar',
|
||||
'fakefile': ('foofile.txt', 'contents of foofile'),
|
||||
'realfile': ('barfile.txt', open('realfile').read()),
|
||||
'typedfile': ('bazfile.bin', open('bazfile').read(),
|
||||
'image/jpeg'),
|
||||
'nonamefile': 'contents of nonamefile field',
|
||||
}
|
||||
|
||||
When uploading a file, providing a filename (the first parameter of the
|
||||
tuple) is optional but recommended to best mimic behavior of browsers.
|
||||
|
||||
Note that if ``headers`` are supplied, the 'Content-Type' header will
|
||||
be overwritten because it depends on the dynamic random boundary string
|
||||
which is used to compose the body of the request. The random boundary
|
||||
string can be explicitly set with the ``multipart_boundary`` parameter.
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)}
|
||||
body: bytes | str
|
||||
|
||||
if fields:
|
||||
if "body" in urlopen_kw:
|
||||
raise TypeError(
|
||||
"request got values for both 'fields' and 'body', can only specify one."
|
||||
)
|
||||
|
||||
if encode_multipart:
|
||||
body, content_type = encode_multipart_formdata(
|
||||
fields, boundary=multipart_boundary
|
||||
)
|
||||
else:
|
||||
body, content_type = (
|
||||
urlencode(fields), # type: ignore[arg-type]
|
||||
"application/x-www-form-urlencoded",
|
||||
)
|
||||
|
||||
extra_kw["body"] = body
|
||||
extra_kw["headers"].setdefault("Content-Type", content_type)
|
||||
|
||||
extra_kw.update(urlopen_kw)
|
||||
|
||||
return self.urlopen(method, url, **extra_kw)
|
@ -1,4 +0,0 @@
|
||||
# This file is protected via CODEOWNERS
|
||||
from __future__ import annotations
|
||||
|
||||
__version__ = "2.1.0"
|
@ -1,905 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import typing
|
||||
import warnings
|
||||
from http.client import HTTPConnection as _HTTPConnection
|
||||
from http.client import HTTPException as HTTPException # noqa: F401
|
||||
from http.client import ResponseNotReady
|
||||
from socket import timeout as SocketTimeout
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing import Literal
|
||||
|
||||
from .response import HTTPResponse
|
||||
from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
|
||||
from .util.ssltransport import SSLTransport
|
||||
|
||||
from ._collections import HTTPHeaderDict
|
||||
from .util.response import assert_header_parsing
|
||||
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
|
||||
from .util.util import to_str
|
||||
from .util.wait import wait_for_read
|
||||
|
||||
try: # Compiled with SSL?
|
||||
import ssl
|
||||
|
||||
BaseSSLError = ssl.SSLError
|
||||
except (ImportError, AttributeError):
|
||||
ssl = None # type: ignore[assignment]
|
||||
|
||||
class BaseSSLError(BaseException): # type: ignore[no-redef]
|
||||
pass
|
||||
|
||||
|
||||
from ._base_connection import _TYPE_BODY
|
||||
from ._base_connection import ProxyConfig as ProxyConfig
|
||||
from ._base_connection import _ResponseOptions as _ResponseOptions
|
||||
from ._version import __version__
|
||||
from .exceptions import (
|
||||
ConnectTimeoutError,
|
||||
HeaderParsingError,
|
||||
NameResolutionError,
|
||||
NewConnectionError,
|
||||
ProxyError,
|
||||
SystemTimeWarning,
|
||||
)
|
||||
from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
|
||||
from .util.request import body_to_chunks
|
||||
from .util.ssl_ import assert_fingerprint as _assert_fingerprint
|
||||
from .util.ssl_ import (
|
||||
create_urllib3_context,
|
||||
is_ipaddress,
|
||||
resolve_cert_reqs,
|
||||
resolve_ssl_version,
|
||||
ssl_wrap_socket,
|
||||
)
|
||||
from .util.ssl_match_hostname import CertificateError, match_hostname
|
||||
from .util.url import Url
|
||||
|
||||
# Not a no-op, we're adding this to the namespace so it can be imported.
|
||||
ConnectionError = ConnectionError
|
||||
BrokenPipeError = BrokenPipeError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
port_by_scheme = {"http": 80, "https": 443}
|
||||
|
||||
# When it comes time to update this value as a part of regular maintenance
|
||||
# (ie test_recent_date is failing) update it to ~6 months before the current date.
|
||||
RECENT_DATE = datetime.date(2022, 1, 1)
|
||||
|
||||
_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
|
||||
|
||||
_HAS_SYS_AUDIT = hasattr(sys, "audit")
|
||||
|
||||
|
||||
class HTTPConnection(_HTTPConnection):
|
||||
"""
|
||||
Based on :class:`http.client.HTTPConnection` but provides an extra constructor
|
||||
backwards-compatibility layer between older and newer Pythons.
|
||||
|
||||
Additional keyword parameters are used to configure attributes of the connection.
|
||||
Accepted parameters include:
|
||||
|
||||
- ``source_address``: Set the source address for the current connection.
|
||||
- ``socket_options``: Set specific options on the underlying socket. If not specified, then
|
||||
defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
|
||||
Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
|
||||
|
||||
For example, if you wish to enable TCP Keep Alive in addition to the defaults,
|
||||
you might pass:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
HTTPConnection.default_socket_options + [
|
||||
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
|
||||
]
|
||||
|
||||
Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
|
||||
"""
|
||||
|
||||
default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
|
||||
|
||||
#: Disable Nagle's algorithm by default.
|
||||
#: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
|
||||
default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [
|
||||
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
]
|
||||
|
||||
#: Whether this connection verifies the host's certificate.
|
||||
is_verified: bool = False
|
||||
|
||||
#: Whether this proxy connection verified the proxy host's certificate.
|
||||
# If no proxy is currently connected to the value will be ``None``.
|
||||
proxy_is_verified: bool | None = None
|
||||
|
||||
blocksize: int
|
||||
source_address: tuple[str, int] | None
|
||||
socket_options: connection._TYPE_SOCKET_OPTIONS | None
|
||||
|
||||
_has_connected_to_proxy: bool
|
||||
_response_options: _ResponseOptions | None
|
||||
_tunnel_host: str | None
|
||||
_tunnel_port: int | None
|
||||
_tunnel_scheme: str | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 16384,
|
||||
socket_options: None
|
||||
| (connection._TYPE_SOCKET_OPTIONS) = default_socket_options,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
host=host,
|
||||
port=port,
|
||||
timeout=Timeout.resolve_default_timeout(timeout),
|
||||
source_address=source_address,
|
||||
blocksize=blocksize,
|
||||
)
|
||||
self.socket_options = socket_options
|
||||
self.proxy = proxy
|
||||
self.proxy_config = proxy_config
|
||||
|
||||
self._has_connected_to_proxy = False
|
||||
self._response_options = None
|
||||
self._tunnel_host: str | None = None
|
||||
self._tunnel_port: int | None = None
|
||||
self._tunnel_scheme: str | None = None
|
||||
|
||||
# https://github.com/python/mypy/issues/4125
|
||||
# Mypy treats this as LSP violation, which is considered a bug.
|
||||
# If `host` is made a property it violates LSP, because a writeable attribute is overridden with a read-only one.
|
||||
# However, there is also a `host` setter so LSP is not violated.
|
||||
# Potentially, a `@host.deleter` might be needed depending on how this issue will be fixed.
|
||||
@property
|
||||
def host(self) -> str:
|
||||
"""
|
||||
Getter method to remove any trailing dots that indicate the hostname is an FQDN.
|
||||
|
||||
In general, SSL certificates don't include the trailing dot indicating a
|
||||
fully-qualified domain name, and thus, they don't validate properly when
|
||||
checked against a domain name that includes the dot. In addition, some
|
||||
servers may not expect to receive the trailing dot when provided.
|
||||
|
||||
However, the hostname with trailing dot is critical to DNS resolution; doing a
|
||||
lookup with the trailing dot will properly only resolve the appropriate FQDN,
|
||||
whereas a lookup without a trailing dot will search the system's search domain
|
||||
list. Thus, it's important to keep the original host around for use only in
|
||||
those cases where it's appropriate (i.e., when doing DNS lookup to establish the
|
||||
actual TCP connection across which we're going to send HTTP requests).
|
||||
"""
|
||||
return self._dns_host.rstrip(".")
|
||||
|
||||
@host.setter
|
||||
def host(self, value: str) -> None:
|
||||
"""
|
||||
Setter for the `host` property.
|
||||
|
||||
We assume that only urllib3 uses the _dns_host attribute; httplib itself
|
||||
only uses `host`, and it seems reasonable that other libraries follow suit.
|
||||
"""
|
||||
self._dns_host = value
|
||||
|
||||
def _new_conn(self) -> socket.socket:
|
||||
"""Establish a socket connection and set nodelay settings on it.
|
||||
|
||||
:return: New socket connection.
|
||||
"""
|
||||
try:
|
||||
sock = connection.create_connection(
|
||||
(self._dns_host, self.port),
|
||||
self.timeout,
|
||||
source_address=self.source_address,
|
||||
socket_options=self.socket_options,
|
||||
)
|
||||
except socket.gaierror as e:
|
||||
raise NameResolutionError(self.host, self, e) from e
|
||||
except SocketTimeout as e:
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
|
||||
except OSError as e:
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
|
||||
# Audit hooks are only available in Python 3.8+
|
||||
if _HAS_SYS_AUDIT:
|
||||
sys.audit("http.client.connect", self, self.host, self.port)
|
||||
|
||||
return sock
|
||||
|
||||
def set_tunnel(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
scheme: str = "http",
|
||||
) -> None:
|
||||
if scheme not in ("http", "https"):
|
||||
raise ValueError(
|
||||
f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
|
||||
)
|
||||
super().set_tunnel(host, port=port, headers=headers)
|
||||
self._tunnel_scheme = scheme
|
||||
|
||||
def connect(self) -> None:
|
||||
self.sock = self._new_conn()
|
||||
if self._tunnel_host:
|
||||
# If we're tunneling it means we're connected to our proxy.
|
||||
self._has_connected_to_proxy = True
|
||||
|
||||
# TODO: Fix tunnel so it doesn't depend on self.sock state.
|
||||
self._tunnel() # type: ignore[attr-defined]
|
||||
|
||||
# If there's a proxy to be connected to we are fully connected.
|
||||
# This is set twice (once above and here) due to forwarding proxies
|
||||
# not using tunnelling.
|
||||
self._has_connected_to_proxy = bool(self.proxy)
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
return self.sock is None
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
if self.sock is None:
|
||||
return False
|
||||
return not wait_for_read(self.sock, timeout=0.0)
|
||||
|
||||
@property
|
||||
def has_connected_to_proxy(self) -> bool:
|
||||
return self._has_connected_to_proxy
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
super().close()
|
||||
finally:
|
||||
# Reset all stateful properties so connection
|
||||
# can be re-used without leaking prior configs.
|
||||
self.sock = None
|
||||
self.is_verified = False
|
||||
self.proxy_is_verified = None
|
||||
self._has_connected_to_proxy = False
|
||||
self._response_options = None
|
||||
self._tunnel_host = None
|
||||
self._tunnel_port = None
|
||||
self._tunnel_scheme = None
|
||||
|
||||
def putrequest(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
skip_host: bool = False,
|
||||
skip_accept_encoding: bool = False,
|
||||
) -> None:
|
||||
""""""
|
||||
# Empty docstring because the indentation of CPython's implementation
|
||||
# is broken but we don't want this method in our documentation.
|
||||
match = _CONTAINS_CONTROL_CHAR_RE.search(method)
|
||||
if match:
|
||||
raise ValueError(
|
||||
f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
|
||||
)
|
||||
|
||||
return super().putrequest(
|
||||
method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
|
||||
)
|
||||
|
||||
def putheader(self, header: str, *values: str) -> None:
|
||||
""""""
|
||||
if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
|
||||
super().putheader(header, *values)
|
||||
elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
|
||||
skippable_headers = "', '".join(
|
||||
[str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
|
||||
)
|
||||
raise ValueError(
|
||||
f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
|
||||
)
|
||||
|
||||
# `request` method's signature intentionally violates LSP.
|
||||
# urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
|
||||
def request( # type: ignore[override]
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
*,
|
||||
chunked: bool = False,
|
||||
preload_content: bool = True,
|
||||
decode_content: bool = True,
|
||||
enforce_content_length: bool = True,
|
||||
) -> None:
|
||||
# Update the inner socket's timeout value to send the request.
|
||||
# This only triggers if the connection is re-used.
|
||||
if self.sock is not None:
|
||||
self.sock.settimeout(self.timeout)
|
||||
|
||||
# Store these values to be fed into the HTTPResponse
|
||||
# object later. TODO: Remove this in favor of a real
|
||||
# HTTP lifecycle mechanism.
|
||||
|
||||
# We have to store these before we call .request()
|
||||
# because sometimes we can still salvage a response
|
||||
# off the wire even if we aren't able to completely
|
||||
# send the request body.
|
||||
self._response_options = _ResponseOptions(
|
||||
request_method=method,
|
||||
request_url=url,
|
||||
preload_content=preload_content,
|
||||
decode_content=decode_content,
|
||||
enforce_content_length=enforce_content_length,
|
||||
)
|
||||
|
||||
if headers is None:
|
||||
headers = {}
|
||||
header_keys = frozenset(to_str(k.lower()) for k in headers)
|
||||
skip_accept_encoding = "accept-encoding" in header_keys
|
||||
skip_host = "host" in header_keys
|
||||
self.putrequest(
|
||||
method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
|
||||
)
|
||||
|
||||
# Transform the body into an iterable of sendall()-able chunks
|
||||
# and detect if an explicit Content-Length is doable.
|
||||
chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
|
||||
chunks = chunks_and_cl.chunks
|
||||
content_length = chunks_and_cl.content_length
|
||||
|
||||
# When chunked is explicit set to 'True' we respect that.
|
||||
if chunked:
|
||||
if "transfer-encoding" not in header_keys:
|
||||
self.putheader("Transfer-Encoding", "chunked")
|
||||
else:
|
||||
# Detect whether a framing mechanism is already in use. If so
|
||||
# we respect that value, otherwise we pick chunked vs content-length
|
||||
# depending on the type of 'body'.
|
||||
if "content-length" in header_keys:
|
||||
chunked = False
|
||||
elif "transfer-encoding" in header_keys:
|
||||
chunked = True
|
||||
|
||||
# Otherwise we go off the recommendation of 'body_to_chunks()'.
|
||||
else:
|
||||
chunked = False
|
||||
if content_length is None:
|
||||
if chunks is not None:
|
||||
chunked = True
|
||||
self.putheader("Transfer-Encoding", "chunked")
|
||||
else:
|
||||
self.putheader("Content-Length", str(content_length))
|
||||
|
||||
# Now that framing headers are out of the way we send all the other headers.
|
||||
if "user-agent" not in header_keys:
|
||||
self.putheader("User-Agent", _get_default_user_agent())
|
||||
for header, value in headers.items():
|
||||
self.putheader(header, value)
|
||||
self.endheaders()
|
||||
|
||||
# If we're given a body we start sending that in chunks.
|
||||
if chunks is not None:
|
||||
for chunk in chunks:
|
||||
# Sending empty chunks isn't allowed for TE: chunked
|
||||
# as it indicates the end of the body.
|
||||
if not chunk:
|
||||
continue
|
||||
if isinstance(chunk, str):
|
||||
chunk = chunk.encode("utf-8")
|
||||
if chunked:
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
else:
|
||||
self.send(chunk)
|
||||
|
||||
# Regardless of whether we have a body or not, if we're in
|
||||
# chunked mode we want to send an explicit empty chunk.
|
||||
if chunked:
|
||||
self.send(b"0\r\n\r\n")
|
||||
|
||||
def request_chunked(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Alternative to the common request method, which sends the
|
||||
body with chunked encoding and not as one block
|
||||
"""
|
||||
warnings.warn(
|
||||
"HTTPConnection.request_chunked() is deprecated and will be removed "
|
||||
"in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.request(method, url, body=body, headers=headers, chunked=True)
|
||||
|
||||
def getresponse( # type: ignore[override]
|
||||
self,
|
||||
) -> HTTPResponse:
|
||||
"""
|
||||
Get the response from the server.
|
||||
|
||||
If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
|
||||
|
||||
If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
|
||||
"""
|
||||
# Raise the same error as http.client.HTTPConnection
|
||||
if self._response_options is None:
|
||||
raise ResponseNotReady()
|
||||
|
||||
# Reset this attribute for being used again.
|
||||
resp_options = self._response_options
|
||||
self._response_options = None
|
||||
|
||||
# Since the connection's timeout value may have been updated
|
||||
# we need to set the timeout on the socket.
|
||||
self.sock.settimeout(self.timeout)
|
||||
|
||||
# This is needed here to avoid circular import errors
|
||||
from .response import HTTPResponse
|
||||
|
||||
# Get the response from http.client.HTTPConnection
|
||||
httplib_response = super().getresponse()
|
||||
|
||||
try:
|
||||
assert_header_parsing(httplib_response.msg)
|
||||
except (HeaderParsingError, TypeError) as hpe:
|
||||
log.warning(
|
||||
"Failed to parse headers (url=%s): %s",
|
||||
_url_from_connection(self, resp_options.request_url),
|
||||
hpe,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
headers = HTTPHeaderDict(httplib_response.msg.items())
|
||||
|
||||
response = HTTPResponse(
|
||||
body=httplib_response,
|
||||
headers=headers,
|
||||
status=httplib_response.status,
|
||||
version=httplib_response.version,
|
||||
reason=httplib_response.reason,
|
||||
preload_content=resp_options.preload_content,
|
||||
decode_content=resp_options.decode_content,
|
||||
original_response=httplib_response,
|
||||
enforce_content_length=resp_options.enforce_content_length,
|
||||
request_method=resp_options.request_method,
|
||||
request_url=resp_options.request_url,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
class HTTPSConnection(HTTPConnection):
|
||||
"""
|
||||
Many of the parameters to this constructor are passed to the underlying SSL
|
||||
socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
|
||||
"""
|
||||
|
||||
default_port = port_by_scheme["https"] # type: ignore[misc]
|
||||
|
||||
cert_reqs: int | str | None = None
|
||||
ca_certs: str | None = None
|
||||
ca_cert_dir: str | None = None
|
||||
ca_cert_data: None | str | bytes = None
|
||||
ssl_version: int | str | None = None
|
||||
ssl_minimum_version: int | None = None
|
||||
ssl_maximum_version: int | None = None
|
||||
assert_fingerprint: str | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 16384,
|
||||
socket_options: None
|
||||
| (connection._TYPE_SOCKET_OPTIONS) = HTTPConnection.default_socket_options,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
cert_reqs: int | str | None = None,
|
||||
assert_hostname: None | str | Literal[False] = None,
|
||||
assert_fingerprint: str | None = None,
|
||||
server_hostname: str | None = None,
|
||||
ssl_context: ssl.SSLContext | None = None,
|
||||
ca_certs: str | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
ssl_minimum_version: int | None = None,
|
||||
ssl_maximum_version: int | None = None,
|
||||
ssl_version: int | str | None = None, # Deprecated
|
||||
cert_file: str | None = None,
|
||||
key_file: str | None = None,
|
||||
key_password: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
host,
|
||||
port=port,
|
||||
timeout=timeout,
|
||||
source_address=source_address,
|
||||
blocksize=blocksize,
|
||||
socket_options=socket_options,
|
||||
proxy=proxy,
|
||||
proxy_config=proxy_config,
|
||||
)
|
||||
|
||||
self.key_file = key_file
|
||||
self.cert_file = cert_file
|
||||
self.key_password = key_password
|
||||
self.ssl_context = ssl_context
|
||||
self.server_hostname = server_hostname
|
||||
self.assert_hostname = assert_hostname
|
||||
self.assert_fingerprint = assert_fingerprint
|
||||
self.ssl_version = ssl_version
|
||||
self.ssl_minimum_version = ssl_minimum_version
|
||||
self.ssl_maximum_version = ssl_maximum_version
|
||||
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
|
||||
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
|
||||
self.ca_cert_data = ca_cert_data
|
||||
|
||||
# cert_reqs depends on ssl_context so calculate last.
|
||||
if cert_reqs is None:
|
||||
if self.ssl_context is not None:
|
||||
cert_reqs = self.ssl_context.verify_mode
|
||||
else:
|
||||
cert_reqs = resolve_cert_reqs(None)
|
||||
self.cert_reqs = cert_reqs
|
||||
|
||||
def set_cert(
|
||||
self,
|
||||
key_file: str | None = None,
|
||||
cert_file: str | None = None,
|
||||
cert_reqs: int | str | None = None,
|
||||
key_password: str | None = None,
|
||||
ca_certs: str | None = None,
|
||||
assert_hostname: None | str | Literal[False] = None,
|
||||
assert_fingerprint: str | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
) -> None:
|
||||
"""
|
||||
This method should only be called once, before the connection is used.
|
||||
"""
|
||||
warnings.warn(
|
||||
"HTTPSConnection.set_cert() is deprecated and will be removed "
|
||||
"in urllib3 v2.1.0. Instead provide the parameters to the "
|
||||
"HTTPSConnection constructor.",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
|
||||
# have an SSLContext object in which case we'll use its verify_mode.
|
||||
if cert_reqs is None:
|
||||
if self.ssl_context is not None:
|
||||
cert_reqs = self.ssl_context.verify_mode
|
||||
else:
|
||||
cert_reqs = resolve_cert_reqs(None)
|
||||
|
||||
self.key_file = key_file
|
||||
self.cert_file = cert_file
|
||||
self.cert_reqs = cert_reqs
|
||||
self.key_password = key_password
|
||||
self.assert_hostname = assert_hostname
|
||||
self.assert_fingerprint = assert_fingerprint
|
||||
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
|
||||
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
|
||||
self.ca_cert_data = ca_cert_data
|
||||
|
||||
def connect(self) -> None:
|
||||
sock: socket.socket | ssl.SSLSocket
|
||||
self.sock = sock = self._new_conn()
|
||||
server_hostname: str = self.host
|
||||
tls_in_tls = False
|
||||
|
||||
# Do we need to establish a tunnel?
|
||||
if self._tunnel_host is not None:
|
||||
# We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
|
||||
if self._tunnel_scheme == "https":
|
||||
self.sock = sock = self._connect_tls_proxy(self.host, sock)
|
||||
tls_in_tls = True
|
||||
|
||||
# If we're tunneling it means we're connected to our proxy.
|
||||
self._has_connected_to_proxy = True
|
||||
|
||||
self._tunnel() # type: ignore[attr-defined]
|
||||
# Override the host with the one we're requesting data from.
|
||||
server_hostname = self._tunnel_host
|
||||
|
||||
if self.server_hostname is not None:
|
||||
server_hostname = self.server_hostname
|
||||
|
||||
is_time_off = datetime.date.today() < RECENT_DATE
|
||||
if is_time_off:
|
||||
warnings.warn(
|
||||
(
|
||||
f"System time is way off (before {RECENT_DATE}). This will probably "
|
||||
"lead to SSL verification errors"
|
||||
),
|
||||
SystemTimeWarning,
|
||||
)
|
||||
|
||||
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
|
||||
sock=sock,
|
||||
cert_reqs=self.cert_reqs,
|
||||
ssl_version=self.ssl_version,
|
||||
ssl_minimum_version=self.ssl_minimum_version,
|
||||
ssl_maximum_version=self.ssl_maximum_version,
|
||||
ca_certs=self.ca_certs,
|
||||
ca_cert_dir=self.ca_cert_dir,
|
||||
ca_cert_data=self.ca_cert_data,
|
||||
cert_file=self.cert_file,
|
||||
key_file=self.key_file,
|
||||
key_password=self.key_password,
|
||||
server_hostname=server_hostname,
|
||||
ssl_context=self.ssl_context,
|
||||
tls_in_tls=tls_in_tls,
|
||||
assert_hostname=self.assert_hostname,
|
||||
assert_fingerprint=self.assert_fingerprint,
|
||||
)
|
||||
self.sock = sock_and_verified.socket
|
||||
self.is_verified = sock_and_verified.is_verified
|
||||
|
||||
# If there's a proxy to be connected to we are fully connected.
|
||||
# This is set twice (once above and here) due to forwarding proxies
|
||||
# not using tunnelling.
|
||||
self._has_connected_to_proxy = bool(self.proxy)
|
||||
|
||||
def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
|
||||
"""
|
||||
Establish a TLS connection to the proxy using the provided SSL context.
|
||||
"""
|
||||
# `_connect_tls_proxy` is called when self._tunnel_host is truthy.
|
||||
proxy_config = typing.cast(ProxyConfig, self.proxy_config)
|
||||
ssl_context = proxy_config.ssl_context
|
||||
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
|
||||
sock,
|
||||
cert_reqs=self.cert_reqs,
|
||||
ssl_version=self.ssl_version,
|
||||
ssl_minimum_version=self.ssl_minimum_version,
|
||||
ssl_maximum_version=self.ssl_maximum_version,
|
||||
ca_certs=self.ca_certs,
|
||||
ca_cert_dir=self.ca_cert_dir,
|
||||
ca_cert_data=self.ca_cert_data,
|
||||
server_hostname=hostname,
|
||||
ssl_context=ssl_context,
|
||||
assert_hostname=proxy_config.assert_hostname,
|
||||
assert_fingerprint=proxy_config.assert_fingerprint,
|
||||
# Features that aren't implemented for proxies yet:
|
||||
cert_file=None,
|
||||
key_file=None,
|
||||
key_password=None,
|
||||
tls_in_tls=False,
|
||||
)
|
||||
self.proxy_is_verified = sock_and_verified.is_verified
|
||||
return sock_and_verified.socket # type: ignore[return-value]
|
||||
|
||||
|
||||
class _WrappedAndVerifiedSocket(typing.NamedTuple):
|
||||
"""
|
||||
Wrapped socket and whether the connection is
|
||||
verified after the TLS handshake
|
||||
"""
|
||||
|
||||
socket: ssl.SSLSocket | SSLTransport
|
||||
is_verified: bool
|
||||
|
||||
|
||||
def _ssl_wrap_socket_and_match_hostname(
|
||||
sock: socket.socket,
|
||||
*,
|
||||
cert_reqs: None | str | int,
|
||||
ssl_version: None | str | int,
|
||||
ssl_minimum_version: int | None,
|
||||
ssl_maximum_version: int | None,
|
||||
cert_file: str | None,
|
||||
key_file: str | None,
|
||||
key_password: str | None,
|
||||
ca_certs: str | None,
|
||||
ca_cert_dir: str | None,
|
||||
ca_cert_data: None | str | bytes,
|
||||
assert_hostname: None | str | Literal[False],
|
||||
assert_fingerprint: str | None,
|
||||
server_hostname: str | None,
|
||||
ssl_context: ssl.SSLContext | None,
|
||||
tls_in_tls: bool = False,
|
||||
) -> _WrappedAndVerifiedSocket:
|
||||
"""Logic for constructing an SSLContext from all TLS parameters, passing
|
||||
that down into ssl_wrap_socket, and then doing certificate verification
|
||||
either via hostname or fingerprint. This function exists to guarantee
|
||||
that both proxies and targets have the same behavior when connecting via TLS.
|
||||
"""
|
||||
default_ssl_context = False
|
||||
if ssl_context is None:
|
||||
default_ssl_context = True
|
||||
context = create_urllib3_context(
|
||||
ssl_version=resolve_ssl_version(ssl_version),
|
||||
ssl_minimum_version=ssl_minimum_version,
|
||||
ssl_maximum_version=ssl_maximum_version,
|
||||
cert_reqs=resolve_cert_reqs(cert_reqs),
|
||||
)
|
||||
else:
|
||||
context = ssl_context
|
||||
|
||||
context.verify_mode = resolve_cert_reqs(cert_reqs)
|
||||
|
||||
# In some cases, we want to verify hostnames ourselves
|
||||
if (
|
||||
# `ssl` can't verify fingerprints or alternate hostnames
|
||||
assert_fingerprint
|
||||
or assert_hostname
|
||||
# assert_hostname can be set to False to disable hostname checking
|
||||
or assert_hostname is False
|
||||
# We still support OpenSSL 1.0.2, which prevents us from verifying
|
||||
# hostnames easily: https://github.com/pyca/pyopenssl/pull/933
|
||||
or ssl_.IS_PYOPENSSL
|
||||
or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
|
||||
):
|
||||
context.check_hostname = False
|
||||
|
||||
# Try to load OS default certs if none are given. We need to do the hasattr() check
|
||||
# for custom pyOpenSSL SSLContext objects because they don't support
|
||||
# load_default_certs().
|
||||
if (
|
||||
not ca_certs
|
||||
and not ca_cert_dir
|
||||
and not ca_cert_data
|
||||
and default_ssl_context
|
||||
and hasattr(context, "load_default_certs")
|
||||
):
|
||||
context.load_default_certs()
|
||||
|
||||
# Ensure that IPv6 addresses are in the proper format and don't have a
|
||||
# scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
|
||||
# and interprets them as DNS hostnames.
|
||||
if server_hostname is not None:
|
||||
normalized = server_hostname.strip("[]")
|
||||
if "%" in normalized:
|
||||
normalized = normalized[: normalized.rfind("%")]
|
||||
if is_ipaddress(normalized):
|
||||
server_hostname = normalized
|
||||
|
||||
ssl_sock = ssl_wrap_socket(
|
||||
sock=sock,
|
||||
keyfile=key_file,
|
||||
certfile=cert_file,
|
||||
key_password=key_password,
|
||||
ca_certs=ca_certs,
|
||||
ca_cert_dir=ca_cert_dir,
|
||||
ca_cert_data=ca_cert_data,
|
||||
server_hostname=server_hostname,
|
||||
ssl_context=context,
|
||||
tls_in_tls=tls_in_tls,
|
||||
)
|
||||
|
||||
try:
|
||||
if assert_fingerprint:
|
||||
_assert_fingerprint(
|
||||
ssl_sock.getpeercert(binary_form=True), assert_fingerprint
|
||||
)
|
||||
elif (
|
||||
context.verify_mode != ssl.CERT_NONE
|
||||
and not context.check_hostname
|
||||
and assert_hostname is not False
|
||||
):
|
||||
cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
|
||||
|
||||
# Need to signal to our match_hostname whether to use 'commonName' or not.
|
||||
# If we're using our own constructed SSLContext we explicitly set 'False'
|
||||
# because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
|
||||
if default_ssl_context:
|
||||
hostname_checks_common_name = False
|
||||
else:
|
||||
hostname_checks_common_name = (
|
||||
getattr(context, "hostname_checks_common_name", False) or False
|
||||
)
|
||||
|
||||
_match_hostname(
|
||||
cert,
|
||||
assert_hostname or server_hostname, # type: ignore[arg-type]
|
||||
hostname_checks_common_name,
|
||||
)
|
||||
|
||||
return _WrappedAndVerifiedSocket(
|
||||
socket=ssl_sock,
|
||||
is_verified=context.verify_mode == ssl.CERT_REQUIRED
|
||||
or bool(assert_fingerprint),
|
||||
)
|
||||
except BaseException:
|
||||
ssl_sock.close()
|
||||
raise
|
||||
|
||||
|
||||
def _match_hostname(
|
||||
cert: _TYPE_PEER_CERT_RET_DICT | None,
|
||||
asserted_hostname: str,
|
||||
hostname_checks_common_name: bool = False,
|
||||
) -> None:
|
||||
# Our upstream implementation of ssl.match_hostname()
|
||||
# only applies this normalization to IP addresses so it doesn't
|
||||
# match DNS SANs so we do the same thing!
|
||||
stripped_hostname = asserted_hostname.strip("[]")
|
||||
if is_ipaddress(stripped_hostname):
|
||||
asserted_hostname = stripped_hostname
|
||||
|
||||
try:
|
||||
match_hostname(cert, asserted_hostname, hostname_checks_common_name)
|
||||
except CertificateError as e:
|
||||
log.warning(
|
||||
"Certificate did not match expected hostname: %s. Certificate: %s",
|
||||
asserted_hostname,
|
||||
cert,
|
||||
)
|
||||
# Add cert to exception and reraise so client code can inspect
|
||||
# the cert when catching the exception, if they want to
|
||||
e._peer_cert = cert # type: ignore[attr-defined]
|
||||
raise
|
||||
|
||||
|
||||
def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
|
||||
# Look for the phrase 'wrong version number', if found
|
||||
# then we should warn the user that we're very sure that
|
||||
# this proxy is HTTP-only and they have a configuration issue.
|
||||
error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
|
||||
is_likely_http_proxy = (
|
||||
"wrong version number" in error_normalized
|
||||
or "unknown protocol" in error_normalized
|
||||
)
|
||||
http_proxy_warning = (
|
||||
". Your proxy appears to only use HTTP and not HTTPS, "
|
||||
"try changing your proxy URL to be HTTP. See: "
|
||||
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
|
||||
"#https-proxy-error-http-proxy"
|
||||
)
|
||||
new_err = ProxyError(
|
||||
f"Unable to connect to proxy"
|
||||
f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
|
||||
err,
|
||||
)
|
||||
new_err.__cause__ = err
|
||||
return new_err
|
||||
|
||||
|
||||
def _get_default_user_agent() -> str:
|
||||
return f"python-urllib3/{__version__}"
|
||||
|
||||
|
||||
class DummyConnection:
|
||||
"""Used to detect a failed ConnectionCls import."""
|
||||
|
||||
|
||||
if not ssl:
|
||||
HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
|
||||
|
||||
|
||||
VerifiedHTTPSConnection = HTTPSConnection
|
||||
|
||||
|
||||
def _url_from_connection(
|
||||
conn: HTTPConnection | HTTPSConnection, path: str | None = None
|
||||
) -> str:
|
||||
"""Returns the URL from a given connection. This is mainly used for testing and logging."""
|
||||
|
||||
scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
|
||||
|
||||
return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url
|
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@ -1,548 +0,0 @@
|
||||
"""
|
||||
Module for using pyOpenSSL as a TLS backend. This module was relevant before
|
||||
the standard library ``ssl`` module supported SNI, but now that we've dropped
|
||||
support for Python 2.7 all relevant Python versions support SNI so
|
||||
**this module is no longer recommended**.
|
||||
|
||||
This needs the following packages installed:
|
||||
|
||||
* `pyOpenSSL`_ (tested with 16.0.0)
|
||||
* `cryptography`_ (minimum 1.3.4, from pyopenssl)
|
||||
* `idna`_ (minimum 2.0)
|
||||
|
||||
However, pyOpenSSL depends on cryptography, so while we use all three directly here we
|
||||
end up having relatively few packages required.
|
||||
|
||||
You can install them with the following command:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ python -m pip install pyopenssl cryptography idna
|
||||
|
||||
To activate certificate checking, call
|
||||
:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
|
||||
before you begin making HTTP requests. This can be done in a ``sitecustomize``
|
||||
module, or at any other time before your application begins using ``urllib3``,
|
||||
like this:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
try:
|
||||
import urllib3.contrib.pyopenssl
|
||||
urllib3.contrib.pyopenssl.inject_into_urllib3()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
.. _pyopenssl: https://www.pyopenssl.org
|
||||
.. _cryptography: https://cryptography.io
|
||||
.. _idna: https://github.com/kjd/idna
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import OpenSSL.SSL # type: ignore[import]
|
||||
from cryptography import x509
|
||||
|
||||
try:
|
||||
from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined]
|
||||
except ImportError:
|
||||
# UnsupportedExtension is gone in cryptography >= 2.1.0
|
||||
class UnsupportedExtension(Exception): # type: ignore[no-redef]
|
||||
pass
|
||||
|
||||
|
||||
import logging
|
||||
import ssl
|
||||
import typing
|
||||
from io import BytesIO
|
||||
from socket import socket as socket_cls
|
||||
from socket import timeout
|
||||
|
||||
from .. import util
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from OpenSSL.crypto import X509 # type: ignore[import]
|
||||
|
||||
|
||||
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
|
||||
|
||||
# Map from urllib3 to PyOpenSSL compatible parameter-values.
|
||||
_openssl_versions = {
|
||||
util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
|
||||
util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
|
||||
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
|
||||
}
|
||||
|
||||
if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
|
||||
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
|
||||
|
||||
if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
|
||||
_openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
|
||||
|
||||
|
||||
_stdlib_to_openssl_verify = {
|
||||
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
|
||||
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
|
||||
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
|
||||
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
|
||||
}
|
||||
_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()}
|
||||
|
||||
# The SSLvX values are the most likely to be missing in the future
|
||||
# but we check them all just to be sure.
|
||||
_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr(
|
||||
OpenSSL.SSL, "OP_NO_SSLv3", 0
|
||||
)
|
||||
_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0)
|
||||
_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0)
|
||||
_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0)
|
||||
_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0)
|
||||
|
||||
_openssl_to_ssl_minimum_version: dict[int, int] = {
|
||||
ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
|
||||
ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3,
|
||||
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1,
|
||||
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1,
|
||||
ssl.TLSVersion.TLSv1_3: (
|
||||
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
|
||||
),
|
||||
ssl.TLSVersion.MAXIMUM_SUPPORTED: (
|
||||
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
|
||||
),
|
||||
}
|
||||
_openssl_to_ssl_maximum_version: dict[int, int] = {
|
||||
ssl.TLSVersion.MINIMUM_SUPPORTED: (
|
||||
_OP_NO_SSLv2_OR_SSLv3
|
||||
| _OP_NO_TLSv1
|
||||
| _OP_NO_TLSv1_1
|
||||
| _OP_NO_TLSv1_2
|
||||
| _OP_NO_TLSv1_3
|
||||
),
|
||||
ssl.TLSVersion.TLSv1: (
|
||||
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3
|
||||
),
|
||||
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3,
|
||||
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3,
|
||||
ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3,
|
||||
ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
|
||||
}
|
||||
|
||||
# OpenSSL will only write 16K at a time
|
||||
SSL_WRITE_BLOCKSIZE = 16384
|
||||
|
||||
orig_util_SSLContext = util.ssl_.SSLContext
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def inject_into_urllib3() -> None:
|
||||
"Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
|
||||
|
||||
_validate_dependencies_met()
|
||||
|
||||
util.SSLContext = PyOpenSSLContext # type: ignore[assignment]
|
||||
util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment]
|
||||
util.IS_PYOPENSSL = True
|
||||
util.ssl_.IS_PYOPENSSL = True
|
||||
|
||||
|
||||
def extract_from_urllib3() -> None:
|
||||
"Undo monkey-patching by :func:`inject_into_urllib3`."
|
||||
|
||||
util.SSLContext = orig_util_SSLContext
|
||||
util.ssl_.SSLContext = orig_util_SSLContext
|
||||
util.IS_PYOPENSSL = False
|
||||
util.ssl_.IS_PYOPENSSL = False
|
||||
|
||||
|
||||
def _validate_dependencies_met() -> None:
|
||||
"""
|
||||
Verifies that PyOpenSSL's package-level dependencies have been met.
|
||||
Throws `ImportError` if they are not met.
|
||||
"""
|
||||
# Method added in `cryptography==1.1`; not available in older versions
|
||||
from cryptography.x509.extensions import Extensions
|
||||
|
||||
if getattr(Extensions, "get_extension_for_class", None) is None:
|
||||
raise ImportError(
|
||||
"'cryptography' module missing required functionality. "
|
||||
"Try upgrading to v1.3.4 or newer."
|
||||
)
|
||||
|
||||
# pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
|
||||
# attribute is only present on those versions.
|
||||
from OpenSSL.crypto import X509
|
||||
|
||||
x509 = X509()
|
||||
if getattr(x509, "_x509", None) is None:
|
||||
raise ImportError(
|
||||
"'pyOpenSSL' module missing required functionality. "
|
||||
"Try upgrading to v0.14 or newer."
|
||||
)
|
||||
|
||||
|
||||
def _dnsname_to_stdlib(name: str) -> str | None:
|
||||
"""
|
||||
Converts a dNSName SubjectAlternativeName field to the form used by the
|
||||
standard library on the given Python version.
|
||||
|
||||
Cryptography produces a dNSName as a unicode string that was idna-decoded
|
||||
from ASCII bytes. We need to idna-encode that string to get it back, and
|
||||
then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
|
||||
uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
|
||||
|
||||
If the name cannot be idna-encoded then we return None signalling that
|
||||
the name given should be skipped.
|
||||
"""
|
||||
|
||||
def idna_encode(name: str) -> bytes | None:
|
||||
"""
|
||||
Borrowed wholesale from the Python Cryptography Project. It turns out
|
||||
that we can't just safely call `idna.encode`: it can explode for
|
||||
wildcard names. This avoids that problem.
|
||||
"""
|
||||
import idna
|
||||
|
||||
try:
|
||||
for prefix in ["*.", "."]:
|
||||
if name.startswith(prefix):
|
||||
name = name[len(prefix) :]
|
||||
return prefix.encode("ascii") + idna.encode(name)
|
||||
return idna.encode(name)
|
||||
except idna.core.IDNAError:
|
||||
return None
|
||||
|
||||
# Don't send IPv6 addresses through the IDNA encoder.
|
||||
if ":" in name:
|
||||
return name
|
||||
|
||||
encoded_name = idna_encode(name)
|
||||
if encoded_name is None:
|
||||
return None
|
||||
return encoded_name.decode("utf-8")
|
||||
|
||||
|
||||
def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
|
||||
"""
|
||||
Given an PyOpenSSL certificate, provides all the subject alternative names.
|
||||
"""
|
||||
cert = peer_cert.to_cryptography()
|
||||
|
||||
# We want to find the SAN extension. Ask Cryptography to locate it (it's
|
||||
# faster than looping in Python)
|
||||
try:
|
||||
ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
|
||||
except x509.ExtensionNotFound:
|
||||
# No such extension, return the empty list.
|
||||
return []
|
||||
except (
|
||||
x509.DuplicateExtension,
|
||||
UnsupportedExtension,
|
||||
x509.UnsupportedGeneralNameType,
|
||||
UnicodeError,
|
||||
) as e:
|
||||
# A problem has been found with the quality of the certificate. Assume
|
||||
# no SAN field is present.
|
||||
log.warning(
|
||||
"A problem was encountered with the certificate that prevented "
|
||||
"urllib3 from finding the SubjectAlternativeName field. This can "
|
||||
"affect certificate validation. The error was %s",
|
||||
e,
|
||||
)
|
||||
return []
|
||||
|
||||
# We want to return dNSName and iPAddress fields. We need to cast the IPs
|
||||
# back to strings because the match_hostname function wants them as
|
||||
# strings.
|
||||
# Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
|
||||
# decoded. This is pretty frustrating, but that's what the standard library
|
||||
# does with certificates, and so we need to attempt to do the same.
|
||||
# We also want to skip over names which cannot be idna encoded.
|
||||
names = [
|
||||
("DNS", name)
|
||||
for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
|
||||
if name is not None
|
||||
]
|
||||
names.extend(
|
||||
("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
|
||||
)
|
||||
|
||||
return names
|
||||
|
||||
|
||||
class WrappedSocket:
|
||||
"""API-compatibility wrapper for Python OpenSSL's Connection-class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
connection: OpenSSL.SSL.Connection,
|
||||
socket: socket_cls,
|
||||
suppress_ragged_eofs: bool = True,
|
||||
) -> None:
|
||||
self.connection = connection
|
||||
self.socket = socket
|
||||
self.suppress_ragged_eofs = suppress_ragged_eofs
|
||||
self._io_refs = 0
|
||||
self._closed = False
|
||||
|
||||
def fileno(self) -> int:
|
||||
return self.socket.fileno()
|
||||
|
||||
# Copy-pasted from Python 3.5 source code
|
||||
def _decref_socketios(self) -> None:
|
||||
if self._io_refs > 0:
|
||||
self._io_refs -= 1
|
||||
if self._closed:
|
||||
self.close()
|
||||
|
||||
def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes:
|
||||
try:
|
||||
data = self.connection.recv(*args, **kwargs)
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
|
||||
return b""
|
||||
else:
|
||||
raise OSError(e.args[0], str(e)) from e
|
||||
except OpenSSL.SSL.ZeroReturnError:
|
||||
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
|
||||
return b""
|
||||
else:
|
||||
raise
|
||||
except OpenSSL.SSL.WantReadError as e:
|
||||
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
|
||||
raise timeout("The read operation timed out") from e
|
||||
else:
|
||||
return self.recv(*args, **kwargs)
|
||||
|
||||
# TLS 1.3 post-handshake authentication
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"read error: {e!r}") from e
|
||||
else:
|
||||
return data # type: ignore[no-any-return]
|
||||
|
||||
def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int:
|
||||
try:
|
||||
return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return]
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
|
||||
return 0
|
||||
else:
|
||||
raise OSError(e.args[0], str(e)) from e
|
||||
except OpenSSL.SSL.ZeroReturnError:
|
||||
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
|
||||
return 0
|
||||
else:
|
||||
raise
|
||||
except OpenSSL.SSL.WantReadError as e:
|
||||
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
|
||||
raise timeout("The read operation timed out") from e
|
||||
else:
|
||||
return self.recv_into(*args, **kwargs)
|
||||
|
||||
# TLS 1.3 post-handshake authentication
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"read error: {e!r}") from e
|
||||
|
||||
def settimeout(self, timeout: float) -> None:
|
||||
return self.socket.settimeout(timeout)
|
||||
|
||||
def _send_until_done(self, data: bytes) -> int:
|
||||
while True:
|
||||
try:
|
||||
return self.connection.send(data) # type: ignore[no-any-return]
|
||||
except OpenSSL.SSL.WantWriteError as e:
|
||||
if not util.wait_for_write(self.socket, self.socket.gettimeout()):
|
||||
raise timeout() from e
|
||||
continue
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
raise OSError(e.args[0], str(e)) from e
|
||||
|
||||
def sendall(self, data: bytes) -> None:
|
||||
total_sent = 0
|
||||
while total_sent < len(data):
|
||||
sent = self._send_until_done(
|
||||
data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
|
||||
)
|
||||
total_sent += sent
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# FIXME rethrow compatible exceptions should we ever use this
|
||||
self.connection.shutdown()
|
||||
|
||||
def close(self) -> None:
|
||||
self._closed = True
|
||||
if self._io_refs <= 0:
|
||||
self._real_close()
|
||||
|
||||
def _real_close(self) -> None:
|
||||
try:
|
||||
return self.connection.close() # type: ignore[no-any-return]
|
||||
except OpenSSL.SSL.Error:
|
||||
return
|
||||
|
||||
def getpeercert(
|
||||
self, binary_form: bool = False
|
||||
) -> dict[str, list[typing.Any]] | None:
|
||||
x509 = self.connection.get_peer_certificate()
|
||||
|
||||
if not x509:
|
||||
return x509 # type: ignore[no-any-return]
|
||||
|
||||
if binary_form:
|
||||
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return]
|
||||
|
||||
return {
|
||||
"subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item]
|
||||
"subjectAltName": get_subj_alt_name(x509),
|
||||
}
|
||||
|
||||
def version(self) -> str:
|
||||
return self.connection.get_protocol_version_name() # type: ignore[no-any-return]
|
||||
|
||||
|
||||
WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined]
|
||||
|
||||
|
||||
class PyOpenSSLContext:
|
||||
"""
|
||||
I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
|
||||
for translating the interface of the standard library ``SSLContext`` object
|
||||
to calls into PyOpenSSL.
|
||||
"""
|
||||
|
||||
def __init__(self, protocol: int) -> None:
|
||||
self.protocol = _openssl_versions[protocol]
|
||||
self._ctx = OpenSSL.SSL.Context(self.protocol)
|
||||
self._options = 0
|
||||
self.check_hostname = False
|
||||
self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED
|
||||
self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED
|
||||
|
||||
@property
|
||||
def options(self) -> int:
|
||||
return self._options
|
||||
|
||||
@options.setter
|
||||
def options(self, value: int) -> None:
|
||||
self._options = value
|
||||
self._set_ctx_options()
|
||||
|
||||
@property
|
||||
def verify_mode(self) -> int:
|
||||
return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
|
||||
|
||||
@verify_mode.setter
|
||||
def verify_mode(self, value: ssl.VerifyMode) -> None:
|
||||
self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
|
||||
|
||||
def set_default_verify_paths(self) -> None:
|
||||
self._ctx.set_default_verify_paths()
|
||||
|
||||
def set_ciphers(self, ciphers: bytes | str) -> None:
|
||||
if isinstance(ciphers, str):
|
||||
ciphers = ciphers.encode("utf-8")
|
||||
self._ctx.set_cipher_list(ciphers)
|
||||
|
||||
def load_verify_locations(
|
||||
self,
|
||||
cafile: str | None = None,
|
||||
capath: str | None = None,
|
||||
cadata: bytes | None = None,
|
||||
) -> None:
|
||||
if cafile is not None:
|
||||
cafile = cafile.encode("utf-8") # type: ignore[assignment]
|
||||
if capath is not None:
|
||||
capath = capath.encode("utf-8") # type: ignore[assignment]
|
||||
try:
|
||||
self._ctx.load_verify_locations(cafile, capath)
|
||||
if cadata is not None:
|
||||
self._ctx.load_verify_locations(BytesIO(cadata))
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e
|
||||
|
||||
def load_cert_chain(
|
||||
self,
|
||||
certfile: str,
|
||||
keyfile: str | None = None,
|
||||
password: str | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
self._ctx.use_certificate_chain_file(certfile)
|
||||
if password is not None:
|
||||
if not isinstance(password, bytes):
|
||||
password = password.encode("utf-8") # type: ignore[assignment]
|
||||
self._ctx.set_passwd_cb(lambda *_: password)
|
||||
self._ctx.use_privatekey_file(keyfile or certfile)
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e
|
||||
|
||||
def set_alpn_protocols(self, protocols: list[bytes | str]) -> None:
|
||||
protocols = [util.util.to_bytes(p, "ascii") for p in protocols]
|
||||
return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return]
|
||||
|
||||
def wrap_socket(
|
||||
self,
|
||||
sock: socket_cls,
|
||||
server_side: bool = False,
|
||||
do_handshake_on_connect: bool = True,
|
||||
suppress_ragged_eofs: bool = True,
|
||||
server_hostname: bytes | str | None = None,
|
||||
) -> WrappedSocket:
|
||||
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
|
||||
|
||||
# If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3
|
||||
if server_hostname and not util.ssl_.is_ipaddress(server_hostname):
|
||||
if isinstance(server_hostname, str):
|
||||
server_hostname = server_hostname.encode("utf-8")
|
||||
cnx.set_tlsext_host_name(server_hostname)
|
||||
|
||||
cnx.set_connect_state()
|
||||
|
||||
while True:
|
||||
try:
|
||||
cnx.do_handshake()
|
||||
except OpenSSL.SSL.WantReadError as e:
|
||||
if not util.wait_for_read(sock, sock.gettimeout()):
|
||||
raise timeout("select timed out") from e
|
||||
continue
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"bad handshake: {e!r}") from e
|
||||
break
|
||||
|
||||
return WrappedSocket(cnx, sock)
|
||||
|
||||
def _set_ctx_options(self) -> None:
|
||||
self._ctx.set_options(
|
||||
self._options
|
||||
| _openssl_to_ssl_minimum_version[self._minimum_version]
|
||||
| _openssl_to_ssl_maximum_version[self._maximum_version]
|
||||
)
|
||||
|
||||
@property
|
||||
def minimum_version(self) -> int:
|
||||
return self._minimum_version
|
||||
|
||||
@minimum_version.setter
|
||||
def minimum_version(self, minimum_version: int) -> None:
|
||||
self._minimum_version = minimum_version
|
||||
self._set_ctx_options()
|
||||
|
||||
@property
|
||||
def maximum_version(self) -> int:
|
||||
return self._maximum_version
|
||||
|
||||
@maximum_version.setter
|
||||
def maximum_version(self, maximum_version: int) -> None:
|
||||
self._maximum_version = maximum_version
|
||||
self._set_ctx_options()
|
||||
|
||||
|
||||
def _verify_callback(
|
||||
cnx: OpenSSL.SSL.Connection,
|
||||
x509: X509,
|
||||
err_no: int,
|
||||
err_depth: int,
|
||||
return_code: int,
|
||||
) -> bool:
|
||||
return err_no == 0
|
@ -1,230 +0,0 @@
|
||||
"""
|
||||
This module contains provisional support for SOCKS proxies from within
|
||||
urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
|
||||
SOCKS5. To enable its functionality, either install PySocks or install this
|
||||
module with the ``socks`` extra.
|
||||
|
||||
The SOCKS implementation supports the full range of urllib3 features. It also
|
||||
supports the following SOCKS features:
|
||||
|
||||
- SOCKS4A (``proxy_url='socks4a://...``)
|
||||
- SOCKS4 (``proxy_url='socks4://...``)
|
||||
- SOCKS5 with remote DNS (``proxy_url='socks5h://...``)
|
||||
- SOCKS5 with local DNS (``proxy_url='socks5://...``)
|
||||
- Usernames and passwords for the SOCKS proxy
|
||||
|
||||
.. note::
|
||||
It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in
|
||||
your ``proxy_url`` to ensure that DNS resolution is done from the remote
|
||||
server instead of client-side when connecting to a domain name.
|
||||
|
||||
SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5
|
||||
supports IPv4, IPv6, and domain names.
|
||||
|
||||
When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url``
|
||||
will be sent as the ``userid`` section of the SOCKS request:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
proxy_url="socks4a://<userid>@proxy-host"
|
||||
|
||||
When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion
|
||||
of the ``proxy_url`` will be sent as the username/password to authenticate
|
||||
with the proxy:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
proxy_url="socks5h://<username>:<password>@proxy-host"
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
try:
|
||||
import socks # type: ignore[import]
|
||||
except ImportError:
|
||||
import warnings
|
||||
|
||||
from ..exceptions import DependencyWarning
|
||||
|
||||
warnings.warn(
|
||||
(
|
||||
"SOCKS support in urllib3 requires the installation of optional "
|
||||
"dependencies: specifically, PySocks. For more information, see "
|
||||
"https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies"
|
||||
),
|
||||
DependencyWarning,
|
||||
)
|
||||
raise
|
||||
|
||||
import typing
|
||||
from socket import timeout as SocketTimeout
|
||||
|
||||
from ..connection import HTTPConnection, HTTPSConnection
|
||||
from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
from ..exceptions import ConnectTimeoutError, NewConnectionError
|
||||
from ..poolmanager import PoolManager
|
||||
from ..util.url import parse_url
|
||||
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
ssl = None # type: ignore[assignment]
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class _TYPE_SOCKS_OPTIONS(TypedDict):
|
||||
socks_version: int
|
||||
proxy_host: str | None
|
||||
proxy_port: str | None
|
||||
username: str | None
|
||||
password: str | None
|
||||
rdns: bool
|
||||
|
||||
|
||||
class SOCKSConnection(HTTPConnection):
|
||||
"""
|
||||
A plain-text HTTP connection that connects via a SOCKS proxy.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
_socks_options: _TYPE_SOCKS_OPTIONS,
|
||||
*args: typing.Any,
|
||||
**kwargs: typing.Any,
|
||||
) -> None:
|
||||
self._socks_options = _socks_options
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def _new_conn(self) -> socks.socksocket:
|
||||
"""
|
||||
Establish a new connection via the SOCKS proxy.
|
||||
"""
|
||||
extra_kw: dict[str, typing.Any] = {}
|
||||
if self.source_address:
|
||||
extra_kw["source_address"] = self.source_address
|
||||
|
||||
if self.socket_options:
|
||||
extra_kw["socket_options"] = self.socket_options
|
||||
|
||||
try:
|
||||
conn = socks.create_connection(
|
||||
(self.host, self.port),
|
||||
proxy_type=self._socks_options["socks_version"],
|
||||
proxy_addr=self._socks_options["proxy_host"],
|
||||
proxy_port=self._socks_options["proxy_port"],
|
||||
proxy_username=self._socks_options["username"],
|
||||
proxy_password=self._socks_options["password"],
|
||||
proxy_rdns=self._socks_options["rdns"],
|
||||
timeout=self.timeout,
|
||||
**extra_kw,
|
||||
)
|
||||
|
||||
except SocketTimeout as e:
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
|
||||
except socks.ProxyError as e:
|
||||
# This is fragile as hell, but it seems to be the only way to raise
|
||||
# useful errors here.
|
||||
if e.socket_err:
|
||||
error = e.socket_err
|
||||
if isinstance(error, SocketTimeout):
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
else:
|
||||
# Adding `from e` messes with coverage somehow, so it's omitted.
|
||||
# See #2386.
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {error}"
|
||||
)
|
||||
else:
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
|
||||
except OSError as e: # Defensive: PySocks should catch all these.
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
|
||||
return conn
|
||||
|
||||
|
||||
# We don't need to duplicate the Verified/Unverified distinction from
|
||||
# urllib3/connection.py here because the HTTPSConnection will already have been
|
||||
# correctly set to either the Verified or Unverified form by that module. This
|
||||
# means the SOCKSHTTPSConnection will automatically be the correct type.
|
||||
class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
|
||||
pass
|
||||
|
||||
|
||||
class SOCKSHTTPConnectionPool(HTTPConnectionPool):
|
||||
ConnectionCls = SOCKSConnection
|
||||
|
||||
|
||||
class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
|
||||
ConnectionCls = SOCKSHTTPSConnection
|
||||
|
||||
|
||||
class SOCKSProxyManager(PoolManager):
|
||||
"""
|
||||
A version of the urllib3 ProxyManager that routes connections via the
|
||||
defined SOCKS proxy.
|
||||
"""
|
||||
|
||||
pool_classes_by_scheme = {
|
||||
"http": SOCKSHTTPConnectionPool,
|
||||
"https": SOCKSHTTPSConnectionPool,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
proxy_url: str,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
):
|
||||
parsed = parse_url(proxy_url)
|
||||
|
||||
if username is None and password is None and parsed.auth is not None:
|
||||
split = parsed.auth.split(":")
|
||||
if len(split) == 2:
|
||||
username, password = split
|
||||
if parsed.scheme == "socks5":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS5
|
||||
rdns = False
|
||||
elif parsed.scheme == "socks5h":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS5
|
||||
rdns = True
|
||||
elif parsed.scheme == "socks4":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS4
|
||||
rdns = False
|
||||
elif parsed.scheme == "socks4a":
|
||||
socks_version = socks.PROXY_TYPE_SOCKS4
|
||||
rdns = True
|
||||
else:
|
||||
raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
|
||||
|
||||
self.proxy_url = proxy_url
|
||||
|
||||
socks_options = {
|
||||
"socks_version": socks_version,
|
||||
"proxy_host": parsed.host,
|
||||
"proxy_port": parsed.port,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"rdns": rdns,
|
||||
}
|
||||
connection_pool_kw["_socks_options"] = socks_options
|
||||
|
||||
super().__init__(num_pools, headers, **connection_pool_kw)
|
||||
|
||||
self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
|
@ -1,318 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import typing
|
||||
import warnings
|
||||
from email.errors import MessageDefect
|
||||
from http.client import IncompleteRead as httplib_IncompleteRead
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from .connection import HTTPConnection
|
||||
from .connectionpool import ConnectionPool
|
||||
from .response import HTTPResponse
|
||||
from .util.retry import Retry
|
||||
|
||||
# Base Exceptions
|
||||
|
||||
|
||||
class HTTPError(Exception):
|
||||
"""Base exception used by this module."""
|
||||
|
||||
|
||||
class HTTPWarning(Warning):
|
||||
"""Base warning used by this module."""
|
||||
|
||||
|
||||
_TYPE_REDUCE_RESULT = typing.Tuple[
|
||||
typing.Callable[..., object], typing.Tuple[object, ...]
|
||||
]
|
||||
|
||||
|
||||
class PoolError(HTTPError):
|
||||
"""Base exception for errors caused within a pool."""
|
||||
|
||||
def __init__(self, pool: ConnectionPool, message: str) -> None:
|
||||
self.pool = pool
|
||||
super().__init__(f"{pool}: {message}")
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
# For pickling purposes.
|
||||
return self.__class__, (None, None)
|
||||
|
||||
|
||||
class RequestError(PoolError):
|
||||
"""Base exception for PoolErrors that have associated URLs."""
|
||||
|
||||
def __init__(self, pool: ConnectionPool, url: str, message: str) -> None:
|
||||
self.url = url
|
||||
super().__init__(pool, message)
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
# For pickling purposes.
|
||||
return self.__class__, (None, self.url, None)
|
||||
|
||||
|
||||
class SSLError(HTTPError):
|
||||
"""Raised when SSL certificate fails in an HTTPS connection."""
|
||||
|
||||
|
||||
class ProxyError(HTTPError):
|
||||
"""Raised when the connection to a proxy fails."""
|
||||
|
||||
# The original error is also available as __cause__.
|
||||
original_error: Exception
|
||||
|
||||
def __init__(self, message: str, error: Exception) -> None:
|
||||
super().__init__(message, error)
|
||||
self.original_error = error
|
||||
|
||||
|
||||
class DecodeError(HTTPError):
|
||||
"""Raised when automatic decoding based on Content-Type fails."""
|
||||
|
||||
|
||||
class ProtocolError(HTTPError):
|
||||
"""Raised when something unexpected happens mid-request/response."""
|
||||
|
||||
|
||||
#: Renamed to ProtocolError but aliased for backwards compatibility.
|
||||
ConnectionError = ProtocolError
|
||||
|
||||
|
||||
# Leaf Exceptions
|
||||
|
||||
|
||||
class MaxRetryError(RequestError):
|
||||
"""Raised when the maximum number of retries is exceeded.
|
||||
|
||||
:param pool: The connection pool
|
||||
:type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
|
||||
:param str url: The requested Url
|
||||
:param reason: The underlying error
|
||||
:type reason: :class:`Exception`
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, pool: ConnectionPool, url: str, reason: Exception | None = None
|
||||
) -> None:
|
||||
self.reason = reason
|
||||
|
||||
message = f"Max retries exceeded with url: {url} (Caused by {reason!r})"
|
||||
|
||||
super().__init__(pool, url, message)
|
||||
|
||||
|
||||
class HostChangedError(RequestError):
|
||||
"""Raised when an existing pool gets a request for a foreign host."""
|
||||
|
||||
def __init__(
|
||||
self, pool: ConnectionPool, url: str, retries: Retry | int = 3
|
||||
) -> None:
|
||||
message = f"Tried to open a foreign host with url: {url}"
|
||||
super().__init__(pool, url, message)
|
||||
self.retries = retries
|
||||
|
||||
|
||||
class TimeoutStateError(HTTPError):
|
||||
"""Raised when passing an invalid state to a timeout"""
|
||||
|
||||
|
||||
class TimeoutError(HTTPError):
|
||||
"""Raised when a socket timeout error occurs.
|
||||
|
||||
Catching this error will catch both :exc:`ReadTimeoutErrors
|
||||
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
|
||||
"""
|
||||
|
||||
|
||||
class ReadTimeoutError(TimeoutError, RequestError):
|
||||
"""Raised when a socket timeout occurs while receiving data from a server"""
|
||||
|
||||
|
||||
# This timeout error does not have a URL attached and needs to inherit from the
|
||||
# base HTTPError
|
||||
class ConnectTimeoutError(TimeoutError):
|
||||
"""Raised when a socket timeout occurs while connecting to a server"""
|
||||
|
||||
|
||||
class NewConnectionError(ConnectTimeoutError, HTTPError):
|
||||
"""Raised when we fail to establish a new connection. Usually ECONNREFUSED."""
|
||||
|
||||
def __init__(self, conn: HTTPConnection, message: str) -> None:
|
||||
self.conn = conn
|
||||
super().__init__(f"{conn}: {message}")
|
||||
|
||||
@property
|
||||
def pool(self) -> HTTPConnection:
|
||||
warnings.warn(
|
||||
"The 'pool' property is deprecated and will be removed "
|
||||
"in urllib3 v2.1.0. Use 'conn' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
return self.conn
|
||||
|
||||
|
||||
class NameResolutionError(NewConnectionError):
|
||||
"""Raised when host name resolution fails."""
|
||||
|
||||
def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror):
|
||||
message = f"Failed to resolve '{host}' ({reason})"
|
||||
super().__init__(conn, message)
|
||||
|
||||
|
||||
class EmptyPoolError(PoolError):
|
||||
"""Raised when a pool runs out of connections and no more are allowed."""
|
||||
|
||||
|
||||
class FullPoolError(PoolError):
|
||||
"""Raised when we try to add a connection to a full pool in blocking mode."""
|
||||
|
||||
|
||||
class ClosedPoolError(PoolError):
|
||||
"""Raised when a request enters a pool after the pool has been closed."""
|
||||
|
||||
|
||||
class LocationValueError(ValueError, HTTPError):
|
||||
"""Raised when there is something wrong with a given URL input."""
|
||||
|
||||
|
||||
class LocationParseError(LocationValueError):
|
||||
"""Raised when get_host or similar fails to parse the URL input."""
|
||||
|
||||
def __init__(self, location: str) -> None:
|
||||
message = f"Failed to parse: {location}"
|
||||
super().__init__(message)
|
||||
|
||||
self.location = location
|
||||
|
||||
|
||||
class URLSchemeUnknown(LocationValueError):
|
||||
"""Raised when a URL input has an unsupported scheme."""
|
||||
|
||||
def __init__(self, scheme: str):
|
||||
message = f"Not supported URL scheme {scheme}"
|
||||
super().__init__(message)
|
||||
|
||||
self.scheme = scheme
|
||||
|
||||
|
||||
class ResponseError(HTTPError):
|
||||
"""Used as a container for an error reason supplied in a MaxRetryError."""
|
||||
|
||||
GENERIC_ERROR = "too many error responses"
|
||||
SPECIFIC_ERROR = "too many {status_code} error responses"
|
||||
|
||||
|
||||
class SecurityWarning(HTTPWarning):
|
||||
"""Warned when performing security reducing actions"""
|
||||
|
||||
|
||||
class InsecureRequestWarning(SecurityWarning):
|
||||
"""Warned when making an unverified HTTPS request."""
|
||||
|
||||
|
||||
class NotOpenSSLWarning(SecurityWarning):
|
||||
"""Warned when using unsupported SSL library"""
|
||||
|
||||
|
||||
class SystemTimeWarning(SecurityWarning):
|
||||
"""Warned when system time is suspected to be wrong"""
|
||||
|
||||
|
||||
class InsecurePlatformWarning(SecurityWarning):
|
||||
"""Warned when certain TLS/SSL configuration is not available on a platform."""
|
||||
|
||||
|
||||
class DependencyWarning(HTTPWarning):
|
||||
"""
|
||||
Warned when an attempt is made to import a module with missing optional
|
||||
dependencies.
|
||||
"""
|
||||
|
||||
|
||||
class ResponseNotChunked(ProtocolError, ValueError):
|
||||
"""Response needs to be chunked in order to read it as chunks."""
|
||||
|
||||
|
||||
class BodyNotHttplibCompatible(HTTPError):
|
||||
"""
|
||||
Body should be :class:`http.client.HTTPResponse` like
|
||||
(have an fp attribute which returns raw chunks) for read_chunked().
|
||||
"""
|
||||
|
||||
|
||||
class IncompleteRead(HTTPError, httplib_IncompleteRead):
|
||||
"""
|
||||
Response length doesn't match expected Content-Length
|
||||
|
||||
Subclass of :class:`http.client.IncompleteRead` to allow int value
|
||||
for ``partial`` to avoid creating large objects on streamed reads.
|
||||
"""
|
||||
|
||||
def __init__(self, partial: int, expected: int) -> None:
|
||||
self.partial = partial # type: ignore[assignment]
|
||||
self.expected = expected
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "IncompleteRead(%i bytes read, %i more expected)" % (
|
||||
self.partial, # type: ignore[str-format]
|
||||
self.expected,
|
||||
)
|
||||
|
||||
|
||||
class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
|
||||
"""Invalid chunk length in a chunked response."""
|
||||
|
||||
def __init__(self, response: HTTPResponse, length: bytes) -> None:
|
||||
self.partial: int = response.tell() # type: ignore[assignment]
|
||||
self.expected: int | None = response.length_remaining
|
||||
self.response = response
|
||||
self.length = length
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "InvalidChunkLength(got length %r, %i bytes read)" % (
|
||||
self.length,
|
||||
self.partial,
|
||||
)
|
||||
|
||||
|
||||
class InvalidHeader(HTTPError):
|
||||
"""The header provided was somehow invalid."""
|
||||
|
||||
|
||||
class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
|
||||
"""ProxyManager does not support the supplied scheme"""
|
||||
|
||||
# TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
|
||||
|
||||
def __init__(self, scheme: str | None) -> None:
|
||||
# 'localhost' is here because our URL parser parses
|
||||
# localhost:8080 -> scheme=localhost, remove if we fix this.
|
||||
if scheme == "localhost":
|
||||
scheme = None
|
||||
if scheme is None:
|
||||
message = "Proxy URL had no scheme, should start with http:// or https://"
|
||||
else:
|
||||
message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://"
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class ProxySchemeUnsupported(ValueError):
|
||||
"""Fetching HTTPS resources through HTTPS proxies is unsupported"""
|
||||
|
||||
|
||||
class HeaderParsingError(HTTPError):
|
||||
"""Raised by assert_header_parsing, but we convert it to a log.warning statement."""
|
||||
|
||||
def __init__(
|
||||
self, defects: list[MessageDefect], unparsed_data: bytes | str | None
|
||||
) -> None:
|
||||
message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}"
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class UnrewindableBodyError(HTTPError):
|
||||
"""urllib3 encountered an error when trying to rewind a body"""
|
@ -1,345 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import email.utils
|
||||
import mimetypes
|
||||
import typing
|
||||
|
||||
_TYPE_FIELD_VALUE = typing.Union[str, bytes]
|
||||
_TYPE_FIELD_VALUE_TUPLE = typing.Union[
|
||||
_TYPE_FIELD_VALUE,
|
||||
typing.Tuple[str, _TYPE_FIELD_VALUE],
|
||||
typing.Tuple[str, _TYPE_FIELD_VALUE, str],
|
||||
]
|
||||
|
||||
|
||||
def guess_content_type(
|
||||
filename: str | None, default: str = "application/octet-stream"
|
||||
) -> str:
|
||||
"""
|
||||
Guess the "Content-Type" of a file.
|
||||
|
||||
:param filename:
|
||||
The filename to guess the "Content-Type" of using :mod:`mimetypes`.
|
||||
:param default:
|
||||
If no "Content-Type" can be guessed, default to `default`.
|
||||
"""
|
||||
if filename:
|
||||
return mimetypes.guess_type(filename)[0] or default
|
||||
return default
|
||||
|
||||
|
||||
def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
Helper function to format and quote a single header parameter using the
|
||||
strategy defined in RFC 2231.
|
||||
|
||||
Particularly useful for header parameters which might contain
|
||||
non-ASCII values, like file names. This follows
|
||||
`RFC 2388 Section 4.4 <https://tools.ietf.org/html/rfc2388#section-4.4>`_.
|
||||
|
||||
:param name:
|
||||
The name of the parameter, a string expected to be ASCII only.
|
||||
:param value:
|
||||
The value of the parameter, provided as ``bytes`` or `str``.
|
||||
:returns:
|
||||
An RFC-2231-formatted unicode string.
|
||||
|
||||
.. deprecated:: 2.0.0
|
||||
Will be removed in urllib3 v2.1.0. This is not valid for
|
||||
``multipart/form-data`` header parameters.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"'format_header_param_rfc2231' is deprecated and will be "
|
||||
"removed in urllib3 v2.1.0. This is not valid for "
|
||||
"multipart/form-data header parameters.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8")
|
||||
|
||||
if not any(ch in value for ch in '"\\\r\n'):
|
||||
result = f'{name}="{value}"'
|
||||
try:
|
||||
result.encode("ascii")
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
pass
|
||||
else:
|
||||
return result
|
||||
|
||||
value = email.utils.encode_rfc2231(value, "utf-8")
|
||||
value = f"{name}*={value}"
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
Format and quote a single multipart header parameter.
|
||||
|
||||
This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching
|
||||
the behavior of current browser and curl versions. Values are
|
||||
assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are
|
||||
percent encoded.
|
||||
|
||||
.. _WHATWG HTML Standard:
|
||||
https://html.spec.whatwg.org/multipage/
|
||||
form-control-infrastructure.html#multipart-form-data
|
||||
|
||||
:param name:
|
||||
The name of the parameter, an ASCII-only ``str``.
|
||||
:param value:
|
||||
The value of the parameter, a ``str`` or UTF-8 encoded
|
||||
``bytes``.
|
||||
:returns:
|
||||
A string ``name="value"`` with the escaped value.
|
||||
|
||||
.. versionchanged:: 2.0.0
|
||||
Matches the WHATWG HTML Standard as of 2021/06/10. Control
|
||||
characters are no longer percent encoded.
|
||||
|
||||
.. versionchanged:: 2.0.0
|
||||
Renamed from ``format_header_param_html5`` and
|
||||
``format_header_param``. The old names will be removed in
|
||||
urllib3 v2.1.0.
|
||||
"""
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8")
|
||||
|
||||
# percent encode \n \r "
|
||||
value = value.translate({10: "%0A", 13: "%0D", 34: "%22"})
|
||||
return f'{name}="{value}"'
|
||||
|
||||
|
||||
def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
.. deprecated:: 2.0.0
|
||||
Renamed to :func:`format_multipart_header_param`. Will be
|
||||
removed in urllib3 v2.1.0.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"'format_header_param_html5' has been renamed to "
|
||||
"'format_multipart_header_param'. The old name will be "
|
||||
"removed in urllib3 v2.1.0.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return format_multipart_header_param(name, value)
|
||||
|
||||
|
||||
def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
.. deprecated:: 2.0.0
|
||||
Renamed to :func:`format_multipart_header_param`. Will be
|
||||
removed in urllib3 v2.1.0.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"'format_header_param' has been renamed to "
|
||||
"'format_multipart_header_param'. The old name will be "
|
||||
"removed in urllib3 v2.1.0.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return format_multipart_header_param(name, value)
|
||||
|
||||
|
||||
class RequestField:
|
||||
"""
|
||||
A data container for request body parameters.
|
||||
|
||||
:param name:
|
||||
The name of this request field. Must be unicode.
|
||||
:param data:
|
||||
The data/value body.
|
||||
:param filename:
|
||||
An optional filename of the request field. Must be unicode.
|
||||
:param headers:
|
||||
An optional dict-like object of headers to initially use for the field.
|
||||
|
||||
.. versionchanged:: 2.0.0
|
||||
The ``header_formatter`` parameter is deprecated and will
|
||||
be removed in urllib3 v2.1.0.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data: _TYPE_FIELD_VALUE,
|
||||
filename: str | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
|
||||
):
|
||||
self._name = name
|
||||
self._filename = filename
|
||||
self.data = data
|
||||
self.headers: dict[str, str | None] = {}
|
||||
if headers:
|
||||
self.headers = dict(headers)
|
||||
|
||||
if header_formatter is not None:
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"The 'header_formatter' parameter is deprecated and "
|
||||
"will be removed in urllib3 v2.1.0.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.header_formatter = header_formatter
|
||||
else:
|
||||
self.header_formatter = format_multipart_header_param
|
||||
|
||||
@classmethod
|
||||
def from_tuples(
|
||||
cls,
|
||||
fieldname: str,
|
||||
value: _TYPE_FIELD_VALUE_TUPLE,
|
||||
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
|
||||
) -> RequestField:
|
||||
"""
|
||||
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
|
||||
|
||||
Supports constructing :class:`~urllib3.fields.RequestField` from
|
||||
parameter of key/value strings AND key/filetuple. A filetuple is a
|
||||
(filename, data, MIME type) tuple where the MIME type is optional.
|
||||
For example::
|
||||
|
||||
'foo': 'bar',
|
||||
'fakefile': ('foofile.txt', 'contents of foofile'),
|
||||
'realfile': ('barfile.txt', open('realfile').read()),
|
||||
'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'),
|
||||
'nonamefile': 'contents of nonamefile field',
|
||||
|
||||
Field names and filenames must be unicode.
|
||||
"""
|
||||
filename: str | None
|
||||
content_type: str | None
|
||||
data: _TYPE_FIELD_VALUE
|
||||
|
||||
if isinstance(value, tuple):
|
||||
if len(value) == 3:
|
||||
filename, data, content_type = typing.cast(
|
||||
typing.Tuple[str, _TYPE_FIELD_VALUE, str], value
|
||||
)
|
||||
else:
|
||||
filename, data = typing.cast(
|
||||
typing.Tuple[str, _TYPE_FIELD_VALUE], value
|
||||
)
|
||||
content_type = guess_content_type(filename)
|
||||
else:
|
||||
filename = None
|
||||
content_type = None
|
||||
data = value
|
||||
|
||||
request_param = cls(
|
||||
fieldname, data, filename=filename, header_formatter=header_formatter
|
||||
)
|
||||
request_param.make_multipart(content_type=content_type)
|
||||
|
||||
return request_param
|
||||
|
||||
def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
Override this method to change how each multipart header
|
||||
parameter is formatted. By default, this calls
|
||||
:func:`format_multipart_header_param`.
|
||||
|
||||
:param name:
|
||||
The name of the parameter, an ASCII-only ``str``.
|
||||
:param value:
|
||||
The value of the parameter, a ``str`` or UTF-8 encoded
|
||||
``bytes``.
|
||||
|
||||
:meta public:
|
||||
"""
|
||||
return self.header_formatter(name, value)
|
||||
|
||||
def _render_parts(
|
||||
self,
|
||||
header_parts: (
|
||||
dict[str, _TYPE_FIELD_VALUE | None]
|
||||
| typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]]
|
||||
),
|
||||
) -> str:
|
||||
"""
|
||||
Helper function to format and quote a single header.
|
||||
|
||||
Useful for single headers that are composed of multiple items. E.g.,
|
||||
'Content-Disposition' fields.
|
||||
|
||||
:param header_parts:
|
||||
A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
|
||||
as `k1="v1"; k2="v2"; ...`.
|
||||
"""
|
||||
iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]]
|
||||
|
||||
parts = []
|
||||
if isinstance(header_parts, dict):
|
||||
iterable = header_parts.items()
|
||||
else:
|
||||
iterable = header_parts
|
||||
|
||||
for name, value in iterable:
|
||||
if value is not None:
|
||||
parts.append(self._render_part(name, value))
|
||||
|
||||
return "; ".join(parts)
|
||||
|
||||
def render_headers(self) -> str:
|
||||
"""
|
||||
Renders the headers for this request field.
|
||||
"""
|
||||
lines = []
|
||||
|
||||
sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
|
||||
for sort_key in sort_keys:
|
||||
if self.headers.get(sort_key, False):
|
||||
lines.append(f"{sort_key}: {self.headers[sort_key]}")
|
||||
|
||||
for header_name, header_value in self.headers.items():
|
||||
if header_name not in sort_keys:
|
||||
if header_value:
|
||||
lines.append(f"{header_name}: {header_value}")
|
||||
|
||||
lines.append("\r\n")
|
||||
return "\r\n".join(lines)
|
||||
|
||||
def make_multipart(
|
||||
self,
|
||||
content_disposition: str | None = None,
|
||||
content_type: str | None = None,
|
||||
content_location: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Makes this request field into a multipart request field.
|
||||
|
||||
This method overrides "Content-Disposition", "Content-Type" and
|
||||
"Content-Location" headers to the request parameter.
|
||||
|
||||
:param content_disposition:
|
||||
The 'Content-Disposition' of the request body. Defaults to 'form-data'
|
||||
:param content_type:
|
||||
The 'Content-Type' of the request body.
|
||||
:param content_location:
|
||||
The 'Content-Location' of the request body.
|
||||
|
||||
"""
|
||||
content_disposition = (content_disposition or "form-data") + "; ".join(
|
||||
[
|
||||
"",
|
||||
self._render_parts(
|
||||
(("name", self._name), ("filename", self._filename))
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
self.headers["Content-Disposition"] = content_disposition
|
||||
self.headers["Content-Type"] = content_type
|
||||
self.headers["Content-Location"] = content_location
|
@ -1,89 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import binascii
|
||||
import codecs
|
||||
import os
|
||||
import typing
|
||||
from io import BytesIO
|
||||
|
||||
from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField
|
||||
|
||||
writer = codecs.lookup("utf-8")[3]
|
||||
|
||||
_TYPE_FIELDS_SEQUENCE = typing.Sequence[
|
||||
typing.Union[typing.Tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField]
|
||||
]
|
||||
_TYPE_FIELDS = typing.Union[
|
||||
_TYPE_FIELDS_SEQUENCE,
|
||||
typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE],
|
||||
]
|
||||
|
||||
|
||||
def choose_boundary() -> str:
|
||||
"""
|
||||
Our embarrassingly-simple replacement for mimetools.choose_boundary.
|
||||
"""
|
||||
return binascii.hexlify(os.urandom(16)).decode()
|
||||
|
||||
|
||||
def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
|
||||
"""
|
||||
Iterate over fields.
|
||||
|
||||
Supports list of (k, v) tuples and dicts, and lists of
|
||||
:class:`~urllib3.fields.RequestField`.
|
||||
|
||||
"""
|
||||
iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]]
|
||||
|
||||
if isinstance(fields, typing.Mapping):
|
||||
iterable = fields.items()
|
||||
else:
|
||||
iterable = fields
|
||||
|
||||
for field in iterable:
|
||||
if isinstance(field, RequestField):
|
||||
yield field
|
||||
else:
|
||||
yield RequestField.from_tuples(*field)
|
||||
|
||||
|
||||
def encode_multipart_formdata(
|
||||
fields: _TYPE_FIELDS, boundary: str | None = None
|
||||
) -> tuple[bytes, str]:
|
||||
"""
|
||||
Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
|
||||
|
||||
:param fields:
|
||||
Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
|
||||
Values are processed by :func:`urllib3.fields.RequestField.from_tuples`.
|
||||
|
||||
:param boundary:
|
||||
If not specified, then a random boundary will be generated using
|
||||
:func:`urllib3.filepost.choose_boundary`.
|
||||
"""
|
||||
body = BytesIO()
|
||||
if boundary is None:
|
||||
boundary = choose_boundary()
|
||||
|
||||
for field in iter_field_objects(fields):
|
||||
body.write(f"--{boundary}\r\n".encode("latin-1"))
|
||||
|
||||
writer(body).write(field.render_headers())
|
||||
data = field.data
|
||||
|
||||
if isinstance(data, int):
|
||||
data = str(data) # Backwards compatibility
|
||||
|
||||
if isinstance(data, str):
|
||||
writer(body).write(data)
|
||||
else:
|
||||
body.write(data)
|
||||
|
||||
body.write(b"\r\n")
|
||||
|
||||
body.write(f"--{boundary}--\r\n".encode("latin-1"))
|
||||
|
||||
content_type = f"multipart/form-data; boundary={boundary}"
|
||||
|
||||
return body.getvalue(), content_type
|
@ -1,638 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import typing
|
||||
import warnings
|
||||
from types import TracebackType
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from ._collections import HTTPHeaderDict, RecentlyUsedContainer
|
||||
from ._request_methods import RequestMethods
|
||||
from .connection import ProxyConfig
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
|
||||
from .exceptions import (
|
||||
LocationValueError,
|
||||
MaxRetryError,
|
||||
ProxySchemeUnknown,
|
||||
URLSchemeUnknown,
|
||||
)
|
||||
from .response import BaseHTTPResponse
|
||||
from .util.connection import _TYPE_SOCKET_OPTIONS
|
||||
from .util.proxy import connection_requires_http_tunnel
|
||||
from .util.retry import Retry
|
||||
from .util.timeout import Timeout
|
||||
from .util.url import Url, parse_url
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
import ssl
|
||||
from typing import Literal
|
||||
|
||||
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SSL_KEYWORDS = (
|
||||
"key_file",
|
||||
"cert_file",
|
||||
"cert_reqs",
|
||||
"ca_certs",
|
||||
"ca_cert_data",
|
||||
"ssl_version",
|
||||
"ssl_minimum_version",
|
||||
"ssl_maximum_version",
|
||||
"ca_cert_dir",
|
||||
"ssl_context",
|
||||
"key_password",
|
||||
"server_hostname",
|
||||
)
|
||||
# Default value for `blocksize` - a new parameter introduced to
|
||||
# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
|
||||
_DEFAULT_BLOCKSIZE = 16384
|
||||
|
||||
_SelfT = typing.TypeVar("_SelfT")
|
||||
|
||||
|
||||
class PoolKey(typing.NamedTuple):
|
||||
"""
|
||||
All known keyword arguments that could be provided to the pool manager, its
|
||||
pools, or the underlying connections.
|
||||
|
||||
All custom key schemes should include the fields in this key at a minimum.
|
||||
"""
|
||||
|
||||
key_scheme: str
|
||||
key_host: str
|
||||
key_port: int | None
|
||||
key_timeout: Timeout | float | int | None
|
||||
key_retries: Retry | bool | int | None
|
||||
key_block: bool | None
|
||||
key_source_address: tuple[str, int] | None
|
||||
key_key_file: str | None
|
||||
key_key_password: str | None
|
||||
key_cert_file: str | None
|
||||
key_cert_reqs: str | None
|
||||
key_ca_certs: str | None
|
||||
key_ca_cert_data: str | bytes | None
|
||||
key_ssl_version: int | str | None
|
||||
key_ssl_minimum_version: ssl.TLSVersion | None
|
||||
key_ssl_maximum_version: ssl.TLSVersion | None
|
||||
key_ca_cert_dir: str | None
|
||||
key_ssl_context: ssl.SSLContext | None
|
||||
key_maxsize: int | None
|
||||
key_headers: frozenset[tuple[str, str]] | None
|
||||
key__proxy: Url | None
|
||||
key__proxy_headers: frozenset[tuple[str, str]] | None
|
||||
key__proxy_config: ProxyConfig | None
|
||||
key_socket_options: _TYPE_SOCKET_OPTIONS | None
|
||||
key__socks_options: frozenset[tuple[str, str]] | None
|
||||
key_assert_hostname: bool | str | None
|
||||
key_assert_fingerprint: str | None
|
||||
key_server_hostname: str | None
|
||||
key_blocksize: int | None
|
||||
|
||||
|
||||
def _default_key_normalizer(
|
||||
key_class: type[PoolKey], request_context: dict[str, typing.Any]
|
||||
) -> PoolKey:
|
||||
"""
|
||||
Create a pool key out of a request context dictionary.
|
||||
|
||||
According to RFC 3986, both the scheme and host are case-insensitive.
|
||||
Therefore, this function normalizes both before constructing the pool
|
||||
key for an HTTPS request. If you wish to change this behaviour, provide
|
||||
alternate callables to ``key_fn_by_scheme``.
|
||||
|
||||
:param key_class:
|
||||
The class to use when constructing the key. This should be a namedtuple
|
||||
with the ``scheme`` and ``host`` keys at a minimum.
|
||||
:type key_class: namedtuple
|
||||
:param request_context:
|
||||
A dictionary-like object that contain the context for a request.
|
||||
:type request_context: dict
|
||||
|
||||
:return: A namedtuple that can be used as a connection pool key.
|
||||
:rtype: PoolKey
|
||||
"""
|
||||
# Since we mutate the dictionary, make a copy first
|
||||
context = request_context.copy()
|
||||
context["scheme"] = context["scheme"].lower()
|
||||
context["host"] = context["host"].lower()
|
||||
|
||||
# These are both dictionaries and need to be transformed into frozensets
|
||||
for key in ("headers", "_proxy_headers", "_socks_options"):
|
||||
if key in context and context[key] is not None:
|
||||
context[key] = frozenset(context[key].items())
|
||||
|
||||
# The socket_options key may be a list and needs to be transformed into a
|
||||
# tuple.
|
||||
socket_opts = context.get("socket_options")
|
||||
if socket_opts is not None:
|
||||
context["socket_options"] = tuple(socket_opts)
|
||||
|
||||
# Map the kwargs to the names in the namedtuple - this is necessary since
|
||||
# namedtuples can't have fields starting with '_'.
|
||||
for key in list(context.keys()):
|
||||
context["key_" + key] = context.pop(key)
|
||||
|
||||
# Default to ``None`` for keys missing from the context
|
||||
for field in key_class._fields:
|
||||
if field not in context:
|
||||
context[field] = None
|
||||
|
||||
# Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
|
||||
if context.get("key_blocksize") is None:
|
||||
context["key_blocksize"] = _DEFAULT_BLOCKSIZE
|
||||
|
||||
return key_class(**context)
|
||||
|
||||
|
||||
#: A dictionary that maps a scheme to a callable that creates a pool key.
|
||||
#: This can be used to alter the way pool keys are constructed, if desired.
|
||||
#: Each PoolManager makes a copy of this dictionary so they can be configured
|
||||
#: globally here, or individually on the instance.
|
||||
key_fn_by_scheme = {
|
||||
"http": functools.partial(_default_key_normalizer, PoolKey),
|
||||
"https": functools.partial(_default_key_normalizer, PoolKey),
|
||||
}
|
||||
|
||||
pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
|
||||
|
||||
|
||||
class PoolManager(RequestMethods):
|
||||
"""
|
||||
Allows for arbitrary requests while transparently keeping track of
|
||||
necessary connection pools for you.
|
||||
|
||||
:param num_pools:
|
||||
Number of connection pools to cache before discarding the least
|
||||
recently used pool.
|
||||
|
||||
:param headers:
|
||||
Headers to include with all requests, unless other headers are given
|
||||
explicitly.
|
||||
|
||||
:param \\**connection_pool_kw:
|
||||
Additional parameters are used to create fresh
|
||||
:class:`urllib3.connectionpool.ConnectionPool` instances.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
http = urllib3.PoolManager(num_pools=2)
|
||||
|
||||
resp1 = http.request("GET", "https://google.com/")
|
||||
resp2 = http.request("GET", "https://google.com/mail")
|
||||
resp3 = http.request("GET", "https://yahoo.com/")
|
||||
|
||||
print(len(http.pools))
|
||||
# 2
|
||||
|
||||
"""
|
||||
|
||||
proxy: Url | None = None
|
||||
proxy_config: ProxyConfig | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
) -> None:
|
||||
super().__init__(headers)
|
||||
self.connection_pool_kw = connection_pool_kw
|
||||
|
||||
self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
|
||||
self.pools = RecentlyUsedContainer(num_pools)
|
||||
|
||||
# Locally set the pool classes and keys so other PoolManagers can
|
||||
# override them.
|
||||
self.pool_classes_by_scheme = pool_classes_by_scheme
|
||||
self.key_fn_by_scheme = key_fn_by_scheme.copy()
|
||||
|
||||
def __enter__(self: _SelfT) -> _SelfT:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_val: BaseException | None,
|
||||
exc_tb: TracebackType | None,
|
||||
) -> Literal[False]:
|
||||
self.clear()
|
||||
# Return False to re-raise any potential exceptions
|
||||
return False
|
||||
|
||||
def _new_pool(
|
||||
self,
|
||||
scheme: str,
|
||||
host: str,
|
||||
port: int,
|
||||
request_context: dict[str, typing.Any] | None = None,
|
||||
) -> HTTPConnectionPool:
|
||||
"""
|
||||
Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
|
||||
any additional pool keyword arguments.
|
||||
|
||||
If ``request_context`` is provided, it is provided as keyword arguments
|
||||
to the pool class used. This method is used to actually create the
|
||||
connection pools handed out by :meth:`connection_from_url` and
|
||||
companion methods. It is intended to be overridden for customization.
|
||||
"""
|
||||
pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
|
||||
if request_context is None:
|
||||
request_context = self.connection_pool_kw.copy()
|
||||
|
||||
# Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
|
||||
# set to 'None' in the request_context.
|
||||
if request_context.get("blocksize") is None:
|
||||
request_context["blocksize"] = _DEFAULT_BLOCKSIZE
|
||||
|
||||
# Although the context has everything necessary to create the pool,
|
||||
# this function has historically only used the scheme, host, and port
|
||||
# in the positional args. When an API change is acceptable these can
|
||||
# be removed.
|
||||
for key in ("scheme", "host", "port"):
|
||||
request_context.pop(key, None)
|
||||
|
||||
if scheme == "http":
|
||||
for kw in SSL_KEYWORDS:
|
||||
request_context.pop(kw, None)
|
||||
|
||||
return pool_cls(host, port, **request_context)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Empty our store of pools and direct them all to close.
|
||||
|
||||
This will not affect in-flight connections, but they will not be
|
||||
re-used after completion.
|
||||
"""
|
||||
self.pools.clear()
|
||||
|
||||
def connection_from_host(
|
||||
self,
|
||||
host: str | None,
|
||||
port: int | None = None,
|
||||
scheme: str | None = "http",
|
||||
pool_kwargs: dict[str, typing.Any] | None = None,
|
||||
) -> HTTPConnectionPool:
|
||||
"""
|
||||
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
|
||||
|
||||
If ``port`` isn't given, it will be derived from the ``scheme`` using
|
||||
``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
|
||||
provided, it is merged with the instance's ``connection_pool_kw``
|
||||
variable and used to create the new connection pool, if one is
|
||||
needed.
|
||||
"""
|
||||
|
||||
if not host:
|
||||
raise LocationValueError("No host specified.")
|
||||
|
||||
request_context = self._merge_pool_kwargs(pool_kwargs)
|
||||
request_context["scheme"] = scheme or "http"
|
||||
if not port:
|
||||
port = port_by_scheme.get(request_context["scheme"].lower(), 80)
|
||||
request_context["port"] = port
|
||||
request_context["host"] = host
|
||||
|
||||
return self.connection_from_context(request_context)
|
||||
|
||||
def connection_from_context(
|
||||
self, request_context: dict[str, typing.Any]
|
||||
) -> HTTPConnectionPool:
|
||||
"""
|
||||
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
|
||||
|
||||
``request_context`` must at least contain the ``scheme`` key and its
|
||||
value must be a key in ``key_fn_by_scheme`` instance variable.
|
||||
"""
|
||||
if "strict" in request_context:
|
||||
warnings.warn(
|
||||
"The 'strict' parameter is no longer needed on Python 3+. "
|
||||
"This will raise an error in urllib3 v2.1.0.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
request_context.pop("strict")
|
||||
|
||||
scheme = request_context["scheme"].lower()
|
||||
pool_key_constructor = self.key_fn_by_scheme.get(scheme)
|
||||
if not pool_key_constructor:
|
||||
raise URLSchemeUnknown(scheme)
|
||||
pool_key = pool_key_constructor(request_context)
|
||||
|
||||
return self.connection_from_pool_key(pool_key, request_context=request_context)
|
||||
|
||||
def connection_from_pool_key(
|
||||
self, pool_key: PoolKey, request_context: dict[str, typing.Any]
|
||||
) -> HTTPConnectionPool:
|
||||
"""
|
||||
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
|
||||
|
||||
``pool_key`` should be a namedtuple that only contains immutable
|
||||
objects. At a minimum it must have the ``scheme``, ``host``, and
|
||||
``port`` fields.
|
||||
"""
|
||||
with self.pools.lock:
|
||||
# If the scheme, host, or port doesn't match existing open
|
||||
# connections, open a new ConnectionPool.
|
||||
pool = self.pools.get(pool_key)
|
||||
if pool:
|
||||
return pool
|
||||
|
||||
# Make a fresh ConnectionPool of the desired type
|
||||
scheme = request_context["scheme"]
|
||||
host = request_context["host"]
|
||||
port = request_context["port"]
|
||||
pool = self._new_pool(scheme, host, port, request_context=request_context)
|
||||
self.pools[pool_key] = pool
|
||||
|
||||
return pool
|
||||
|
||||
def connection_from_url(
|
||||
self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
|
||||
) -> HTTPConnectionPool:
|
||||
"""
|
||||
Similar to :func:`urllib3.connectionpool.connection_from_url`.
|
||||
|
||||
If ``pool_kwargs`` is not provided and a new pool needs to be
|
||||
constructed, ``self.connection_pool_kw`` is used to initialize
|
||||
the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
|
||||
is provided, it is used instead. Note that if a new pool does not
|
||||
need to be created for the request, the provided ``pool_kwargs`` are
|
||||
not used.
|
||||
"""
|
||||
u = parse_url(url)
|
||||
return self.connection_from_host(
|
||||
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
|
||||
)
|
||||
|
||||
def _merge_pool_kwargs(
|
||||
self, override: dict[str, typing.Any] | None
|
||||
) -> dict[str, typing.Any]:
|
||||
"""
|
||||
Merge a dictionary of override values for self.connection_pool_kw.
|
||||
|
||||
This does not modify self.connection_pool_kw and returns a new dict.
|
||||
Any keys in the override dictionary with a value of ``None`` are
|
||||
removed from the merged dictionary.
|
||||
"""
|
||||
base_pool_kwargs = self.connection_pool_kw.copy()
|
||||
if override:
|
||||
for key, value in override.items():
|
||||
if value is None:
|
||||
try:
|
||||
del base_pool_kwargs[key]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
base_pool_kwargs[key] = value
|
||||
return base_pool_kwargs
|
||||
|
||||
def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
|
||||
"""
|
||||
Indicates if the proxy requires the complete destination URL in the
|
||||
request. Normally this is only needed when not using an HTTP CONNECT
|
||||
tunnel.
|
||||
"""
|
||||
if self.proxy is None:
|
||||
return False
|
||||
|
||||
return not connection_requires_http_tunnel(
|
||||
self.proxy, self.proxy_config, parsed_url.scheme
|
||||
)
|
||||
|
||||
def urlopen( # type: ignore[override]
|
||||
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
|
||||
with custom cross-host redirect logic and only sends the request-uri
|
||||
portion of the ``url``.
|
||||
|
||||
The given ``url`` parameter must be absolute, such that an appropriate
|
||||
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
|
||||
"""
|
||||
u = parse_url(url)
|
||||
|
||||
if u.scheme is None:
|
||||
warnings.warn(
|
||||
"URLs without a scheme (ie 'https://') are deprecated and will raise an error "
|
||||
"in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs "
|
||||
"start with 'https://' or 'http://'. Read more in this issue: "
|
||||
"https://github.com/urllib3/urllib3/issues/2920",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
|
||||
|
||||
kw["assert_same_host"] = False
|
||||
kw["redirect"] = False
|
||||
|
||||
if "headers" not in kw:
|
||||
kw["headers"] = self.headers
|
||||
|
||||
if self._proxy_requires_url_absolute_form(u):
|
||||
response = conn.urlopen(method, url, **kw)
|
||||
else:
|
||||
response = conn.urlopen(method, u.request_uri, **kw)
|
||||
|
||||
redirect_location = redirect and response.get_redirect_location()
|
||||
if not redirect_location:
|
||||
return response
|
||||
|
||||
# Support relative URLs for redirecting.
|
||||
redirect_location = urljoin(url, redirect_location)
|
||||
|
||||
if response.status == 303:
|
||||
# Change the method according to RFC 9110, Section 15.4.4.
|
||||
method = "GET"
|
||||
# And lose the body not to transfer anything sensitive.
|
||||
kw["body"] = None
|
||||
kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
|
||||
|
||||
retries = kw.get("retries")
|
||||
if not isinstance(retries, Retry):
|
||||
retries = Retry.from_int(retries, redirect=redirect)
|
||||
|
||||
# Strip headers marked as unsafe to forward to the redirected location.
|
||||
# Check remove_headers_on_redirect to avoid a potential network call within
|
||||
# conn.is_same_host() which may use socket.gethostbyname() in the future.
|
||||
if retries.remove_headers_on_redirect and not conn.is_same_host(
|
||||
redirect_location
|
||||
):
|
||||
new_headers = kw["headers"].copy()
|
||||
for header in kw["headers"]:
|
||||
if header.lower() in retries.remove_headers_on_redirect:
|
||||
new_headers.pop(header, None)
|
||||
kw["headers"] = new_headers
|
||||
|
||||
try:
|
||||
retries = retries.increment(method, url, response=response, _pool=conn)
|
||||
except MaxRetryError:
|
||||
if retries.raise_on_redirect:
|
||||
response.drain_conn()
|
||||
raise
|
||||
return response
|
||||
|
||||
kw["retries"] = retries
|
||||
kw["redirect"] = redirect
|
||||
|
||||
log.info("Redirecting %s -> %s", url, redirect_location)
|
||||
|
||||
response.drain_conn()
|
||||
return self.urlopen(method, redirect_location, **kw)
|
||||
|
||||
|
||||
class ProxyManager(PoolManager):
|
||||
"""
|
||||
Behaves just like :class:`PoolManager`, but sends all requests through
|
||||
the defined proxy, using the CONNECT method for HTTPS URLs.
|
||||
|
||||
:param proxy_url:
|
||||
The URL of the proxy to be used.
|
||||
|
||||
:param proxy_headers:
|
||||
A dictionary containing headers that will be sent to the proxy. In case
|
||||
of HTTP they are being sent with each request, while in the
|
||||
HTTPS/CONNECT case they are sent only once. Could be used for proxy
|
||||
authentication.
|
||||
|
||||
:param proxy_ssl_context:
|
||||
The proxy SSL context is used to establish the TLS connection to the
|
||||
proxy when using HTTPS proxies.
|
||||
|
||||
:param use_forwarding_for_https:
|
||||
(Defaults to False) If set to True will forward requests to the HTTPS
|
||||
proxy to be made on behalf of the client instead of creating a TLS
|
||||
tunnel via the CONNECT method. **Enabling this flag means that request
|
||||
and response headers and content will be visible from the HTTPS proxy**
|
||||
whereas tunneling keeps request and response headers and content
|
||||
private. IP address, target hostname, SNI, and port are always visible
|
||||
to an HTTPS proxy even when this flag is disabled.
|
||||
|
||||
:param proxy_assert_hostname:
|
||||
The hostname of the certificate to verify against.
|
||||
|
||||
:param proxy_assert_fingerprint:
|
||||
The fingerprint of the certificate to verify against.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
proxy = urllib3.ProxyManager("https://localhost:3128/")
|
||||
|
||||
resp1 = proxy.request("GET", "https://google.com/")
|
||||
resp2 = proxy.request("GET", "https://httpbin.org/")
|
||||
|
||||
print(len(proxy.pools))
|
||||
# 1
|
||||
|
||||
resp3 = proxy.request("GET", "https://httpbin.org/")
|
||||
resp4 = proxy.request("GET", "https://twitter.com/")
|
||||
|
||||
print(len(proxy.pools))
|
||||
# 3
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
proxy_url: str,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
proxy_headers: typing.Mapping[str, str] | None = None,
|
||||
proxy_ssl_context: ssl.SSLContext | None = None,
|
||||
use_forwarding_for_https: bool = False,
|
||||
proxy_assert_hostname: None | str | Literal[False] = None,
|
||||
proxy_assert_fingerprint: str | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
) -> None:
|
||||
if isinstance(proxy_url, HTTPConnectionPool):
|
||||
str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
|
||||
else:
|
||||
str_proxy_url = proxy_url
|
||||
proxy = parse_url(str_proxy_url)
|
||||
|
||||
if proxy.scheme not in ("http", "https"):
|
||||
raise ProxySchemeUnknown(proxy.scheme)
|
||||
|
||||
if not proxy.port:
|
||||
port = port_by_scheme.get(proxy.scheme, 80)
|
||||
proxy = proxy._replace(port=port)
|
||||
|
||||
self.proxy = proxy
|
||||
self.proxy_headers = proxy_headers or {}
|
||||
self.proxy_ssl_context = proxy_ssl_context
|
||||
self.proxy_config = ProxyConfig(
|
||||
proxy_ssl_context,
|
||||
use_forwarding_for_https,
|
||||
proxy_assert_hostname,
|
||||
proxy_assert_fingerprint,
|
||||
)
|
||||
|
||||
connection_pool_kw["_proxy"] = self.proxy
|
||||
connection_pool_kw["_proxy_headers"] = self.proxy_headers
|
||||
connection_pool_kw["_proxy_config"] = self.proxy_config
|
||||
|
||||
super().__init__(num_pools, headers, **connection_pool_kw)
|
||||
|
||||
def connection_from_host(
|
||||
self,
|
||||
host: str | None,
|
||||
port: int | None = None,
|
||||
scheme: str | None = "http",
|
||||
pool_kwargs: dict[str, typing.Any] | None = None,
|
||||
) -> HTTPConnectionPool:
|
||||
if scheme == "https":
|
||||
return super().connection_from_host(
|
||||
host, port, scheme, pool_kwargs=pool_kwargs
|
||||
)
|
||||
|
||||
return super().connection_from_host(
|
||||
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
|
||||
)
|
||||
|
||||
def _set_proxy_headers(
|
||||
self, url: str, headers: typing.Mapping[str, str] | None = None
|
||||
) -> typing.Mapping[str, str]:
|
||||
"""
|
||||
Sets headers needed by proxies: specifically, the Accept and Host
|
||||
headers. Only sets headers not provided by the user.
|
||||
"""
|
||||
headers_ = {"Accept": "*/*"}
|
||||
|
||||
netloc = parse_url(url).netloc
|
||||
if netloc:
|
||||
headers_["Host"] = netloc
|
||||
|
||||
if headers:
|
||||
headers_.update(headers)
|
||||
return headers_
|
||||
|
||||
def urlopen( # type: ignore[override]
|
||||
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
|
||||
) -> BaseHTTPResponse:
|
||||
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
|
||||
u = parse_url(url)
|
||||
if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
|
||||
# For connections using HTTP CONNECT, httplib sets the necessary
|
||||
# headers on the CONNECT to the proxy. If we're not using CONNECT,
|
||||
# we'll definitely need to set 'Host' at the very least.
|
||||
headers = kw.get("headers", self.headers)
|
||||
kw["headers"] = self._set_proxy_headers(url, headers)
|
||||
|
||||
return super().urlopen(method, url, redirect=redirect, **kw)
|
||||
|
||||
|
||||
def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
|
||||
return ProxyManager(proxy_url=url, **kw)
|
@ -1,2 +0,0 @@
|
||||
# Instruct type checkers to look for inline type annotations in this package.
|
||||
# See PEP 561.
|
File diff suppressed because it is too large
Load Diff
@ -1,42 +0,0 @@
|
||||
# For backwards compatibility, provide imports that used to be here.
|
||||
from __future__ import annotations
|
||||
|
||||
from .connection import is_connection_dropped
|
||||
from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
|
||||
from .response import is_fp_closed
|
||||
from .retry import Retry
|
||||
from .ssl_ import (
|
||||
ALPN_PROTOCOLS,
|
||||
IS_PYOPENSSL,
|
||||
SSLContext,
|
||||
assert_fingerprint,
|
||||
create_urllib3_context,
|
||||
resolve_cert_reqs,
|
||||
resolve_ssl_version,
|
||||
ssl_wrap_socket,
|
||||
)
|
||||
from .timeout import Timeout
|
||||
from .url import Url, parse_url
|
||||
from .wait import wait_for_read, wait_for_write
|
||||
|
||||
__all__ = (
|
||||
"IS_PYOPENSSL",
|
||||
"SSLContext",
|
||||
"ALPN_PROTOCOLS",
|
||||
"Retry",
|
||||
"Timeout",
|
||||
"Url",
|
||||
"assert_fingerprint",
|
||||
"create_urllib3_context",
|
||||
"is_connection_dropped",
|
||||
"is_fp_closed",
|
||||
"parse_url",
|
||||
"make_headers",
|
||||
"resolve_cert_reqs",
|
||||
"resolve_ssl_version",
|
||||
"ssl_wrap_socket",
|
||||
"wait_for_read",
|
||||
"wait_for_write",
|
||||
"SKIP_HEADER",
|
||||
"SKIPPABLE_HEADERS",
|
||||
)
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,137 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import typing
|
||||
|
||||
from ..exceptions import LocationParseError
|
||||
from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
||||
|
||||
_TYPE_SOCKET_OPTIONS = typing.Sequence[typing.Tuple[int, int, typing.Union[int, bytes]]]
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from .._base_connection import BaseHTTPConnection
|
||||
|
||||
|
||||
def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific
|
||||
"""
|
||||
Returns True if the connection is dropped and should be closed.
|
||||
:param conn: :class:`urllib3.connection.HTTPConnection` object.
|
||||
"""
|
||||
return not conn.is_connected
|
||||
|
||||
|
||||
# This function is copied from socket.py in the Python 2.7 standard
|
||||
# library test suite. Added to its signature is only `socket_options`.
|
||||
# One additional modification is that we avoid binding to IPv6 servers
|
||||
# discovered in DNS if the system doesn't have IPv6 functionality.
|
||||
def create_connection(
|
||||
address: tuple[str, int],
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
|
||||
) -> socket.socket:
|
||||
"""Connect to *address* and return the socket object.
|
||||
|
||||
Convenience function. Connect to *address* (a 2-tuple ``(host,
|
||||
port)``) and return the socket object. Passing the optional
|
||||
*timeout* parameter will set the timeout on the socket instance
|
||||
before attempting to connect. If no *timeout* is supplied, the
|
||||
global default timeout setting returned by :func:`socket.getdefaulttimeout`
|
||||
is used. If *source_address* is set it must be a tuple of (host, port)
|
||||
for the socket to bind as a source address before making the connection.
|
||||
An host of '' or port 0 tells the OS to use the default.
|
||||
"""
|
||||
|
||||
host, port = address
|
||||
if host.startswith("["):
|
||||
host = host.strip("[]")
|
||||
err = None
|
||||
|
||||
# Using the value from allowed_gai_family() in the context of getaddrinfo lets
|
||||
# us select whether to work with IPv4 DNS records, IPv6 records, or both.
|
||||
# The original create_connection function always returns all records.
|
||||
family = allowed_gai_family()
|
||||
|
||||
try:
|
||||
host.encode("idna")
|
||||
except UnicodeError:
|
||||
raise LocationParseError(f"'{host}', label empty or too long") from None
|
||||
|
||||
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
sock = None
|
||||
try:
|
||||
sock = socket.socket(af, socktype, proto)
|
||||
|
||||
# If provided, set socket level options before connecting.
|
||||
_set_socket_options(sock, socket_options)
|
||||
|
||||
if timeout is not _DEFAULT_TIMEOUT:
|
||||
sock.settimeout(timeout)
|
||||
if source_address:
|
||||
sock.bind(source_address)
|
||||
sock.connect(sa)
|
||||
# Break explicitly a reference cycle
|
||||
err = None
|
||||
return sock
|
||||
|
||||
except OSError as _:
|
||||
err = _
|
||||
if sock is not None:
|
||||
sock.close()
|
||||
|
||||
if err is not None:
|
||||
try:
|
||||
raise err
|
||||
finally:
|
||||
# Break explicitly a reference cycle
|
||||
err = None
|
||||
else:
|
||||
raise OSError("getaddrinfo returns an empty list")
|
||||
|
||||
|
||||
def _set_socket_options(
|
||||
sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None
|
||||
) -> None:
|
||||
if options is None:
|
||||
return
|
||||
|
||||
for opt in options:
|
||||
sock.setsockopt(*opt)
|
||||
|
||||
|
||||
def allowed_gai_family() -> socket.AddressFamily:
|
||||
"""This function is designed to work in the context of
|
||||
getaddrinfo, where family=socket.AF_UNSPEC is the default and
|
||||
will perform a DNS search for both IPv6 and IPv4 records."""
|
||||
|
||||
family = socket.AF_INET
|
||||
if HAS_IPV6:
|
||||
family = socket.AF_UNSPEC
|
||||
return family
|
||||
|
||||
|
||||
def _has_ipv6(host: str) -> bool:
|
||||
"""Returns True if the system can bind an IPv6 address."""
|
||||
sock = None
|
||||
has_ipv6 = False
|
||||
|
||||
if socket.has_ipv6:
|
||||
# has_ipv6 returns true if cPython was compiled with IPv6 support.
|
||||
# It does not tell us if the system has IPv6 support enabled. To
|
||||
# determine that we must bind to an IPv6 address.
|
||||
# https://github.com/urllib3/urllib3/pull/611
|
||||
# https://bugs.python.org/issue658327
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET6)
|
||||
sock.bind((host, 0))
|
||||
has_ipv6 = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if sock:
|
||||
sock.close()
|
||||
return has_ipv6
|
||||
|
||||
|
||||
HAS_IPV6 = _has_ipv6("::1")
|
@ -1,43 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from .url import Url
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ..connection import ProxyConfig
|
||||
|
||||
|
||||
def connection_requires_http_tunnel(
|
||||
proxy_url: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
destination_scheme: str | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Returns True if the connection requires an HTTP CONNECT through the proxy.
|
||||
|
||||
:param URL proxy_url:
|
||||
URL of the proxy.
|
||||
:param ProxyConfig proxy_config:
|
||||
Proxy configuration from poolmanager.py
|
||||
:param str destination_scheme:
|
||||
The scheme of the destination. (i.e https, http, etc)
|
||||
"""
|
||||
# If we're not using a proxy, no way to use a tunnel.
|
||||
if proxy_url is None:
|
||||
return False
|
||||
|
||||
# HTTP destinations never require tunneling, we always forward.
|
||||
if destination_scheme == "http":
|
||||
return False
|
||||
|
||||
# Support for forwarding with HTTPS proxies and HTTPS destinations.
|
||||
if (
|
||||
proxy_url.scheme == "https"
|
||||
and proxy_config
|
||||
and proxy_config.use_forwarding_for_https
|
||||
):
|
||||
return False
|
||||
|
||||
# Otherwise always use a tunnel.
|
||||
return True
|
@ -1,256 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import typing
|
||||
from base64 import b64encode
|
||||
from enum import Enum
|
||||
|
||||
from ..exceptions import UnrewindableBodyError
|
||||
from .util import to_bytes
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing import Final
|
||||
|
||||
# Pass as a value within ``headers`` to skip
|
||||
# emitting some HTTP headers that are added automatically.
|
||||
# The only headers that are supported are ``Accept-Encoding``,
|
||||
# ``Host``, and ``User-Agent``.
|
||||
SKIP_HEADER = "@@@SKIP_HEADER@@@"
|
||||
SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"])
|
||||
|
||||
ACCEPT_ENCODING = "gzip,deflate"
|
||||
try:
|
||||
try:
|
||||
import brotlicffi as _unused_module_brotli # type: ignore[import] # noqa: F401
|
||||
except ImportError:
|
||||
import brotli as _unused_module_brotli # type: ignore[import] # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ACCEPT_ENCODING += ",br"
|
||||
try:
|
||||
import zstandard as _unused_module_zstd # type: ignore[import] # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ACCEPT_ENCODING += ",zstd"
|
||||
|
||||
|
||||
class _TYPE_FAILEDTELL(Enum):
|
||||
token = 0
|
||||
|
||||
|
||||
_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token
|
||||
|
||||
_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL]
|
||||
|
||||
# When sending a request with these methods we aren't expecting
|
||||
# a body so don't need to set an explicit 'Content-Length: 0'
|
||||
# The reason we do this in the negative instead of tracking methods
|
||||
# which 'should' have a body is because unknown methods should be
|
||||
# treated as if they were 'POST' which *does* expect a body.
|
||||
_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"}
|
||||
|
||||
|
||||
def make_headers(
|
||||
keep_alive: bool | None = None,
|
||||
accept_encoding: bool | list[str] | str | None = None,
|
||||
user_agent: str | None = None,
|
||||
basic_auth: str | None = None,
|
||||
proxy_basic_auth: str | None = None,
|
||||
disable_cache: bool | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Shortcuts for generating request headers.
|
||||
|
||||
:param keep_alive:
|
||||
If ``True``, adds 'connection: keep-alive' header.
|
||||
|
||||
:param accept_encoding:
|
||||
Can be a boolean, list, or string.
|
||||
``True`` translates to 'gzip,deflate'. If either the ``brotli`` or
|
||||
``brotlicffi`` package is installed 'gzip,deflate,br' is used instead.
|
||||
List will get joined by comma.
|
||||
String will be used as provided.
|
||||
|
||||
:param user_agent:
|
||||
String representing the user-agent you want, such as
|
||||
"python-urllib3/0.6"
|
||||
|
||||
:param basic_auth:
|
||||
Colon-separated username:password string for 'authorization: basic ...'
|
||||
auth header.
|
||||
|
||||
:param proxy_basic_auth:
|
||||
Colon-separated username:password string for 'proxy-authorization: basic ...'
|
||||
auth header.
|
||||
|
||||
:param disable_cache:
|
||||
If ``True``, adds 'cache-control: no-cache' header.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0"))
|
||||
# {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
|
||||
print(urllib3.util.make_headers(accept_encoding=True))
|
||||
# {'accept-encoding': 'gzip,deflate'}
|
||||
"""
|
||||
headers: dict[str, str] = {}
|
||||
if accept_encoding:
|
||||
if isinstance(accept_encoding, str):
|
||||
pass
|
||||
elif isinstance(accept_encoding, list):
|
||||
accept_encoding = ",".join(accept_encoding)
|
||||
else:
|
||||
accept_encoding = ACCEPT_ENCODING
|
||||
headers["accept-encoding"] = accept_encoding
|
||||
|
||||
if user_agent:
|
||||
headers["user-agent"] = user_agent
|
||||
|
||||
if keep_alive:
|
||||
headers["connection"] = "keep-alive"
|
||||
|
||||
if basic_auth:
|
||||
headers[
|
||||
"authorization"
|
||||
] = f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}"
|
||||
|
||||
if proxy_basic_auth:
|
||||
headers[
|
||||
"proxy-authorization"
|
||||
] = f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}"
|
||||
|
||||
if disable_cache:
|
||||
headers["cache-control"] = "no-cache"
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
def set_file_position(
|
||||
body: typing.Any, pos: _TYPE_BODY_POSITION | None
|
||||
) -> _TYPE_BODY_POSITION | None:
|
||||
"""
|
||||
If a position is provided, move file to that point.
|
||||
Otherwise, we'll attempt to record a position for future use.
|
||||
"""
|
||||
if pos is not None:
|
||||
rewind_body(body, pos)
|
||||
elif getattr(body, "tell", None) is not None:
|
||||
try:
|
||||
pos = body.tell()
|
||||
except OSError:
|
||||
# This differentiates from None, allowing us to catch
|
||||
# a failed `tell()` later when trying to rewind the body.
|
||||
pos = _FAILEDTELL
|
||||
|
||||
return pos
|
||||
|
||||
|
||||
def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None:
|
||||
"""
|
||||
Attempt to rewind body to a certain position.
|
||||
Primarily used for request redirects and retries.
|
||||
|
||||
:param body:
|
||||
File-like object that supports seek.
|
||||
|
||||
:param int pos:
|
||||
Position to seek to in file.
|
||||
"""
|
||||
body_seek = getattr(body, "seek", None)
|
||||
if body_seek is not None and isinstance(body_pos, int):
|
||||
try:
|
||||
body_seek(body_pos)
|
||||
except OSError as e:
|
||||
raise UnrewindableBodyError(
|
||||
"An error occurred when rewinding request body for redirect/retry."
|
||||
) from e
|
||||
elif body_pos is _FAILEDTELL:
|
||||
raise UnrewindableBodyError(
|
||||
"Unable to record file position for rewinding "
|
||||
"request body during a redirect/retry."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"body_pos must be of type integer, instead it was {type(body_pos)}."
|
||||
)
|
||||
|
||||
|
||||
class ChunksAndContentLength(typing.NamedTuple):
|
||||
chunks: typing.Iterable[bytes] | None
|
||||
content_length: int | None
|
||||
|
||||
|
||||
def body_to_chunks(
|
||||
body: typing.Any | None, method: str, blocksize: int
|
||||
) -> ChunksAndContentLength:
|
||||
"""Takes the HTTP request method, body, and blocksize and
|
||||
transforms them into an iterable of chunks to pass to
|
||||
socket.sendall() and an optional 'Content-Length' header.
|
||||
|
||||
A 'Content-Length' of 'None' indicates the length of the body
|
||||
can't be determined so should use 'Transfer-Encoding: chunked'
|
||||
for framing instead.
|
||||
"""
|
||||
|
||||
chunks: typing.Iterable[bytes] | None
|
||||
content_length: int | None
|
||||
|
||||
# No body, we need to make a recommendation on 'Content-Length'
|
||||
# based on whether that request method is expected to have
|
||||
# a body or not.
|
||||
if body is None:
|
||||
chunks = None
|
||||
if method.upper() not in _METHODS_NOT_EXPECTING_BODY:
|
||||
content_length = 0
|
||||
else:
|
||||
content_length = None
|
||||
|
||||
# Bytes or strings become bytes
|
||||
elif isinstance(body, (str, bytes)):
|
||||
chunks = (to_bytes(body),)
|
||||
content_length = len(chunks[0])
|
||||
|
||||
# File-like object, TODO: use seek() and tell() for length?
|
||||
elif hasattr(body, "read"):
|
||||
|
||||
def chunk_readable() -> typing.Iterable[bytes]:
|
||||
nonlocal body, blocksize
|
||||
encode = isinstance(body, io.TextIOBase)
|
||||
while True:
|
||||
datablock = body.read(blocksize)
|
||||
if not datablock:
|
||||
break
|
||||
if encode:
|
||||
datablock = datablock.encode("iso-8859-1")
|
||||
yield datablock
|
||||
|
||||
chunks = chunk_readable()
|
||||
content_length = None
|
||||
|
||||
# Otherwise we need to start checking via duck-typing.
|
||||
else:
|
||||
try:
|
||||
# Check if the body implements the buffer API.
|
||||
mv = memoryview(body)
|
||||
except TypeError:
|
||||
try:
|
||||
# Check if the body is an iterable
|
||||
chunks = iter(body)
|
||||
content_length = None
|
||||
except TypeError:
|
||||
raise TypeError(
|
||||
f"'body' must be a bytes-like object, file-like "
|
||||
f"object, or iterable. Instead was {body!r}"
|
||||
) from None
|
||||
else:
|
||||
# Since it implements the buffer API can be passed directly to socket.sendall()
|
||||
chunks = (body,)
|
||||
content_length = mv.nbytes
|
||||
|
||||
return ChunksAndContentLength(chunks=chunks, content_length=content_length)
|
@ -1,101 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client as httplib
|
||||
from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect
|
||||
|
||||
from ..exceptions import HeaderParsingError
|
||||
|
||||
|
||||
def is_fp_closed(obj: object) -> bool:
|
||||
"""
|
||||
Checks whether a given file-like object is closed.
|
||||
|
||||
:param obj:
|
||||
The file-like object to check.
|
||||
"""
|
||||
|
||||
try:
|
||||
# Check `isclosed()` first, in case Python3 doesn't set `closed`.
|
||||
# GH Issue #928
|
||||
return obj.isclosed() # type: ignore[no-any-return, attr-defined]
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Check via the official file-like-object way.
|
||||
return obj.closed # type: ignore[no-any-return, attr-defined]
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Check if the object is a container for another file-like object that
|
||||
# gets released on exhaustion (e.g. HTTPResponse).
|
||||
return obj.fp is None # type: ignore[attr-defined]
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
raise ValueError("Unable to determine whether fp is closed.")
|
||||
|
||||
|
||||
def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
|
||||
"""
|
||||
Asserts whether all headers have been successfully parsed.
|
||||
Extracts encountered errors from the result of parsing headers.
|
||||
|
||||
Only works on Python 3.
|
||||
|
||||
:param http.client.HTTPMessage headers: Headers to verify.
|
||||
|
||||
:raises urllib3.exceptions.HeaderParsingError:
|
||||
If parsing errors are found.
|
||||
"""
|
||||
|
||||
# This will fail silently if we pass in the wrong kind of parameter.
|
||||
# To make debugging easier add an explicit check.
|
||||
if not isinstance(headers, httplib.HTTPMessage):
|
||||
raise TypeError(f"expected httplib.Message, got {type(headers)}.")
|
||||
|
||||
unparsed_data = None
|
||||
|
||||
# get_payload is actually email.message.Message.get_payload;
|
||||
# we're only interested in the result if it's not a multipart message
|
||||
if not headers.is_multipart():
|
||||
payload = headers.get_payload()
|
||||
|
||||
if isinstance(payload, (bytes, str)):
|
||||
unparsed_data = payload
|
||||
|
||||
# httplib is assuming a response body is available
|
||||
# when parsing headers even when httplib only sends
|
||||
# header data to parse_headers() This results in
|
||||
# defects on multipart responses in particular.
|
||||
# See: https://github.com/urllib3/urllib3/issues/800
|
||||
|
||||
# So we ignore the following defects:
|
||||
# - StartBoundaryNotFoundDefect:
|
||||
# The claimed start boundary was never found.
|
||||
# - MultipartInvariantViolationDefect:
|
||||
# A message claimed to be a multipart but no subparts were found.
|
||||
defects = [
|
||||
defect
|
||||
for defect in headers.defects
|
||||
if not isinstance(
|
||||
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
|
||||
)
|
||||
]
|
||||
|
||||
if defects or unparsed_data:
|
||||
raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
|
||||
|
||||
|
||||
def is_response_to_head(response: httplib.HTTPResponse) -> bool:
|
||||
"""
|
||||
Checks whether the request of a response has been a HEAD-request.
|
||||
|
||||
:param http.client.HTTPResponse response:
|
||||
Response to check if the originating request
|
||||
used 'HEAD' as a method.
|
||||
"""
|
||||
# FIXME: Can we do this somehow without accessing private httplib _method?
|
||||
method_str = response._method # type: str # type: ignore[attr-defined]
|
||||
return method_str.upper() == "HEAD"
|
@ -1,529 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import email
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import typing
|
||||
from itertools import takewhile
|
||||
from types import TracebackType
|
||||
|
||||
from ..exceptions import (
|
||||
ConnectTimeoutError,
|
||||
InvalidHeader,
|
||||
MaxRetryError,
|
||||
ProtocolError,
|
||||
ProxyError,
|
||||
ReadTimeoutError,
|
||||
ResponseError,
|
||||
)
|
||||
from .util import reraise
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ..connectionpool import ConnectionPool
|
||||
from ..response import BaseHTTPResponse
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Data structure for representing the metadata of requests that result in a retry.
|
||||
class RequestHistory(typing.NamedTuple):
|
||||
method: str | None
|
||||
url: str | None
|
||||
error: Exception | None
|
||||
status: int | None
|
||||
redirect_location: str | None
|
||||
|
||||
|
||||
class Retry:
|
||||
"""Retry configuration.
|
||||
|
||||
Each retry attempt will create a new Retry object with updated values, so
|
||||
they can be safely reused.
|
||||
|
||||
Retries can be defined as a default for a pool:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
retries = Retry(connect=5, read=2, redirect=5)
|
||||
http = PoolManager(retries=retries)
|
||||
response = http.request("GET", "https://example.com/")
|
||||
|
||||
Or per-request (which overrides the default for the pool):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response = http.request("GET", "https://example.com/", retries=Retry(10))
|
||||
|
||||
Retries can be disabled by passing ``False``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response = http.request("GET", "https://example.com/", retries=False)
|
||||
|
||||
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
|
||||
retries are disabled, in which case the causing exception will be raised.
|
||||
|
||||
:param int total:
|
||||
Total number of retries to allow. Takes precedence over other counts.
|
||||
|
||||
Set to ``None`` to remove this constraint and fall back on other
|
||||
counts.
|
||||
|
||||
Set to ``0`` to fail on the first retry.
|
||||
|
||||
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
||||
|
||||
:param int connect:
|
||||
How many connection-related errors to retry on.
|
||||
|
||||
These are errors raised before the request is sent to the remote server,
|
||||
which we assume has not triggered the server to process the request.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
:param int read:
|
||||
How many times to retry on read errors.
|
||||
|
||||
These errors are raised after the request was sent to the server, so the
|
||||
request may have side-effects.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
:param int redirect:
|
||||
How many redirects to perform. Limit this to avoid infinite redirect
|
||||
loops.
|
||||
|
||||
A redirect is a HTTP response with a status code 301, 302, 303, 307 or
|
||||
308.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
||||
|
||||
:param int status:
|
||||
How many times to retry on bad status codes.
|
||||
|
||||
These are retries made on responses, where status code matches
|
||||
``status_forcelist``.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
:param int other:
|
||||
How many times to retry on other errors.
|
||||
|
||||
Other errors are errors that are not connect, read, redirect or status errors.
|
||||
These errors might be raised after the request was sent to the server, so the
|
||||
request might have side-effects.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
If ``total`` is not set, it's a good idea to set this to 0 to account
|
||||
for unexpected edge cases and avoid infinite retry loops.
|
||||
|
||||
:param Collection allowed_methods:
|
||||
Set of uppercased HTTP method verbs that we should retry on.
|
||||
|
||||
By default, we only retry on methods which are considered to be
|
||||
idempotent (multiple requests with the same parameters end with the
|
||||
same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
|
||||
|
||||
Set to a ``None`` value to retry on any verb.
|
||||
|
||||
:param Collection status_forcelist:
|
||||
A set of integer HTTP status codes that we should force a retry on.
|
||||
A retry is initiated if the request method is in ``allowed_methods``
|
||||
and the response status code is in ``status_forcelist``.
|
||||
|
||||
By default, this is disabled with ``None``.
|
||||
|
||||
:param float backoff_factor:
|
||||
A backoff factor to apply between attempts after the second try
|
||||
(most errors are resolved immediately by a second try without a
|
||||
delay). urllib3 will sleep for::
|
||||
|
||||
{backoff factor} * (2 ** ({number of previous retries}))
|
||||
|
||||
seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
|
||||
|
||||
random.uniform(0, {backoff jitter})
|
||||
|
||||
seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
|
||||
sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
|
||||
be longer than `backoff_max`.
|
||||
|
||||
By default, backoff is disabled (factor set to 0).
|
||||
|
||||
:param bool raise_on_redirect: Whether, if the number of redirects is
|
||||
exhausted, to raise a MaxRetryError, or to return a response with a
|
||||
response code in the 3xx range.
|
||||
|
||||
:param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
|
||||
whether we should raise an exception, or return a response,
|
||||
if status falls in ``status_forcelist`` range and retries have
|
||||
been exhausted.
|
||||
|
||||
:param tuple history: The history of the request encountered during
|
||||
each call to :meth:`~Retry.increment`. The list is in the order
|
||||
the requests occurred. Each list item is of class :class:`RequestHistory`.
|
||||
|
||||
:param bool respect_retry_after_header:
|
||||
Whether to respect Retry-After header on status codes defined as
|
||||
:attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
|
||||
|
||||
:param Collection remove_headers_on_redirect:
|
||||
Sequence of headers to remove from the request when a response
|
||||
indicating a redirect is returned before firing off the redirected
|
||||
request.
|
||||
"""
|
||||
|
||||
#: Default methods to be used for ``allowed_methods``
|
||||
DEFAULT_ALLOWED_METHODS = frozenset(
|
||||
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
|
||||
)
|
||||
|
||||
#: Default status codes to be used for ``status_forcelist``
|
||||
RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
|
||||
|
||||
#: Default headers to be used for ``remove_headers_on_redirect``
|
||||
DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])
|
||||
|
||||
#: Default maximum backoff time.
|
||||
DEFAULT_BACKOFF_MAX = 120
|
||||
|
||||
# Backward compatibility; assigned outside of the class.
|
||||
DEFAULT: typing.ClassVar[Retry]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
total: bool | int | None = 10,
|
||||
connect: int | None = None,
|
||||
read: int | None = None,
|
||||
redirect: bool | int | None = None,
|
||||
status: int | None = None,
|
||||
other: int | None = None,
|
||||
allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
|
||||
status_forcelist: typing.Collection[int] | None = None,
|
||||
backoff_factor: float = 0,
|
||||
backoff_max: float = DEFAULT_BACKOFF_MAX,
|
||||
raise_on_redirect: bool = True,
|
||||
raise_on_status: bool = True,
|
||||
history: tuple[RequestHistory, ...] | None = None,
|
||||
respect_retry_after_header: bool = True,
|
||||
remove_headers_on_redirect: typing.Collection[
|
||||
str
|
||||
] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
|
||||
backoff_jitter: float = 0.0,
|
||||
) -> None:
|
||||
self.total = total
|
||||
self.connect = connect
|
||||
self.read = read
|
||||
self.status = status
|
||||
self.other = other
|
||||
|
||||
if redirect is False or total is False:
|
||||
redirect = 0
|
||||
raise_on_redirect = False
|
||||
|
||||
self.redirect = redirect
|
||||
self.status_forcelist = status_forcelist or set()
|
||||
self.allowed_methods = allowed_methods
|
||||
self.backoff_factor = backoff_factor
|
||||
self.backoff_max = backoff_max
|
||||
self.raise_on_redirect = raise_on_redirect
|
||||
self.raise_on_status = raise_on_status
|
||||
self.history = history or ()
|
||||
self.respect_retry_after_header = respect_retry_after_header
|
||||
self.remove_headers_on_redirect = frozenset(
|
||||
h.lower() for h in remove_headers_on_redirect
|
||||
)
|
||||
self.backoff_jitter = backoff_jitter
|
||||
|
||||
def new(self, **kw: typing.Any) -> Retry:
|
||||
params = dict(
|
||||
total=self.total,
|
||||
connect=self.connect,
|
||||
read=self.read,
|
||||
redirect=self.redirect,
|
||||
status=self.status,
|
||||
other=self.other,
|
||||
allowed_methods=self.allowed_methods,
|
||||
status_forcelist=self.status_forcelist,
|
||||
backoff_factor=self.backoff_factor,
|
||||
backoff_max=self.backoff_max,
|
||||
raise_on_redirect=self.raise_on_redirect,
|
||||
raise_on_status=self.raise_on_status,
|
||||
history=self.history,
|
||||
remove_headers_on_redirect=self.remove_headers_on_redirect,
|
||||
respect_retry_after_header=self.respect_retry_after_header,
|
||||
backoff_jitter=self.backoff_jitter,
|
||||
)
|
||||
|
||||
params.update(kw)
|
||||
return type(self)(**params) # type: ignore[arg-type]
|
||||
|
||||
@classmethod
|
||||
def from_int(
|
||||
cls,
|
||||
retries: Retry | bool | int | None,
|
||||
redirect: bool | int | None = True,
|
||||
default: Retry | bool | int | None = None,
|
||||
) -> Retry:
|
||||
"""Backwards-compatibility for the old retries format."""
|
||||
if retries is None:
|
||||
retries = default if default is not None else cls.DEFAULT
|
||||
|
||||
if isinstance(retries, Retry):
|
||||
return retries
|
||||
|
||||
redirect = bool(redirect) and None
|
||||
new_retries = cls(retries, redirect=redirect)
|
||||
log.debug("Converted retries value: %r -> %r", retries, new_retries)
|
||||
return new_retries
|
||||
|
||||
def get_backoff_time(self) -> float:
|
||||
"""Formula for computing the current backoff
|
||||
|
||||
:rtype: float
|
||||
"""
|
||||
# We want to consider only the last consecutive errors sequence (Ignore redirects).
|
||||
consecutive_errors_len = len(
|
||||
list(
|
||||
takewhile(lambda x: x.redirect_location is None, reversed(self.history))
|
||||
)
|
||||
)
|
||||
if consecutive_errors_len <= 1:
|
||||
return 0
|
||||
|
||||
backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
|
||||
if self.backoff_jitter != 0.0:
|
||||
backoff_value += random.random() * self.backoff_jitter
|
||||
return float(max(0, min(self.backoff_max, backoff_value)))
|
||||
|
||||
def parse_retry_after(self, retry_after: str) -> float:
|
||||
seconds: float
|
||||
# Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
|
||||
if re.match(r"^\s*[0-9]+\s*$", retry_after):
|
||||
seconds = int(retry_after)
|
||||
else:
|
||||
retry_date_tuple = email.utils.parsedate_tz(retry_after)
|
||||
if retry_date_tuple is None:
|
||||
raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
|
||||
|
||||
retry_date = email.utils.mktime_tz(retry_date_tuple)
|
||||
seconds = retry_date - time.time()
|
||||
|
||||
seconds = max(seconds, 0)
|
||||
|
||||
return seconds
|
||||
|
||||
def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
|
||||
"""Get the value of Retry-After in seconds."""
|
||||
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
|
||||
if retry_after is None:
|
||||
return None
|
||||
|
||||
return self.parse_retry_after(retry_after)
|
||||
|
||||
def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
|
||||
retry_after = self.get_retry_after(response)
|
||||
if retry_after:
|
||||
time.sleep(retry_after)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _sleep_backoff(self) -> None:
|
||||
backoff = self.get_backoff_time()
|
||||
if backoff <= 0:
|
||||
return
|
||||
time.sleep(backoff)
|
||||
|
||||
def sleep(self, response: BaseHTTPResponse | None = None) -> None:
|
||||
"""Sleep between retry attempts.
|
||||
|
||||
This method will respect a server's ``Retry-After`` response header
|
||||
and sleep the duration of the time requested. If that is not present, it
|
||||
will use an exponential backoff. By default, the backoff factor is 0 and
|
||||
this method will return immediately.
|
||||
"""
|
||||
|
||||
if self.respect_retry_after_header and response:
|
||||
slept = self.sleep_for_retry(response)
|
||||
if slept:
|
||||
return
|
||||
|
||||
self._sleep_backoff()
|
||||
|
||||
def _is_connection_error(self, err: Exception) -> bool:
|
||||
"""Errors when we're fairly sure that the server did not receive the
|
||||
request, so it should be safe to retry.
|
||||
"""
|
||||
if isinstance(err, ProxyError):
|
||||
err = err.original_error
|
||||
return isinstance(err, ConnectTimeoutError)
|
||||
|
||||
def _is_read_error(self, err: Exception) -> bool:
|
||||
"""Errors that occur after the request has been started, so we should
|
||||
assume that the server began processing it.
|
||||
"""
|
||||
return isinstance(err, (ReadTimeoutError, ProtocolError))
|
||||
|
||||
def _is_method_retryable(self, method: str) -> bool:
|
||||
"""Checks if a given HTTP method should be retried upon, depending if
|
||||
it is included in the allowed_methods
|
||||
"""
|
||||
if self.allowed_methods and method.upper() not in self.allowed_methods:
|
||||
return False
|
||||
return True
|
||||
|
||||
def is_retry(
|
||||
self, method: str, status_code: int, has_retry_after: bool = False
|
||||
) -> bool:
|
||||
"""Is this method/status code retryable? (Based on allowlists and control
|
||||
variables such as the number of total retries to allow, whether to
|
||||
respect the Retry-After header, whether this header is present, and
|
||||
whether the returned status code is on the list of status codes to
|
||||
be retried upon on the presence of the aforementioned header)
|
||||
"""
|
||||
if not self._is_method_retryable(method):
|
||||
return False
|
||||
|
||||
if self.status_forcelist and status_code in self.status_forcelist:
|
||||
return True
|
||||
|
||||
return bool(
|
||||
self.total
|
||||
and self.respect_retry_after_header
|
||||
and has_retry_after
|
||||
and (status_code in self.RETRY_AFTER_STATUS_CODES)
|
||||
)
|
||||
|
||||
def is_exhausted(self) -> bool:
|
||||
"""Are we out of retries?"""
|
||||
retry_counts = [
|
||||
x
|
||||
for x in (
|
||||
self.total,
|
||||
self.connect,
|
||||
self.read,
|
||||
self.redirect,
|
||||
self.status,
|
||||
self.other,
|
||||
)
|
||||
if x
|
||||
]
|
||||
if not retry_counts:
|
||||
return False
|
||||
|
||||
return min(retry_counts) < 0
|
||||
|
||||
def increment(
|
||||
self,
|
||||
method: str | None = None,
|
||||
url: str | None = None,
|
||||
response: BaseHTTPResponse | None = None,
|
||||
error: Exception | None = None,
|
||||
_pool: ConnectionPool | None = None,
|
||||
_stacktrace: TracebackType | None = None,
|
||||
) -> Retry:
|
||||
"""Return a new Retry object with incremented retry counters.
|
||||
|
||||
:param response: A response object, or None, if the server did not
|
||||
return a response.
|
||||
:type response: :class:`~urllib3.response.BaseHTTPResponse`
|
||||
:param Exception error: An error encountered during the request, or
|
||||
None if the response was received successfully.
|
||||
|
||||
:return: A new ``Retry`` object.
|
||||
"""
|
||||
if self.total is False and error:
|
||||
# Disabled, indicate to re-raise the error.
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
|
||||
total = self.total
|
||||
if total is not None:
|
||||
total -= 1
|
||||
|
||||
connect = self.connect
|
||||
read = self.read
|
||||
redirect = self.redirect
|
||||
status_count = self.status
|
||||
other = self.other
|
||||
cause = "unknown"
|
||||
status = None
|
||||
redirect_location = None
|
||||
|
||||
if error and self._is_connection_error(error):
|
||||
# Connect retry?
|
||||
if connect is False:
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
elif connect is not None:
|
||||
connect -= 1
|
||||
|
||||
elif error and self._is_read_error(error):
|
||||
# Read retry?
|
||||
if read is False or method is None or not self._is_method_retryable(method):
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
elif read is not None:
|
||||
read -= 1
|
||||
|
||||
elif error:
|
||||
# Other retry?
|
||||
if other is not None:
|
||||
other -= 1
|
||||
|
||||
elif response and response.get_redirect_location():
|
||||
# Redirect retry?
|
||||
if redirect is not None:
|
||||
redirect -= 1
|
||||
cause = "too many redirects"
|
||||
response_redirect_location = response.get_redirect_location()
|
||||
if response_redirect_location:
|
||||
redirect_location = response_redirect_location
|
||||
status = response.status
|
||||
|
||||
else:
|
||||
# Incrementing because of a server error like a 500 in
|
||||
# status_forcelist and the given method is in the allowed_methods
|
||||
cause = ResponseError.GENERIC_ERROR
|
||||
if response and response.status:
|
||||
if status_count is not None:
|
||||
status_count -= 1
|
||||
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
|
||||
status = response.status
|
||||
|
||||
history = self.history + (
|
||||
RequestHistory(method, url, error, status, redirect_location),
|
||||
)
|
||||
|
||||
new_retry = self.new(
|
||||
total=total,
|
||||
connect=connect,
|
||||
read=read,
|
||||
redirect=redirect,
|
||||
status=status_count,
|
||||
other=other,
|
||||
history=history,
|
||||
)
|
||||
|
||||
if new_retry.is_exhausted():
|
||||
reason = error or ResponseError(cause)
|
||||
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
|
||||
|
||||
log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
|
||||
|
||||
return new_retry
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
|
||||
f"read={self.read}, redirect={self.redirect}, status={self.status})"
|
||||
)
|
||||
|
||||
|
||||
# For backwards compatibility (equivalent to pre-v1.9):
|
||||
Retry.DEFAULT = Retry(3)
|
@ -1,514 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import typing
|
||||
import warnings
|
||||
from binascii import unhexlify
|
||||
from hashlib import md5, sha1, sha256
|
||||
|
||||
from ..exceptions import ProxySchemeUnsupported, SSLError
|
||||
from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE
|
||||
|
||||
SSLContext = None
|
||||
SSLTransport = None
|
||||
HAS_NEVER_CHECK_COMMON_NAME = False
|
||||
IS_PYOPENSSL = False
|
||||
ALPN_PROTOCOLS = ["http/1.1"]
|
||||
|
||||
_TYPE_VERSION_INFO = typing.Tuple[int, int, int, str, int]
|
||||
|
||||
# Maps the length of a digest to a possible hash function producing this digest
|
||||
HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256}
|
||||
|
||||
|
||||
def _is_bpo_43522_fixed(
|
||||
implementation_name: str,
|
||||
version_info: _TYPE_VERSION_INFO,
|
||||
pypy_version_info: _TYPE_VERSION_INFO | None,
|
||||
) -> bool:
|
||||
"""Return True for CPython 3.8.9+, 3.9.3+ or 3.10+ and PyPy 7.3.8+ where
|
||||
setting SSLContext.hostname_checks_common_name to False works.
|
||||
|
||||
Outside of CPython and PyPy we don't know which implementations work
|
||||
or not so we conservatively use our hostname matching as we know that works
|
||||
on all implementations.
|
||||
|
||||
https://github.com/urllib3/urllib3/issues/2192#issuecomment-821832963
|
||||
https://foss.heptapod.net/pypy/pypy/-/issues/3539
|
||||
"""
|
||||
if implementation_name == "pypy":
|
||||
# https://foss.heptapod.net/pypy/pypy/-/issues/3129
|
||||
return pypy_version_info >= (7, 3, 8) # type: ignore[operator]
|
||||
elif implementation_name == "cpython":
|
||||
major_minor = version_info[:2]
|
||||
micro = version_info[2]
|
||||
return (
|
||||
(major_minor == (3, 8) and micro >= 9)
|
||||
or (major_minor == (3, 9) and micro >= 3)
|
||||
or major_minor >= (3, 10)
|
||||
)
|
||||
else: # Defensive:
|
||||
return False
|
||||
|
||||
|
||||
def _is_has_never_check_common_name_reliable(
|
||||
openssl_version: str,
|
||||
openssl_version_number: int,
|
||||
implementation_name: str,
|
||||
version_info: _TYPE_VERSION_INFO,
|
||||
pypy_version_info: _TYPE_VERSION_INFO | None,
|
||||
) -> bool:
|
||||
# As of May 2023, all released versions of LibreSSL fail to reject certificates with
|
||||
# only common names, see https://github.com/urllib3/urllib3/pull/3024
|
||||
is_openssl = openssl_version.startswith("OpenSSL ")
|
||||
# Before fixing OpenSSL issue #14579, the SSL_new() API was not copying hostflags
|
||||
# like X509_CHECK_FLAG_NEVER_CHECK_SUBJECT, which tripped up CPython.
|
||||
# https://github.com/openssl/openssl/issues/14579
|
||||
# This was released in OpenSSL 1.1.1l+ (>=0x101010cf)
|
||||
is_openssl_issue_14579_fixed = openssl_version_number >= 0x101010CF
|
||||
|
||||
return is_openssl and (
|
||||
is_openssl_issue_14579_fixed
|
||||
or _is_bpo_43522_fixed(implementation_name, version_info, pypy_version_info)
|
||||
)
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ssl import VerifyMode
|
||||
from typing import Literal, TypedDict
|
||||
|
||||
from .ssltransport import SSLTransport as SSLTransportType
|
||||
|
||||
class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False):
|
||||
subjectAltName: tuple[tuple[str, str], ...]
|
||||
subject: tuple[tuple[tuple[str, str], ...], ...]
|
||||
serialNumber: str
|
||||
|
||||
|
||||
# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X'
|
||||
_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {}
|
||||
|
||||
try: # Do we have ssl at all?
|
||||
import ssl
|
||||
from ssl import ( # type: ignore[assignment]
|
||||
CERT_REQUIRED,
|
||||
HAS_NEVER_CHECK_COMMON_NAME,
|
||||
OP_NO_COMPRESSION,
|
||||
OP_NO_TICKET,
|
||||
OPENSSL_VERSION,
|
||||
OPENSSL_VERSION_NUMBER,
|
||||
PROTOCOL_TLS,
|
||||
PROTOCOL_TLS_CLIENT,
|
||||
OP_NO_SSLv2,
|
||||
OP_NO_SSLv3,
|
||||
SSLContext,
|
||||
TLSVersion,
|
||||
)
|
||||
|
||||
PROTOCOL_SSLv23 = PROTOCOL_TLS
|
||||
|
||||
# Setting SSLContext.hostname_checks_common_name = False didn't work before CPython
|
||||
# 3.8.9, 3.9.3, and 3.10 (but OK on PyPy) or OpenSSL 1.1.1l+
|
||||
if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable(
|
||||
OPENSSL_VERSION,
|
||||
OPENSSL_VERSION_NUMBER,
|
||||
sys.implementation.name,
|
||||
sys.version_info,
|
||||
sys.pypy_version_info if sys.implementation.name == "pypy" else None, # type: ignore[attr-defined]
|
||||
):
|
||||
HAS_NEVER_CHECK_COMMON_NAME = False
|
||||
|
||||
# Need to be careful here in case old TLS versions get
|
||||
# removed in future 'ssl' module implementations.
|
||||
for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"):
|
||||
try:
|
||||
_SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr(
|
||||
TLSVersion, attr
|
||||
)
|
||||
except AttributeError: # Defensive:
|
||||
continue
|
||||
|
||||
from .ssltransport import SSLTransport # type: ignore[assignment]
|
||||
except ImportError:
|
||||
OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment]
|
||||
OP_NO_TICKET = 0x4000 # type: ignore[assignment]
|
||||
OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment]
|
||||
OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment]
|
||||
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment]
|
||||
PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment]
|
||||
|
||||
|
||||
_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None]
|
||||
|
||||
|
||||
def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
|
||||
"""
|
||||
Checks if given fingerprint matches the supplied certificate.
|
||||
|
||||
:param cert:
|
||||
Certificate as bytes object.
|
||||
:param fingerprint:
|
||||
Fingerprint as string of hexdigits, can be interspersed by colons.
|
||||
"""
|
||||
|
||||
if cert is None:
|
||||
raise SSLError("No certificate for the peer.")
|
||||
|
||||
fingerprint = fingerprint.replace(":", "").lower()
|
||||
digest_length = len(fingerprint)
|
||||
hashfunc = HASHFUNC_MAP.get(digest_length)
|
||||
if not hashfunc:
|
||||
raise SSLError(f"Fingerprint of invalid length: {fingerprint}")
|
||||
|
||||
# We need encode() here for py32; works on py2 and p33.
|
||||
fingerprint_bytes = unhexlify(fingerprint.encode())
|
||||
|
||||
cert_digest = hashfunc(cert).digest()
|
||||
|
||||
if not hmac.compare_digest(cert_digest, fingerprint_bytes):
|
||||
raise SSLError(
|
||||
f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"'
|
||||
)
|
||||
|
||||
|
||||
def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
|
||||
"""
|
||||
Resolves the argument to a numeric constant, which can be passed to
|
||||
the wrap_socket function/method from the ssl module.
|
||||
Defaults to :data:`ssl.CERT_REQUIRED`.
|
||||
If given a string it is assumed to be the name of the constant in the
|
||||
:mod:`ssl` module or its abbreviation.
|
||||
(So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
|
||||
If it's neither `None` nor a string we assume it is already the numeric
|
||||
constant which can directly be passed to wrap_socket.
|
||||
"""
|
||||
if candidate is None:
|
||||
return CERT_REQUIRED
|
||||
|
||||
if isinstance(candidate, str):
|
||||
res = getattr(ssl, candidate, None)
|
||||
if res is None:
|
||||
res = getattr(ssl, "CERT_" + candidate)
|
||||
return res # type: ignore[no-any-return]
|
||||
|
||||
return candidate # type: ignore[return-value]
|
||||
|
||||
|
||||
def resolve_ssl_version(candidate: None | int | str) -> int:
|
||||
"""
|
||||
like resolve_cert_reqs
|
||||
"""
|
||||
if candidate is None:
|
||||
return PROTOCOL_TLS
|
||||
|
||||
if isinstance(candidate, str):
|
||||
res = getattr(ssl, candidate, None)
|
||||
if res is None:
|
||||
res = getattr(ssl, "PROTOCOL_" + candidate)
|
||||
return typing.cast(int, res)
|
||||
|
||||
return candidate
|
||||
|
||||
|
||||
def create_urllib3_context(
|
||||
ssl_version: int | None = None,
|
||||
cert_reqs: int | None = None,
|
||||
options: int | None = None,
|
||||
ciphers: str | None = None,
|
||||
ssl_minimum_version: int | None = None,
|
||||
ssl_maximum_version: int | None = None,
|
||||
) -> ssl.SSLContext:
|
||||
"""Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3.
|
||||
|
||||
:param ssl_version:
|
||||
The desired protocol version to use. This will default to
|
||||
PROTOCOL_SSLv23 which will negotiate the highest protocol that both
|
||||
the server and your installation of OpenSSL support.
|
||||
|
||||
This parameter is deprecated instead use 'ssl_minimum_version'.
|
||||
:param ssl_minimum_version:
|
||||
The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
|
||||
:param ssl_maximum_version:
|
||||
The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
|
||||
Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the
|
||||
default value.
|
||||
:param cert_reqs:
|
||||
Whether to require the certificate verification. This defaults to
|
||||
``ssl.CERT_REQUIRED``.
|
||||
:param options:
|
||||
Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
|
||||
``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``.
|
||||
:param ciphers:
|
||||
Which cipher suites to allow the server to select. Defaults to either system configured
|
||||
ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers.
|
||||
:returns:
|
||||
Constructed SSLContext object with specified options
|
||||
:rtype: SSLContext
|
||||
"""
|
||||
if SSLContext is None:
|
||||
raise TypeError("Can't create an SSLContext object without an ssl module")
|
||||
|
||||
# This means 'ssl_version' was specified as an exact value.
|
||||
if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT):
|
||||
# Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version'
|
||||
# to avoid conflicts.
|
||||
if ssl_minimum_version is not None or ssl_maximum_version is not None:
|
||||
raise ValueError(
|
||||
"Can't specify both 'ssl_version' and either "
|
||||
"'ssl_minimum_version' or 'ssl_maximum_version'"
|
||||
)
|
||||
|
||||
# 'ssl_version' is deprecated and will be removed in the future.
|
||||
else:
|
||||
# Use 'ssl_minimum_version' and 'ssl_maximum_version' instead.
|
||||
ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get(
|
||||
ssl_version, TLSVersion.MINIMUM_SUPPORTED
|
||||
)
|
||||
ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get(
|
||||
ssl_version, TLSVersion.MAXIMUM_SUPPORTED
|
||||
)
|
||||
|
||||
# This warning message is pushing users to use 'ssl_minimum_version'
|
||||
# instead of both min/max. Best practice is to only set the minimum version and
|
||||
# keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED'
|
||||
warnings.warn(
|
||||
"'ssl_version' option is deprecated and will be "
|
||||
"removed in urllib3 v2.1.0. Instead use 'ssl_minimum_version'",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# PROTOCOL_TLS is deprecated in Python 3.10 so we always use PROTOCOL_TLS_CLIENT
|
||||
context = SSLContext(PROTOCOL_TLS_CLIENT)
|
||||
|
||||
if ssl_minimum_version is not None:
|
||||
context.minimum_version = ssl_minimum_version
|
||||
else: # Python <3.10 defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here
|
||||
context.minimum_version = TLSVersion.TLSv1_2
|
||||
|
||||
if ssl_maximum_version is not None:
|
||||
context.maximum_version = ssl_maximum_version
|
||||
|
||||
# Unless we're given ciphers defer to either system ciphers in
|
||||
# the case of OpenSSL 1.1.1+ or use our own secure default ciphers.
|
||||
if ciphers:
|
||||
context.set_ciphers(ciphers)
|
||||
|
||||
# Setting the default here, as we may have no ssl module on import
|
||||
cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
|
||||
|
||||
if options is None:
|
||||
options = 0
|
||||
# SSLv2 is easily broken and is considered harmful and dangerous
|
||||
options |= OP_NO_SSLv2
|
||||
# SSLv3 has several problems and is now dangerous
|
||||
options |= OP_NO_SSLv3
|
||||
# Disable compression to prevent CRIME attacks for OpenSSL 1.0+
|
||||
# (issue #309)
|
||||
options |= OP_NO_COMPRESSION
|
||||
# TLSv1.2 only. Unless set explicitly, do not request tickets.
|
||||
# This may save some bandwidth on wire, and although the ticket is encrypted,
|
||||
# there is a risk associated with it being on wire,
|
||||
# if the server is not rotating its ticketing keys properly.
|
||||
options |= OP_NO_TICKET
|
||||
|
||||
context.options |= options
|
||||
|
||||
# Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
|
||||
# necessary for conditional client cert authentication with TLS 1.3.
|
||||
# The attribute is None for OpenSSL <= 1.1.0 or does not exist in older
|
||||
# versions of Python. We only enable if certificate verification is enabled to work
|
||||
# around Python issue #37428
|
||||
# See: https://bugs.python.org/issue37428
|
||||
if (
|
||||
cert_reqs == ssl.CERT_REQUIRED
|
||||
and getattr(context, "post_handshake_auth", None) is not None
|
||||
):
|
||||
context.post_handshake_auth = True
|
||||
|
||||
# The order of the below lines setting verify_mode and check_hostname
|
||||
# matter due to safe-guards SSLContext has to prevent an SSLContext with
|
||||
# check_hostname=True, verify_mode=NONE/OPTIONAL.
|
||||
# We always set 'check_hostname=False' for pyOpenSSL so we rely on our own
|
||||
# 'ssl.match_hostname()' implementation.
|
||||
if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL:
|
||||
context.verify_mode = cert_reqs
|
||||
context.check_hostname = True
|
||||
else:
|
||||
context.check_hostname = False
|
||||
context.verify_mode = cert_reqs
|
||||
|
||||
try:
|
||||
context.hostname_checks_common_name = False
|
||||
except AttributeError: # Defensive: for CPython < 3.8.9 and 3.9.3; for PyPy < 7.3.8
|
||||
pass
|
||||
|
||||
# Enable logging of TLS session keys via defacto standard environment variable
|
||||
# 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.
|
||||
if hasattr(context, "keylog_filename"):
|
||||
sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
|
||||
if sslkeylogfile:
|
||||
context.keylog_filename = sslkeylogfile
|
||||
|
||||
return context
|
||||
|
||||
|
||||
@typing.overload
|
||||
def ssl_wrap_socket(
|
||||
sock: socket.socket,
|
||||
keyfile: str | None = ...,
|
||||
certfile: str | None = ...,
|
||||
cert_reqs: int | None = ...,
|
||||
ca_certs: str | None = ...,
|
||||
server_hostname: str | None = ...,
|
||||
ssl_version: int | None = ...,
|
||||
ciphers: str | None = ...,
|
||||
ssl_context: ssl.SSLContext | None = ...,
|
||||
ca_cert_dir: str | None = ...,
|
||||
key_password: str | None = ...,
|
||||
ca_cert_data: None | str | bytes = ...,
|
||||
tls_in_tls: Literal[False] = ...,
|
||||
) -> ssl.SSLSocket:
|
||||
...
|
||||
|
||||
|
||||
@typing.overload
|
||||
def ssl_wrap_socket(
|
||||
sock: socket.socket,
|
||||
keyfile: str | None = ...,
|
||||
certfile: str | None = ...,
|
||||
cert_reqs: int | None = ...,
|
||||
ca_certs: str | None = ...,
|
||||
server_hostname: str | None = ...,
|
||||
ssl_version: int | None = ...,
|
||||
ciphers: str | None = ...,
|
||||
ssl_context: ssl.SSLContext | None = ...,
|
||||
ca_cert_dir: str | None = ...,
|
||||
key_password: str | None = ...,
|
||||
ca_cert_data: None | str | bytes = ...,
|
||||
tls_in_tls: bool = ...,
|
||||
) -> ssl.SSLSocket | SSLTransportType:
|
||||
...
|
||||
|
||||
|
||||
def ssl_wrap_socket(
|
||||
sock: socket.socket,
|
||||
keyfile: str | None = None,
|
||||
certfile: str | None = None,
|
||||
cert_reqs: int | None = None,
|
||||
ca_certs: str | None = None,
|
||||
server_hostname: str | None = None,
|
||||
ssl_version: int | None = None,
|
||||
ciphers: str | None = None,
|
||||
ssl_context: ssl.SSLContext | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
key_password: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
tls_in_tls: bool = False,
|
||||
) -> ssl.SSLSocket | SSLTransportType:
|
||||
"""
|
||||
All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and
|
||||
ca_cert_dir have the same meaning as they do when using
|
||||
:func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`,
|
||||
:meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`.
|
||||
|
||||
:param server_hostname:
|
||||
When SNI is supported, the expected hostname of the certificate
|
||||
:param ssl_context:
|
||||
A pre-made :class:`SSLContext` object. If none is provided, one will
|
||||
be created using :func:`create_urllib3_context`.
|
||||
:param ciphers:
|
||||
A string of ciphers we wish the client to support.
|
||||
:param ca_cert_dir:
|
||||
A directory containing CA certificates in multiple separate files, as
|
||||
supported by OpenSSL's -CApath flag or the capath argument to
|
||||
SSLContext.load_verify_locations().
|
||||
:param key_password:
|
||||
Optional password if the keyfile is encrypted.
|
||||
:param ca_cert_data:
|
||||
Optional string containing CA certificates in PEM format suitable for
|
||||
passing as the cadata parameter to SSLContext.load_verify_locations()
|
||||
:param tls_in_tls:
|
||||
Use SSLTransport to wrap the existing socket.
|
||||
"""
|
||||
context = ssl_context
|
||||
if context is None:
|
||||
# Note: This branch of code and all the variables in it are only used in tests.
|
||||
# We should consider deprecating and removing this code.
|
||||
context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
|
||||
|
||||
if ca_certs or ca_cert_dir or ca_cert_data:
|
||||
try:
|
||||
context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data)
|
||||
except OSError as e:
|
||||
raise SSLError(e) from e
|
||||
|
||||
elif ssl_context is None and hasattr(context, "load_default_certs"):
|
||||
# try to load OS default certs; works well on Windows.
|
||||
context.load_default_certs()
|
||||
|
||||
# Attempt to detect if we get the goofy behavior of the
|
||||
# keyfile being encrypted and OpenSSL asking for the
|
||||
# passphrase via the terminal and instead error out.
|
||||
if keyfile and key_password is None and _is_key_file_encrypted(keyfile):
|
||||
raise SSLError("Client private key is encrypted, password is required")
|
||||
|
||||
if certfile:
|
||||
if key_password is None:
|
||||
context.load_cert_chain(certfile, keyfile)
|
||||
else:
|
||||
context.load_cert_chain(certfile, keyfile, key_password)
|
||||
|
||||
try:
|
||||
context.set_alpn_protocols(ALPN_PROTOCOLS)
|
||||
except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols
|
||||
pass
|
||||
|
||||
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
|
||||
return ssl_sock
|
||||
|
||||
|
||||
def is_ipaddress(hostname: str | bytes) -> bool:
|
||||
"""Detects whether the hostname given is an IPv4 or IPv6 address.
|
||||
Also detects IPv6 addresses with Zone IDs.
|
||||
|
||||
:param str hostname: Hostname to examine.
|
||||
:return: True if the hostname is an IP address, False otherwise.
|
||||
"""
|
||||
if isinstance(hostname, bytes):
|
||||
# IDN A-label bytes are ASCII compatible.
|
||||
hostname = hostname.decode("ascii")
|
||||
return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname))
|
||||
|
||||
|
||||
def _is_key_file_encrypted(key_file: str) -> bool:
|
||||
"""Detects if a key file is encrypted or not."""
|
||||
with open(key_file) as f:
|
||||
for line in f:
|
||||
# Look for Proc-Type: 4,ENCRYPTED
|
||||
if "ENCRYPTED" in line:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _ssl_wrap_socket_impl(
|
||||
sock: socket.socket,
|
||||
ssl_context: ssl.SSLContext,
|
||||
tls_in_tls: bool,
|
||||
server_hostname: str | None = None,
|
||||
) -> ssl.SSLSocket | SSLTransportType:
|
||||
if tls_in_tls:
|
||||
if not SSLTransport:
|
||||
# Import error, ssl is not available.
|
||||
raise ProxySchemeUnsupported(
|
||||
"TLS in TLS requires support for the 'ssl' module"
|
||||
)
|
||||
|
||||
SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
|
||||
return SSLTransport(sock, ssl_context, server_hostname)
|
||||
|
||||
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
|
@ -1,159 +0,0 @@
|
||||
"""The match_hostname() function from Python 3.5, essential when using SSL."""
|
||||
|
||||
# Note: This file is under the PSF license as the code comes from the python
|
||||
# stdlib. http://docs.python.org/3/license.html
|
||||
# It is modified to remove commonName support.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import typing
|
||||
from ipaddress import IPv4Address, IPv6Address
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from .ssl_ import _TYPE_PEER_CERT_RET_DICT
|
||||
|
||||
__version__ = "3.5.0.1"
|
||||
|
||||
|
||||
class CertificateError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _dnsname_match(
|
||||
dn: typing.Any, hostname: str, max_wildcards: int = 1
|
||||
) -> typing.Match[str] | None | bool:
|
||||
"""Matching according to RFC 6125, section 6.4.3
|
||||
|
||||
http://tools.ietf.org/html/rfc6125#section-6.4.3
|
||||
"""
|
||||
pats = []
|
||||
if not dn:
|
||||
return False
|
||||
|
||||
# Ported from python3-syntax:
|
||||
# leftmost, *remainder = dn.split(r'.')
|
||||
parts = dn.split(r".")
|
||||
leftmost = parts[0]
|
||||
remainder = parts[1:]
|
||||
|
||||
wildcards = leftmost.count("*")
|
||||
if wildcards > max_wildcards:
|
||||
# Issue #17980: avoid denials of service by refusing more
|
||||
# than one wildcard per fragment. A survey of established
|
||||
# policy among SSL implementations showed it to be a
|
||||
# reasonable choice.
|
||||
raise CertificateError(
|
||||
"too many wildcards in certificate DNS name: " + repr(dn)
|
||||
)
|
||||
|
||||
# speed up common case w/o wildcards
|
||||
if not wildcards:
|
||||
return bool(dn.lower() == hostname.lower())
|
||||
|
||||
# RFC 6125, section 6.4.3, subitem 1.
|
||||
# The client SHOULD NOT attempt to match a presented identifier in which
|
||||
# the wildcard character comprises a label other than the left-most label.
|
||||
if leftmost == "*":
|
||||
# When '*' is a fragment by itself, it matches a non-empty dotless
|
||||
# fragment.
|
||||
pats.append("[^.]+")
|
||||
elif leftmost.startswith("xn--") or hostname.startswith("xn--"):
|
||||
# RFC 6125, section 6.4.3, subitem 3.
|
||||
# The client SHOULD NOT attempt to match a presented identifier
|
||||
# where the wildcard character is embedded within an A-label or
|
||||
# U-label of an internationalized domain name.
|
||||
pats.append(re.escape(leftmost))
|
||||
else:
|
||||
# Otherwise, '*' matches any dotless string, e.g. www*
|
||||
pats.append(re.escape(leftmost).replace(r"\*", "[^.]*"))
|
||||
|
||||
# add the remaining fragments, ignore any wildcards
|
||||
for frag in remainder:
|
||||
pats.append(re.escape(frag))
|
||||
|
||||
pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE)
|
||||
return pat.match(hostname)
|
||||
|
||||
|
||||
def _ipaddress_match(ipname: str, host_ip: IPv4Address | IPv6Address) -> bool:
|
||||
"""Exact matching of IP addresses.
|
||||
|
||||
RFC 9110 section 4.3.5: "A reference identity of IP-ID contains the decoded
|
||||
bytes of the IP address. An IP version 4 address is 4 octets, and an IP
|
||||
version 6 address is 16 octets. [...] A reference identity of type IP-ID
|
||||
matches if the address is identical to an iPAddress value of the
|
||||
subjectAltName extension of the certificate."
|
||||
"""
|
||||
# OpenSSL may add a trailing newline to a subjectAltName's IP address
|
||||
# Divergence from upstream: ipaddress can't handle byte str
|
||||
ip = ipaddress.ip_address(ipname.rstrip())
|
||||
return bool(ip.packed == host_ip.packed)
|
||||
|
||||
|
||||
def match_hostname(
|
||||
cert: _TYPE_PEER_CERT_RET_DICT | None,
|
||||
hostname: str,
|
||||
hostname_checks_common_name: bool = False,
|
||||
) -> None:
|
||||
"""Verify that *cert* (in decoded format as returned by
|
||||
SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125
|
||||
rules are followed, but IP addresses are not accepted for *hostname*.
|
||||
|
||||
CertificateError is raised on failure. On success, the function
|
||||
returns nothing.
|
||||
"""
|
||||
if not cert:
|
||||
raise ValueError(
|
||||
"empty or no certificate, match_hostname needs a "
|
||||
"SSL socket or SSL context with either "
|
||||
"CERT_OPTIONAL or CERT_REQUIRED"
|
||||
)
|
||||
try:
|
||||
# Divergence from upstream: ipaddress can't handle byte str
|
||||
#
|
||||
# The ipaddress module shipped with Python < 3.9 does not support
|
||||
# scoped IPv6 addresses so we unconditionally strip the Zone IDs for
|
||||
# now. Once we drop support for Python 3.9 we can remove this branch.
|
||||
if "%" in hostname:
|
||||
host_ip = ipaddress.ip_address(hostname[: hostname.rfind("%")])
|
||||
else:
|
||||
host_ip = ipaddress.ip_address(hostname)
|
||||
|
||||
except ValueError:
|
||||
# Not an IP address (common case)
|
||||
host_ip = None
|
||||
dnsnames = []
|
||||
san: tuple[tuple[str, str], ...] = cert.get("subjectAltName", ())
|
||||
key: str
|
||||
value: str
|
||||
for key, value in san:
|
||||
if key == "DNS":
|
||||
if host_ip is None and _dnsname_match(value, hostname):
|
||||
return
|
||||
dnsnames.append(value)
|
||||
elif key == "IP Address":
|
||||
if host_ip is not None and _ipaddress_match(value, host_ip):
|
||||
return
|
||||
dnsnames.append(value)
|
||||
|
||||
# We only check 'commonName' if it's enabled and we're not verifying
|
||||
# an IP address. IP addresses aren't valid within 'commonName'.
|
||||
if hostname_checks_common_name and host_ip is None and not dnsnames:
|
||||
for sub in cert.get("subject", ()):
|
||||
for key, value in sub:
|
||||
if key == "commonName":
|
||||
if _dnsname_match(value, hostname):
|
||||
return
|
||||
dnsnames.append(value)
|
||||
|
||||
if len(dnsnames) > 1:
|
||||
raise CertificateError(
|
||||
"hostname %r "
|
||||
"doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames)))
|
||||
)
|
||||
elif len(dnsnames) == 1:
|
||||
raise CertificateError(f"hostname {hostname!r} doesn't match {dnsnames[0]!r}")
|
||||
else:
|
||||
raise CertificateError("no appropriate subjectAltName fields were found")
|
@ -1,280 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import socket
|
||||
import ssl
|
||||
import typing
|
||||
|
||||
from ..exceptions import ProxySchemeUnsupported
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing import Literal
|
||||
|
||||
from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT
|
||||
|
||||
|
||||
_SelfT = typing.TypeVar("_SelfT", bound="SSLTransport")
|
||||
_WriteBuffer = typing.Union[bytearray, memoryview]
|
||||
_ReturnValue = typing.TypeVar("_ReturnValue")
|
||||
|
||||
SSL_BLOCKSIZE = 16384
|
||||
|
||||
|
||||
class SSLTransport:
|
||||
"""
|
||||
The SSLTransport wraps an existing socket and establishes an SSL connection.
|
||||
|
||||
Contrary to Python's implementation of SSLSocket, it allows you to chain
|
||||
multiple TLS connections together. It's particularly useful if you need to
|
||||
implement TLS within TLS.
|
||||
|
||||
The class supports most of the socket API operations.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _validate_ssl_context_for_tls_in_tls(ssl_context: ssl.SSLContext) -> None:
|
||||
"""
|
||||
Raises a ProxySchemeUnsupported if the provided ssl_context can't be used
|
||||
for TLS in TLS.
|
||||
|
||||
The only requirement is that the ssl_context provides the 'wrap_bio'
|
||||
methods.
|
||||
"""
|
||||
|
||||
if not hasattr(ssl_context, "wrap_bio"):
|
||||
raise ProxySchemeUnsupported(
|
||||
"TLS in TLS requires SSLContext.wrap_bio() which isn't "
|
||||
"available on non-native SSLContext"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
socket: socket.socket,
|
||||
ssl_context: ssl.SSLContext,
|
||||
server_hostname: str | None = None,
|
||||
suppress_ragged_eofs: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Create an SSLTransport around socket using the provided ssl_context.
|
||||
"""
|
||||
self.incoming = ssl.MemoryBIO()
|
||||
self.outgoing = ssl.MemoryBIO()
|
||||
|
||||
self.suppress_ragged_eofs = suppress_ragged_eofs
|
||||
self.socket = socket
|
||||
|
||||
self.sslobj = ssl_context.wrap_bio(
|
||||
self.incoming, self.outgoing, server_hostname=server_hostname
|
||||
)
|
||||
|
||||
# Perform initial handshake.
|
||||
self._ssl_io_loop(self.sslobj.do_handshake)
|
||||
|
||||
def __enter__(self: _SelfT) -> _SelfT:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: typing.Any) -> None:
|
||||
self.close()
|
||||
|
||||
def fileno(self) -> int:
|
||||
return self.socket.fileno()
|
||||
|
||||
def read(self, len: int = 1024, buffer: typing.Any | None = None) -> int | bytes:
|
||||
return self._wrap_ssl_read(len, buffer)
|
||||
|
||||
def recv(self, buflen: int = 1024, flags: int = 0) -> int | bytes:
|
||||
if flags != 0:
|
||||
raise ValueError("non-zero flags not allowed in calls to recv")
|
||||
return self._wrap_ssl_read(buflen)
|
||||
|
||||
def recv_into(
|
||||
self,
|
||||
buffer: _WriteBuffer,
|
||||
nbytes: int | None = None,
|
||||
flags: int = 0,
|
||||
) -> None | int | bytes:
|
||||
if flags != 0:
|
||||
raise ValueError("non-zero flags not allowed in calls to recv_into")
|
||||
if nbytes is None:
|
||||
nbytes = len(buffer)
|
||||
return self.read(nbytes, buffer)
|
||||
|
||||
def sendall(self, data: bytes, flags: int = 0) -> None:
|
||||
if flags != 0:
|
||||
raise ValueError("non-zero flags not allowed in calls to sendall")
|
||||
count = 0
|
||||
with memoryview(data) as view, view.cast("B") as byte_view:
|
||||
amount = len(byte_view)
|
||||
while count < amount:
|
||||
v = self.send(byte_view[count:])
|
||||
count += v
|
||||
|
||||
def send(self, data: bytes, flags: int = 0) -> int:
|
||||
if flags != 0:
|
||||
raise ValueError("non-zero flags not allowed in calls to send")
|
||||
return self._ssl_io_loop(self.sslobj.write, data)
|
||||
|
||||
def makefile(
|
||||
self,
|
||||
mode: str,
|
||||
buffering: int | None = None,
|
||||
*,
|
||||
encoding: str | None = None,
|
||||
errors: str | None = None,
|
||||
newline: str | None = None,
|
||||
) -> typing.BinaryIO | typing.TextIO | socket.SocketIO:
|
||||
"""
|
||||
Python's httpclient uses makefile and buffered io when reading HTTP
|
||||
messages and we need to support it.
|
||||
|
||||
This is unfortunately a copy and paste of socket.py makefile with small
|
||||
changes to point to the socket directly.
|
||||
"""
|
||||
if not set(mode) <= {"r", "w", "b"}:
|
||||
raise ValueError(f"invalid mode {mode!r} (only r, w, b allowed)")
|
||||
|
||||
writing = "w" in mode
|
||||
reading = "r" in mode or not writing
|
||||
assert reading or writing
|
||||
binary = "b" in mode
|
||||
rawmode = ""
|
||||
if reading:
|
||||
rawmode += "r"
|
||||
if writing:
|
||||
rawmode += "w"
|
||||
raw = socket.SocketIO(self, rawmode) # type: ignore[arg-type]
|
||||
self.socket._io_refs += 1 # type: ignore[attr-defined]
|
||||
if buffering is None:
|
||||
buffering = -1
|
||||
if buffering < 0:
|
||||
buffering = io.DEFAULT_BUFFER_SIZE
|
||||
if buffering == 0:
|
||||
if not binary:
|
||||
raise ValueError("unbuffered streams must be binary")
|
||||
return raw
|
||||
buffer: typing.BinaryIO
|
||||
if reading and writing:
|
||||
buffer = io.BufferedRWPair(raw, raw, buffering) # type: ignore[assignment]
|
||||
elif reading:
|
||||
buffer = io.BufferedReader(raw, buffering)
|
||||
else:
|
||||
assert writing
|
||||
buffer = io.BufferedWriter(raw, buffering)
|
||||
if binary:
|
||||
return buffer
|
||||
text = io.TextIOWrapper(buffer, encoding, errors, newline)
|
||||
text.mode = mode # type: ignore[misc]
|
||||
return text
|
||||
|
||||
def unwrap(self) -> None:
|
||||
self._ssl_io_loop(self.sslobj.unwrap)
|
||||
|
||||
def close(self) -> None:
|
||||
self.socket.close()
|
||||
|
||||
@typing.overload
|
||||
def getpeercert(
|
||||
self, binary_form: Literal[False] = ...
|
||||
) -> _TYPE_PEER_CERT_RET_DICT | None:
|
||||
...
|
||||
|
||||
@typing.overload
|
||||
def getpeercert(self, binary_form: Literal[True]) -> bytes | None:
|
||||
...
|
||||
|
||||
def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET:
|
||||
return self.sslobj.getpeercert(binary_form) # type: ignore[return-value]
|
||||
|
||||
def version(self) -> str | None:
|
||||
return self.sslobj.version()
|
||||
|
||||
def cipher(self) -> tuple[str, str, int] | None:
|
||||
return self.sslobj.cipher()
|
||||
|
||||
def selected_alpn_protocol(self) -> str | None:
|
||||
return self.sslobj.selected_alpn_protocol()
|
||||
|
||||
def selected_npn_protocol(self) -> str | None:
|
||||
return self.sslobj.selected_npn_protocol()
|
||||
|
||||
def shared_ciphers(self) -> list[tuple[str, str, int]] | None:
|
||||
return self.sslobj.shared_ciphers()
|
||||
|
||||
def compression(self) -> str | None:
|
||||
return self.sslobj.compression()
|
||||
|
||||
def settimeout(self, value: float | None) -> None:
|
||||
self.socket.settimeout(value)
|
||||
|
||||
def gettimeout(self) -> float | None:
|
||||
return self.socket.gettimeout()
|
||||
|
||||
def _decref_socketios(self) -> None:
|
||||
self.socket._decref_socketios() # type: ignore[attr-defined]
|
||||
|
||||
def _wrap_ssl_read(self, len: int, buffer: bytearray | None = None) -> int | bytes:
|
||||
try:
|
||||
return self._ssl_io_loop(self.sslobj.read, len, buffer)
|
||||
except ssl.SSLError as e:
|
||||
if e.errno == ssl.SSL_ERROR_EOF and self.suppress_ragged_eofs:
|
||||
return 0 # eof, return 0.
|
||||
else:
|
||||
raise
|
||||
|
||||
# func is sslobj.do_handshake or sslobj.unwrap
|
||||
@typing.overload
|
||||
def _ssl_io_loop(self, func: typing.Callable[[], None]) -> None:
|
||||
...
|
||||
|
||||
# func is sslobj.write, arg1 is data
|
||||
@typing.overload
|
||||
def _ssl_io_loop(self, func: typing.Callable[[bytes], int], arg1: bytes) -> int:
|
||||
...
|
||||
|
||||
# func is sslobj.read, arg1 is len, arg2 is buffer
|
||||
@typing.overload
|
||||
def _ssl_io_loop(
|
||||
self,
|
||||
func: typing.Callable[[int, bytearray | None], bytes],
|
||||
arg1: int,
|
||||
arg2: bytearray | None,
|
||||
) -> bytes:
|
||||
...
|
||||
|
||||
def _ssl_io_loop(
|
||||
self,
|
||||
func: typing.Callable[..., _ReturnValue],
|
||||
arg1: None | bytes | int = None,
|
||||
arg2: bytearray | None = None,
|
||||
) -> _ReturnValue:
|
||||
"""Performs an I/O loop between incoming/outgoing and the socket."""
|
||||
should_loop = True
|
||||
ret = None
|
||||
|
||||
while should_loop:
|
||||
errno = None
|
||||
try:
|
||||
if arg1 is None and arg2 is None:
|
||||
ret = func()
|
||||
elif arg2 is None:
|
||||
ret = func(arg1)
|
||||
else:
|
||||
ret = func(arg1, arg2)
|
||||
except ssl.SSLError as e:
|
||||
if e.errno not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE):
|
||||
# WANT_READ, and WANT_WRITE are expected, others are not.
|
||||
raise e
|
||||
errno = e.errno
|
||||
|
||||
buf = self.outgoing.read()
|
||||
self.socket.sendall(buf)
|
||||
|
||||
if errno is None:
|
||||
should_loop = False
|
||||
elif errno == ssl.SSL_ERROR_WANT_READ:
|
||||
buf = self.socket.recv(SSL_BLOCKSIZE)
|
||||
if buf:
|
||||
self.incoming.write(buf)
|
||||
else:
|
||||
self.incoming.write_eof()
|
||||
return typing.cast(_ReturnValue, ret)
|
@ -1,279 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import typing
|
||||
from enum import Enum
|
||||
from socket import getdefaulttimeout
|
||||
|
||||
from ..exceptions import TimeoutStateError
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing import Final
|
||||
|
||||
|
||||
class _TYPE_DEFAULT(Enum):
|
||||
# This value should never be passed to socket.settimeout() so for safety we use a -1.
|
||||
# socket.settimout() raises a ValueError for negative values.
|
||||
token = -1
|
||||
|
||||
|
||||
_DEFAULT_TIMEOUT: Final[_TYPE_DEFAULT] = _TYPE_DEFAULT.token
|
||||
|
||||
_TYPE_TIMEOUT = typing.Optional[typing.Union[float, _TYPE_DEFAULT]]
|
||||
|
||||
|
||||
class Timeout:
|
||||
"""Timeout configuration.
|
||||
|
||||
Timeouts can be defined as a default for a pool:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
timeout = urllib3.util.Timeout(connect=2.0, read=7.0)
|
||||
|
||||
http = urllib3.PoolManager(timeout=timeout)
|
||||
|
||||
resp = http.request("GET", "https://example.com/")
|
||||
|
||||
print(resp.status)
|
||||
|
||||
Or per-request (which overrides the default for the pool):
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response = http.request("GET", "https://example.com/", timeout=Timeout(10))
|
||||
|
||||
Timeouts can be disabled by setting all the parameters to ``None``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
no_timeout = Timeout(connect=None, read=None)
|
||||
response = http.request("GET", "https://example.com/", timeout=no_timeout)
|
||||
|
||||
|
||||
:param total:
|
||||
This combines the connect and read timeouts into one; the read timeout
|
||||
will be set to the time leftover from the connect attempt. In the
|
||||
event that both a connect timeout and a total are specified, or a read
|
||||
timeout and a total are specified, the shorter timeout will be applied.
|
||||
|
||||
Defaults to None.
|
||||
|
||||
:type total: int, float, or None
|
||||
|
||||
:param connect:
|
||||
The maximum amount of time (in seconds) to wait for a connection
|
||||
attempt to a server to succeed. Omitting the parameter will default the
|
||||
connect timeout to the system default, probably `the global default
|
||||
timeout in socket.py
|
||||
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
|
||||
None will set an infinite timeout for connection attempts.
|
||||
|
||||
:type connect: int, float, or None
|
||||
|
||||
:param read:
|
||||
The maximum amount of time (in seconds) to wait between consecutive
|
||||
read operations for a response from the server. Omitting the parameter
|
||||
will default the read timeout to the system default, probably `the
|
||||
global default timeout in socket.py
|
||||
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
|
||||
None will set an infinite timeout.
|
||||
|
||||
:type read: int, float, or None
|
||||
|
||||
.. note::
|
||||
|
||||
Many factors can affect the total amount of time for urllib3 to return
|
||||
an HTTP response.
|
||||
|
||||
For example, Python's DNS resolver does not obey the timeout specified
|
||||
on the socket. Other factors that can affect total request time include
|
||||
high CPU load, high swap, the program running at a low priority level,
|
||||
or other behaviors.
|
||||
|
||||
In addition, the read and total timeouts only measure the time between
|
||||
read operations on the socket connecting the client and the server,
|
||||
not the total amount of time for the request to return a complete
|
||||
response. For most requests, the timeout is raised because the server
|
||||
has not sent the first byte in the specified time. This is not always
|
||||
the case; if a server streams one byte every fifteen seconds, a timeout
|
||||
of 20 seconds will not trigger, even though the request will take
|
||||
several minutes to complete.
|
||||
|
||||
If your goal is to cut off any request after a set amount of wall clock
|
||||
time, consider having a second "watcher" thread to cut off a slow
|
||||
request.
|
||||
"""
|
||||
|
||||
#: A sentinel object representing the default timeout value
|
||||
DEFAULT_TIMEOUT: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
total: _TYPE_TIMEOUT = None,
|
||||
connect: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
read: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
) -> None:
|
||||
self._connect = self._validate_timeout(connect, "connect")
|
||||
self._read = self._validate_timeout(read, "read")
|
||||
self.total = self._validate_timeout(total, "total")
|
||||
self._start_connect: float | None = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{type(self).__name__}(connect={self._connect!r}, read={self._read!r}, total={self.total!r})"
|
||||
|
||||
# __str__ provided for backwards compatibility
|
||||
__str__ = __repr__
|
||||
|
||||
@staticmethod
|
||||
def resolve_default_timeout(timeout: _TYPE_TIMEOUT) -> float | None:
|
||||
return getdefaulttimeout() if timeout is _DEFAULT_TIMEOUT else timeout
|
||||
|
||||
@classmethod
|
||||
def _validate_timeout(cls, value: _TYPE_TIMEOUT, name: str) -> _TYPE_TIMEOUT:
|
||||
"""Check that a timeout attribute is valid.
|
||||
|
||||
:param value: The timeout value to validate
|
||||
:param name: The name of the timeout attribute to validate. This is
|
||||
used to specify in error messages.
|
||||
:return: The validated and casted version of the given value.
|
||||
:raises ValueError: If it is a numeric value less than or equal to
|
||||
zero, or the type is not an integer, float, or None.
|
||||
"""
|
||||
if value is None or value is _DEFAULT_TIMEOUT:
|
||||
return value
|
||||
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(
|
||||
"Timeout cannot be a boolean value. It must "
|
||||
"be an int, float or None."
|
||||
)
|
||||
try:
|
||||
float(value)
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(
|
||||
"Timeout value %s was %s, but it must be an "
|
||||
"int, float or None." % (name, value)
|
||||
) from None
|
||||
|
||||
try:
|
||||
if value <= 0:
|
||||
raise ValueError(
|
||||
"Attempted to set %s timeout to %s, but the "
|
||||
"timeout cannot be set to a value less "
|
||||
"than or equal to 0." % (name, value)
|
||||
)
|
||||
except TypeError:
|
||||
raise ValueError(
|
||||
"Timeout value %s was %s, but it must be an "
|
||||
"int, float or None." % (name, value)
|
||||
) from None
|
||||
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def from_float(cls, timeout: _TYPE_TIMEOUT) -> Timeout:
|
||||
"""Create a new Timeout from a legacy timeout value.
|
||||
|
||||
The timeout value used by httplib.py sets the same timeout on the
|
||||
connect(), and recv() socket requests. This creates a :class:`Timeout`
|
||||
object that sets the individual timeouts to the ``timeout`` value
|
||||
passed to this function.
|
||||
|
||||
:param timeout: The legacy timeout value.
|
||||
:type timeout: integer, float, :attr:`urllib3.util.Timeout.DEFAULT_TIMEOUT`, or None
|
||||
:return: Timeout object
|
||||
:rtype: :class:`Timeout`
|
||||
"""
|
||||
return Timeout(read=timeout, connect=timeout)
|
||||
|
||||
def clone(self) -> Timeout:
|
||||
"""Create a copy of the timeout object
|
||||
|
||||
Timeout properties are stored per-pool but each request needs a fresh
|
||||
Timeout object to ensure each one has its own start/stop configured.
|
||||
|
||||
:return: a copy of the timeout object
|
||||
:rtype: :class:`Timeout`
|
||||
"""
|
||||
# We can't use copy.deepcopy because that will also create a new object
|
||||
# for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
|
||||
# detect the user default.
|
||||
return Timeout(connect=self._connect, read=self._read, total=self.total)
|
||||
|
||||
def start_connect(self) -> float:
|
||||
"""Start the timeout clock, used during a connect() attempt
|
||||
|
||||
:raises urllib3.exceptions.TimeoutStateError: if you attempt
|
||||
to start a timer that has been started already.
|
||||
"""
|
||||
if self._start_connect is not None:
|
||||
raise TimeoutStateError("Timeout timer has already been started.")
|
||||
self._start_connect = time.monotonic()
|
||||
return self._start_connect
|
||||
|
||||
def get_connect_duration(self) -> float:
|
||||
"""Gets the time elapsed since the call to :meth:`start_connect`.
|
||||
|
||||
:return: Elapsed time in seconds.
|
||||
:rtype: float
|
||||
:raises urllib3.exceptions.TimeoutStateError: if you attempt
|
||||
to get duration for a timer that hasn't been started.
|
||||
"""
|
||||
if self._start_connect is None:
|
||||
raise TimeoutStateError(
|
||||
"Can't get connect duration for timer that has not started."
|
||||
)
|
||||
return time.monotonic() - self._start_connect
|
||||
|
||||
@property
|
||||
def connect_timeout(self) -> _TYPE_TIMEOUT:
|
||||
"""Get the value to use when setting a connection timeout.
|
||||
|
||||
This will be a positive float or integer, the value None
|
||||
(never timeout), or the default system timeout.
|
||||
|
||||
:return: Connect timeout.
|
||||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
|
||||
"""
|
||||
if self.total is None:
|
||||
return self._connect
|
||||
|
||||
if self._connect is None or self._connect is _DEFAULT_TIMEOUT:
|
||||
return self.total
|
||||
|
||||
return min(self._connect, self.total) # type: ignore[type-var]
|
||||
|
||||
@property
|
||||
def read_timeout(self) -> float | None:
|
||||
"""Get the value for the read timeout.
|
||||
|
||||
This assumes some time has elapsed in the connection timeout and
|
||||
computes the read timeout appropriately.
|
||||
|
||||
If self.total is set, the read timeout is dependent on the amount of
|
||||
time taken by the connect timeout. If the connection time has not been
|
||||
established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be
|
||||
raised.
|
||||
|
||||
:return: Value to use for the read timeout.
|
||||
:rtype: int, float or None
|
||||
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
|
||||
has not yet been called on this object.
|
||||
"""
|
||||
if (
|
||||
self.total is not None
|
||||
and self.total is not _DEFAULT_TIMEOUT
|
||||
and self._read is not None
|
||||
and self._read is not _DEFAULT_TIMEOUT
|
||||
):
|
||||
# In case the connect timeout has not yet been established.
|
||||
if self._start_connect is None:
|
||||
return self._read
|
||||
return max(0, min(self.total - self.get_connect_duration(), self._read))
|
||||
elif self.total is not None and self.total is not _DEFAULT_TIMEOUT:
|
||||
return max(0, self.total - self.get_connect_duration())
|
||||
else:
|
||||
return self.resolve_default_timeout(self._read)
|
@ -1,471 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import typing
|
||||
|
||||
from ..exceptions import LocationParseError
|
||||
from .util import to_str
|
||||
|
||||
# We only want to normalize urls with an HTTP(S) scheme.
|
||||
# urllib3 infers URLs without a scheme (None) to be http.
|
||||
_NORMALIZABLE_SCHEMES = ("http", "https", None)
|
||||
|
||||
# Almost all of these patterns were derived from the
|
||||
# 'rfc3986' module: https://github.com/python-hyper/rfc3986
|
||||
_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")
|
||||
_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")
|
||||
_URI_RE = re.compile(
|
||||
r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"
|
||||
r"(?://([^\\/?#]*))?"
|
||||
r"([^?#]*)"
|
||||
r"(?:\?([^#]*))?"
|
||||
r"(?:#(.*))?$",
|
||||
re.UNICODE | re.DOTALL,
|
||||
)
|
||||
|
||||
_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"
|
||||
_HEX_PAT = "[0-9A-Fa-f]{1,4}"
|
||||
_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)
|
||||
_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}
|
||||
_variations = [
|
||||
# 6( h16 ":" ) ls32
|
||||
"(?:%(hex)s:){6}%(ls32)s",
|
||||
# "::" 5( h16 ":" ) ls32
|
||||
"::(?:%(hex)s:){5}%(ls32)s",
|
||||
# [ h16 ] "::" 4( h16 ":" ) ls32
|
||||
"(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",
|
||||
# [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
|
||||
"(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",
|
||||
# [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
|
||||
"(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",
|
||||
# [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
|
||||
"(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",
|
||||
# [ *4( h16 ":" ) h16 ] "::" ls32
|
||||
"(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",
|
||||
# [ *5( h16 ":" ) h16 ] "::" h16
|
||||
"(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",
|
||||
# [ *6( h16 ":" ) h16 ] "::"
|
||||
"(?:(?:%(hex)s:){0,6}%(hex)s)?::",
|
||||
]
|
||||
|
||||
_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"
|
||||
_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"
|
||||
_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"
|
||||
_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"
|
||||
_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"
|
||||
_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")
|
||||
|
||||
_IPV4_RE = re.compile("^" + _IPV4_PAT + "$")
|
||||
_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")
|
||||
_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")
|
||||
_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")
|
||||
_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")
|
||||
|
||||
_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (
|
||||
_REG_NAME_PAT,
|
||||
_IPV4_PAT,
|
||||
_IPV6_ADDRZ_PAT,
|
||||
)
|
||||
_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)
|
||||
|
||||
_UNRESERVED_CHARS = set(
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"
|
||||
)
|
||||
_SUB_DELIM_CHARS = set("!$&'()*+,;=")
|
||||
_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}
|
||||
_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}
|
||||
_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}
|
||||
|
||||
|
||||
class Url(
|
||||
typing.NamedTuple(
|
||||
"Url",
|
||||
[
|
||||
("scheme", typing.Optional[str]),
|
||||
("auth", typing.Optional[str]),
|
||||
("host", typing.Optional[str]),
|
||||
("port", typing.Optional[int]),
|
||||
("path", typing.Optional[str]),
|
||||
("query", typing.Optional[str]),
|
||||
("fragment", typing.Optional[str]),
|
||||
],
|
||||
)
|
||||
):
|
||||
"""
|
||||
Data structure for representing an HTTP URL. Used as a return value for
|
||||
:func:`parse_url`. Both the scheme and host are normalized as they are
|
||||
both case-insensitive according to RFC 3986.
|
||||
"""
|
||||
|
||||
def __new__( # type: ignore[no-untyped-def]
|
||||
cls,
|
||||
scheme: str | None = None,
|
||||
auth: str | None = None,
|
||||
host: str | None = None,
|
||||
port: int | None = None,
|
||||
path: str | None = None,
|
||||
query: str | None = None,
|
||||
fragment: str | None = None,
|
||||
):
|
||||
if path and not path.startswith("/"):
|
||||
path = "/" + path
|
||||
if scheme is not None:
|
||||
scheme = scheme.lower()
|
||||
return super().__new__(cls, scheme, auth, host, port, path, query, fragment)
|
||||
|
||||
@property
|
||||
def hostname(self) -> str | None:
|
||||
"""For backwards-compatibility with urlparse. We're nice like that."""
|
||||
return self.host
|
||||
|
||||
@property
|
||||
def request_uri(self) -> str:
|
||||
"""Absolute path including the query string."""
|
||||
uri = self.path or "/"
|
||||
|
||||
if self.query is not None:
|
||||
uri += "?" + self.query
|
||||
|
||||
return uri
|
||||
|
||||
@property
|
||||
def authority(self) -> str | None:
|
||||
"""
|
||||
Authority component as defined in RFC 3986 3.2.
|
||||
This includes userinfo (auth), host and port.
|
||||
|
||||
i.e.
|
||||
userinfo@host:port
|
||||
"""
|
||||
userinfo = self.auth
|
||||
netloc = self.netloc
|
||||
if netloc is None or userinfo is None:
|
||||
return netloc
|
||||
else:
|
||||
return f"{userinfo}@{netloc}"
|
||||
|
||||
@property
|
||||
def netloc(self) -> str | None:
|
||||
"""
|
||||
Network location including host and port.
|
||||
|
||||
If you need the equivalent of urllib.parse's ``netloc``,
|
||||
use the ``authority`` property instead.
|
||||
"""
|
||||
if self.host is None:
|
||||
return None
|
||||
if self.port:
|
||||
return f"{self.host}:{self.port}"
|
||||
return self.host
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""
|
||||
Convert self into a url
|
||||
|
||||
This function should more or less round-trip with :func:`.parse_url`. The
|
||||
returned url may not be exactly the same as the url inputted to
|
||||
:func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
|
||||
with a blank port will have : removed).
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
U = urllib3.util.parse_url("https://google.com/mail/")
|
||||
|
||||
print(U.url)
|
||||
# "https://google.com/mail/"
|
||||
|
||||
print( urllib3.util.Url("https", "username:password",
|
||||
"host.com", 80, "/path", "query", "fragment"
|
||||
).url
|
||||
)
|
||||
# "https://username:password@host.com:80/path?query#fragment"
|
||||
"""
|
||||
scheme, auth, host, port, path, query, fragment = self
|
||||
url = ""
|
||||
|
||||
# We use "is not None" we want things to happen with empty strings (or 0 port)
|
||||
if scheme is not None:
|
||||
url += scheme + "://"
|
||||
if auth is not None:
|
||||
url += auth + "@"
|
||||
if host is not None:
|
||||
url += host
|
||||
if port is not None:
|
||||
url += ":" + str(port)
|
||||
if path is not None:
|
||||
url += path
|
||||
if query is not None:
|
||||
url += "?" + query
|
||||
if fragment is not None:
|
||||
url += "#" + fragment
|
||||
|
||||
return url
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.url
|
||||
|
||||
|
||||
@typing.overload
|
||||
def _encode_invalid_chars(
|
||||
component: str, allowed_chars: typing.Container[str]
|
||||
) -> str: # Abstract
|
||||
...
|
||||
|
||||
|
||||
@typing.overload
|
||||
def _encode_invalid_chars(
|
||||
component: None, allowed_chars: typing.Container[str]
|
||||
) -> None: # Abstract
|
||||
...
|
||||
|
||||
|
||||
def _encode_invalid_chars(
|
||||
component: str | None, allowed_chars: typing.Container[str]
|
||||
) -> str | None:
|
||||
"""Percent-encodes a URI component without reapplying
|
||||
onto an already percent-encoded component.
|
||||
"""
|
||||
if component is None:
|
||||
return component
|
||||
|
||||
component = to_str(component)
|
||||
|
||||
# Normalize existing percent-encoded bytes.
|
||||
# Try to see if the component we're encoding is already percent-encoded
|
||||
# so we can skip all '%' characters but still encode all others.
|
||||
component, percent_encodings = _PERCENT_RE.subn(
|
||||
lambda match: match.group(0).upper(), component
|
||||
)
|
||||
|
||||
uri_bytes = component.encode("utf-8", "surrogatepass")
|
||||
is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
|
||||
encoded_component = bytearray()
|
||||
|
||||
for i in range(0, len(uri_bytes)):
|
||||
# Will return a single character bytestring
|
||||
byte = uri_bytes[i : i + 1]
|
||||
byte_ord = ord(byte)
|
||||
if (is_percent_encoded and byte == b"%") or (
|
||||
byte_ord < 128 and byte.decode() in allowed_chars
|
||||
):
|
||||
encoded_component += byte
|
||||
continue
|
||||
encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper()))
|
||||
|
||||
return encoded_component.decode()
|
||||
|
||||
|
||||
def _remove_path_dot_segments(path: str) -> str:
|
||||
# See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
|
||||
segments = path.split("/") # Turn the path into a list of segments
|
||||
output = [] # Initialize the variable to use to store output
|
||||
|
||||
for segment in segments:
|
||||
# '.' is the current directory, so ignore it, it is superfluous
|
||||
if segment == ".":
|
||||
continue
|
||||
# Anything other than '..', should be appended to the output
|
||||
if segment != "..":
|
||||
output.append(segment)
|
||||
# In this case segment == '..', if we can, we should pop the last
|
||||
# element
|
||||
elif output:
|
||||
output.pop()
|
||||
|
||||
# If the path starts with '/' and the output is empty or the first string
|
||||
# is non-empty
|
||||
if path.startswith("/") and (not output or output[0]):
|
||||
output.insert(0, "")
|
||||
|
||||
# If the path starts with '/.' or '/..' ensure we add one more empty
|
||||
# string to add a trailing '/'
|
||||
if path.endswith(("/.", "/..")):
|
||||
output.append("")
|
||||
|
||||
return "/".join(output)
|
||||
|
||||
|
||||
@typing.overload
|
||||
def _normalize_host(host: None, scheme: str | None) -> None:
|
||||
...
|
||||
|
||||
|
||||
@typing.overload
|
||||
def _normalize_host(host: str, scheme: str | None) -> str:
|
||||
...
|
||||
|
||||
|
||||
def _normalize_host(host: str | None, scheme: str | None) -> str | None:
|
||||
if host:
|
||||
if scheme in _NORMALIZABLE_SCHEMES:
|
||||
is_ipv6 = _IPV6_ADDRZ_RE.match(host)
|
||||
if is_ipv6:
|
||||
# IPv6 hosts of the form 'a::b%zone' are encoded in a URL as
|
||||
# such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID
|
||||
# separator as necessary to return a valid RFC 4007 scoped IP.
|
||||
match = _ZONE_ID_RE.search(host)
|
||||
if match:
|
||||
start, end = match.span(1)
|
||||
zone_id = host[start:end]
|
||||
|
||||
if zone_id.startswith("%25") and zone_id != "%25":
|
||||
zone_id = zone_id[3:]
|
||||
else:
|
||||
zone_id = zone_id[1:]
|
||||
zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)
|
||||
return f"{host[:start].lower()}%{zone_id}{host[end:]}"
|
||||
else:
|
||||
return host.lower()
|
||||
elif not _IPV4_RE.match(host):
|
||||
return to_str(
|
||||
b".".join([_idna_encode(label) for label in host.split(".")]),
|
||||
"ascii",
|
||||
)
|
||||
return host
|
||||
|
||||
|
||||
def _idna_encode(name: str) -> bytes:
|
||||
if not name.isascii():
|
||||
try:
|
||||
import idna
|
||||
except ImportError:
|
||||
raise LocationParseError(
|
||||
"Unable to parse URL without the 'idna' module"
|
||||
) from None
|
||||
|
||||
try:
|
||||
return idna.encode(name.lower(), strict=True, std3_rules=True)
|
||||
except idna.IDNAError:
|
||||
raise LocationParseError(
|
||||
f"Name '{name}' is not a valid IDNA label"
|
||||
) from None
|
||||
|
||||
return name.lower().encode("ascii")
|
||||
|
||||
|
||||
def _encode_target(target: str) -> str:
|
||||
"""Percent-encodes a request target so that there are no invalid characters
|
||||
|
||||
Pre-condition for this function is that 'target' must start with '/'.
|
||||
If that is the case then _TARGET_RE will always produce a match.
|
||||
"""
|
||||
match = _TARGET_RE.match(target)
|
||||
if not match: # Defensive:
|
||||
raise LocationParseError(f"{target!r} is not a valid request URI")
|
||||
|
||||
path, query = match.groups()
|
||||
encoded_target = _encode_invalid_chars(path, _PATH_CHARS)
|
||||
if query is not None:
|
||||
query = _encode_invalid_chars(query, _QUERY_CHARS)
|
||||
encoded_target += "?" + query
|
||||
return encoded_target
|
||||
|
||||
|
||||
def parse_url(url: str) -> Url:
|
||||
"""
|
||||
Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
|
||||
performed to parse incomplete urls. Fields not provided will be None.
|
||||
This parser is RFC 3986 and RFC 6874 compliant.
|
||||
|
||||
The parser logic and helper functions are based heavily on
|
||||
work done in the ``rfc3986`` module.
|
||||
|
||||
:param str url: URL to parse into a :class:`.Url` namedtuple.
|
||||
|
||||
Partly backwards-compatible with :mod:`urllib.parse`.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
print( urllib3.util.parse_url('http://google.com/mail/'))
|
||||
# Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
|
||||
|
||||
print( urllib3.util.parse_url('google.com:80'))
|
||||
# Url(scheme=None, host='google.com', port=80, path=None, ...)
|
||||
|
||||
print( urllib3.util.parse_url('/foo?bar'))
|
||||
# Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
|
||||
"""
|
||||
if not url:
|
||||
# Empty
|
||||
return Url()
|
||||
|
||||
source_url = url
|
||||
if not _SCHEME_RE.search(url):
|
||||
url = "//" + url
|
||||
|
||||
scheme: str | None
|
||||
authority: str | None
|
||||
auth: str | None
|
||||
host: str | None
|
||||
port: str | None
|
||||
port_int: int | None
|
||||
path: str | None
|
||||
query: str | None
|
||||
fragment: str | None
|
||||
|
||||
try:
|
||||
scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]
|
||||
normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES
|
||||
|
||||
if scheme:
|
||||
scheme = scheme.lower()
|
||||
|
||||
if authority:
|
||||
auth, _, host_port = authority.rpartition("@")
|
||||
auth = auth or None
|
||||
host, port = _HOST_PORT_RE.match(host_port).groups() # type: ignore[union-attr]
|
||||
if auth and normalize_uri:
|
||||
auth = _encode_invalid_chars(auth, _USERINFO_CHARS)
|
||||
if port == "":
|
||||
port = None
|
||||
else:
|
||||
auth, host, port = None, None, None
|
||||
|
||||
if port is not None:
|
||||
port_int = int(port)
|
||||
if not (0 <= port_int <= 65535):
|
||||
raise LocationParseError(url)
|
||||
else:
|
||||
port_int = None
|
||||
|
||||
host = _normalize_host(host, scheme)
|
||||
|
||||
if normalize_uri and path:
|
||||
path = _remove_path_dot_segments(path)
|
||||
path = _encode_invalid_chars(path, _PATH_CHARS)
|
||||
if normalize_uri and query:
|
||||
query = _encode_invalid_chars(query, _QUERY_CHARS)
|
||||
if normalize_uri and fragment:
|
||||
fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)
|
||||
|
||||
except (ValueError, AttributeError) as e:
|
||||
raise LocationParseError(source_url) from e
|
||||
|
||||
# For the sake of backwards compatibility we put empty
|
||||
# string values for path if there are any defined values
|
||||
# beyond the path in the URL.
|
||||
# TODO: Remove this when we break backwards compatibility.
|
||||
if not path:
|
||||
if query is not None or fragment is not None:
|
||||
path = ""
|
||||
else:
|
||||
path = None
|
||||
|
||||
return Url(
|
||||
scheme=scheme,
|
||||
auth=auth,
|
||||
host=host,
|
||||
port=port_int,
|
||||
path=path,
|
||||
query=query,
|
||||
fragment=fragment,
|
||||
)
|
@ -1,42 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
from types import TracebackType
|
||||
|
||||
|
||||
def to_bytes(
|
||||
x: str | bytes, encoding: str | None = None, errors: str | None = None
|
||||
) -> bytes:
|
||||
if isinstance(x, bytes):
|
||||
return x
|
||||
elif not isinstance(x, str):
|
||||
raise TypeError(f"not expecting type {type(x).__name__}")
|
||||
if encoding or errors:
|
||||
return x.encode(encoding or "utf-8", errors=errors or "strict")
|
||||
return x.encode()
|
||||
|
||||
|
||||
def to_str(
|
||||
x: str | bytes, encoding: str | None = None, errors: str | None = None
|
||||
) -> str:
|
||||
if isinstance(x, str):
|
||||
return x
|
||||
elif not isinstance(x, bytes):
|
||||
raise TypeError(f"not expecting type {type(x).__name__}")
|
||||
if encoding or errors:
|
||||
return x.decode(encoding or "utf-8", errors=errors or "strict")
|
||||
return x.decode()
|
||||
|
||||
|
||||
def reraise(
|
||||
tp: type[BaseException] | None,
|
||||
value: BaseException,
|
||||
tb: TracebackType | None = None,
|
||||
) -> typing.NoReturn:
|
||||
try:
|
||||
if value.__traceback__ is not tb:
|
||||
raise value.with_traceback(tb)
|
||||
raise value
|
||||
finally:
|
||||
value = None # type: ignore[assignment]
|
||||
tb = None
|
@ -1,124 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import select
|
||||
import socket
|
||||
from functools import partial
|
||||
|
||||
__all__ = ["wait_for_read", "wait_for_write"]
|
||||
|
||||
|
||||
# How should we wait on sockets?
|
||||
#
|
||||
# There are two types of APIs you can use for waiting on sockets: the fancy
|
||||
# modern stateful APIs like epoll/kqueue, and the older stateless APIs like
|
||||
# select/poll. The stateful APIs are more efficient when you have a lots of
|
||||
# sockets to keep track of, because you can set them up once and then use them
|
||||
# lots of times. But we only ever want to wait on a single socket at a time
|
||||
# and don't want to keep track of state, so the stateless APIs are actually
|
||||
# more efficient. So we want to use select() or poll().
|
||||
#
|
||||
# Now, how do we choose between select() and poll()? On traditional Unixes,
|
||||
# select() has a strange calling convention that makes it slow, or fail
|
||||
# altogether, for high-numbered file descriptors. The point of poll() is to fix
|
||||
# that, so on Unixes, we prefer poll().
|
||||
#
|
||||
# On Windows, there is no poll() (or at least Python doesn't provide a wrapper
|
||||
# for it), but that's OK, because on Windows, select() doesn't have this
|
||||
# strange calling convention; plain select() works fine.
|
||||
#
|
||||
# So: on Windows we use select(), and everywhere else we use poll(). We also
|
||||
# fall back to select() in case poll() is somehow broken or missing.
|
||||
|
||||
|
||||
def select_wait_for_socket(
|
||||
sock: socket.socket,
|
||||
read: bool = False,
|
||||
write: bool = False,
|
||||
timeout: float | None = None,
|
||||
) -> bool:
|
||||
if not read and not write:
|
||||
raise RuntimeError("must specify at least one of read=True, write=True")
|
||||
rcheck = []
|
||||
wcheck = []
|
||||
if read:
|
||||
rcheck.append(sock)
|
||||
if write:
|
||||
wcheck.append(sock)
|
||||
# When doing a non-blocking connect, most systems signal success by
|
||||
# marking the socket writable. Windows, though, signals success by marked
|
||||
# it as "exceptional". We paper over the difference by checking the write
|
||||
# sockets for both conditions. (The stdlib selectors module does the same
|
||||
# thing.)
|
||||
fn = partial(select.select, rcheck, wcheck, wcheck)
|
||||
rready, wready, xready = fn(timeout)
|
||||
return bool(rready or wready or xready)
|
||||
|
||||
|
||||
def poll_wait_for_socket(
|
||||
sock: socket.socket,
|
||||
read: bool = False,
|
||||
write: bool = False,
|
||||
timeout: float | None = None,
|
||||
) -> bool:
|
||||
if not read and not write:
|
||||
raise RuntimeError("must specify at least one of read=True, write=True")
|
||||
mask = 0
|
||||
if read:
|
||||
mask |= select.POLLIN
|
||||
if write:
|
||||
mask |= select.POLLOUT
|
||||
poll_obj = select.poll()
|
||||
poll_obj.register(sock, mask)
|
||||
|
||||
# For some reason, poll() takes timeout in milliseconds
|
||||
def do_poll(t: float | None) -> list[tuple[int, int]]:
|
||||
if t is not None:
|
||||
t *= 1000
|
||||
return poll_obj.poll(t)
|
||||
|
||||
return bool(do_poll(timeout))
|
||||
|
||||
|
||||
def _have_working_poll() -> bool:
|
||||
# Apparently some systems have a select.poll that fails as soon as you try
|
||||
# to use it, either due to strange configuration or broken monkeypatching
|
||||
# from libraries like eventlet/greenlet.
|
||||
try:
|
||||
poll_obj = select.poll()
|
||||
poll_obj.poll(0)
|
||||
except (AttributeError, OSError):
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def wait_for_socket(
|
||||
sock: socket.socket,
|
||||
read: bool = False,
|
||||
write: bool = False,
|
||||
timeout: float | None = None,
|
||||
) -> bool:
|
||||
# We delay choosing which implementation to use until the first time we're
|
||||
# called. We could do it at import time, but then we might make the wrong
|
||||
# decision if someone goes wild with monkeypatching select.poll after
|
||||
# we're imported.
|
||||
global wait_for_socket
|
||||
if _have_working_poll():
|
||||
wait_for_socket = poll_wait_for_socket
|
||||
elif hasattr(select, "select"):
|
||||
wait_for_socket = select_wait_for_socket
|
||||
return wait_for_socket(sock, read, write, timeout)
|
||||
|
||||
|
||||
def wait_for_read(sock: socket.socket, timeout: float | None = None) -> bool:
|
||||
"""Waits for reading to be available on a given socket.
|
||||
Returns True if the socket is readable, or False if the timeout expired.
|
||||
"""
|
||||
return wait_for_socket(sock, read=True, timeout=timeout)
|
||||
|
||||
|
||||
def wait_for_write(sock: socket.socket, timeout: float | None = None) -> bool:
|
||||
"""Waits for writing to be available on a given socket.
|
||||
Returns True if the socket is readable, or False if the timeout expired.
|
||||
"""
|
||||
return wait_for_socket(sock, write=True, timeout=timeout)
|
@ -1,19 +0,0 @@
|
||||
Copyright (c) 2015 Rossen Georgiev <rossen@rgp.io>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
@ -1,519 +0,0 @@
|
||||
"""
|
||||
Module for deserializing/serializing to and from VDF
|
||||
"""
|
||||
__version__ = "3.4"
|
||||
__author__ = "Rossen Georgiev"
|
||||
|
||||
import re
|
||||
import sys
|
||||
import struct
|
||||
from binascii import crc32
|
||||
from io import BytesIO
|
||||
from io import StringIO as unicodeIO
|
||||
|
||||
try:
|
||||
from collections.abc import Mapping
|
||||
except:
|
||||
from collections import Mapping
|
||||
|
||||
# Py2 & Py3 compatibility
|
||||
if sys.version_info[0] >= 3:
|
||||
string_type = str
|
||||
int_type = int
|
||||
BOMS = '\ufffe\ufeff'
|
||||
|
||||
def strip_bom(line):
|
||||
return line.lstrip(BOMS)
|
||||
else:
|
||||
from StringIO import StringIO as strIO
|
||||
string_type = basestring
|
||||
int_type = long
|
||||
BOMS = '\xef\xbb\xbf\xff\xfe\xfe\xff'
|
||||
BOMS_UNICODE = '\\ufffe\\ufeff'.decode('unicode-escape')
|
||||
|
||||
def strip_bom(line):
|
||||
return line.lstrip(BOMS if isinstance(line, str) else BOMS_UNICODE)
|
||||
|
||||
# string escaping
|
||||
_unescape_char_map = {
|
||||
r"\n": "\n",
|
||||
r"\t": "\t",
|
||||
r"\v": "\v",
|
||||
r"\b": "\b",
|
||||
r"\r": "\r",
|
||||
r"\f": "\f",
|
||||
r"\a": "\a",
|
||||
r"\\": "\\",
|
||||
r"\?": "?",
|
||||
r"\"": "\"",
|
||||
r"\'": "\'",
|
||||
}
|
||||
_escape_char_map = {v: k for k, v in _unescape_char_map.items()}
|
||||
|
||||
def _re_escape_match(m):
|
||||
return _escape_char_map[m.group()]
|
||||
|
||||
def _re_unescape_match(m):
|
||||
return _unescape_char_map[m.group()]
|
||||
|
||||
def _escape(text):
|
||||
return re.sub(r"[\n\t\v\b\r\f\a\\\?\"']", _re_escape_match, text)
|
||||
|
||||
def _unescape(text):
|
||||
return re.sub(r"(\\n|\\t|\\v|\\b|\\r|\\f|\\a|\\\\|\\\?|\\\"|\\')", _re_unescape_match, text)
|
||||
|
||||
# parsing and dumping for KV1
|
||||
def parse(fp, mapper=dict, merge_duplicate_keys=True, escaped=True):
|
||||
"""
|
||||
Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a VDF)
|
||||
to a Python object.
|
||||
|
||||
``mapper`` specifies the Python object used after deserializetion. ``dict` is
|
||||
used by default. Alternatively, ``collections.OrderedDict`` can be used if you
|
||||
wish to preserve key order. Or any object that acts like a ``dict``.
|
||||
|
||||
``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the
|
||||
same key into one instead of overwriting. You can se this to ``False`` if you are
|
||||
using ``VDFDict`` and need to preserve the duplicates.
|
||||
"""
|
||||
if not issubclass(mapper, Mapping):
|
||||
raise TypeError("Expected mapper to be subclass of dict, got %s" % type(mapper))
|
||||
if not hasattr(fp, 'readline'):
|
||||
raise TypeError("Expected fp to be a file-like object supporting line iteration")
|
||||
|
||||
stack = [mapper()]
|
||||
expect_bracket = False
|
||||
|
||||
re_keyvalue = re.compile(r'^("(?P<qkey>(?:\\.|[^\\"])*)"|(?P<key>#?[a-z0-9\-\_\\\?$%<>]+))'
|
||||
r'([ \t]*('
|
||||
r'"(?P<qval>(?:\\.|[^\\"])*)(?P<vq_end>")?'
|
||||
r'|(?P<val>(?:(?<!/)/(?!/)|[a-z0-9\-\_\\\?\*\.$<> ])+)'
|
||||
r'|(?P<sblock>{[ \t]*)(?P<eblock>})?'
|
||||
r'))?',
|
||||
flags=re.I)
|
||||
|
||||
for lineno, line in enumerate(fp, 1):
|
||||
if lineno == 1:
|
||||
line = strip_bom(line)
|
||||
|
||||
line = line.lstrip()
|
||||
|
||||
# skip empty and comment lines
|
||||
if line == "" or line[0] == '/':
|
||||
continue
|
||||
|
||||
# one level deeper
|
||||
if line[0] == "{":
|
||||
expect_bracket = False
|
||||
continue
|
||||
|
||||
if expect_bracket:
|
||||
raise SyntaxError("vdf.parse: expected openning bracket",
|
||||
(getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 1, line))
|
||||
|
||||
# one level back
|
||||
if line[0] == "}":
|
||||
if len(stack) > 1:
|
||||
stack.pop()
|
||||
continue
|
||||
|
||||
raise SyntaxError("vdf.parse: one too many closing parenthasis",
|
||||
(getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line))
|
||||
|
||||
# parse keyvalue pairs
|
||||
while True:
|
||||
match = re_keyvalue.match(line)
|
||||
|
||||
if not match:
|
||||
try:
|
||||
line += next(fp)
|
||||
continue
|
||||
except StopIteration:
|
||||
raise SyntaxError("vdf.parse: unexpected EOF (open key quote?)",
|
||||
(getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line))
|
||||
|
||||
key = match.group('key') if match.group('qkey') is None else match.group('qkey')
|
||||
val = match.group('qval')
|
||||
if val is None:
|
||||
val = match.group('val')
|
||||
if val is not None:
|
||||
val = val.rstrip()
|
||||
if val == "":
|
||||
val = None
|
||||
|
||||
if escaped:
|
||||
key = _unescape(key)
|
||||
|
||||
# we have a key with value in parenthesis, so we make a new dict obj (level deeper)
|
||||
if val is None:
|
||||
if merge_duplicate_keys and key in stack[-1]:
|
||||
_m = stack[-1][key]
|
||||
# we've descended a level deeper, if value is str, we have to overwrite it to mapper
|
||||
if not isinstance(_m, mapper):
|
||||
_m = stack[-1][key] = mapper()
|
||||
else:
|
||||
_m = mapper()
|
||||
stack[-1][key] = _m
|
||||
|
||||
if match.group('eblock') is None:
|
||||
# only expect a bracket if it's not already closed or on the same line
|
||||
stack.append(_m)
|
||||
if match.group('sblock') is None:
|
||||
expect_bracket = True
|
||||
|
||||
# we've matched a simple keyvalue pair, map it to the last dict obj in the stack
|
||||
else:
|
||||
# if the value is line consume one more line and try to match again,
|
||||
# until we get the KeyValue pair
|
||||
if match.group('vq_end') is None and match.group('qval') is not None:
|
||||
try:
|
||||
line += next(fp)
|
||||
continue
|
||||
except StopIteration:
|
||||
raise SyntaxError("vdf.parse: unexpected EOF (open quote for value?)",
|
||||
(getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line))
|
||||
|
||||
stack[-1][key] = _unescape(val) if escaped else val
|
||||
|
||||
# exit the loop
|
||||
break
|
||||
|
||||
if len(stack) != 1:
|
||||
raise SyntaxError("vdf.parse: unclosed parenthasis or quotes (EOF)",
|
||||
(getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line))
|
||||
|
||||
return stack.pop()
|
||||
|
||||
|
||||
def loads(s, **kwargs):
|
||||
"""
|
||||
Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
|
||||
document) to a Python object.
|
||||
"""
|
||||
if not isinstance(s, string_type):
|
||||
raise TypeError("Expected s to be a str, got %s" % type(s))
|
||||
|
||||
try:
|
||||
fp = unicodeIO(s)
|
||||
except TypeError:
|
||||
fp = strIO(s)
|
||||
|
||||
return parse(fp, **kwargs)
|
||||
|
||||
|
||||
def load(fp, **kwargs):
|
||||
"""
|
||||
Deserialize ``fp`` (a ``.readline()``-supporting file-like object containing
|
||||
a JSON document) to a Python object.
|
||||
"""
|
||||
return parse(fp, **kwargs)
|
||||
|
||||
|
||||
def dumps(obj, pretty=False, escaped=True):
|
||||
"""
|
||||
Serialize ``obj`` to a VDF formatted ``str``.
|
||||
"""
|
||||
if not isinstance(obj, Mapping):
|
||||
raise TypeError("Expected data to be an instance of``dict``")
|
||||
if not isinstance(pretty, bool):
|
||||
raise TypeError("Expected pretty to be of type bool")
|
||||
if not isinstance(escaped, bool):
|
||||
raise TypeError("Expected escaped to be of type bool")
|
||||
|
||||
return ''.join(_dump_gen(obj, pretty, escaped))
|
||||
|
||||
|
||||
def dump(obj, fp, pretty=False, escaped=True):
|
||||
"""
|
||||
Serialize ``obj`` as a VDF formatted stream to ``fp`` (a
|
||||
``.write()``-supporting file-like object).
|
||||
"""
|
||||
if not isinstance(obj, Mapping):
|
||||
raise TypeError("Expected data to be an instance of``dict``")
|
||||
if not hasattr(fp, 'write'):
|
||||
raise TypeError("Expected fp to have write() method")
|
||||
if not isinstance(pretty, bool):
|
||||
raise TypeError("Expected pretty to be of type bool")
|
||||
if not isinstance(escaped, bool):
|
||||
raise TypeError("Expected escaped to be of type bool")
|
||||
|
||||
for chunk in _dump_gen(obj, pretty, escaped):
|
||||
fp.write(chunk)
|
||||
|
||||
|
||||
def _dump_gen(data, pretty=False, escaped=True, level=0):
|
||||
indent = "\t"
|
||||
line_indent = ""
|
||||
|
||||
if pretty:
|
||||
line_indent = indent * level
|
||||
|
||||
for key, value in data.items():
|
||||
if escaped and isinstance(key, string_type):
|
||||
key = _escape(key)
|
||||
|
||||
if isinstance(value, Mapping):
|
||||
yield '%s"%s"\n%s{\n' % (line_indent, key, line_indent)
|
||||
for chunk in _dump_gen(value, pretty, escaped, level+1):
|
||||
yield chunk
|
||||
yield "%s}\n" % line_indent
|
||||
else:
|
||||
if escaped and isinstance(value, string_type):
|
||||
value = _escape(value)
|
||||
|
||||
yield '%s"%s" "%s"\n' % (line_indent, key, value)
|
||||
|
||||
|
||||
# binary VDF
|
||||
class BASE_INT(int_type):
|
||||
def __repr__(self):
|
||||
return "%s(%d)" % (self.__class__.__name__, self)
|
||||
|
||||
class UINT_64(BASE_INT):
|
||||
pass
|
||||
|
||||
class INT_64(BASE_INT):
|
||||
pass
|
||||
|
||||
class POINTER(BASE_INT):
|
||||
pass
|
||||
|
||||
class COLOR(BASE_INT):
|
||||
pass
|
||||
|
||||
BIN_NONE = b'\x00'
|
||||
BIN_STRING = b'\x01'
|
||||
BIN_INT32 = b'\x02'
|
||||
BIN_FLOAT32 = b'\x03'
|
||||
BIN_POINTER = b'\x04'
|
||||
BIN_WIDESTRING = b'\x05'
|
||||
BIN_COLOR = b'\x06'
|
||||
BIN_UINT64 = b'\x07'
|
||||
BIN_END = b'\x08'
|
||||
BIN_INT64 = b'\x0A'
|
||||
BIN_END_ALT = b'\x0B'
|
||||
|
||||
def binary_loads(b, mapper=dict, merge_duplicate_keys=True, alt_format=False, raise_on_remaining=True):
|
||||
"""
|
||||
Deserialize ``b`` (``bytes`` containing a VDF in "binary form")
|
||||
to a Python object.
|
||||
|
||||
``mapper`` specifies the Python object used after deserializetion. ``dict` is
|
||||
used by default. Alternatively, ``collections.OrderedDict`` can be used if you
|
||||
wish to preserve key order. Or any object that acts like a ``dict``.
|
||||
|
||||
``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the
|
||||
same key into one instead of overwriting. You can se this to ``False`` if you are
|
||||
using ``VDFDict`` and need to preserve the duplicates.
|
||||
"""
|
||||
if not isinstance(b, bytes):
|
||||
raise TypeError("Expected s to be bytes, got %s" % type(b))
|
||||
|
||||
return binary_load(BytesIO(b), mapper, merge_duplicate_keys, alt_format, raise_on_remaining)
|
||||
|
||||
def binary_load(fp, mapper=dict, merge_duplicate_keys=True, alt_format=False, raise_on_remaining=False):
|
||||
"""
|
||||
Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
|
||||
binary VDF) to a Python object.
|
||||
|
||||
``mapper`` specifies the Python object used after deserializetion. ``dict` is
|
||||
used by default. Alternatively, ``collections.OrderedDict`` can be used if you
|
||||
wish to preserve key order. Or any object that acts like a ``dict``.
|
||||
|
||||
``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the
|
||||
same key into one instead of overwriting. You can se this to ``False`` if you are
|
||||
using ``VDFDict`` and need to preserve the duplicates.
|
||||
"""
|
||||
if not hasattr(fp, 'read') or not hasattr(fp, 'tell') or not hasattr(fp, 'seek'):
|
||||
raise TypeError("Expected fp to be a file-like object with tell()/seek() and read() returning bytes")
|
||||
if not issubclass(mapper, Mapping):
|
||||
raise TypeError("Expected mapper to be subclass of dict, got %s" % type(mapper))
|
||||
|
||||
# helpers
|
||||
int32 = struct.Struct('<i')
|
||||
uint64 = struct.Struct('<Q')
|
||||
int64 = struct.Struct('<q')
|
||||
float32 = struct.Struct('<f')
|
||||
|
||||
def read_string(fp, wide=False):
|
||||
buf, end = b'', -1
|
||||
offset = fp.tell()
|
||||
|
||||
# locate string end
|
||||
while end == -1:
|
||||
chunk = fp.read(64)
|
||||
|
||||
if chunk == b'':
|
||||
raise SyntaxError("Unterminated cstring (offset: %d)" % offset)
|
||||
|
||||
buf += chunk
|
||||
end = buf.find(b'\x00\x00' if wide else b'\x00')
|
||||
|
||||
if wide:
|
||||
end += end % 2
|
||||
|
||||
# rewind fp
|
||||
fp.seek(end - len(buf) + (2 if wide else 1), 1)
|
||||
|
||||
# decode string
|
||||
result = buf[:end]
|
||||
|
||||
if wide:
|
||||
result = result.decode('utf-16')
|
||||
elif bytes is not str:
|
||||
result = result.decode('utf-8', 'replace')
|
||||
else:
|
||||
try:
|
||||
result.decode('ascii')
|
||||
except:
|
||||
result = result.decode('utf-8', 'replace')
|
||||
|
||||
return result
|
||||
|
||||
stack = [mapper()]
|
||||
CURRENT_BIN_END = BIN_END if not alt_format else BIN_END_ALT
|
||||
|
||||
for t in iter(lambda: fp.read(1), b''):
|
||||
if t == CURRENT_BIN_END:
|
||||
if len(stack) > 1:
|
||||
stack.pop()
|
||||
continue
|
||||
break
|
||||
|
||||
key = read_string(fp)
|
||||
|
||||
if t == BIN_NONE:
|
||||
if merge_duplicate_keys and key in stack[-1]:
|
||||
_m = stack[-1][key]
|
||||
else:
|
||||
_m = mapper()
|
||||
stack[-1][key] = _m
|
||||
stack.append(_m)
|
||||
elif t == BIN_STRING:
|
||||
stack[-1][key] = read_string(fp)
|
||||
elif t == BIN_WIDESTRING:
|
||||
stack[-1][key] = read_string(fp, wide=True)
|
||||
elif t in (BIN_INT32, BIN_POINTER, BIN_COLOR):
|
||||
val = int32.unpack(fp.read(int32.size))[0]
|
||||
|
||||
if t == BIN_POINTER:
|
||||
val = POINTER(val)
|
||||
elif t == BIN_COLOR:
|
||||
val = COLOR(val)
|
||||
|
||||
stack[-1][key] = val
|
||||
elif t == BIN_UINT64:
|
||||
stack[-1][key] = UINT_64(uint64.unpack(fp.read(int64.size))[0])
|
||||
elif t == BIN_INT64:
|
||||
stack[-1][key] = INT_64(int64.unpack(fp.read(int64.size))[0])
|
||||
elif t == BIN_FLOAT32:
|
||||
stack[-1][key] = float32.unpack(fp.read(float32.size))[0]
|
||||
else:
|
||||
raise SyntaxError("Unknown data type at offset %d: %s" % (fp.tell() - 1, repr(t)))
|
||||
|
||||
if len(stack) != 1:
|
||||
raise SyntaxError("Reached EOF, but Binary VDF is incomplete")
|
||||
if raise_on_remaining and fp.read(1) != b'':
|
||||
fp.seek(-1, 1)
|
||||
raise SyntaxError("Binary VDF ended at offset %d, but there is more data remaining" % (fp.tell() - 1))
|
||||
|
||||
return stack.pop()
|
||||
|
||||
def binary_dumps(obj, alt_format=False):
|
||||
"""
|
||||
Serialize ``obj`` to a binary VDF formatted ``bytes``.
|
||||
"""
|
||||
buf = BytesIO()
|
||||
binary_dump(obj, buf, alt_format)
|
||||
return buf.getvalue()
|
||||
|
||||
def binary_dump(obj, fp, alt_format=False):
|
||||
"""
|
||||
Serialize ``obj`` to a binary VDF formatted ``bytes`` and write it to ``fp`` filelike object
|
||||
"""
|
||||
if not isinstance(obj, Mapping):
|
||||
raise TypeError("Expected obj to be type of Mapping")
|
||||
if not hasattr(fp, 'write'):
|
||||
raise TypeError("Expected fp to have write() method")
|
||||
|
||||
for chunk in _binary_dump_gen(obj, alt_format=alt_format):
|
||||
fp.write(chunk)
|
||||
|
||||
def _binary_dump_gen(obj, level=0, alt_format=False):
|
||||
if level == 0 and len(obj) == 0:
|
||||
return
|
||||
|
||||
int32 = struct.Struct('<i')
|
||||
uint64 = struct.Struct('<Q')
|
||||
int64 = struct.Struct('<q')
|
||||
float32 = struct.Struct('<f')
|
||||
|
||||
for key, value in obj.items():
|
||||
if isinstance(key, string_type):
|
||||
key = key.encode('utf-8')
|
||||
else:
|
||||
raise TypeError("dict keys must be of type str, got %s" % type(key))
|
||||
|
||||
if isinstance(value, Mapping):
|
||||
yield BIN_NONE + key + BIN_NONE
|
||||
for chunk in _binary_dump_gen(value, level+1, alt_format=alt_format):
|
||||
yield chunk
|
||||
elif isinstance(value, UINT_64):
|
||||
yield BIN_UINT64 + key + BIN_NONE + uint64.pack(value)
|
||||
elif isinstance(value, INT_64):
|
||||
yield BIN_INT64 + key + BIN_NONE + int64.pack(value)
|
||||
elif isinstance(value, string_type):
|
||||
try:
|
||||
value = value.encode('utf-8') + BIN_NONE
|
||||
yield BIN_STRING
|
||||
except:
|
||||
value = value.encode('utf-16') + BIN_NONE*2
|
||||
yield BIN_WIDESTRING
|
||||
yield key + BIN_NONE + value
|
||||
elif isinstance(value, float):
|
||||
yield BIN_FLOAT32 + key + BIN_NONE + float32.pack(value)
|
||||
elif isinstance(value, (COLOR, POINTER, int, int_type)):
|
||||
if isinstance(value, COLOR):
|
||||
yield BIN_COLOR
|
||||
elif isinstance(value, POINTER):
|
||||
yield BIN_POINTER
|
||||
else:
|
||||
yield BIN_INT32
|
||||
yield key + BIN_NONE
|
||||
yield int32.pack(value)
|
||||
else:
|
||||
raise TypeError("Unsupported type: %s" % type(value))
|
||||
|
||||
yield BIN_END if not alt_format else BIN_END_ALT
|
||||
|
||||
|
||||
def vbkv_loads(s, mapper=dict, merge_duplicate_keys=True):
|
||||
"""
|
||||
Deserialize ``s`` (``bytes`` containing a VBKV to a Python object.
|
||||
|
||||
``mapper`` specifies the Python object used after deserializetion. ``dict` is
|
||||
used by default. Alternatively, ``collections.OrderedDict`` can be used if you
|
||||
wish to preserve key order. Or any object that acts like a ``dict``.
|
||||
|
||||
``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the
|
||||
same key into one instead of overwriting. You can se this to ``False`` if you are
|
||||
using ``VDFDict`` and need to preserve the duplicates.
|
||||
"""
|
||||
if s[:4] != b'VBKV':
|
||||
raise ValueError("Invalid header")
|
||||
|
||||
checksum, = struct.unpack('<i', s[4:8])
|
||||
|
||||
if checksum != crc32(s[8:]):
|
||||
raise ValueError("Invalid checksum")
|
||||
|
||||
return binary_loads(s[8:], mapper, merge_duplicate_keys, alt_format=True)
|
||||
|
||||
def vbkv_dumps(obj):
|
||||
"""
|
||||
Serialize ``obj`` to a VBKV formatted ``bytes``.
|
||||
"""
|
||||
data = b''.join(_binary_dump_gen(obj, alt_format=True))
|
||||
checksum = crc32(data)
|
||||
|
||||
return b'VBKV' + struct.pack('<i', checksum) + data
|
Binary file not shown.
Loading…
Reference in New Issue
Block a user