plexpy/lib/cheroot/test/test_conn.py

"""Tests for TCP connection handling, including proper and timely close."""

import errno
from re import match as _matches_pattern
import socket
import sys
import time
import logging
import traceback as traceback_
from collections import namedtuple
import http.client
import urllib.request

import pytest
from jaraco.text import trim, unwrap

from cheroot.test import helper, webtest
from cheroot._compat import IS_CI, IS_MACOS, IS_PYPY, IS_WINDOWS
import cheroot.server


IS_PY36 = sys.version_info[:2] == (3, 6)
IS_SLOW_ENV = IS_MACOS or IS_WINDOWS


timeout = 1
pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN'


class Controller(helper.Controller):
    """Controller for serving WSGI apps."""

    def hello(req, resp):
        """Render Hello world."""
        return 'Hello, world!'

    def pov(req, resp):
        """Render ``pov`` value."""
        return pov

    def stream(req, resp):
        """Render streaming response."""
        if 'set_cl' in req.environ['QUERY_STRING']:
            resp.headers['Content-Length'] = str(10)

        def content():
            for x in range(10):
                yield str(x)

        return content()

    def upload(req, resp):
        """Process file upload and render thank."""
        if not req.environ['REQUEST_METHOD'] == 'POST':
            raise AssertionError(
                "'POST' != request.method %r" %
                req.environ['REQUEST_METHOD'],
            )
        input_contents = req.environ['wsgi.input'].read().decode('utf-8')
        return f"thanks for '{input_contents !s}'"

    def custom_204(req, resp):
        """Render response with status 204."""
        resp.status = '204'
        return 'Code = 204'

    def custom_304(req, resp):
        """Render response with status 304."""
        resp.status = '304'
        return 'Code = 304'

    def err_before_read(req, resp):
        """Render response with status 500."""
        resp.status = '500 Internal Server Error'
        return 'ok'

    def one_megabyte_of_a(req, resp):
        """Render 1MB response."""
        return ['a' * 1024] * 1024

    def wrong_cl_buffered(req, resp):
        """Render buffered response with invalid length value."""
        resp.headers['Content-Length'] = '5'
        return 'I have too many bytes'

    def wrong_cl_unbuffered(req, resp):
        """Render unbuffered response with invalid length value."""
        resp.headers['Content-Length'] = '5'
        return ['I too', ' have too many bytes']

    def _munge(string):
        """Encode PATH_INFO correctly depending on Python version.

        WSGI 1.0 is a mess around unicode. Create endpoints
        that match the PATH_INFO that it produces.
        """
        return string.encode('utf-8').decode('latin-1')

    handlers = {
        '/hello': hello,
        '/pov': pov,
        '/page1': pov,
        '/page2': pov,
        '/page3': pov,
        '/stream': stream,
        '/upload': upload,
        '/custom/204': custom_204,
        '/custom/304': custom_304,
        '/err_before_read': err_before_read,
        '/one_megabyte_of_a': one_megabyte_of_a,
        '/wrong_cl_buffered': wrong_cl_buffered,
        '/wrong_cl_unbuffered': wrong_cl_unbuffered,
    }


class ErrorLogMonitor:
    """Mock class to access the server error_log calls made by the server."""

    ErrorLogCall = namedtuple('ErrorLogCall', ['msg', 'level', 'traceback'])

    def __init__(self):
        """Initialize the server error log monitor/interceptor.

        If you need to ignore a particular error message use the property
        ``ignored_msgs`` by appending to the list the expected error messages.
        """
        self.calls = []
        # to be used the the teardown validation
        self.ignored_msgs = []

    def __call__(self, msg='', level=logging.INFO, traceback=False):
        """Intercept the call to the server error_log method."""
        if traceback:
            tblines = traceback_.format_exc()
        else:
            tblines = ''
        self.calls.append(ErrorLogMonitor.ErrorLogCall(msg, level, tblines))


@pytest.fixture
def raw_testing_server(wsgi_server_client):
    """Attach a WSGI app to the given server and preconfigure it."""
    app = Controller()

    def _timeout(req, resp):
        return str(wsgi_server.timeout)
    app.handlers['/timeout'] = _timeout
    wsgi_server = wsgi_server_client.server_instance
    wsgi_server.wsgi_app = app
    wsgi_server.max_request_body_size = 1001
    wsgi_server.timeout = timeout
    wsgi_server.server_client = wsgi_server_client
    wsgi_server.keep_alive_conn_limit = 2

    return wsgi_server


@pytest.fixture
def testing_server(raw_testing_server, monkeypatch):
    """Modify the "raw" base server to monitor the error_log messages.

    If you need to ignore a particular error message use the property
    ``testing_server.error_log.ignored_msgs`` by appending to the list
    the expected error messages.
    """
    # patch the error_log calls of the server instance
    monkeypatch.setattr(raw_testing_server, 'error_log', ErrorLogMonitor())

    yield raw_testing_server

    # Teardown verification, in case that the server logged an
    # error that wasn't notified to the client or we just made a mistake.
    # pylint: disable=possibly-unused-variable
    for c_msg, c_level, c_traceback in raw_testing_server.error_log.calls:
        if c_level <= logging.WARNING:
            continue

        assert c_msg in raw_testing_server.error_log.ignored_msgs, (
            'Found error in the error log: '
            "message = '{c_msg}', level = '{c_level}'\n"
            '{c_traceback}'.format(**locals()),
        )


@pytest.fixture
def test_client(testing_server):
    """Get and return a test client out of the given server."""
    return testing_server.server_client


def header_exists(header_name, headers):
    """Check that a header is present."""
    return header_name.lower() in (k.lower() for (k, _) in headers)


def header_has_value(header_name, header_value, headers):
    """Check that a header with a given value is present."""
    return header_name.lower() in (
        k.lower() for (k, v) in headers
        if v == header_value
    )


def test_HTTP11_persistent_connections(test_client):
    """Test persistent HTTP/1.1 connections."""
    # Initialize a persistent HTTP connection
    http_connection = test_client.get_connection()
    http_connection.auto_open = False
    http_connection.connect()

    # Make the first request and assert there's no "Connection: close".
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/pov', http_conn=http_connection,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert not header_exists('Connection', actual_headers)

    # Make another request on the same connection.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/page1', http_conn=http_connection,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert not header_exists('Connection', actual_headers)

    # Test client-side close.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/page2', http_conn=http_connection,
        headers=[('Connection', 'close')],
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert header_has_value('Connection', 'close', actual_headers)

    # Make another request on the same connection, which should error.
    with pytest.raises(http.client.NotConnected):
        test_client.get('/pov', http_conn=http_connection)


@pytest.mark.parametrize(
    'set_cl',
    (
        False,  # Without Content-Length
        True,  # With Content-Length
    ),
)
def test_streaming_11(test_client, set_cl):
    """Test serving of streaming responses with HTTP/1.1 protocol."""
    # Initialize a persistent HTTP connection
    http_connection = test_client.get_connection()
    http_connection.auto_open = False
    http_connection.connect()

    # Make the first request and assert there's no "Connection: close".
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/pov', http_conn=http_connection,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert not header_exists('Connection', actual_headers)

    # Make another, streamed request on the same connection.
    if set_cl:
        # When a Content-Length is provided, the content should stream
        # without closing the connection.
        status_line, actual_headers, actual_resp_body = test_client.get(
            '/stream?set_cl=Yes', http_conn=http_connection,
        )
        assert header_exists('Content-Length', actual_headers)
        assert not header_has_value('Connection', 'close', actual_headers)
        assert not header_exists('Transfer-Encoding', actual_headers)

        assert actual_status == 200
        assert status_line[4:] == 'OK'
        assert actual_resp_body == b'0123456789'
    else:
        # When no Content-Length response header is provided,
        # streamed output will either close the connection, or use
        # chunked encoding, to determine transfer-length.
        status_line, actual_headers, actual_resp_body = test_client.get(
            '/stream', http_conn=http_connection,
        )
        assert not header_exists('Content-Length', actual_headers)
        assert actual_status == 200
        assert status_line[4:] == 'OK'
        assert actual_resp_body == b'0123456789'

        chunked_response = False
        for k, v in actual_headers:
            if k.lower() == 'transfer-encoding':
                if str(v) == 'chunked':
                    chunked_response = True

        if chunked_response:
            assert not header_has_value('Connection', 'close', actual_headers)
        else:
            assert header_has_value('Connection', 'close', actual_headers)

            # Make another request on the same connection, which should
            # error.
            with pytest.raises(http.client.NotConnected):
                test_client.get('/pov', http_conn=http_connection)

        # Try HEAD.
        # See https://www.bitbucket.org/cherrypy/cherrypy/issue/864.
        # TODO: figure out how can this be possible on an closed connection
        # (chunked_response case)
        status_line, actual_headers, actual_resp_body = test_client.head(
            '/stream', http_conn=http_connection,
        )
        assert actual_status == 200
        assert status_line[4:] == 'OK'
        assert actual_resp_body == b''
        assert not header_exists('Transfer-Encoding', actual_headers)

    # Prevent the resource warnings:
    http_connection.close()


@pytest.mark.parametrize(
    'set_cl',
    (
        False,  # Without Content-Length
        True,  # With Content-Length
    ),
)
def test_streaming_10(test_client, set_cl):
    """Test serving of streaming responses with HTTP/1.0 protocol."""
    original_server_protocol = test_client.server_instance.protocol
    test_client.server_instance.protocol = 'HTTP/1.0'

    # Initialize a persistent HTTP connection
    http_connection = test_client.get_connection()
    http_connection.auto_open = False
    http_connection.connect()

    # Make the first request and assert Keep-Alive.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/pov', http_conn=http_connection,
        headers=[('Connection', 'Keep-Alive')],
        protocol='HTTP/1.0',
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert header_has_value('Connection', 'Keep-Alive', actual_headers)

    # Make another, streamed request on the same connection.
    if set_cl:
        # When a Content-Length is provided, the content should
        # stream without closing the connection.
        status_line, actual_headers, actual_resp_body = test_client.get(
            '/stream?set_cl=Yes', http_conn=http_connection,
            headers=[('Connection', 'Keep-Alive')],
            protocol='HTTP/1.0',
        )
        actual_status = int(status_line[:3])
        assert actual_status == 200
        assert status_line[4:] == 'OK'
        assert actual_resp_body == b'0123456789'

        assert header_exists('Content-Length', actual_headers)
        assert header_has_value('Connection', 'Keep-Alive', actual_headers)
        assert not header_exists('Transfer-Encoding', actual_headers)
    else:
        # When a Content-Length is not provided,
        # the server should close the connection.
        status_line, actual_headers, actual_resp_body = test_client.get(
            '/stream', http_conn=http_connection,
            headers=[('Connection', 'Keep-Alive')],
            protocol='HTTP/1.0',
        )
        actual_status = int(status_line[:3])
        assert actual_status == 200
        assert status_line[4:] == 'OK'
        assert actual_resp_body == b'0123456789'

        assert not header_exists('Content-Length', actual_headers)
        assert not header_has_value('Connection', 'Keep-Alive', actual_headers)
        assert not header_exists('Transfer-Encoding', actual_headers)

        # Make another request on the same connection, which should error.
        with pytest.raises(http.client.NotConnected):
            test_client.get(
                '/pov', http_conn=http_connection,
                protocol='HTTP/1.0',
            )

    test_client.server_instance.protocol = original_server_protocol

    # Prevent the resource warnings:
    http_connection.close()


@pytest.mark.parametrize(
    'http_server_protocol',
    (
        'HTTP/1.0',
        pytest.param(
            'HTTP/1.1',
            marks=pytest.mark.xfail(
                IS_PYPY and IS_CI,
                reason='Fails under PyPy in CI for unknown reason',
                strict=False,
            ),
        ),
    ),
)
def test_keepalive(test_client, http_server_protocol):
    """Test Keep-Alive enabled connections."""
    original_server_protocol = test_client.server_instance.protocol
    test_client.server_instance.protocol = http_server_protocol

    http_client_protocol = 'HTTP/1.0'

    # Initialize a persistent HTTP connection
    http_connection = test_client.get_connection()
    http_connection.auto_open = False
    http_connection.connect()

    # Test a normal HTTP/1.0 request.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/page2',
        protocol=http_client_protocol,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert not header_exists('Connection', actual_headers)

    # Test a keep-alive HTTP/1.0 request.

    status_line, actual_headers, actual_resp_body = test_client.get(
        '/page3', headers=[('Connection', 'Keep-Alive')],
        http_conn=http_connection, protocol=http_client_protocol,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert header_has_value('Connection', 'Keep-Alive', actual_headers)
    assert header_has_value(
        'Keep-Alive',
        'timeout={test_client.server_instance.timeout}'.format(**locals()),
        actual_headers,
    )

    # Remove the keep-alive header again.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/page3', http_conn=http_connection,
        protocol=http_client_protocol,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert not header_exists('Connection', actual_headers)
    assert not header_exists('Keep-Alive', actual_headers)

    test_client.server_instance.protocol = original_server_protocol

    # Prevent the resource warnings:
    http_connection.close()


def test_keepalive_conn_management(test_client):
    """Test management of Keep-Alive connections."""
    test_client.server_instance.timeout = 2

    def connection():
        # Initialize a persistent HTTP connection
        http_connection = test_client.get_connection()
        http_connection.auto_open = False
        http_connection.connect()
        return http_connection

    def request(conn, keepalive=True):
        status_line, actual_headers, actual_resp_body = test_client.get(
            '/page3', headers=[('Connection', 'Keep-Alive')],
            http_conn=conn, protocol='HTTP/1.0',
        )
        actual_status = int(status_line[:3])
        assert actual_status == 200
        assert status_line[4:] == 'OK'
        assert actual_resp_body == pov.encode()
        if keepalive:
            assert header_has_value('Connection', 'Keep-Alive', actual_headers)
            assert header_has_value(
                'Keep-Alive',
                'timeout={test_client.server_instance.timeout}'.
                format(**locals()),
                actual_headers,
            )
        else:
            assert not header_exists('Connection', actual_headers)
            assert not header_exists('Keep-Alive', actual_headers)

    def check_server_idle_conn_count(count, timeout=1.0):
        deadline = time.time() + timeout
        while True:
            n = test_client.server_instance._connections._num_connections
            if n == count:
                return
            assert time.time() <= deadline, (
                'idle conn count mismatch, wanted {count}, got {n}'.
                format(**locals()),
            )

    disconnect_errors = (
        http.client.BadStatusLine,
        http.client.CannotSendRequest,
        http.client.NotConnected,
    )

    # Make a new connection.
    c1 = connection()
    request(c1)
    check_server_idle_conn_count(1)

    # Make a second one.
    c2 = connection()
    request(c2)
    check_server_idle_conn_count(2)

    # Reusing the first connection should still work.
    request(c1)
    check_server_idle_conn_count(2)

    # Creating a new connection should still work, but we should
    # have run out of available connections to keep alive, so the
    # server should tell us to close.
    c3 = connection()
    request(c3, keepalive=False)
    check_server_idle_conn_count(2)

    # Show that the third connection was closed.
    with pytest.raises(disconnect_errors):
        request(c3)
    check_server_idle_conn_count(2)

    # Wait for some of our timeout.
    time.sleep(1.2)

    # Refresh the second connection.
    request(c2)
    check_server_idle_conn_count(2)

    # Wait for the remainder of our timeout, plus one tick.
    time.sleep(1.2)
    check_server_idle_conn_count(1)

    # First connection should now be expired.
    with pytest.raises(disconnect_errors):
        request(c1)
    check_server_idle_conn_count(1)

    # But the second one should still be valid.
    request(c2)
    check_server_idle_conn_count(1)

    # Restore original timeout.
    test_client.server_instance.timeout = timeout

    # Prevent the resource warnings:
    c1.close()
    c2.close()
    c3.close()


@pytest.mark.parametrize(
    ('simulated_exception', 'error_number', 'exception_leaks'),
    (
        pytest.param(
            socket.error, errno.ECONNRESET, False,
            id='socket.error(ECONNRESET)',
        ),
        pytest.param(
            socket.error, errno.EPIPE, False,
            id='socket.error(EPIPE)',
        ),
        pytest.param(
            socket.error, errno.ENOTCONN, False,
            id='simulated socket.error(ENOTCONN)',
        ),
        pytest.param(
            None,  # <-- don't raise an artificial exception
            errno.ENOTCONN, False,
            id='real socket.error(ENOTCONN)',
            marks=pytest.mark.xfail(
                IS_WINDOWS,
                reason='Now reproducible this way on Windows',
            ),
        ),
        pytest.param(
            socket.error, errno.ESHUTDOWN, False,
            id='socket.error(ESHUTDOWN)',
        ),
        pytest.param(RuntimeError, 666, True, id='RuntimeError(666)'),
        pytest.param(socket.error, -1, True, id='socket.error(-1)'),
    ) + (
        pytest.param(
            ConnectionResetError, errno.ECONNRESET, False,
            id='ConnectionResetError(ECONNRESET)',
        ),
        pytest.param(
            BrokenPipeError, errno.EPIPE, False,
            id='BrokenPipeError(EPIPE)',
        ),
        pytest.param(
            BrokenPipeError, errno.ESHUTDOWN, False,
            id='BrokenPipeError(ESHUTDOWN)',
        ),
    ),
)
def test_broken_connection_during_tcp_fin(
        error_number, exception_leaks,
        mocker, monkeypatch,
        simulated_exception, test_client,
):
    """Test there's no traceback on broken connection during close.

    It artificially causes :py:data:`~errno.ECONNRESET` /
    :py:data:`~errno.EPIPE` / :py:data:`~errno.ESHUTDOWN` /
    :py:data:`~errno.ENOTCONN` as well as unrelated :py:exc:`RuntimeError`
    and :py:exc:`socket.error(-1) <socket.error>` on the server socket when
    :py:meth:`socket.shutdown() <socket.socket.shutdown>` is called. It's
    triggered by closing the client socket before the server had a chance
    to respond.

    The expectation is that only :py:exc:`RuntimeError` and a
    :py:exc:`socket.error` with an unusual error code would leak.

    With the :py:data:`None`-parameter, a real non-simulated
    :py:exc:`OSError(107, 'Transport endpoint is not connected')
    <OSError>` happens.
    """
    exc_instance = (
        None if simulated_exception is None
        else simulated_exception(error_number, 'Simulated socket error')
    )
    old_close_kernel_socket = (
        test_client.server_instance.
        ConnectionClass._close_kernel_socket
    )

    def _close_kernel_socket(self):
        monkeypatch.setattr(  # `socket.shutdown` is read-only otherwise
            self, 'socket',
            mocker.mock_module.Mock(wraps=self.socket),
        )
        if exc_instance is not None:
            monkeypatch.setattr(
                self.socket, 'shutdown',
                mocker.mock_module.Mock(side_effect=exc_instance),
            )
        _close_kernel_socket.fin_spy = mocker.spy(self.socket, 'shutdown')

        try:
            old_close_kernel_socket(self)
        except simulated_exception:
            _close_kernel_socket.exception_leaked = True
        else:
            _close_kernel_socket.exception_leaked = False

    monkeypatch.setattr(
        test_client.server_instance.ConnectionClass,
        '_close_kernel_socket',
        _close_kernel_socket,
    )

    conn = test_client.get_connection()
    conn.auto_open = False
    conn.connect()
    conn.send(b'GET /hello HTTP/1.1')
    conn.send(('Host: %s' % conn.host).encode('ascii'))
    conn.close()

    # Let the server attempt TCP shutdown:
    for _ in range(10 * (2 if IS_SLOW_ENV else 1)):
        time.sleep(0.1)
        if hasattr(_close_kernel_socket, 'exception_leaked'):
            break

    if exc_instance is not None:  # simulated by us
        assert _close_kernel_socket.fin_spy.spy_exception is exc_instance
    else:  # real
        assert isinstance(
            _close_kernel_socket.fin_spy.spy_exception, socket.error,
        )
        assert _close_kernel_socket.fin_spy.spy_exception.errno == error_number

    assert _close_kernel_socket.exception_leaked is exception_leaks


def test_broken_connection_during_http_communication_fallback(  # noqa: WPS118
        monkeypatch,
        test_client,
        testing_server,
        wsgi_server_thread,
):
    """Test that unhandled internal error cascades into shutdown."""
    def _raise_connection_reset(*_args, **_kwargs):
        raise ConnectionResetError(666)

    def _read_request_line(self):
        monkeypatch.setattr(self.conn.rfile, 'close', _raise_connection_reset)
        monkeypatch.setattr(self.conn.wfile, 'write', _raise_connection_reset)
        _raise_connection_reset()

    monkeypatch.setattr(
        test_client.server_instance.ConnectionClass.RequestHandlerClass,
        'read_request_line',
        _read_request_line,
    )

    test_client.get_connection().send(b'GET / HTTP/1.1')
    wsgi_server_thread.join()  # no extra logs upon server termination

    actual_log_entries = testing_server.error_log.calls[:]
    testing_server.error_log.calls.clear()  # prevent post-test assertions

    expected_log_entries = (
        (logging.WARNING, r'^socket\.error 666$'),
        (
            logging.INFO,
            '^Got a connection error while handling a connection '
            r'from .*:\d{1,5} \(666\)',
        ),
        (
            logging.CRITICAL,
            r'A fatal exception happened\. Setting the server interrupt flag '
            r'to ConnectionResetError\(666,?\) and giving up\.\n\nPlease, '
            'report this on the Cheroot tracker at '
            r'<https://github\.com/cherrypy/cheroot/issues/new/choose>, '
            'providing a full reproducer with as much context and details '
            r'as possible\.$',
        ),
    )

    assert len(actual_log_entries) == len(expected_log_entries)

    for (  # noqa: WPS352
            (expected_log_level, expected_msg_regex),
            (actual_msg, actual_log_level, _tb),
    ) in zip(expected_log_entries, actual_log_entries):
        assert expected_log_level == actual_log_level
        assert _matches_pattern(expected_msg_regex, actual_msg) is not None, (
            f'{actual_msg !r} does not match {expected_msg_regex !r}'
        )


def test_kb_int_from_http_handler(
        test_client,
        testing_server,
        wsgi_server_thread,
):
    """Test that a keyboard interrupt from HTTP handler causes shutdown."""
    def _trigger_kb_intr(_req, _resp):
        raise KeyboardInterrupt('simulated test handler keyboard interrupt')
    testing_server.wsgi_app.handlers['/kb_intr'] = _trigger_kb_intr

    http_conn = test_client.get_connection()
    http_conn.putrequest('GET', '/kb_intr', skip_host=True)
    http_conn.putheader('Host', http_conn.host)
    http_conn.endheaders()
    wsgi_server_thread.join()  # no extra logs upon server termination

    actual_log_entries = testing_server.error_log.calls[:]
    testing_server.error_log.calls.clear()  # prevent post-test assertions

    expected_log_entries = (
        (
            logging.DEBUG,
            '^Got a server shutdown request while handling a connection '
            r'from .*:\d{1,5} \(simulated test handler keyboard interrupt\)$',
        ),
        (
            logging.DEBUG,
            '^Setting the server interrupt flag to KeyboardInterrupt'
            r"\('simulated test handler keyboard interrupt',?\)$",
        ),
        (
            logging.INFO,
            '^Keyboard Interrupt: shutting down$',
        ),
    )

    assert len(actual_log_entries) == len(expected_log_entries)

    for (  # noqa: WPS352
            (expected_log_level, expected_msg_regex),
            (actual_msg, actual_log_level, _tb),
    ) in zip(expected_log_entries, actual_log_entries):
        assert expected_log_level == actual_log_level
        assert _matches_pattern(expected_msg_regex, actual_msg) is not None, (
            f'{actual_msg !r} does not match {expected_msg_regex !r}'
        )


@pytest.mark.xfail(
    IS_CI and IS_PYPY and IS_PY36 and not IS_SLOW_ENV,
    reason='Fails under PyPy 3.6 under Ubuntu 20.04 in CI for unknown reason',
    # NOTE: Actually covers any Linux
    strict=False,
)
def test_unhandled_exception_in_request_handler(
        mocker,
        monkeypatch,
        test_client,
        testing_server,
        wsgi_server_thread,
):
    """Ensure worker threads are resilient to in-handler exceptions."""

    class SillyMistake(BaseException):  # noqa: WPS418, WPS431
        """A simulated crash within an HTTP handler."""

    def _trigger_scary_exc(_req, _resp):
        raise SillyMistake('simulated unhandled exception 💣 in test handler')

    testing_server.wsgi_app.handlers['/scary_exc'] = _trigger_scary_exc

    server_connection_close_spy = mocker.spy(
        test_client.server_instance.ConnectionClass,
        'close',
    )

    http_conn = test_client.get_connection()
    http_conn.putrequest('GET', '/scary_exc', skip_host=True)
    http_conn.putheader('Host', http_conn.host)
    http_conn.endheaders()

    # NOTE: This spy ensure the log entry gets recorded before we're testing
    # NOTE: them and before server shutdown, preserving their order and making
    # NOTE: the log entry presence non-flaky.
    while not server_connection_close_spy.called:  # noqa: WPS328
        pass

    assert len(testing_server.requests._threads) == 10
    while testing_server.requests.idle < 10:  # noqa: WPS328
        pass
    assert len(testing_server.requests._threads) == 10
    testing_server.interrupt = SystemExit('test requesting shutdown')
    assert not testing_server.requests._threads
    wsgi_server_thread.join()  # no extra logs upon server termination

    actual_log_entries = testing_server.error_log.calls[:]
    testing_server.error_log.calls.clear()  # prevent post-test assertions

    expected_log_entries = (
        (
            logging.ERROR,
            '^Unhandled error while processing an incoming connection '
            'SillyMistake'
            r"\('simulated unhandled exception 💣 in test handler',?\)$",
        ),
        (
            logging.INFO,
            '^SystemExit raised: shutting down$',
        ),
    )

    assert len(actual_log_entries) == len(expected_log_entries)

    for (  # noqa: WPS352
            (expected_log_level, expected_msg_regex),
            (actual_msg, actual_log_level, _tb),
    ) in zip(expected_log_entries, actual_log_entries):
        assert expected_log_level == actual_log_level
        assert _matches_pattern(expected_msg_regex, actual_msg) is not None, (
            f'{actual_msg !r} does not match {expected_msg_regex !r}'
        )


@pytest.mark.xfail(
    IS_CI and IS_PYPY and IS_PY36 and not IS_SLOW_ENV,
    reason='Fails under PyPy 3.6 under Ubuntu 20.04 in CI for unknown reason',
    # NOTE: Actually covers any Linux
    strict=False,
)
def test_remains_alive_post_unhandled_exception(
        mocker,
        monkeypatch,
        test_client,
        testing_server,
        wsgi_server_thread,
):
    """Ensure worker threads are resilient to unhandled exceptions."""

    class ScaryCrash(BaseException):  # noqa: WPS418, WPS431
        """A simulated crash during HTTP parsing."""

    _orig_read_request_line = (
        test_client.server_instance.
        ConnectionClass.RequestHandlerClass.
        read_request_line
    )

    def _read_request_line(self):
        _orig_read_request_line(self)
        raise ScaryCrash(666)

    monkeypatch.setattr(
        test_client.server_instance.ConnectionClass.RequestHandlerClass,
        'read_request_line',
        _read_request_line,
    )

    server_connection_close_spy = mocker.spy(
        test_client.server_instance.ConnectionClass,
        'close',
    )

    # NOTE: The initial worker thread count is 10.
    assert len(testing_server.requests._threads) == 10

    test_client.get_connection().send(b'GET / HTTP/1.1')

    # NOTE: This spy ensure the log entry gets recorded before we're testing
    # NOTE: them and before server shutdown, preserving their order and making
    # NOTE: the log entry presence non-flaky.
    while not server_connection_close_spy.called:  # noqa: WPS328
        pass

    # NOTE: This checks for whether there's any crashed threads
    while testing_server.requests.idle < 10:  # noqa: WPS328
        pass
    assert len(testing_server.requests._threads) == 10
    assert all(
        worker_thread.is_alive()
        for worker_thread in testing_server.requests._threads
    )
    testing_server.interrupt = SystemExit('test requesting shutdown')
    assert not testing_server.requests._threads
    wsgi_server_thread.join()  # no extra logs upon server termination

    actual_log_entries = testing_server.error_log.calls[:]
    testing_server.error_log.calls.clear()  # prevent post-test assertions

    expected_log_entries = (
        (
            logging.ERROR,
            '^Unhandled error while processing an incoming connection '
            r'ScaryCrash\(666,?\)$',
        ),
        (
            logging.INFO,
            '^SystemExit raised: shutting down$',
        ),
    )

    assert len(actual_log_entries) == len(expected_log_entries)

    for (  # noqa: WPS352
            (expected_log_level, expected_msg_regex),
            (actual_msg, actual_log_level, _tb),
    ) in zip(expected_log_entries, actual_log_entries):
        assert expected_log_level == actual_log_level
        assert _matches_pattern(expected_msg_regex, actual_msg) is not None, (
            f'{actual_msg !r} does not match {expected_msg_regex !r}'
        )


@pytest.mark.parametrize(
    'timeout_before_headers',
    (
        True,
        False,
    ),
)
def test_HTTP11_Timeout(test_client, timeout_before_headers):
    """Check timeout without sending any data.

    The server will close the connection with a 408.
    """
    conn = test_client.get_connection()
    conn.auto_open = False
    conn.connect()

    if not timeout_before_headers:
        # Connect but send half the headers only.
        conn.send(b'GET /hello HTTP/1.1')
        conn.send(('Host: %s' % conn.host).encode('ascii'))
    # else: Connect but send nothing.

    # Wait for our socket timeout
    time.sleep(timeout * 2)

    # The request should have returned 408 already.
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    assert response.status == 408
    conn.close()


def test_HTTP11_Timeout_after_request(test_client):
    """Check timeout after at least one request has succeeded.

    The server should close the connection without 408.
    """
    fail_msg = "Writing to timed out socket didn't fail as it should have: %s"

    # Make an initial request
    conn = test_client.get_connection()
    conn.putrequest('GET', '/timeout?t=%s' % timeout, skip_host=True)
    conn.putheader('Host', conn.host)
    conn.endheaders()
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    assert response.status == 200
    actual_body = response.read()
    expected_body = str(timeout).encode()
    assert actual_body == expected_body

    # Make a second request on the same socket
    conn._output(b'GET /hello HTTP/1.1')
    conn._output(('Host: %s' % conn.host).encode('ascii'))
    conn._send_output()
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    assert response.status == 200
    actual_body = response.read()
    expected_body = b'Hello, world!'
    assert actual_body == expected_body

    # Wait for our socket timeout
    time.sleep(timeout * 2)

    # Make another request on the same socket, which should error
    conn._output(b'GET /hello HTTP/1.1')
    conn._output(('Host: %s' % conn.host).encode('ascii'))
    conn._send_output()
    response = conn.response_class(conn.sock, method='GET')
    try:
        response.begin()
    except (socket.error, http.client.BadStatusLine):
        pass
    except Exception as ex:
        pytest.fail(fail_msg % ex)
    else:
        if response.status != 408:
            pytest.fail(fail_msg % response.read())

    conn.close()

    # Make another request on a new socket, which should work
    conn = test_client.get_connection()
    conn.putrequest('GET', '/pov', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.endheaders()
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    assert response.status == 200
    actual_body = response.read()
    expected_body = pov.encode()
    assert actual_body == expected_body

    # Make another request on the same socket,
    # but timeout on the headers
    conn.send(b'GET /hello HTTP/1.1')
    # Wait for our socket timeout
    time.sleep(timeout * 2)
    response = conn.response_class(conn.sock, method='GET')
    try:
        response.begin()
    except (socket.error, http.client.BadStatusLine):
        pass
    except Exception as ex:
        pytest.fail(fail_msg % ex)
    else:
        if response.status != 408:
            pytest.fail(fail_msg % response.read())

    conn.close()

    # Retry the request on a new connection, which should work
    conn = test_client.get_connection()
    conn.putrequest('GET', '/pov', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.endheaders()
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    assert response.status == 200
    actual_body = response.read()
    expected_body = pov.encode()
    assert actual_body == expected_body
    conn.close()


def test_HTTP11_pipelining(test_client):
    """Test HTTP/1.1 pipelining.

    :py:mod:`http.client` doesn't support this directly.
    """
    conn = test_client.get_connection()

    # Put request 1
    conn.putrequest('GET', '/hello', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.endheaders()

    for trial in range(5):
        # Put next request
        conn._output(
            ('GET /hello?%s HTTP/1.1' % trial).encode('iso-8859-1'),
        )
        conn._output(('Host: %s' % conn.host).encode('ascii'))
        conn._send_output()

        # Retrieve previous response
        response = conn.response_class(conn.sock, method='GET')
        # there is a bug in python3 regarding the buffering of
        # ``conn.sock``. Until that bug get's fixed we will
        # monkey patch the ``response`` instance.
        # https://bugs.python.org/issue23377
        response.fp = conn.sock.makefile('rb', 0)
        response.begin()
        body = response.read(13)
        assert response.status == 200
        assert body == b'Hello, world!'

    # Retrieve final response
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    body = response.read()
    assert response.status == 200
    assert body == b'Hello, world!'

    conn.close()


def test_100_Continue(test_client):
    """Test 100-continue header processing."""
    conn = test_client.get_connection()

    # Try a page without an Expect request header first.
    # Note that http.client's response.begin automatically ignores
    # 100 Continue responses, so we must manually check for it.
    conn.putrequest('POST', '/upload', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.putheader('Content-Type', 'text/plain')
    conn.putheader('Content-Length', '4')
    conn.endheaders()
    conn.send(b"d'oh")
    response = conn.response_class(conn.sock, method='POST')
    _version, status, _reason = response._read_status()
    assert status != 100
    conn.close()

    # Now try a page with an Expect header...
    conn.connect()
    conn.putrequest('POST', '/upload', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.putheader('Content-Type', 'text/plain')
    conn.putheader('Content-Length', '17')
    conn.putheader('Expect', '100-continue')
    conn.endheaders()
    response = conn.response_class(conn.sock, method='POST')

    # ...assert and then skip the 100 response
    version, status, reason = response._read_status()
    assert status == 100
    while True:
        line = response.fp.readline().strip()
        if line:
            pytest.fail(
                '100 Continue should not output any headers. Got %r' %
                line,
            )
        else:
            break

    # ...send the body
    body = b'I am a small file'
    conn.send(body)

    # ...get the final response
    response.begin()
    status_line, _actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])
    assert actual_status == 200
    expected_resp_body = f"thanks for '{body.decode() !s}'".encode()
    assert actual_resp_body == expected_resp_body
    conn.close()


@pytest.mark.parametrize(
    'max_request_body_size',
    (
        0,
        1001,
    ),
)
def test_readall_or_close(test_client, max_request_body_size):
    """Test a max_request_body_size of 0 (the default) and 1001."""
    old_max = test_client.server_instance.max_request_body_size

    test_client.server_instance.max_request_body_size = max_request_body_size

    conn = test_client.get_connection()

    # Get a POST page with an error
    conn.putrequest('POST', '/err_before_read', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.putheader('Content-Type', 'text/plain')
    conn.putheader('Content-Length', '1000')
    conn.putheader('Expect', '100-continue')
    conn.endheaders()
    response = conn.response_class(conn.sock, method='POST')

    # ...assert and then skip the 100 response
    _version, status, _reason = response._read_status()
    assert status == 100
    skip = True
    while skip:
        skip = response.fp.readline().strip()

    # ...send the body
    conn.send(b'x' * 1000)

    # ...get the final response
    response.begin()
    status_line, _actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])
    assert actual_status == 500

    # Now try a working page with an Expect header...
    conn._output(b'POST /upload HTTP/1.1')
    conn._output(('Host: %s' % conn.host).encode('ascii'))
    conn._output(b'Content-Type: text/plain')
    conn._output(b'Content-Length: 17')
    conn._output(b'Expect: 100-continue')
    conn._send_output()
    response = conn.response_class(conn.sock, method='POST')

    # ...assert and then skip the 100 response
    version, status, reason = response._read_status()
    assert status == 100
    skip = True
    while skip:
        skip = response.fp.readline().strip()

    # ...send the body
    body = b'I am a small file'
    conn.send(body)

    # ...get the final response
    response.begin()
    status_line, actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])
    assert actual_status == 200
    expected_resp_body = f"thanks for '{body.decode() !s}'".encode()
    assert actual_resp_body == expected_resp_body
    conn.close()

    test_client.server_instance.max_request_body_size = old_max


def test_No_Message_Body(test_client):
    """Test HTTP queries with an empty response body."""
    # Initialize a persistent HTTP connection
    http_connection = test_client.get_connection()
    http_connection.auto_open = False
    http_connection.connect()

    # Make the first request and assert there's no "Connection: close".
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/pov', http_conn=http_connection,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    assert actual_resp_body == pov.encode()
    assert not header_exists('Connection', actual_headers)

    # Make a 204 request on the same connection.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/custom/204', http_conn=http_connection,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 204
    assert not header_exists('Content-Length', actual_headers)
    assert actual_resp_body == b''
    assert not header_exists('Connection', actual_headers)

    # Make a 304 request on the same connection.
    status_line, actual_headers, actual_resp_body = test_client.get(
        '/custom/304', http_conn=http_connection,
    )
    actual_status = int(status_line[:3])
    assert actual_status == 304
    assert not header_exists('Content-Length', actual_headers)
    assert actual_resp_body == b''
    assert not header_exists('Connection', actual_headers)

    # Prevent the resource warnings:
    http_connection.close()


@pytest.mark.xfail(
    reason=unwrap(
        trim("""
        Headers from earlier request leak into the request
        line for a subsequent request, resulting in 400
        instead of 413. See cherrypy/cheroot#69 for details.
        """),
    ),
)
def test_Chunked_Encoding(test_client):
    """Test HTTP uploads with chunked transfer-encoding."""
    # Initialize a persistent HTTP connection
    conn = test_client.get_connection()

    # Try a normal chunked request (with extensions)
    body = (
        b'8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n'
        b'Content-Type: application/json\r\n'
        b'\r\n'
    )
    conn.putrequest('POST', '/upload', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.putheader('Transfer-Encoding', 'chunked')
    conn.putheader('Trailer', 'Content-Type')
    # Note that this is somewhat malformed:
    # we shouldn't be sending Content-Length.
    # RFC 2616 says the server should ignore it.
    conn.putheader('Content-Length', '3')
    conn.endheaders()
    conn.send(body)
    response = conn.getresponse()
    status_line, _actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])
    assert actual_status == 200
    assert status_line[4:] == 'OK'
    expected_resp_body = ("thanks for '%s'" % b'xx\r\nxxxxyyyyy').encode()
    assert actual_resp_body == expected_resp_body

    # Try a chunked request that exceeds server.max_request_body_size.
    # Note that the delimiters and trailer are included.
    body = b'\r\n'.join((b'3e3', b'x' * 995, b'0', b'', b''))
    conn.putrequest('POST', '/upload', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.putheader('Transfer-Encoding', 'chunked')
    conn.putheader('Content-Type', 'text/plain')
    # Chunked requests don't need a content-length
    # conn.putheader("Content-Length", len(body))
    conn.endheaders()
    conn.send(body)
    response = conn.getresponse()
    status_line, actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])
    assert actual_status == 413
    conn.close()


def test_Content_Length_in(test_client):
    """Try a non-chunked request where Content-Length exceeds limit.

    (server.max_request_body_size).
    Assert error before body send.
    """
    # Initialize a persistent HTTP connection
    conn = test_client.get_connection()

    conn.putrequest('POST', '/upload', skip_host=True)
    conn.putheader('Host', conn.host)
    conn.putheader('Content-Type', 'text/plain')
    conn.putheader('Content-Length', '9999')
    conn.endheaders()
    response = conn.getresponse()
    status_line, _actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])
    assert actual_status == 413
    expected_resp_body = (
        b'The entity sent with the request exceeds '
        b'the maximum allowed bytes.'
    )
    assert actual_resp_body == expected_resp_body
    conn.close()


def test_Content_Length_not_int(test_client):
    """Test that malicious Content-Length header returns 400."""
    status_line, _actual_headers, actual_resp_body = test_client.post(
        '/upload',
        headers=[
            ('Content-Type', 'text/plain'),
            ('Content-Length', 'not-an-integer'),
        ],
    )
    actual_status = int(status_line[:3])

    assert actual_status == 400
    assert actual_resp_body == b'Malformed Content-Length Header.'


@pytest.mark.parametrize(
    ('uri', 'expected_resp_status', 'expected_resp_body'),
    (
        (
            '/wrong_cl_buffered', 500,
            (
                b'The requested resource returned more bytes than the '
                b'declared Content-Length.'
            ),
        ),
        ('/wrong_cl_unbuffered', 200, b'I too'),
    ),
)
def test_Content_Length_out(
    test_client,
    uri, expected_resp_status, expected_resp_body,
):
    """Test response with Content-Length less than the response body.

    (non-chunked response)
    """
    conn = test_client.get_connection()
    conn.putrequest('GET', uri, skip_host=True)
    conn.putheader('Host', conn.host)
    conn.endheaders()

    response = conn.getresponse()
    status_line, _actual_headers, actual_resp_body = webtest.shb(response)
    actual_status = int(status_line[:3])

    assert actual_status == expected_resp_status
    assert actual_resp_body == expected_resp_body

    conn.close()

    # the server logs the exception that we had verified from the
    # client perspective. Tell the error_log verification that
    # it can ignore that message.
    test_client.server_instance.error_log.ignored_msgs.extend((
        # Python 3.7+:
        "ValueError('Response body exceeds the declared Content-Length.')",
        # Python 2.7-3.6 (macOS?):
        "ValueError('Response body exceeds the declared Content-Length.',)",
    ))


@pytest.mark.xfail(
    reason='Sometimes this test fails due to low timeout. '
           'Ref: https://github.com/cherrypy/cherrypy/issues/598',
)
def test_598(test_client):
    """Test serving large file with a read timeout in place."""
    # Initialize a persistent HTTP connection
    conn = test_client.get_connection()
    remote_data_conn = urllib.request.urlopen(
        '%s://%s:%s/one_megabyte_of_a'
        % ('http', conn.host, conn.port),
    )
    buf = remote_data_conn.read(512)
    time.sleep(timeout * 0.6)
    remaining = (1024 * 1024) - 512
    while remaining:
        data = remote_data_conn.read(remaining)
        if not data:
            break
        buf += data
        remaining -= len(data)

    assert len(buf) == 1024 * 1024
    assert buf == b'a' * 1024 * 1024
    assert remaining == 0
    remote_data_conn.close()


@pytest.mark.parametrize(
    'invalid_terminator',
    (
        b'\n\n',
        b'\r\n\n',
    ),
)
def test_No_CRLF(test_client, invalid_terminator):
    """Test HTTP queries with no valid CRLF terminators."""
    # Initialize a persistent HTTP connection
    conn = test_client.get_connection()

    conn.send(b'GET /hello HTTP/1.1%s' % invalid_terminator)
    response = conn.response_class(conn.sock, method='GET')
    response.begin()
    actual_resp_body = response.read()
    expected_resp_body = b'HTTP requires CRLF terminators'
    assert actual_resp_body == expected_resp_body
    conn.close()


class FaultySelect:
    """Mock class to insert errors in the selector.select method."""

    def __init__(self, original_select):
        """Initilize helper class to wrap the selector.select method."""
        self.original_select = original_select
        self.request_served = False
        self.os_error_triggered = False

    def __call__(self, timeout):
        """Intercept the calls to selector.select."""
        if self.request_served:
            self.os_error_triggered = True
            raise OSError('Error while selecting the client socket.')

        return self.original_select(timeout)


class FaultyGetMap:
    """Mock class to insert errors in the selector.get_map method."""

    def __init__(self, original_get_map):
        """Initilize helper class to wrap the selector.get_map method."""
        self.original_get_map = original_get_map
        self.sabotage_conn = False
        self.conn_closed = False

    def __call__(self):
        """Intercept the calls to selector.get_map."""
        sabotage_targets = (
            conn for _, (_, _, _, conn) in self.original_get_map().items()
            if isinstance(conn, cheroot.server.HTTPConnection)
        ) if self.sabotage_conn and not self.conn_closed else ()

        for conn in sabotage_targets:
            # close the socket to cause OSError
            conn.close()
            self.conn_closed = True

        return self.original_get_map()


def test_invalid_selected_connection(test_client, monkeypatch):
    """Test the error handling segment of HTTP connection selection.

    See :py:meth:`cheroot.connections.ConnectionManager.get_conn`.
    """
    # patch the select method
    faux_select = FaultySelect(
        test_client.server_instance._connections._selector.select,
    )
    monkeypatch.setattr(
        test_client.server_instance._connections._selector,
        'select',
        faux_select,
    )

    # patch the get_map method
    faux_get_map = FaultyGetMap(
        test_client.server_instance._connections._selector._selector.get_map,
    )

    monkeypatch.setattr(
        test_client.server_instance._connections._selector._selector,
        'get_map',
        faux_get_map,
    )

    # request a page with connection keep-alive to make sure
    # we'll have a connection to be modified.
    resp_status, _resp_headers, _resp_body = test_client.request(
        '/page1', headers=[('Connection', 'Keep-Alive')],
    )

    assert resp_status == '200 OK'
    # trigger the internal errors
    faux_get_map.sabotage_conn = faux_select.request_served = True
    # give time to make sure the error gets handled
    time.sleep(test_client.server_instance.expiration_interval * 2)
    assert faux_select.os_error_triggered
    assert faux_get_map.conn_closed