test/modules/metadata/test_004_remoteip.py

import os
import re
import socket
import time

import pytest

from pyhttpd.backport import not_backported, skip_not_backported
from pyhttpd.depends import needs_dependency

from pyhttpd.conf import HttpdConf
from .env import MetadataTestEnv

# on the cases that fail without it (the others pass there as well)
PER_VHOST = not_backported("r1938908", "mod_remoteip per-vhost lists")


@needs_dependency("mod_remoteip", MetadataTestEnv.has_shared_module("remoteip"),
                  reason="mod_remoteip not available")
class TestRemoteIp:
    LOG_FILE = "test_remoteip.log"
    PEER = "127.0.0.1"
    CLIENT = "203.0.113.7"
    CDN = "198.51.100.9"

    # The test requests come from the internal proxy PEER and carry
    # "X-Forwarded-For: CLIENT, CDN". The proxies are given directly or in
    # a list file, in the server config or in the vhost. Each case expects
    # the client and the (external) proxies seen by mod_remoteip.
    @pytest.mark.parametrize(["scope", "directives", "listed", "proxies"], [
        ["vhost", [f"RemoteIPInternalProxy {PEER}", f"RemoteIPTrustedProxy {CDN}"],
         [], CDN],
        pytest.param("vhost", [f"RemoteIPInternalProxy {PEER}",
                               "RemoteIPTrustedProxyList {list}"],
                     [CDN], CDN, marks=PER_VHOST),
        pytest.param("vhost", ["RemoteIPTrustedProxyList {list}",
                               f"RemoteIPInternalProxy {PEER}"],
                     [CDN], CDN, marks=PER_VHOST),
        pytest.param("vhost", ["RemoteIPTrustedProxy 10.142.1.99",
                               "RemoteIPInternalProxyList {list}"],
                     [PEER, CDN], "-", marks=PER_VHOST),
        ["server", [f"RemoteIPInternalProxy {PEER}", "RemoteIPTrustedProxyList {list}"],
         [CDN], CDN],
    ])
    def test_metadata_004_01(self, env, scope, directives, listed, proxies):
        list_file = os.path.join(env.gen_dir, "remoteip_proxies.lst")
        with open(list_file, "w") as f:
            f.write("".join(f"{ip}\n" for ip in listed))
        lines = [d.format(list=list_file) for d in directives]
        conf = HttpdConf(env, extras={
            "base": [
                "RemoteIPHeader X-Forwarded-For",
                f'CustomLog logs/{self.LOG_FILE} "%a %{{remoteip-proxy-ip-list}}n"',
            ] + (lines if scope == "server" else [])
        })
        conf.start_vhost(domains=[f"test1.{env.http_tld}"], doc_root="htdocs/test1")
        if scope == "vhost":
            conf.add(lines)
        conf.end_vhost()
        conf.install()
        assert env.apache_restart() == 0

        log_path = os.path.join(env.server_logs_dir, self.LOG_FILE)
        open(log_path, 'w').close()
        r = env.curl_get(env.mkurl("https", "test1", "/"), options=[
            '-H', f"X-Forwarded-For: {self.CLIENT}, {self.CDN}"])
        assert r.response["status"] == 200
        with open(log_path) as f:
            lines = [l.strip() for l in f if l.strip()
                     and not l.startswith(f"{self.PEER} -")]
            assert len(lines) == 1, f"expected 1 test line, got {lines}"
            assert lines[0] == f"{self.CLIENT} {proxies}"

    WARNING = r".*both the main server and the virtual host"

    # The proxy entries of the main server and of a vhost are merged, the
    # ones of the vhost taking precedence where subnets overlap, and a
    # warning tells when both have some.
    @pytest.mark.parametrize(["main", "vhost", "proxies", "warning"], [
        # only one of the scopes
        [[f"RemoteIPInternalProxy {PEER}", f"RemoteIPTrustedProxy {CDN}"],
         [], CDN, False],
        [[], [f"RemoteIPInternalProxy {PEER}", f"RemoteIPTrustedProxy {CDN}"],
         CDN, False],
        # (both scopes: merged, and warned about, only with the change)
        # RemoteIPInternalProxy in the main server, a list in the vhost
        pytest.param([f"RemoteIPInternalProxy {PEER}"], ["RemoteIPInternalProxyList {cdn}"],
                     "-", True, marks=PER_VHOST),
        pytest.param([f"RemoteIPInternalProxy {PEER}"], ["RemoteIPTrustedProxyList {cdn}"],
                     CDN, True, marks=PER_VHOST),
        # a list in the main server, a direct entry in the vhost
        pytest.param(["RemoteIPInternalProxyList {peer}"], [f"RemoteIPTrustedProxy {CDN}"],
                     CDN, True, marks=PER_VHOST),
        # direct entries in both
        pytest.param([f"RemoteIPInternalProxy {PEER}"], [f"RemoteIPTrustedProxy {CDN}"],
                     CDN, True, marks=PER_VHOST),
        # the CDN is trusted by the main server and internal for the vhost
        pytest.param([f"RemoteIPInternalProxy {PEER}", f"RemoteIPTrustedProxy {CDN}"],
                     [f"RemoteIPInternalProxy {CDN}"], "-", True, marks=PER_VHOST),
        pytest.param([f"RemoteIPInternalProxy {PEER}", f"RemoteIPTrustedProxy {CDN}"],
                     ["RemoteIPInternalProxyList {cdn}"], "-", True, marks=PER_VHOST),
    ])
    def test_metadata_004_02(self, env, main, vhost, proxies, warning):
        files = {"cdn": self.CDN, "peer": self.PEER}
        for name, ip in files.items():
            with open(os.path.join(env.gen_dir, f"remoteip_{name}.lst"), "w") as f:
                f.write(f"{ip}\n")
        paths = {n: os.path.join(env.gen_dir, f"remoteip_{n}.lst") for n in files}
        conf = HttpdConf(env, extras={
            "base": [
                "RemoteIPHeader X-Forwarded-For",
                f'CustomLog logs/{self.LOG_FILE} "%a %{{remoteip-proxy-ip-list}}n"',
            ] + [d.format(**paths) for d in main]
        })
        conf.start_vhost(domains=[f"test1.{env.http_tld}"], doc_root="htdocs/test1")
        conf.add([d.format(**paths) for d in vhost])
        conf.end_vhost()
        conf.install()
        pos = env.httpd_error_log.current_pos()
        assert env.apache_restart() == 0
        warned = env.httpd_error_log.wait_for(re.compile(self.WARNING), pos, timeout=1)
        assert warned == warning
        if warned:
            env.httpd_error_log.ignore_recent(matches=[self.WARNING])

        log_path = os.path.join(env.server_logs_dir, self.LOG_FILE)
        open(log_path, 'w').close()
        r = env.curl_get(env.mkurl("https", "test1", "/"), options=[
            '-H', f"X-Forwarded-For: {self.CLIENT}, {self.CDN}"])
        assert r.response["status"] == 200
        with open(log_path) as f:
            lines = [l.strip() for l in f if l.strip()
                     and not l.startswith(f"{self.PEER} -")]
            assert len(lines) == 1, f"expected 1 test line, got {lines}"
            assert lines[0] == f"{self.CLIENT} {proxies}"

    # -- PROXY protocol --------------------------------------------------

    # A v2 header: the signature, then version/command, family/transport,
    # the length of what follows, and the addresses.  The commands are LOCAL
    # (0) and PROXY (1); a health checker such as an AWS NLB sends LOCAL,
    # for which the receiver must keep the connection's own address and
    # ignore any address information present (bug 63893).
    PP2_SIG = b"\x0d\x0a\x0d\x0a\x00\x0d\x0a\x51\x55\x49\x54\x0a"
    PP2_LOCAL = 0x20
    PP2_PROXY = 0x21
    PP2_UNSPEC = 0x00
    PP2_TCP4 = 0x11
    # CLIENT:4321 -> CDN:80, as a PROXY protocol TCPv4 address block
    PP2_ADDRS = (socket.inet_aton(CLIENT) + socket.inet_aton(CDN)
                 + (4321).to_bytes(2, "big") + (80).to_bytes(2, "big"))

    @classmethod
    def pp2_header(cls, cmd, fam=0x00, addrs=b""):
        return cls.PP2_SIG + bytes([cmd, fam]) + len(addrs).to_bytes(2, "big") + addrs

    def pp_install(self, env):
        """PROXY protocol on its own port, so the harness's own requests
        on the usual ports are unaffected; log the client address seen."""
        conf = HttpdConf(env, extras={
            "base": [
                f"Listen {env.http_port2}",
            ]
        })
        conf.start_vhost(domains=[f"test1.{env.http_tld}"], port=env.http_port2,
                         doc_root="htdocs/test1")
        conf.add(["RemoteIPProxyProtocol On",
                  f'CustomLog logs/{self.LOG_FILE} "%a"'])
        conf.end_vhost()
        conf.install()
        assert env.apache_restart() == 0

    def pp_request(self, env, header, request=True):
        """Send a PROXY protocol header, then a request unless told not to;
        return (status, client address logged) - None for each if nothing
        came back."""
        log_path = os.path.join(env.server_logs_dir, self.LOG_FILE)
        open(log_path, 'w').close()
        sock = socket.create_connection(("127.0.0.1", env.http_port2), timeout=5)
        sock.sendall(header)
        if request:
            sock.sendall(f"GET / HTTP/1.1\r\nHost: test1.{env.http_tld}\r\n"
                         f"Connection: close\r\n\r\n".encode())
        data = b""
        try:
            while True:
                chunk = sock.recv(65536)
                if not chunk:
                    break
                data += chunk
        except (socket.timeout, ConnectionError):
            pass
        sock.close()
        status = int(data.split(b" ", 2)[1]) if data.startswith(b"HTTP/") else None
        logged = None
        for _ in range(20):
            with open(log_path) as f:
                logged = f.read().strip() or None
            if logged or status is None:
                break
            time.sleep(0.1)
        return status, logged

    # Control: a PROXY command is honoured, the client is the one it names.
    def test_metadata_004_03(self, env):
        self.pp_install(env)
        hdr = self.pp2_header(self.PP2_PROXY, self.PP2_TCP4, self.PP2_ADDRS)
        assert self.pp_request(env, hdr) == (200, self.CLIENT)

    # A LOCAL command (ver_cmd 0x20) is valid: the request is served as
    # coming from the connection's own address, and nothing is logged as
    # an error.  Bug 63893.
    @skip_not_backported("r1938927", "PROXY protocol EOF is not an error")
    def test_metadata_004_04(self, env):
        self.pp_install(env)
        hdr = self.pp2_header(self.PP2_LOCAL, self.PP2_UNSPEC)
        assert self.pp_request(env, hdr) == (200, self.PEER)

    # LOCAL with an address block present: it must be ignored.
    @skip_not_backported("r1938927", "PROXY protocol EOF is not an error")
    def test_metadata_004_05(self, env):
        self.pp_install(env)
        hdr = self.pp2_header(self.PP2_LOCAL, self.PP2_TCP4, self.PP2_ADDRS)
        assert self.pp_request(env, hdr) == (200, self.PEER)

    # A health check which connects, sends an incomplete header and goes
    # away is not an error worth logging (bug 63893 comment 12).
    @skip_not_backported("r1938927", "PROXY protocol EOF is not an error")
    def test_metadata_004_06(self, env):
        self.pp_install(env)
        pos = env.httpd_error_log.current_pos()
        assert self.pp_request(env, self.PP2_SIG, request=False) == (None, None)
        logged = env.httpd_error_log.wait_for(re.compile(r".*AH10184"), pos,
                                              timeout=1)
        assert not logged, "peer going away mid-header was logged as an error"