numa/tests/integration.sh

#!/usr/bin/env bash
# Integration test suite for Numa
# Runs a test instance on port 5354, validates all features, exits with status.
# Usage:
#   ./tests/integration.sh [release|debug]       # all suites
#   SUITES=7 ./tests/integration.sh              # only Suite 7
#   SUITES=1,3,7 ./tests/integration.sh          # Suites 1, 3, and 7

set -euo pipefail

MODE="${1:-release}"
BINARY="./target/$MODE/numa"
PORT=5354
API_PORT=5381
CONFIG="/tmp/numa-integration-test.toml"
LOG="/tmp/numa-integration-test.log"
PASSED=0
FAILED=0

# Suite filter: empty runs all; comma list runs a subset.
SUITES="${SUITES:-}"
should_run_suite() {
    [ -z "$SUITES" ] && return 0
    case ",$SUITES," in *",$1,"*) return 0;; esac
    return 1
}

# Colors
GREEN="\033[32m"
RED="\033[31m"
DIM="\033[90m"
RESET="\033[0m"

check() {
    local name="$1"
    local expected="$2"
    local actual="$3"

    if echo "$actual" | grep -q "$expected"; then
        PASSED=$((PASSED + 1))
        printf "  ${GREEN}✓${RESET} %s\n" "$name"
    else
        FAILED=$((FAILED + 1))
        printf "  ${RED}✗${RESET} %s\n" "$name"
        printf "    ${DIM}expected: %s${RESET}\n" "$expected"
        printf "    ${DIM}     got: %s${RESET}\n" "$actual"
    fi
}

# Build if needed
if [ ! -f "$BINARY" ]; then
    echo "Building $MODE..."
    cargo build --$MODE
fi

run_test_suite() {
    local SUITE_NAME="$1"
    local SUITE_CONFIG="$2"

    cat > "$CONFIG" << CONF
$SUITE_CONFIG
CONF

    echo "Starting Numa on :$PORT ($SUITE_NAME)..."
    RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
    NUMA_PID=$!
    sleep 2

    # Wait for blocklist to load (if blocking is enabled in this suite)
    if echo "$SUITE_CONFIG" | grep -q 'enabled = true'; then
        for i in $(seq 1 20); do
            LOADED=$(curl -sf http://127.0.0.1:$API_PORT/blocking/stats 2>/dev/null \
                | grep -o '"domains_loaded":[0-9]*' | cut -d: -f2)
            if [ "${LOADED:-0}" -gt 0 ]; then break; fi
            sleep 1
        done
    fi

    if ! kill -0 "$NUMA_PID" 2>/dev/null; then
        echo "Failed to start Numa:"
        tail -5 "$LOG"
        return 1
    fi

    DIG="dig @127.0.0.1 -p $PORT +time=5 +tries=1"

    echo ""
    echo "=== Resolution ==="

    check "A record (google.com)" \
        "." \
        "$($DIG google.com A +short)"

    check "AAAA record (google.com)" \
        ":" \
        "$($DIG google.com AAAA +short)"

    check "CNAME chasing (www.github.com)" \
        "github.com" \
        "$($DIG www.github.com A +short)"

    check "MX records (gmail.com)" \
        "gmail-smtp-in" \
        "$($DIG gmail.com MX +short)"

    check "NS records (cloudflare.com)" \
        "cloudflare.com" \
        "$($DIG cloudflare.com NS +short)"

    check "NXDOMAIN" \
        "NXDOMAIN" \
        "$($DIG nope12345678.com A 2>&1 | grep status:)"

    echo ""
    echo "=== Ad Blocking ==="

    if echo "$SUITE_CONFIG" | grep -q 'enabled = true'; then
        check "Blocked domain → 0.0.0.0" \
            "0.0.0.0" \
            "$($DIG ads.google.com A +short)"
    else
        local ADS=$($DIG ads.google.com A +short 2>/dev/null)
        if echo "$ADS" | grep -q "0.0.0.0"; then
            check "Blocking disabled but domain blocked" "should-resolve" "0.0.0.0"
        else
            check "Blocking disabled — domain resolves normally" "." "$ADS"
        fi
    fi

    echo ""
    echo "=== Cache ==="

    $DIG example.com A +short > /dev/null 2>&1
    sleep 1
    check "Cache hit returns result" \
        "." \
        "$($DIG example.com A +short)"

    echo ""
    echo "=== Connectivity ==="

    # Apple captive portal can be slow/flaky on some networks
    local CAPTIVE
    CAPTIVE=$($DIG captive.apple.com A +short 2>/dev/null || echo "timeout")
    if echo "$CAPTIVE" | grep -q "apple\|17\.\|timeout"; then
        check "Apple captive portal" "." "$CAPTIVE"
    else
        check "Apple captive portal" "apple" "$CAPTIVE"
    fi

    check "CDN (jsdelivr)" \
        "." \
        "$($DIG cdn.jsdelivr.net A +short)"

    echo ""
    echo "=== API ==="

    check "Health endpoint" \
        "ok" \
        "$(curl -s http://127.0.0.1:$API_PORT/health)"

    check "Stats endpoint" \
        "uptime_secs" \
        "$(curl -s http://127.0.0.1:$API_PORT/stats)"

    echo ""
    echo "=== Log Health ==="

    ERRORS=$(grep -c 'RECURSIVE ERROR\|PARSE ERROR\|HANDLER ERROR\|panic' "$LOG" 2>/dev/null || echo 0)
    check "No critical errors in log" \
        "0" \
        "$ERRORS"

    kill "$NUMA_PID" 2>/dev/null || true
    wait "$NUMA_PID" 2>/dev/null || true
    sleep 1
}

# ---- Suite 1: Recursive mode + DNSSEC ----
if should_run_suite 1; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 1: Recursive + DNSSEC + Blocking  ║"
echo "╚══════════════════════════════════════════╝"

run_test_suite "recursive + DNSSEC + blocking" "
[server]
bind_addr = \"127.0.0.1:5354\"
api_port = 5381

[upstream]
mode = \"recursive\"

[cache]
max_entries = 10000
min_ttl = 60
max_ttl = 86400

[blocking]
enabled = true

[proxy]
enabled = false

[dnssec]
enabled = true
"

DIG="dig @127.0.0.1 -p $PORT +time=5 +tries=1"

echo ""
echo "=== DNSSEC (recursive only) ==="

# Re-start for DNSSEC checks (suite 1 instance was killed)
RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
sleep 4

check "AD bit set (cloudflare.com)" \
    " ad" \
    "$($DIG cloudflare.com A +dnssec 2>&1 | grep flags:)"

check "EDNS DO bit echoed" \
    "flags: do" \
    "$($DIG cloudflare.com A +dnssec 2>&1 | grep 'EDNS:')"

echo ""
echo "=== TCP wire format (real servers) ==="

# Microsoft's Azure DNS servers require length+message in a single TCP segment.
# This test catches the split-write bug that caused early-eof SERVFAILs.
check "Microsoft domain (update.code.visualstudio.com)" \
    "NOERROR" \
    "$($DIG update.code.visualstudio.com A 2>&1 | grep status:)"

check "Office domain (ecs.office.com)" \
    "NOERROR" \
    "$($DIG ecs.office.com A 2>&1 | grep status:)"

# Azure Application Insights — another strict TCP server
check "Azure telemetry (eastus2-3.in.applicationinsights.azure.com)" \
    "." \
    "$($DIG eastus2-3.in.applicationinsights.azure.com A +short 2>/dev/null || echo 'timeout')"

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

fi  # end Suite 1

# ---- Suite 2: Forward mode (backward compat) ----
if should_run_suite 2; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 2: Forward (DoH) + Blocking       ║"
echo "╚══════════════════════════════════════════╝"

run_test_suite "forward DoH + blocking" "
[server]
bind_addr = \"127.0.0.1:5354\"
api_port = 5381

[upstream]
mode = \"forward\"
address = \"https://9.9.9.9/dns-query\"

[cache]
max_entries = 10000
min_ttl = 60
max_ttl = 86400

[blocking]
enabled = true

[proxy]
enabled = false
"

fi  # end Suite 2

# ---- Suite 3: Forward UDP (plain, no DoH) ----
if should_run_suite 3; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 3: Forward (UDP) + No Blocking    ║"
echo "╚══════════════════════════════════════════╝"

run_test_suite "forward UDP, no blocking" "
[server]
bind_addr = \"127.0.0.1:5354\"
api_port = 5381

[upstream]
mode = \"forward\"
address = \"9.9.9.9\"
port = 53

[cache]
max_entries = 10000
min_ttl = 60
max_ttl = 86400

[blocking]
enabled = false

[proxy]
enabled = false
"

# Verify blocking is actually off
RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
sleep 3

echo ""
echo "=== Blocking disabled ==="
ADS_RESULT=$($DIG ads.google.com A +short 2>/dev/null)
if echo "$ADS_RESULT" | grep -q "0.0.0.0"; then
    check "ads.google.com NOT blocked (blocking disabled)" "not-0.0.0.0" "0.0.0.0"
else
    check "ads.google.com NOT blocked (blocking disabled)" "." "$ADS_RESULT"
fi

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

fi  # end Suite 3

# ---- Suite 4: Local zones + Overrides API ----
if should_run_suite 4; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 4: Local Zones + Overrides API    ║"
echo "╚══════════════════════════════════════════╝"

cat > "$CONFIG" << 'CONF'
[server]
bind_addr = "127.0.0.1:5354"
api_port = 5381

[upstream]
mode = "forward"
address = "9.9.9.9"
port = 53

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = false

[[zones]]
domain = "test.local"
record_type = "A"
value = "10.0.0.1"
ttl = 60

[[zones]]
domain = "mail.local"
record_type = "MX"
value = "10 smtp.local"
ttl = 60
CONF

RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
sleep 3

echo ""
echo "=== Local Zones ==="

check "Local A record (test.local)" \
    "10.0.0.1" \
    "$($DIG test.local A +short)"

check "Local MX record (mail.local)" \
    "smtp.local" \
    "$($DIG mail.local MX +short)"

check "Non-local domain still resolves" \
    "." \
    "$($DIG example.com A +short)"

echo ""
echo "=== Overrides API ==="

# Create override
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:$API_PORT/overrides \
    -H 'Content-Type: application/json' \
    -d '{"domain":"override.test","target":"192.168.1.100","duration_secs":60}')
check "Create override (HTTP 200/201)" \
    "20" \
    "$HTTP_CODE"

sleep 1

check "Override resolves" \
    "192.168.1.100" \
    "$($DIG override.test A +short)"

# List overrides
check "List overrides" \
    "override.test" \
    "$(curl -s http://127.0.0.1:$API_PORT/overrides)"

# Delete override
curl -s -X DELETE http://127.0.0.1:$API_PORT/overrides/override.test > /dev/null

sleep 1

# After delete, should not resolve to override
AFTER_DELETE=$($DIG override.test A +short 2>/dev/null)
if echo "$AFTER_DELETE" | grep -q "192.168.1.100"; then
    check "Override deleted" "not-192.168.1.100" "$AFTER_DELETE"
else
    check "Override deleted" "." "deleted"
fi

echo ""
echo "=== Cache API ==="

check "Cache list" \
    "domain" \
    "$(curl -s http://127.0.0.1:$API_PORT/cache)"

# Flush cache
curl -s -X DELETE http://127.0.0.1:$API_PORT/cache > /dev/null
check "Cache flushed" \
    "0" \
    "$(curl -s http://127.0.0.1:$API_PORT/stats | grep -o '"entries":[0-9]*' | grep -o '[0-9]*')"

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

fi  # end Suite 4

# ---- Suite 5: DNS-over-TLS (RFC 7858) ----
if should_run_suite 5; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 5: DNS-over-TLS (RFC 7858)        ║"
echo "╚══════════════════════════════════════════╝"

if ! command -v kdig >/dev/null 2>&1; then
    printf "  ${DIM}skipped — install 'knot' for kdig${RESET}\n"
elif ! command -v openssl >/dev/null 2>&1; then
    printf "  ${DIM}skipped — openssl not found${RESET}\n"
else
    DOT_PORT=8853
    DOT_CERT=/tmp/numa-integration-dot.crt
    DOT_KEY=/tmp/numa-integration-dot.key

    # Generate a test cert mirroring production self_signed_tls SAN shape
    # (*.numa wildcard + explicit numa.numa apex).
    openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
        -keyout "$DOT_KEY" -out "$DOT_CERT" \
        -subj "/CN=Numa .numa services" \
        -addext "subjectAltName=DNS:*.numa,DNS:numa.numa" \
        >/dev/null 2>&1

    # Suite 5 uses a local zone so it's upstream-independent — the point is
    # to exercise the DoT transport layer (handshake, ALPN, framing,
    # persistent connections), not re-test recursive resolution.
    cat > "$CONFIG" << CONF
[server]
bind_addr = "127.0.0.1:$PORT"
api_port = $API_PORT

[upstream]
mode = "forward"
address = "127.0.0.1"
port = 65535

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = false

[dot]
enabled = true
port = $DOT_PORT
bind_addr = "127.0.0.1"
cert_path = "$DOT_CERT"
key_path = "$DOT_KEY"

[[zones]]
domain = "dot-test.example"
record_type = "A"
value = "10.0.0.1"
ttl = 60
CONF

    RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
    NUMA_PID=$!
    sleep 4

    if ! kill -0 "$NUMA_PID" 2>/dev/null; then
        FAILED=$((FAILED + 1))
        printf "  ${RED}✗${RESET} DoT startup\n"
        printf "    ${DIM}%s${RESET}\n" "$(tail -5 "$LOG")"
    else
        echo ""
        echo "=== Listener ==="

        check "DoT bound on 127.0.0.1:$DOT_PORT" \
            "DoT listening on 127.0.0.1:$DOT_PORT" \
            "$(grep 'DoT listening' "$LOG")"

        KDIG="kdig @127.0.0.1 -p $DOT_PORT +tls +tls-ca=$DOT_CERT +tls-hostname=numa.numa +time=5 +retry=0"

        echo ""
        echo "=== Queries over DoT ==="

        check "DoT local zone A record" \
            "10.0.0.1" \
            "$($KDIG +short dot-test.example A 2>/dev/null)"

        # +keepopen reuses one TLS connection for multiple queries — tests
        # persistent connection handling. kdig applies options left-to-right,
        # so +short and +keepopen must come before the query specs.
        check "DoT persistent connection (3 queries, 1 handshake)" \
            "10.0.0.1" \
            "$($KDIG +keepopen +short dot-test.example A dot-test.example A dot-test.example A 2>/dev/null | head -1)"

        echo ""
        echo "=== ALPN ==="

        # Positive case: client offers "dot", server picks it.
        ALPN_OK=$(echo "" | openssl s_client -connect "127.0.0.1:$DOT_PORT" \
            -servername numa.numa -alpn dot -CAfile "$DOT_CERT" 2>&1 </dev/null || true)
        check "DoT negotiates ALPN \"dot\"" \
            "ALPN protocol: dot" \
            "$ALPN_OK"

        # Negative case: client offers only "h2", server must reject the
        # handshake with no_application_protocol alert (cross-protocol
        # confusion defense, RFC 7858bis §3.2).
        if echo "" | openssl s_client -connect "127.0.0.1:$DOT_PORT" \
            -servername numa.numa -alpn h2 -CAfile "$DOT_CERT" \
            </dev/null >/dev/null 2>&1; then
            ALPN_MISMATCH="handshake unexpectedly succeeded"
        else
            ALPN_MISMATCH="rejected"
        fi
        check "DoT rejects non-dot ALPN" \
            "rejected" \
            "$ALPN_MISMATCH"
    fi

    kill "$NUMA_PID" 2>/dev/null || true
    wait "$NUMA_PID" 2>/dev/null || true
    rm -f "$DOT_CERT" "$DOT_KEY"
fi
sleep 1

fi  # end Suite 5

# ---- Suite 6: Proxy + DoT coexistence ----
if should_run_suite 6; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 6: Proxy + DoT Coexistence        ║"
echo "╚══════════════════════════════════════════╝"

if ! command -v kdig >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
    printf "  ${DIM}skipped — needs kdig + openssl${RESET}\n"
else
    DOT_PORT=8853
    PROXY_HTTP_PORT=8080
    PROXY_HTTPS_PORT=8443
    NUMA_DATA=/tmp/numa-integration-data

    # Fresh data dir so we generate a fresh CA for this suite. Path is set
    # via [server] data_dir in the TOML below, not an env var — numa treats
    # its config file as the single source of truth for all knobs.
    rm -rf "$NUMA_DATA"
    mkdir -p "$NUMA_DATA"

    cat > "$CONFIG" << CONF
[server]
bind_addr = "127.0.0.1:$PORT"
api_port = $API_PORT
data_dir = "$NUMA_DATA"

[upstream]
mode = "forward"
address = "127.0.0.1"
port = 65535

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = true
port = $PROXY_HTTP_PORT
tls_port = $PROXY_HTTPS_PORT
tld = "numa"
bind_addr = "127.0.0.1"

[dot]
enabled = true
port = $DOT_PORT
bind_addr = "127.0.0.1"

[[zones]]
domain = "dot-test.example"
record_type = "A"
value = "10.0.0.1"
ttl = 60
CONF

    RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
    NUMA_PID=$!
    sleep 4

    if ! kill -0 "$NUMA_PID" 2>/dev/null; then
        FAILED=$((FAILED + 1))
        printf "  ${RED}✗${RESET} Startup with proxy + DoT\n"
        printf "    ${DIM}%s${RESET}\n" "$(tail -5 "$LOG")"
    else
        echo ""
        echo "=== Both listeners ==="

        check "DoT listener bound" \
            "DoT listening on 127.0.0.1:$DOT_PORT" \
            "$(grep 'DoT listening' "$LOG")"

        check "HTTPS proxy listener bound" \
            "HTTPS proxy listening on 127.0.0.1:$PROXY_HTTPS_PORT" \
            "$(grep 'HTTPS proxy listening' "$LOG")"

        PANIC_COUNT=$(grep -c 'panicked' "$LOG" 2>/dev/null || echo 0)
        check "No startup panics in log" \
            "^0$" \
            "$PANIC_COUNT"

        echo ""
        echo "=== DoT works with proxy enabled ==="

        # Proxy's build_tls_config runs first and creates the CA in
        # $NUMA_DATA_DIR. DoT self_signed_tls then loads the same CA and
        # issues its own leaf cert. One CA trusts both listeners.
        CA="$NUMA_DATA/ca.pem"
        KDIG="kdig @127.0.0.1 -p $DOT_PORT +tls +tls-ca=$CA +tls-hostname=numa.numa +time=5 +retry=0"

        check "DoT local zone A (with proxy on)" \
            "10.0.0.1" \
            "$($KDIG +short dot-test.example A 2>/dev/null)"

        echo ""
        echo "=== DNS-over-HTTPS (RFC 8484) ==="

        DOH_QUERY_FILE=/tmp/numa-doh-query.bin
        DOH_RESP_FILE=/tmp/numa-doh-resp.bin

        # Build DNS wire-format query for dot-test.example A
        printf '\x00\x01\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x08dot-test\x07example\x00\x00\x01\x00\x01' > "$DOH_QUERY_FILE"

        # POST valid DoH query
        DOH_CODE=$(curl -sk -X POST \
            --resolve "numa.numa:$PROXY_HTTPS_PORT:127.0.0.1" \
            -H "Content-Type: application/dns-message" \
            --data-binary @"$DOH_QUERY_FILE" \
            --cacert "$CA" \
            -o "$DOH_RESP_FILE" \
            -w "%{http_code}" \
            "https://numa.numa:$PROXY_HTTPS_PORT/dns-query")
        check "DoH POST returns HTTP 200" "200" "$DOH_CODE"

        # Check response contains IP 10.0.0.1 (hex: 0a000001)
        DOH_HEX=$(xxd -p "$DOH_RESP_FILE" | tr -d '\n')
        if echo "$DOH_HEX" | grep -q "0a000001"; then
            check "DoH response resolves dot-test.example → 10.0.0.1" "found" "found"
        else
            check "DoH response resolves dot-test.example → 10.0.0.1" "0a000001" "$DOH_HEX"
        fi

        # Wrong Content-Type → 415
        DOH_CT_CODE=$(curl -sk -X POST \
            -H "Host: numa.numa" \
            -H "Content-Type: text/plain" \
            --data-binary @"$DOH_QUERY_FILE" \
            -o /dev/null -w "%{http_code}" \
            "https://127.0.0.1:$PROXY_HTTPS_PORT/dns-query")
        check "DoH wrong Content-Type → 415" "415" "$DOH_CT_CODE"

        # Wrong host → 404 (DoH only serves numa.numa)
        DOH_HOST_CODE=$(curl -sk -X POST \
            -H "Host: foo.numa" \
            -H "Content-Type: application/dns-message" \
            --data-binary @"$DOH_QUERY_FILE" \
            -o /dev/null -w "%{http_code}" \
            "https://127.0.0.1:$PROXY_HTTPS_PORT/dns-query")
        check "DoH wrong host → 404" "404" "$DOH_HOST_CODE"

        rm -f "$DOH_QUERY_FILE" "$DOH_RESP_FILE"

        echo ""
        echo "=== Proxy TLS works with DoT enabled ==="

        # Proxy cert has SAN numa.numa (auto-added "numa" service). A
        # successful handshake validates that the proxy's separate
        # ServerConfig wasn't disturbed by DoT's own cert generation.
        PROXY_TLS=$(echo "" | openssl s_client -connect "127.0.0.1:$PROXY_HTTPS_PORT" \
            -servername numa.numa -CAfile "$CA" 2>&1 </dev/null || true)
        check "Proxy HTTPS TLS handshake succeeds" \
            "Verify return code: 0 (ok)" \
            "$PROXY_TLS"
    fi

    kill "$NUMA_PID" 2>/dev/null || true
    wait "$NUMA_PID" 2>/dev/null || true
    rm -rf "$NUMA_DATA"
fi

fi  # end Suite 6

# ---- Suite 7: filter_aaaa (IPv4-only networks) ----
if should_run_suite 7; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 7: filter_aaaa                    ║"
echo "╚══════════════════════════════════════════╝"

# Config A — filter on, with a local AAAA zone to prove local data bypass.
cat > "$CONFIG" << 'CONF'
[server]
bind_addr = "127.0.0.1:5354"
api_port = 5381
filter_aaaa = true

[upstream]
mode = "forward"
address = "9.9.9.9"
port = 53

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = false

[[zones]]
domain = "v6.test"
record_type = "AAAA"
value = "2001:db8::1"
ttl = 60
CONF

RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
sleep 3

DIG="dig @127.0.0.1 -p $PORT +time=5 +tries=1"

echo ""
echo "=== filter_aaaa = true ==="

# A queries must be untouched.
check "A record resolves under filter_aaaa" \
    "." \
    "$($DIG google.com A +short | head -1)"

# AAAA must be NOERROR (NODATA), not NXDOMAIN, not SERVFAIL.
check "AAAA returns NOERROR (not NXDOMAIN)" \
    "status: NOERROR" \
    "$($DIG google.com AAAA 2>&1 | grep 'status:')"

check "AAAA returns zero answers (NODATA shape)" \
    "ANSWER: 0" \
    "$($DIG google.com AAAA 2>&1 | grep -oE 'ANSWER: [0-9]+' | head -1)"

# Local zone AAAA must survive the filter (PR claim: local data bypasses).
check "Local [[zones]] AAAA bypasses filter" \
    "2001:db8::1" \
    "$($DIG v6.test AAAA +short)"

# HTTPS RR: ipv6hint (SvcParamKey 6) must be stripped. Query as `type65`
# because dig 9.10.6 (macOS) misparses `HTTPS` as a domain name; `type65`
# works on both 9.10.6 and 9.18. Assert on the raw rdata hex (RFC 3597
# generic format), since dig 9.10.6 doesn't pretty-print HTTPS params.
# cloudflare.com's ipv6hint values sit under the 2606:4700 prefix —
# checking that `26064700` is absent from the rdata hex is a precise,
# upstream-stable signal that the TLV was stripped.
HTTPS_OUT=$($DIG cloudflare.com type65 2>&1)
if echo "$HTTPS_OUT" | grep -qE "cloudflare\.com\..*IN[[:space:]]+TYPE65"; then
    HTTPS_HEX=$(echo "$HTTPS_OUT" | grep -A5 "IN[[:space:]]*TYPE65" | tr -d " \t\n")
    if echo "$HTTPS_HEX" | grep -qi "26064700"; then
        check "HTTPS ipv6hint stripped (2606:4700 absent from rdata)" "absent" "present"
    else
        check "HTTPS ipv6hint stripped (2606:4700 absent from rdata)" "absent" "absent"
    fi
else
    # Upstream didn't return an HTTPS record — skip rather than false-pass.
    printf "  ${DIM}~ HTTPS ipv6hint stripped (skipped: no HTTPS RR returned by upstream)${RESET}\n"
fi

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

# Config B — filter off. Regression guard: prove AAAA answers come back
# when the flag isn't set, so a network failure in Config A can't silently
# pass as "filter working".
cat > "$CONFIG" << 'CONF'
[server]
bind_addr = "127.0.0.1:5354"
api_port = 5381

[upstream]
mode = "forward"
address = "9.9.9.9"
port = 53

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = false
CONF

RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
sleep 3

echo ""
echo "=== filter_aaaa unset (regression guard) ==="

check "AAAA returns real answers with filter off" \
    ":" \
    "$($DIG google.com AAAA +short | head -1)"

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

fi  # end Suite 7

# ---- Suite 8: ODoH (Oblivious DoH via public relay + target) ----
# Exercises the full client pipeline: /.well-known/odohconfigs fetch,
# HPKE seal/unseal, URL-query target routing (RFC 9230 §5), dashboard
# QueryPath::Odoh counter. Depends on the public ecosystem being up —
# the probe-odoh-ecosystem.sh script guards against flaky runs.
if should_run_suite 8; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 8: ODoH (Anonymous DNS)           ║"
echo "╚══════════════════════════════════════════╝"

run_test_suite "ODoH via edgecompute.app relay → Cloudflare target" "
[server]
bind_addr = \"127.0.0.1:5354\"
api_port = 5381

[upstream]
mode = \"odoh\"
relay = \"https://odoh-relay.edgecompute.app/proxy\"
target = \"https://odoh.cloudflare-dns.com/dns-query\"

[cache]
max_entries = 10000
min_ttl = 60
max_ttl = 86400

[blocking]
enabled = false

[proxy]
enabled = false
"

# Re-start briefly to assert ODoH-specific observability: the odoh counter
# has to tick above zero after a query, and the stats label has to reflect
# the oblivious path. These guard against silent regressions in the
# QueryPath::Odoh tagging and the /stats serialisation.
RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
for _ in $(seq 1 30); do
    curl -sf "http://127.0.0.1:$API_PORT/health" >/dev/null 2>&1 && break
    sleep 0.1
done

$DIG example.com A +short > /dev/null 2>&1 || true
sleep 1

STATS=$(curl -sf http://127.0.0.1:$API_PORT/stats 2>/dev/null)
# upstream_transport.odoh lives inside the upstream_transport object.
ODOH_COUNT=$(echo "$STATS" | grep -o '"upstream_transport":{[^}]*}' \
    | grep -o '"odoh":[0-9]*' | cut -d: -f2)
check "upstream_transport.odoh > 0 after a query" "[1-9]" "${ODOH_COUNT:-0}"

check "Upstream label advertises odoh://" \
    "odoh://" \
    "$(echo "$STATS" | grep -o '"upstream":"[^"]*"')"

check "Stats mode field is 'odoh'" \
    '"mode":"odoh"' \
    "$(echo "$STATS" | grep -o '"mode":"odoh"')"

# Strict-mode failure path: a clearly-unreachable relay must produce
# SERVFAIL without silent downgrade. We hijack the config to point at
# an .invalid host so we don't rely on external uptime.
kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

cat > "$CONFIG" << 'CONF'
[server]
bind_addr = "127.0.0.1:5354"
api_port = 5381

[upstream]
mode = "odoh"
relay = "https://relay.invalid/proxy"
target = "https://odoh.cloudflare-dns.com/dns-query"
strict = true

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = false
CONF

RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
for _ in $(seq 1 30); do
    curl -sf "http://127.0.0.1:$API_PORT/health" >/dev/null 2>&1 && break
    sleep 0.1
done

check "Strict-mode relay outage returns SERVFAIL" \
    "SERVFAIL" \
    "$($DIG example.com A 2>&1 | grep 'status:')"

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

# Negative: relay and target on the same host must be rejected at startup.
cat > "$CONFIG" << 'CONF'
[server]
bind_addr = "127.0.0.1:5354"
api_port = 5381

[upstream]
mode = "odoh"
relay = "https://odoh.cloudflare-dns.com/proxy"
target = "https://odoh.cloudflare-dns.com/dns-query"
CONF

STARTUP_OUT=$("$BINARY" "$CONFIG" 2>&1 || true)
check "Same-host relay+target rejected at startup" \
    "same host" \
    "$STARTUP_OUT"

# relay_ip / target_ip must land in the bootstrap resolver's override map,
# so reqwest connects direct to the configured IPs instead of resolving the
# hostnames via plain DNS (ODoH's zero-plain-DNS-leak property). Using
# RFC 5737 TEST-NET-1 IPs — never routable, so the OdohConfigCache won't
# actually connect, but the override-map wiring is visible in the startup log.
cat > "$CONFIG" << 'CONF'
[server]
bind_addr = "127.0.0.1:5354"
api_port = 5381

[upstream]
mode = "odoh"
relay = "https://odoh-relay.example.com/proxy"
target = "https://odoh-target.example.org/dns-query"
relay_ip = "192.0.2.1"
target_ip = "192.0.2.2"

[cache]
max_entries = 10000

[blocking]
enabled = false

[proxy]
enabled = false
CONF

RUST_LOG=info "$BINARY" "$CONFIG" > "$LOG" 2>&1 &
NUMA_PID=$!
for _ in $(seq 1 30); do
    curl -sf "http://127.0.0.1:$API_PORT/health" >/dev/null 2>&1 && break
    sleep 0.1
done

OVERRIDE_LOG=$(grep 'bootstrap resolver: host overrides' "$LOG" || true)
check "relay_ip wired into bootstrap override map" \
    "odoh-relay.example.com=192.0.2.1" \
    "$OVERRIDE_LOG"
check "target_ip wired into bootstrap override map" \
    "odoh-target.example.org=192.0.2.2" \
    "$OVERRIDE_LOG"

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

fi  # end Suite 8

# ---- Suite 9: Numa's own ODoH relay (--relay-mode) ----
# Exercises `numa relay PORT` as a forwarding proxy to a real ODoH target.
# Validates the RFC 9230 §5 relay behaviour: URL-query routing, content-type
# gating, body-size cap, and /health observability.
if should_run_suite 9; then
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  Suite 9: Numa ODoH Relay (own)          ║"
echo "╚══════════════════════════════════════════╝"

RELAY_PORT=18443
"$BINARY" relay $RELAY_PORT > "$LOG" 2>&1 &
NUMA_PID=$!
for _ in $(seq 1 30); do
    curl -sf "http://127.0.0.1:$RELAY_PORT/health" >/dev/null 2>&1 && break
    sleep 0.1
done

echo ""
echo "=== Relay Endpoints ==="

check "Health endpoint returns ok" \
    "ok" \
    "$(curl -sf http://127.0.0.1:$RELAY_PORT/health | head -1)"

# Happy path: forwards arbitrary body to Cloudflare's ODoH target. The
# target will reject the garbage envelope with HTTP 400 — which is exactly
# what proves our relay faithfully forwarded (otherwise we'd see our own
# 4xx from the relay itself).
HAPPY_STATUS=$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
    -H "Content-Type: application/oblivious-dns-message" \
    --data-binary "garbage-forwarded-end-to-end" \
    "http://127.0.0.1:$RELAY_PORT/relay?targethost=odoh.cloudflare-dns.com&targetpath=/dns-query")
check "Relay forwards to target (target rejects garbage → 400)" \
    "400" \
    "$HAPPY_STATUS"

echo ""
echo "=== Guards ==="

check "Missing content-type → 415" \
    "415" \
    "$(curl -sS -o /dev/null -w '%{http_code}' -X POST --data-binary 'x' \
        'http://127.0.0.1:'$RELAY_PORT'/relay?targethost=odoh.cloudflare-dns.com&targetpath=/dns-query')"

check "Oversized body (>4 KiB) → 413" \
    "413" \
    "$(head -c 5000 /dev/urandom | curl -sS -o /dev/null -w '%{http_code}' -X POST \
        -H 'Content-Type: application/oblivious-dns-message' --data-binary @- \
        'http://127.0.0.1:'$RELAY_PORT'/relay?targethost=odoh.cloudflare-dns.com&targetpath=/dns-query')"

check "Invalid targethost (no dot) → 400" \
    "400" \
    "$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
        -H 'Content-Type: application/oblivious-dns-message' --data-binary 'x' \
        'http://127.0.0.1:'$RELAY_PORT'/relay?targethost=invalid&targetpath=/dns-query')"

echo ""
echo "=== Counters ==="

HEALTH=$(curl -sf "http://127.0.0.1:$RELAY_PORT/health")
check "Relay counted at least one forwarded_ok" \
    "[1-9]" \
    "$(echo "$HEALTH" | grep 'forwarded_ok' | awk '{print $2}')"
check "Relay counted at least one rejected_bad_request" \
    "[1-9]" \
    "$(echo "$HEALTH" | grep 'rejected_bad_request' | awk '{print $2}')"

kill "$NUMA_PID" 2>/dev/null || true
wait "$NUMA_PID" 2>/dev/null || true
sleep 1

fi  # end Suite 9

# Summary
echo ""
TOTAL=$((PASSED + FAILED))
if [ "$FAILED" -eq 0 ]; then
    printf "${GREEN}All %d tests passed.${RESET}\n" "$TOTAL"
    exit 0
else
    printf "${RED}%d/%d tests failed.${RESET}\n" "$FAILED" "$TOTAL"
    echo ""
    echo "Log: $LOG"
    exit 1
fi