numa.toml

@@ -66,6 +66,13 @@ api_port = 5380
 # [[forwarding]]                                    # DoH upstream: full https:// URL
 # suffix = "example.corp"
 # upstream = "https://dns.quad9.net/dns-query"
+#
+# [[forwarding]]                                    # array of upstreams → SRTT-aware failover
+# suffix = ["google.com", "goog"]                   # fastest-healthy first, dead one skipped
+# upstream = [
+#     "tls://9.9.9.9#dns.quad9.net",
+#     "tls://149.112.112.112#dns.quad9.net",
+# ]
 
 # [blocking]
 # enabled = true               # set to false to disable ad blocking

src/config.rs

@@ -41,17 +41,30 @@ pub struct Config {
 pub struct ForwardingRuleConfig {
     #[serde(deserialize_with = "string_or_vec")]
     pub suffix: Vec<String>,
-    pub upstream: String,
+    #[serde(deserialize_with = "string_or_vec")]
+    pub upstream: Vec<String>,
 }
 
 impl ForwardingRuleConfig {
     fn to_runtime_rules(&self) -> Result<Vec<crate::system_dns::ForwardingRule>> {
-        let upstream = crate::forward::parse_upstream(&self.upstream, 53)
-            .map_err(|e| format!("forwarding rule for upstream '{}': {}", self.upstream, e))?;
+        if self.upstream.is_empty() {
+            return Err(format!(
+                "forwarding rule for suffix {:?}: upstream must not be empty",
+                self.suffix
+            )
+            .into());
+        }
+        let mut primary = Vec::with_capacity(self.upstream.len());
+        for s in &self.upstream {
+            let u = crate::forward::parse_upstream(s, 53)
+                .map_err(|e| format!("forwarding rule for upstream '{}': {}", s, e))?;
+            primary.push(u);
+        }
+        let pool = crate::forward::UpstreamPool::new(primary, vec![]);
         Ok(self
             .suffix
             .iter()
-            .map(|s| crate::system_dns::ForwardingRule::new(s.clone(), upstream.clone()))
+            .map(|s| crate::system_dns::ForwardingRule::new(s.clone(), pool.clone()))
             .collect())
     }
 }
@@ -643,7 +656,7 @@ mod tests {
         let config: Config = toml::from_str(toml).unwrap();
         assert_eq!(config.forwarding.len(), 1);
         assert_eq!(config.forwarding[0].suffix, &["home.local"]);
-        assert_eq!(config.forwarding[0].upstream, "100.90.1.63:5361");
+        assert_eq!(config.forwarding[0].upstream, vec!["100.90.1.63:5361"]);
     }
 
     #[test]
@@ -671,7 +684,7 @@ mod tests {
         "#;
         let config: Config = toml::from_str(toml).unwrap();
         assert_eq!(config.forwarding.len(), 2);
-        assert_eq!(config.forwarding[1].upstream, "10.0.0.1");
+        assert_eq!(config.forwarding[1].upstream, vec!["10.0.0.1"]);
     }
 
     #[test]
@@ -693,28 +706,29 @@ mod tests {
     fn forwarding_suffix_array_expands_to_multiple_runtime_rules() {
         let rule = ForwardingRuleConfig {
             suffix: vec!["168.192.in-addr.arpa".to_string(), "onsite".to_string()],
-            upstream: "192.168.88.1".to_string(),
+            upstream: vec!["192.168.88.1".to_string()],
         };
         let runtime = rule.to_runtime_rules().unwrap();
         assert_eq!(runtime.len(), 2);
         assert_eq!(runtime[0].suffix, "168.192.in-addr.arpa");
         assert_eq!(runtime[1].suffix, "onsite");
-        assert_eq!(runtime[0].upstream, runtime[1].upstream);
+        assert_eq!(
+            runtime[0].upstream.preferred(),
+            runtime[1].upstream.preferred()
+        );
     }
 
     #[test]
     fn forwarding_upstream_with_explicit_port() {
         let rule = ForwardingRuleConfig {
             suffix: vec!["home.local".to_string()],
-            upstream: "100.90.1.63:5361".to_string(),
+            upstream: vec!["100.90.1.63:5361".to_string()],
         };
         let runtime = rule.to_runtime_rules().unwrap();
         assert_eq!(runtime.len(), 1);
-        assert!(matches!(
-            runtime[0].upstream,
-            crate::forward::Upstream::Udp(_)
-        ));
-        assert_eq!(runtime[0].upstream.to_string(), "100.90.1.63:5361");
+        let preferred = runtime[0].upstream.preferred().unwrap();
+        assert!(matches!(preferred, crate::forward::Upstream::Udp(_)));
+        assert_eq!(preferred.to_string(), "100.90.1.63:5361");
         assert_eq!(runtime[0].suffix, "home.local");
     }
 
@@ -722,17 +736,20 @@ mod tests {
     fn forwarding_upstream_defaults_to_port_53() {
         let rule = ForwardingRuleConfig {
             suffix: vec!["home.local".to_string()],
-            upstream: "100.90.1.63".to_string(),
+            upstream: vec!["100.90.1.63".to_string()],
         };
         let runtime = rule.to_runtime_rules().unwrap();
-        assert_eq!(runtime[0].upstream.to_string(), "100.90.1.63:53");
+        assert_eq!(
+            runtime[0].upstream.preferred().unwrap().to_string(),
+            "100.90.1.63:53"
+        );
     }
 
     #[test]
     fn forwarding_invalid_upstream_returns_error() {
         let rule = ForwardingRuleConfig {
             suffix: vec!["home.local".to_string()],
-            upstream: "not-a-valid-host".to_string(),
+            upstream: vec!["not-a-valid-host".to_string()],
         };
         assert!(rule.to_runtime_rules().is_err());
     }
@@ -741,14 +758,14 @@ mod tests {
     fn forwarding_upstream_accepts_dot_scheme() {
         let rule = ForwardingRuleConfig {
             suffix: vec!["google.com".to_string()],
-            upstream: "tls://9.9.9.9#dns.quad9.net".to_string(),
+            upstream: vec!["tls://9.9.9.9#dns.quad9.net".to_string()],
         };
         let runtime = rule
             .to_runtime_rules()
             .expect("tls:// upstream should parse");
         assert_eq!(runtime.len(), 1);
         assert_eq!(
-            runtime[0].upstream.to_string(),
+            runtime[0].upstream.preferred().unwrap().to_string(),
             "tls://9.9.9.9:853#dns.quad9.net"
         );
     }
@@ -757,14 +774,14 @@ mod tests {
     fn forwarding_upstream_accepts_doh_scheme() {
         let rule = ForwardingRuleConfig {
             suffix: vec!["goog".to_string()],
-            upstream: "https://dns.quad9.net/dns-query".to_string(),
+            upstream: vec!["https://dns.quad9.net/dns-query".to_string()],
         };
         let runtime = rule
             .to_runtime_rules()
             .expect("https:// upstream should parse");
         assert_eq!(runtime.len(), 1);
         assert_eq!(
-            runtime[0].upstream.to_string(),
+            runtime[0].upstream.preferred().unwrap().to_string(),
             "https://dns.quad9.net/dns-query"
         );
     }
@@ -773,44 +790,90 @@ mod tests {
     fn forwarding_config_rules_take_precedence_over_discovered() {
         let config_rules = vec![ForwardingRuleConfig {
             suffix: vec!["home.local".to_string()],
-            upstream: "10.0.0.1:53".to_string(),
+            upstream: vec!["10.0.0.1:53".to_string()],
         }];
         let discovered = vec![crate::system_dns::ForwardingRule::new(
             "home.local".to_string(),
-            crate::forward::Upstream::Udp("192.168.1.1:53".parse().unwrap()),
+            crate::forward::UpstreamPool::new(
+                vec![crate::forward::Upstream::Udp(
+                    "192.168.1.1:53".parse().unwrap(),
+                )],
+                vec![],
+            ),
         )];
         let merged = merge_forwarding_rules(&config_rules, discovered).unwrap();
         let picked = crate::system_dns::match_forwarding_rule("host.home.local", &merged)
             .expect("rule should match");
-        assert_eq!(picked.to_string(), "10.0.0.1:53");
+        assert_eq!(picked.preferred().unwrap().to_string(), "10.0.0.1:53");
     }
 
     #[test]
     fn forwarding_merge_preserves_non_overlapping_discovered() {
         let config_rules = vec![ForwardingRuleConfig {
             suffix: vec!["home.local".to_string()],
-            upstream: "10.0.0.1:53".to_string(),
+            upstream: vec!["10.0.0.1:53".to_string()],
         }];
         let discovered = vec![crate::system_dns::ForwardingRule::new(
             "corp.example".to_string(),
-            crate::forward::Upstream::Udp("192.168.1.1:53".parse().unwrap()),
+            crate::forward::UpstreamPool::new(
+                vec![crate::forward::Upstream::Udp(
+                    "192.168.1.1:53".parse().unwrap(),
+                )],
+                vec![],
+            ),
         )];
         let merged = merge_forwarding_rules(&config_rules, discovered).unwrap();
         assert_eq!(merged.len(), 2);
         let picked = crate::system_dns::match_forwarding_rule("host.corp.example", &merged)
             .expect("discovered rule should still match");
-        assert_eq!(picked.to_string(), "192.168.1.1:53");
+        assert_eq!(picked.preferred().unwrap().to_string(), "192.168.1.1:53");
     }
 
     #[test]
     fn forwarding_merge_suffix_array_expands_to_multiple_rules() {
         let config_rules = vec![ForwardingRuleConfig {
             suffix: vec!["a.local".to_string(), "b.local".to_string()],
-            upstream: "10.0.0.1:53".to_string(),
+            upstream: vec!["10.0.0.1:53".to_string()],
         }];
         let merged = merge_forwarding_rules(&config_rules, vec![]).unwrap();
         assert_eq!(merged.len(), 2);
     }
+
+    #[test]
+    fn forwarding_parses_upstream_array() {
+        let toml = r#"
+            [[forwarding]]
+            suffix = "google.com"
+            upstream = ["tls://9.9.9.9#dns.quad9.net", "tls://149.112.112.112#dns.quad9.net"]
+        "#;
+        let config: Config = toml::from_str(toml).unwrap();
+        assert_eq!(config.forwarding.len(), 1);
+        assert_eq!(config.forwarding[0].upstream.len(), 2);
+    }
+
+    #[test]
+    fn forwarding_upstream_array_builds_pool_with_multiple_primaries() {
+        let rule = ForwardingRuleConfig {
+            suffix: vec!["google.com".to_string()],
+            upstream: vec![
+                "tls://9.9.9.9#dns.quad9.net".to_string(),
+                "tls://149.112.112.112#dns.quad9.net".to_string(),
+            ],
+        };
+        let runtime = rule.to_runtime_rules().unwrap();
+        assert_eq!(runtime.len(), 1);
+        let label = runtime[0].upstream.label();
+        assert!(label.contains("+1 more"), "label was: {}", label);
+    }
+
+    #[test]
+    fn forwarding_empty_upstream_array_errors() {
+        let rule = ForwardingRuleConfig {
+            suffix: vec!["home.local".to_string()],
+            upstream: vec![],
+        };
+        assert!(rule.to_runtime_rules().is_err());
+    }
 }
 
 pub struct ConfigLoad {

src/ctx.rs

@@ -16,7 +16,9 @@ use crate::blocklist::BlocklistStore;
 use crate::buffer::BytePacketBuffer;
 use crate::cache::{DnsCache, DnssecStatus};
 use crate::config::{UpstreamMode, ZoneMap};
-use crate::forward::{forward_query_raw, forward_with_failover_raw, Upstream, UpstreamPool};
+#[cfg(test)]
+use crate::forward::Upstream;
+use crate::forward::{forward_with_failover_raw, UpstreamPool};
 use crate::header::ResultCode;
 use crate::health::HealthMeta;
 use crate::lan::PeerStore;
@@ -190,13 +192,31 @@ pub async fn resolve_query(
                     resp.header.authed_data = true;
                 }
                 (resp, QueryPath::Cached, cached_dnssec)
-            } else if let Some(upstream) =
+            } else if let Some(pool) =
                 crate::system_dns::match_forwarding_rule(&qname, &ctx.forwarding_rules)
             {
                 // Conditional forwarding takes priority over recursive mode
                 // (e.g. Tailscale .ts.net, VPC private zones)
-                match forward_and_cache(raw_wire, upstream, ctx, &qname, qtype).await {
+                match forward_with_failover_raw(
+                    raw_wire,
+                    pool,
+                    &ctx.srtt,
+                    ctx.timeout,
+                    ctx.hedge_delay,
+                )
+                .await
+                {
+                    Ok(resp_wire) => match cache_and_parse(ctx, &qname, qtype, &resp_wire) {
                         Ok(resp) => (resp, QueryPath::Forwarded, DnssecStatus::Indeterminate),
+                        Err(e) => {
+                            error!("{} | {:?} {} | PARSE ERROR | {}", src_addr, qtype, qname, e);
+                            (
+                                DnsPacket::response_from(&query, ResultCode::SERVFAIL),
+                                QueryPath::UpstreamError,
+                                DnssecStatus::Indeterminate,
+                            )
+                        }
+                    },
                     Err(e) => {
                         error!(
                             "{} | {:?} {} | FORWARD ERROR | {}",
@@ -433,17 +453,6 @@ pub async fn refresh_entry(ctx: &ServerCtx, qname: &str, qtype: QueryType) {
     }
 }
 
-async fn forward_and_cache(
-    wire: &[u8],
-    upstream: &Upstream,
-    ctx: &ServerCtx,
-    qname: &str,
-    qtype: QueryType,
-) -> crate::Result<DnsPacket> {
-    let resp_wire = forward_query_raw(wire, upstream, ctx.timeout).await?;
-    cache_and_parse(ctx, qname, qtype, &resp_wire)
-}
-
 pub async fn handle_query(
     mut buffer: BytePacketBuffer,
     raw_len: usize,
@@ -1082,7 +1091,7 @@ mod tests {
         let mut ctx = crate::testutil::test_ctx().await;
         ctx.forwarding_rules = vec![ForwardingRule::new(
             "168.192.in-addr.arpa".to_string(),
-            Upstream::Udp(upstream_addr),
+            UpstreamPool::new(vec![Upstream::Udp(upstream_addr)], vec![]),
         )];
         let ctx = Arc::new(ctx);
 
@@ -1237,7 +1246,7 @@ mod tests {
         let mut ctx = crate::testutil::test_ctx().await;
         ctx.forwarding_rules = vec![ForwardingRule::new(
             "corp".to_string(),
-            Upstream::Udp(upstream_addr),
+            UpstreamPool::new(vec![Upstream::Udp(upstream_addr)], vec![]),
         )];
         let ctx = Arc::new(ctx);
 
@@ -1254,6 +1263,37 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn pipeline_forwarding_fails_over_to_second_upstream() {
+        let dead = crate::testutil::blackhole_upstream();
+
+        let mut live_resp = DnsPacket::new();
+        live_resp.header.response = true;
+        live_resp.header.rescode = ResultCode::NOERROR;
+        live_resp.answers.push(DnsRecord::A {
+            domain: "internal.corp".to_string(),
+            addr: Ipv4Addr::new(10, 9, 9, 9),
+            ttl: 600,
+        });
+        let live = crate::testutil::mock_upstream(live_resp).await;
+
+        let mut ctx = crate::testutil::test_ctx().await;
+        ctx.forwarding_rules = vec![ForwardingRule::new(
+            "corp".to_string(),
+            UpstreamPool::new(vec![Upstream::Udp(dead), Upstream::Udp(live)], vec![]),
+        )];
+        let ctx = Arc::new(ctx);
+
+        let (resp, path) = resolve_in_test(&ctx, "internal.corp", QueryType::A).await;
+        assert_eq!(path, QueryPath::Forwarded);
+        assert_eq!(resp.header.rescode, ResultCode::NOERROR);
+        assert_eq!(resp.answers.len(), 1);
+        match &resp.answers[0] {
+            DnsRecord::A { addr, .. } => assert_eq!(*addr, Ipv4Addr::new(10, 9, 9, 9)),
+            other => panic!("expected A record, got {:?}", other),
+        }
+    }
+
     #[tokio::test]
     async fn pipeline_default_pool_reports_upstream_path() {
         let mut upstream_resp = DnsPacket::new();

src/forward.rs

@@ -25,6 +25,18 @@ pub enum Upstream {
     },
 }
 
+impl Upstream {
+    /// IP address to key SRTT tracking on, if the upstream has a stable one.
+    /// `Doh` routes through a URL + connection pool, so there's no single IP
+    /// to track; SRTT is skipped for it.
+    pub fn tracked_ip(&self) -> Option<IpAddr> {
+        match self {
+            Upstream::Udp(addr) | Upstream::Dot { addr, .. } => Some(addr.ip()),
+            Upstream::Doh { .. } => None,
+        }
+    }
+}
+
 impl PartialEq for Upstream {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
@@ -345,18 +357,17 @@ pub async fn forward_with_failover_raw(
     timeout_duration: Duration,
     hedge_delay: Duration,
 ) -> Result<Vec<u8>> {
-    let mut candidates: Vec<(usize, u64)> = pool
-        .primary
+    let mut candidates: Vec<(usize, u64)> = {
+        let srtt_read = srtt.read().unwrap();
+        pool.primary
             .iter()
             .enumerate()
             .map(|(i, u)| {
-            let rtt = match u {
-                Upstream::Udp(addr) => srtt.read().unwrap().get(addr.ip()),
-                _ => 0,
-            };
+                let rtt = u.tracked_ip().map(|ip| srtt_read.get(ip)).unwrap_or(0);
                 (i, rtt)
             })
-        .collect();
+            .collect()
+    };
     candidates.sort_by_key(|&(_, rtt)| rtt);
 
     let all_upstreams: Vec<&Upstream> = candidates
@@ -380,15 +391,15 @@ pub async fn forward_with_failover_raw(
         };
         match result {
             Ok(resp) => {
-                if let Upstream::Udp(addr) = upstream {
+                if let Some(ip) = upstream.tracked_ip() {
                     let rtt_ms = start.elapsed().as_millis() as u64;
-                    srtt.write().unwrap().record_rtt(addr.ip(), rtt_ms, false);
+                    srtt.write().unwrap().record_rtt(ip, rtt_ms, false);
                 }
                 return Ok(resp);
             }
             Err(e) => {
-                if let Upstream::Udp(addr) = upstream {
-                    srtt.write().unwrap().record_failure(addr.ip());
+                if let Some(ip) = upstream.tracked_ip() {
+                    srtt.write().unwrap().record_failure(ip);
                 }
                 log::debug!("upstream {} failed: {}", upstream, e);
                 last_err = Some(e);
@@ -707,4 +718,62 @@ mod tests {
         assert!(!pool.maybe_update_primary("not-an-ip", 53));
         assert_eq!(pool.preferred().unwrap().to_string(), "1.2.3.4:53");
     }
+
+    fn tcp_closed_port() -> SocketAddr {
+        // Bind a TCP listener, grab the port, drop → kernel returns RST on connect.
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let addr = listener.local_addr().unwrap();
+        drop(listener);
+        addr
+    }
+
+    #[tokio::test]
+    async fn udp_failure_records_in_srtt() {
+        let blackhole = crate::testutil::blackhole_upstream();
+        let pool = UpstreamPool::new(vec![Upstream::Udp(blackhole)], vec![]);
+        let srtt = RwLock::new(SrttCache::new(true));
+        let _ = forward_with_failover_raw(
+            &[0u8; 12],
+            &pool,
+            &srtt,
+            Duration::from_millis(100),
+            Duration::ZERO,
+        )
+        .await;
+        assert!(srtt.read().unwrap().is_known(blackhole.ip()));
+    }
+
+    #[tokio::test]
+    async fn dot_failure_records_in_srtt() {
+        let dead1 = tcp_closed_port();
+        let dead2 = tcp_closed_port();
+        let connector = build_dot_connector().unwrap();
+        let pool = UpstreamPool::new(
+            vec![
+                Upstream::Dot {
+                    addr: dead1,
+                    tls_name: Some("dns.quad9.net".to_string()),
+                    connector: connector.clone(),
+                },
+                Upstream::Dot {
+                    addr: dead2,
+                    tls_name: Some("dns.quad9.net".to_string()),
+                    connector,
+                },
+            ],
+            vec![],
+        );
+        let srtt = RwLock::new(SrttCache::new(true));
+        let _ = forward_with_failover_raw(
+            &[0u8; 12],
+            &pool,
+            &srtt,
+            Duration::from_millis(500),
+            Duration::ZERO,
+        )
+        .await;
+        let cache = srtt.read().unwrap();
+        assert!(cache.is_known(dead1.ip()));
+        assert!(cache.is_known(dead2.ip()));
+    }
 }

src/serve.rs

@@ -123,7 +123,11 @@ pub async fn run(config_path: String) -> crate::Result<()> {
 
     for fwd in &config.forwarding {
         for suffix in &fwd.suffix {
-            info!("forwarding .{} to {} (config rule)", suffix, fwd.upstream);
+            info!(
+                "forwarding .{} to {} (config rule)",
+                suffix,
+                fwd.upstream.join(", ")
+            );
         }
     }
     let forwarding_rules =

src/system_dns.rs

@@ -2,7 +2,9 @@ use std::net::SocketAddr;
 
 use log::info;
 
+#[cfg(any(target_os = "macos", target_os = "linux"))]
 use crate::forward::Upstream;
+use crate::forward::UpstreamPool;
 
 fn print_recursive_hint() {
     let is_recursive = crate::config::load_config("numa.toml")
@@ -20,15 +22,15 @@ fn is_loopback_or_stub(addr: &str) -> bool {
 }
 
 /// A conditional forwarding rule: domains matching `suffix` are forwarded to `upstream`.
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct ForwardingRule {
     pub suffix: String,
     dot_suffix: String, // pre-computed ".suffix" for zero-alloc matching
-    pub upstream: Upstream,
+    pub upstream: UpstreamPool,
 }
 
 impl ForwardingRule {
-    pub fn new(suffix: String, upstream: Upstream) -> Self {
+    pub fn new(suffix: String, upstream: UpstreamPool) -> Self {
         let dot_suffix = format!(".{}", suffix);
         Self {
             suffix,
@@ -216,7 +218,8 @@ fn discover_macos() -> SystemDnsInfo {
     for rule in &rules {
         info!(
             "auto-discovered forwarding: *.{} -> {}",
-            rule.suffix, rule.upstream
+            rule.suffix,
+            rule.upstream.label()
         );
     }
     if rules.is_empty() {
@@ -235,7 +238,8 @@ fn discover_macos() -> SystemDnsInfo {
 #[cfg(any(target_os = "macos", target_os = "linux"))]
 fn make_rule(domain: &str, nameserver: &str) -> Option<ForwardingRule> {
     let addr = crate::forward::parse_upstream_addr(nameserver, 53).ok()?;
-    Some(ForwardingRule::new(domain.to_string(), Upstream::Udp(addr)))
+    let pool = UpstreamPool::new(vec![Upstream::Udp(addr)], vec![]);
+    Some(ForwardingRule::new(domain.to_string(), pool))
 }
 
 #[cfg(target_os = "linux")]
@@ -1033,7 +1037,7 @@ fn uninstall_windows() -> Result<(), String> {
 pub fn match_forwarding_rule<'a>(
     domain: &str,
     rules: &'a [ForwardingRule],
-) -> Option<&'a Upstream> {
+) -> Option<&'a UpstreamPool> {
     for rule in rules {
         if domain == rule.suffix || domain.ends_with(&rule.dot_suffix) {
             return Some(&rule.upstream);