fix(cluster): resolve DNS failures on systemd-resolved hosts

Docker's embedded DNS at 127.0.0.11 is only reachable from the
container's own network namespace. k3s pods in child namespaces
cannot reach it, causing silent DNS failures on Ubuntu and other
systemd-resolved hosts where /etc/resolv.conf contains 127.0.0.53.

Sniff upstream DNS resolvers from the host in the Rust bootstrap
crate (reading /run/systemd/resolve/resolv.conf then /etc/resolv.conf),
filter loopback addresses (127.x.x.x and ::1), and pass the result
to the container as the UPSTREAM_DNS env var. The entrypoint checks
this env var first, falling back to /etc/resolv.conf for manual
container launches.

This follows the existing pattern used by registry config, SSH
gateway, GPU support, and image tags: the bootstrap discovers
host state and passes it as env vars.

Closes #437

Signed-off-by: Brian Taylor <brian.taylor818@gmail.com>

Author: brianwtaylor

crates/openshell-bootstrap/src/docker.rs

@@ -236,6 +236,49 @@ fn home_dir() -> Option<String> {
     std::env::var("HOME").ok()
 }
 
+/// Discover upstream DNS resolvers from the host's resolver configuration.
+///
+/// Reads systemd-resolved's upstream config first (`/run/systemd/resolve/resolv.conf`),
+/// then falls back to `/etc/resolv.conf`. Filters out loopback addresses (127.x.x.x
+/// and ::1) that are unreachable from k3s pod network namespaces.
+///
+/// Returns an empty vec if no usable resolvers are found.
+fn resolve_upstream_dns() -> Vec<String> {
+    let paths = ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"];
+
+    for path in &paths {
+        if let Ok(contents) = std::fs::read_to_string(path) {
+            let resolvers: Vec<String> = contents
+                .lines()
+                .filter_map(|line| {
+                    let line = line.trim();
+                    if !line.starts_with("nameserver") {
+                        return None;
+                    }
+                    let ip = line.split_whitespace().nth(1)?;
+                    if ip.starts_with("127.") || ip == "::1" {
+                        return None;
+                    }
+                    Some(ip.to_string())
+                })
+                .collect();
+
+            if !resolvers.is_empty() {
+                tracing::debug!(
+                    "Discovered {} upstream DNS resolver(s) from {}: {}",
+                    resolvers.len(),
+                    path,
+                    resolvers.join(", "),
+                );
+                return resolvers;
+            }
+        }
+    }
+
+    tracing::debug!("No upstream DNS resolvers found in host resolver config");
+    Vec::new()
+}
+
 /// Create an SSH Docker client from remote options.
 pub async fn create_ssh_docker_client(remote: &RemoteOptions) -> Result<Docker> {
     // Ensure destination has ssh:// prefix
@@ -675,6 +718,13 @@ pub async fn ensure_container(
         env_vars.push("GPU_ENABLED=true".to_string());
     }
 
+    // Pass upstream DNS resolvers discovered on the host so the entrypoint
+    // can configure k3s without probing files inside the container.
+    let upstream_dns = resolve_upstream_dns();
+    if !upstream_dns.is_empty() {
+        env_vars.push(format!("UPSTREAM_DNS={}", upstream_dns.join(",")));
+    }
+
     let env = Some(env_vars);
 
     let config = ContainerCreateBody {
@@ -1195,4 +1245,112 @@ mod tests {
             "should return a reasonable number of sockets"
         );
     }
+
+    #[test]
+    fn resolve_upstream_dns_filters_loopback() {
+        // This test validates the function runs without panic on the current host.
+        // The exact output depends on the host's DNS config, but loopback
+        // addresses must never appear in the result.
+        let resolvers = resolve_upstream_dns();
+        for r in &resolvers {
+            assert!(
+                !r.starts_with("127."),
+                "IPv4 loopback should be filtered: {r}"
+            );
+            assert_ne!(r, "::1", "IPv6 loopback should be filtered");
+        }
+    }
+
+    #[test]
+    fn resolve_upstream_dns_returns_vec() {
+        // Verify the function returns a vec (may be empty in some CI environments
+        // where no resolv.conf exists, but should never panic).
+        let resolvers = resolve_upstream_dns();
+        assert!(
+            resolvers.len() <= 20,
+            "should return a reasonable number of resolvers"
+        );
+    }
+
+    /// Helper: parse resolv.conf content using the same logic as resolve_upstream_dns().
+    /// Allows deterministic testing without depending on host DNS config.
+    fn parse_resolv_conf(contents: &str) -> Vec<String> {
+        contents
+            .lines()
+            .filter_map(|line| {
+                let line = line.trim();
+                if !line.starts_with("nameserver") {
+                    return None;
+                }
+                let ip = line.split_whitespace().nth(1)?;
+                if ip.starts_with("127.") || ip == "::1" {
+                    return None;
+                }
+                Some(ip.to_string())
+            })
+            .collect()
+    }
+
+    #[test]
+    fn parse_resolv_conf_filters_ipv4_loopback() {
+        let input = "nameserver 127.0.0.1\nnameserver 127.0.0.53\nnameserver 127.0.0.11\n";
+        assert!(parse_resolv_conf(input).is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_filters_ipv6_loopback() {
+        let input = "nameserver ::1\n";
+        assert!(parse_resolv_conf(input).is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_passes_real_resolvers() {
+        let input = "nameserver 8.8.8.8\nnameserver 1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_mixed_loopback_and_real() {
+        let input =
+            "nameserver 127.0.0.53\nnameserver ::1\nnameserver 10.0.0.1\nnameserver 172.16.0.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["10.0.0.1", "172.16.0.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_ignores_comments_and_other_lines() {
+        let input =
+            "# nameserver 8.8.8.8\nsearch example.com\noptions ndots:5\nnameserver 1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_handles_tabs_and_extra_spaces() {
+        let input = "nameserver\t8.8.8.8\nnameserver     1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_empty_input() {
+        assert!(parse_resolv_conf("").is_empty());
+        assert!(parse_resolv_conf("   \n\n").is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_bare_nameserver_keyword() {
+        assert!(parse_resolv_conf("nameserver\n").is_empty());
+        assert!(parse_resolv_conf("nameserver   \n").is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_systemd_resolved_typical() {
+        let input =
+            "# This is /run/systemd/resolve/resolv.conf\nnameserver 192.168.1.1\nsearch lan\n";
+        assert_eq!(parse_resolv_conf(input), vec!["192.168.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_crlf_line_endings() {
+        let input = "nameserver 8.8.8.8\r\nnameserver 1.1.1.1\r\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
 }

deploy/docker/cluster-entrypoint.sh

@@ -69,7 +69,46 @@ wait_for_default_route() {
 #   3. Adding DNAT rules so traffic to <eth0_ip>:53 reaches Docker's DNS
 #   4. Writing that IP into the k3s resolv.conf
 
+# Extract upstream DNS resolvers reachable from k3s pod namespaces.
+# Docker's embedded DNS (127.0.0.11) is namespace-local — DNAT to it from
+# pod traffic is dropped as a martian packet. Use real upstream servers instead.
+#
+# Priority:
+#   1. UPSTREAM_DNS env var (set by bootstrap, comma-separated)
+#   2. /etc/resolv.conf (fallback for non-bootstrap launches)
+get_upstream_resolvers() {
+    local resolvers=""
+
+    # Bootstrap-provided resolvers (sniffed from host by the Rust bootstrap crate)
+    if [ -n "${UPSTREAM_DNS:-}" ]; then
+        resolvers=$(printf '%s\n' "$UPSTREAM_DNS" | tr ',' '\n' | \
+            awk '{ip=$1; if(ip !~ /^127\./ && ip != "::1" && ip != "") print ip}')
+    fi
+
+    # Fallback: Docker-generated resolv.conf may have non-loopback servers
+    if [ -z "$resolvers" ]; then
+        resolvers=$(awk '/^nameserver/{ip=$2; gsub(/\r/,"",ip); if(ip !~ /^127\./ && ip != "::1") print ip}' \
+            /etc/resolv.conf)
+    fi
+
+    echo "$resolvers"
+}
+
 setup_dns_proxy() {
+    # Prefer upstream resolvers that work across network namespaces.
+    # This avoids the DNAT-to-loopback problem on systemd-resolved hosts.
+    UPSTREAM_DNS=$(get_upstream_resolvers)
+    if [ -n "$UPSTREAM_DNS" ]; then
+        : > "$RESOLV_CONF"
+        echo "$UPSTREAM_DNS" | while read -r ns; do
+            [ -n "$ns" ] && echo "nameserver $ns" >> "$RESOLV_CONF"
+        done
+        echo "DNS: using upstream resolvers directly (avoids cross-namespace DNAT)"
+        cat "$RESOLV_CONF"
+        return 0
+    fi
+
+    # Fall back to DNAT proxy when no upstream resolvers are available.
     # Extract Docker's actual DNS listener ports from the DOCKER_OUTPUT chain.
     # Docker sets up rules like:
     #   -A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp --dport 53 -j DNAT --to-destination 127.0.0.11:<port>
@@ -160,6 +199,8 @@ verify_dns() {
         sleep 1
         i=$((i + 1))
     done
+    echo "Warning: DNS verification failed for $lookup_host after $attempts attempts"
+    echo "  resolv.conf: $(head -3 "$RESOLV_CONF" 2>/dev/null)"
     return 1
 }