USHIFT-6796: C2CC: DNS forwarding between clusters

assets/components/openshift-dns/dns/configmap.yaml

@@ -1,6 +1,7 @@
 apiVersion: v1
 data:
   Corefile: |
+    {{- .C2CCDNSBlocks }}
     .:5353 {
         bufsize 1232
         errors

pkg/components/controllers.go

@@ -304,8 +304,12 @@ func startDNSController(ctx context.Context, cfg *config.Config, kubeconfigPath
 
 	hostsEnabled := cfg.DNS.Hosts.Status == config.HostsStatusEnabled
 	extraParams := assets.RenderParams{
-		"ClusterIP":    cfg.Network.DNS,
-		"HostsEnabled": hostsEnabled,
+		"ClusterIP":     cfg.Network.DNS,
+		"HostsEnabled":  hostsEnabled,
+		"C2CCDNSBlocks": "",
+	}
+	if cfg.C2CC.IsEnabled() {
+		extraParams["C2CCDNSBlocks"] = config.RenderC2CCDNSBlocks(cfg.C2CC.Resolved)
 	}
 
 	if err := assets.ApplyServices(ctx, svc, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {

pkg/config/c2cc.go

@@ -39,6 +39,7 @@ type ResolvedRemoteCluster struct {
 	ClusterNetwork []*net.IPNet
 	ServiceNetwork []*net.IPNet
 	Domain         string
+	DNSIP          string // 10th IP of ServiceNetwork[0], computed during validation when Domain is set
 }
 
 func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet {
@@ -264,6 +265,14 @@ func validateRemoteCluster(
 		} else {
 			seenRemoteDomains[rc.Domain] = i
 		}
+		if len(rc.ServiceNetwork) > 0 {
+			dnsIP, err := getClusterDNS(rc.ServiceNetwork[0])
+			if err != nil {
+				errs = append(errs, fmt.Errorf("%s: failed to compute DNS IP from serviceNetwork[0] %q: %w", label, rc.ServiceNetwork[0], err))
+			} else {
+				res.DNSIP = dnsIP
+			}
+		}
 	}
 
 	errs = append(errs, validateIPFamilyConsistencyNets(res.ClusterNetwork, label+".clusterNetwork")...)
@@ -345,3 +354,33 @@ func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labe
 func cidrsOverlap(a, b *net.IPNet) bool {
 	return a.Contains(b.IP) || b.Contains(a.IP)
 }
+
+// RenderC2CCDNSBlocks generates CoreDNS server blocks for cross-cluster DNS.
+func RenderC2CCDNSBlocks(resolved []ResolvedRemoteCluster) string {
+	var blocks []string
+	for _, rc := range resolved {
+		if rc.Domain == "" {
+			continue
+		}
+		blocks = append(blocks, formatDNSBlock(rc.Domain, rc.DNSIP))
+	}
+	if len(blocks) == 0 {
+		return ""
+	}
+	return "\n" + strings.Join(blocks, "\n")
+}
+
+func formatDNSBlock(domain, dnsIP string) string {
+	return fmt.Sprintf(`    %s:5353 {
+        bufsize 1232
+        errors
+        log . {
+            class error
+        }
+        rewrite stop name suffix .%s .cluster.local answer auto
+        forward . %s
+        cache 10 {
+            denial 9984 10
+        }
+    }`, domain, domain, dnsIP)
+}

pkg/config/c2cc_test.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"net"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestC2CC_IsEnabled(t *testing.T) {
@@ -541,3 +543,113 @@ func TestC2CC_ValidateDualStack(t *testing.T) {
 		assert.NoError(t, cfg.C2CC.validate(cfg))
 	})
 }
+
+func TestC2CC_DNSIP(t *testing.T) {
+	stubHostIPs(t, nil)
+
+	t.Run("DNSIP populated when domain is set", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			RemoteClusters: []RemoteCluster{{
+				NextHop:        "10.100.0.2",
+				ClusterNetwork: []string{"10.45.0.0/16"},
+				ServiceNetwork: []string{"10.46.0.0/16"},
+				Domain:         "cluster-b.remote",
+			}},
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, "10.46.0.10", cfg.C2CC.Resolved[0].DNSIP)
+	})
+
+	t.Run("DNSIP empty when domain is not set", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			RemoteClusters: []RemoteCluster{{
+				NextHop:        "10.100.0.2",
+				ClusterNetwork: []string{"10.45.0.0/16"},
+				ServiceNetwork: []string{"10.46.0.0/16"},
+			}},
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Empty(t, cfg.C2CC.Resolved[0].DNSIP)
+	})
+
+	t.Run("DNSIP for IPv6 service network", func(t *testing.T) {
+		cfg := mkIPv6OnlyC2CCConfig(C2CC{
+			RemoteClusters: []RemoteCluster{{
+				NextHop:        "fd00::2",
+				ClusterNetwork: []string{"fd03::/48"},
+				ServiceNetwork: []string{"fd04::/112"},
+				Domain:         "cluster-b.remote",
+			}},
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, "fd04::a", cfg.C2CC.Resolved[0].DNSIP)
+	})
+}
+
+func parseCIDR(t *testing.T, s string) *net.IPNet {
+	t.Helper()
+	_, ipNet, err := net.ParseCIDR(s)
+	require.NoError(t, err)
+	return ipNet
+}
+
+func TestRenderC2CCDNSBlocks(t *testing.T) {
+	t.Run("no domains configured", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{{
+			NextHop:        net.ParseIP("10.100.0.2"),
+			ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")},
+			ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")},
+		}}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.Empty(t, result)
+	})
+
+	t.Run("single domain", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{{
+			NextHop:        net.ParseIP("10.100.0.2"),
+			ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")},
+			ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")},
+			Domain:         "cluster-b.remote",
+			DNSIP:          "10.46.0.10",
+		}}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.True(t, strings.HasPrefix(result, "\n"), "result should start with newline for YAML block scalar")
+		assert.Contains(t, result, "cluster-b.remote:5353")
+		assert.Contains(t, result, "rewrite stop name suffix .cluster-b.remote .cluster.local answer auto")
+		assert.Contains(t, result, "forward . 10.46.0.10")
+		assert.Contains(t, result, "denial 9984 10")
+	})
+
+	t.Run("multiple domains", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{
+			{
+				Domain: "cluster-b.remote",
+				DNSIP:  "10.46.0.10",
+			},
+			{
+				Domain: "cluster-c.remote",
+				DNSIP:  "10.56.0.10",
+			},
+		}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.Contains(t, result, "cluster-b.remote:5353")
+		assert.Contains(t, result, "forward . 10.46.0.10")
+		assert.Contains(t, result, "cluster-c.remote:5353")
+		assert.Contains(t, result, "forward . 10.56.0.10")
+	})
+
+	t.Run("mixed domain and no-domain", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{
+			{
+				Domain: "cluster-b.remote",
+				DNSIP:  "10.46.0.10",
+			},
+			{
+				Domain: "",
+			},
+		}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.Contains(t, result, "cluster-b.remote:5353")
+		assert.NotContains(t, result, "cluster-c")
+	})
+}

pkg/controllers/c2cc/helpers_test.go

@@ -12,6 +12,7 @@ type testRemoteConfig struct {
 	nextHop        string
 	clusterNetwork []string
 	serviceNetwork []string
+	domain         string
 }
 
 func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRemoteConfig {
@@ -22,6 +23,15 @@ func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRem
 	}
 }
 
+func testRemoteWithDomain(nextHop string, clusterNetwork, serviceNetwork []string, domain string) testRemoteConfig {
+	return testRemoteConfig{
+		nextHop:        nextHop,
+		clusterNetwork: clusterNetwork,
+		serviceNetwork: serviceNetwork,
+		domain:         domain,
+	}
+}
+
 func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Config {
 	t.Helper()
 
@@ -32,6 +42,7 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co
 	for _, r := range remotes {
 		resolved := config.ResolvedRemoteCluster{
 			NextHop: net.ParseIP(r.nextHop),
+			Domain:  r.domain,
 		}
 		require.NotNil(t, resolved.NextHop, "invalid nextHop: %s", r.nextHop)
 

test/assets/c2cc/hello-microshift.yaml

@@ -10,10 +10,27 @@ spec:
   - name: hello-microshift
     image: quay.io/microshift/busybox:1.36
     command: ["/bin/sh"]
-    args: ["-c", "while true; do echo -ne \"HTTP/1.0 200 OK\r\nContent-Length: 16\r\n\r\nHello MicroShift\" | nc -l -p 8080 ; done"]
+    args:
+    - -c
+    - |
+      mkdir -p /tmp/www/cgi-bin
+      cat > /tmp/www/cgi-bin/hello <<SCRIPT
+      #!/bin/sh
+      echo "Content-Type: text/plain"
+      echo ""
+      SRC=\$(echo "\${REMOTE_ADDR}" | sed 's/\[//;s/\]//;s/^::ffff://')
+      echo "Hello from ${MY_POD_IP}, source: \${SRC}"
+      SCRIPT
+      chmod +x /tmp/www/cgi-bin/hello
+      httpd -f -p 8080 -h /tmp/www
     ports:
     - containerPort: 8080
       protocol: TCP
+    env:
+    - name: MY_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:

test/resources/c2cc.resource

@@ -222,3 +222,18 @@ Remove Foreign Subnet From SNAT Annotation
     ...    Triggers a reconcile by briefly corrupting the annotation.
     [Arguments]    ${alias}
     Corrupt Node SNAT Annotation On Cluster    ${alias}
+
+Verify Corefile Contains C2CC Server Block
+    [Documentation]    Check that the CoreDNS Corefile configmap contains a server block for the given domain.
+    [Arguments]    ${alias}    ${domain}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc get configmap dns-default -n openshift-dns -o jsonpath='{.data.Corefile}'
+    Should Contain    ${stdout}    ${domain}:5353
+    Should Contain    ${stdout}    rewrite stop name suffix .${domain} .cluster.local answer auto
+
+Verify Corefile Does Not Contain C2CC Server Block
+    [Documentation]    Check that the CoreDNS Corefile does NOT contain a server block for the given domain.
+    [Arguments]    ${alias}    ${domain}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc get configmap dns-default -n openshift-dns -o jsonpath='{.data.Corefile}'
+    Should Not Contain    ${stdout}    ${domain}:5353

test/scenarios-bootc/el9/presubmits/el98-src@c2cc.sh

@@ -115,6 +115,7 @@ scenario_run_tests() {
         suites/c2cc/sanity.robot \
         suites/c2cc/infrastructure.robot \
         suites/c2cc/connectivity.robot \
+        suites/c2cc/dns.robot \
         suites/c2cc/reconciliation.robot \
         suites/c2cc/cleanup.robot
 }

test/suites/c2cc/connectivity.robot

@@ -23,25 +23,53 @@ Pod To Pod From Cluster A To Cluster B
     [Documentation]    Verify pod on Cluster A can reach pod IP on Cluster B.
     ${pod_ip_b}=    Get Hello Pod IP    cluster-b
     ${stdout}=    Curl From Cluster    cluster-a    ${pod_ip_b}    8080
-    Should Contain    ${stdout}    Hello MicroShift
+    Should Contain    ${stdout}    Hello from
 
 Pod To Pod From Cluster B To Cluster A
     [Documentation]    Verify pod on Cluster B can reach pod IP on Cluster A.
     ${pod_ip_a}=    Get Hello Pod IP    cluster-a
     ${stdout}=    Curl From Cluster    cluster-b    ${pod_ip_a}    8080
-    Should Contain    ${stdout}    Hello MicroShift
+    Should Contain    ${stdout}    Hello from
 
 Pod To Service From Cluster A To Cluster B
     [Documentation]    Verify pod on Cluster A can reach service ClusterIP on Cluster B.
     ${svc_ip_b}=    Get Hello Service IP    cluster-b
     ${stdout}=    Curl From Cluster    cluster-a    ${svc_ip_b}    8080
-    Should Contain    ${stdout}    Hello MicroShift
+    Should Contain    ${stdout}    Hello from
 
 Pod To Service From Cluster B To Cluster A
     [Documentation]    Verify pod on Cluster B can reach service ClusterIP on Cluster A.
     ${svc_ip_a}=    Get Hello Service IP    cluster-a
     ${stdout}=    Curl From Cluster    cluster-b    ${svc_ip_a}    8080
-    Should Contain    ${stdout}    Hello MicroShift
+    Should Contain    ${stdout}    Hello from
+
+Source IP Preserved Pod To Pod From Cluster A To Cluster B
+    [Documentation]    Verify cross-cluster pod-to-pod traffic preserves the source pod IP (no SNAT).
+    ${curl_pod_ip}=    Get Curl Pod IP    cluster-a
+    ${pod_ip_b}=    Get Hello Pod IP    cluster-b
+    ${stdout}=    Curl From Cluster    cluster-a    ${pod_ip_b}    8080
+    Should Contain    ${stdout}    source: ${curl_pod_ip}
+
+Source IP Preserved Pod To Pod From Cluster B To Cluster A
+    [Documentation]    Verify cross-cluster pod-to-pod traffic preserves the source pod IP (no SNAT).
+    ${curl_pod_ip}=    Get Curl Pod IP    cluster-b
+    ${pod_ip_a}=    Get Hello Pod IP    cluster-a
+    ${stdout}=    Curl From Cluster    cluster-b    ${pod_ip_a}    8080
+    Should Contain    ${stdout}    source: ${curl_pod_ip}
+
+Source IP Preserved Pod To Service From Cluster A To Cluster B
+    [Documentation]    Verify cross-cluster pod-to-service traffic preserves the source pod IP (no SNAT).
+    ${curl_pod_ip}=    Get Curl Pod IP    cluster-a
+    ${svc_ip_b}=    Get Hello Service IP    cluster-b
+    ${stdout}=    Curl From Cluster    cluster-a    ${svc_ip_b}    8080
+    Should Contain    ${stdout}    source: ${curl_pod_ip}
+
+Source IP Preserved Pod To Service From Cluster B To Cluster A
+    [Documentation]    Verify cross-cluster pod-to-service traffic preserves the source pod IP (no SNAT).
+    ${curl_pod_ip}=    Get Curl Pod IP    cluster-b
+    ${svc_ip_a}=    Get Hello Service IP    cluster-a
+    ${stdout}=    Curl From Cluster    cluster-b    ${svc_ip_a}    8080
+    Should Contain    ${stdout}    source: ${curl_pod_ip}
 
 
 *** Keywords ***
@@ -99,9 +127,16 @@ Get Hello Service IP
     ...    oc get svc hello-microshift -n ${NAMESPACE} -o jsonpath='{.spec.clusterIP}'
     RETURN    ${ip}
 
+Get Curl Pod IP
+    [Documentation]    Get the pod IP of curl-pod on the given cluster.
+    [Arguments]    ${alias}
+    ${ip}=    Oc On Cluster    ${alias}
+    ...    oc get pod curl-pod -n ${NAMESPACE} -o jsonpath='{.status.podIP}'
+    RETURN    ${ip}
+
 Curl From Cluster
     [Documentation]    Exec curl from curl-pod on the given cluster to the target IP and port.
     [Arguments]    ${alias}    ${ip}    ${port}
     ${stdout}=    Oc On Cluster    ${alias}
-    ...    oc exec curl-pod -n ${NAMESPACE} -- curl -sS --max-time 10 http://${ip}:${port}
+    ...    oc exec curl-pod -n ${NAMESPACE} -- curl -sS --max-time 10 http://${ip}:${port}/cgi-bin/hello
     RETURN    ${stdout}