test(e2e): add cross mesh granularity

Makefile

@@ -87,7 +87,7 @@ unit:
 test: lint unit e2e
 
 e2e:
-	KILO_IMAGE=squat/kilo:test bash_unit $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/multi-cluster.sh ./e2e/handlers.sh ./e2e/kgctl.sh ./e2e/teardown.sh
+	KILO_IMAGE=squat/kilo:test bash_unit $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/cross-mesh.sh ./e2e/multi-cluster.sh ./e2e/handlers.sh ./e2e/kgctl.sh ./e2e/teardown.sh
 
 docs/kg.md:
 	go run ./cmd/kg/... --help | head -n -2 > help.txt

cmd/kg/main.go

@@ -77,6 +77,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.CrossGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -237,6 +238,7 @@ func runRoot(_ *cobra.Command, _ []string) error {
 	switch gr {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.CrossGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", granularity, availableGranularities)
 	}

cmd/kgctl/main.go

@@ -49,6 +49,7 @@ var (
 	availableGranularities = []string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.CrossGranularity),
 		string(mesh.AutoGranularity),
 	}
 	availableLogLevels = []string{
@@ -91,6 +92,7 @@ func runRoot(c *cobra.Command, _ []string) error {
 	switch opts.granularity {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.CrossGranularity:
 	case mesh.AutoGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %s unknown; posible values are: %s", granularity, availableGranularities)
@@ -164,8 +166,9 @@ func determineGranularity(gr mesh.Granularity, ns []*mesh.Node) (mesh.Granularit
 		switch ret {
 		case mesh.LogicalGranularity:
 		case mesh.FullGranularity:
+		case mesh.CrossGranularity:
 		default:
-			return ret, fmt.Errorf("mesh granularity %s is not supported", opts.granularity)
+			return ret, fmt.Errorf("mesh granularity %s is not supported", ret)
 		}
 		return ret, nil
 	}

docs/kg.md

@@ -50,7 +50,7 @@ Flags:
       --local                          Should Kilo manage routes within a location? (default true)
       --log-level string               Log level to use. Possible values: all, debug, info, warn, error, none (default "info")
       --master string                  The address of the Kubernetes API server (overrides any value in kubeconfig).
-      --mesh-granularity string        The granularity of the network mesh to create. Possible values: location, full (default "location")
+      --mesh-granularity string        The granularity of the network mesh to create. Possible values: location, full, cross (default "location")
       --mtu string                     The MTU of the WireGuard interface created by Kilo. Set to 'auto' to detect from the underlay interface. (default "auto")
       --port int                       The port over which WireGuard peers should communicate. (default 51820)
       --prioritise-private-addresses   Prefer to assign a private IP address to the node's endpoint.

docs/topology.md

@@ -47,6 +47,11 @@ kgctl graph | circo -Tsvg > cluster.svg
 
 <img src="./graphs/full-mesh.svg" />
 
+# Cross Mesh
+
+In this topology all nodes within the same location are not encrypted. Traffic to any other node outside of current location is encrypted
+with direct node-to-node encryption. To use this mesh specify `--mesh-granularity=cross`.
+
 ## Mixed 
 
 The `kilo.squat.ai/location` annotation can be used to create cluster mixing some fully meshed nodes and some nodes grouped by logical location.

e2e/cross-mesh.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# This suite exercises --mesh-granularity=cross on the bridge-CNI test
+# cluster. Cross drops the WireGuard tunnel between nodes that share a
+# location and expects the underlying CNI to handle intra-location
+# traffic over its own overlay (e.g. Cilium VXLAN). The Kilo bridge CNI
+# used by this kind cluster has no such overlay, so cross-location peer
+# topology can be validated here but pod-to-pod connectivity cannot —
+# that lives in the Cilium-CNI suite (e2e/cilium-cross-mesh.sh).
+
+setup_suite() {
+	# Place control-plane and the first worker into one location, and the
+	# second worker into another, so that "cross" produces tunnels only
+	# between the two locations and not within a single location.
+	_kubectl annotate node "$KIND_CLUSTER-control-plane" kilo.squat.ai/location=loc-a --overwrite
+	_kubectl annotate node "$KIND_CLUSTER-worker"        kilo.squat.ai/location=loc-a --overwrite
+	_kubectl annotate node "$KIND_CLUSTER-worker2"       kilo.squat.ai/location=loc-b --overwrite
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=cross"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace
+}
+
+# Restore the cluster to a clean state for the suites that follow
+# (multi-cluster.sh, handlers.sh, kgctl.sh): remove the location
+# annotations this suite added and roll the DaemonSet back to
+# --mesh-granularity=location, matching the state location-mesh.sh
+# leaves behind.
+teardown_suite() {
+	_kubectl annotate node "$KIND_CLUSTER-control-plane" kilo.squat.ai/location- 2>/dev/null || true
+	_kubectl annotate node "$KIND_CLUSTER-worker"        kilo.squat.ai/location- 2>/dev/null || true
+	_kubectl annotate node "$KIND_CLUSTER-worker2"       kilo.squat.ai/location- 2>/dev/null || true
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace
+}
+
+test_cross_mesh_peer() {
+	check_peer wg99 e2e 10.5.0.1/32 cross
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity cross)"
+}
+
+# In "cross" granularity, every node in another location must appear as a
+# WireGuard peer (direct tunnels across locations), while nodes in the same
+# location must NOT appear as peers (intra-location traffic stays on the CNI).
+# In "location" the same-location worker would not have any [Peer] entry at
+# all (it is a non-leader); in "full" both same- and cross-location nodes
+# would appear as peers. This sanity-checks that "cross" sits in between.
+test_cross_peer_topology() {
+	local CP_PEERS WORKER_PEERS WORKER2_PEERS
+	CP_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-control-plane" | grep -c '^\[Peer\]')
+	WORKER_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker"        | grep -c '^\[Peer\]')
+	WORKER2_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker2"      | grep -c '^\[Peer\]')
+	# Each loc-a node should peer only with the single loc-b node.
+	assert_equals "1" "$CP_PEERS"      "control-plane (loc-a) should have 1 peer (the loc-b node)"
+	assert_equals "1" "$WORKER_PEERS"  "worker (loc-a) should have 1 peer (the loc-b node)"
+	# The loc-b node should peer with both loc-a nodes.
+	assert_equals "2" "$WORKER2_PEERS" "worker2 (loc-b) should have 2 peers (both loc-a nodes)"
+}

pkg/k8s/backend.go

@@ -340,6 +340,7 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 		switch meshGranularity {
 		case mesh.LogicalGranularity:
 		case mesh.FullGranularity:
+		case mesh.CrossGranularity:
 		default:
 			meshGranularity = ""
 		}

pkg/mesh/backend.go

@@ -50,6 +50,9 @@ const (
 	// FullGranularity indicates that the network should create
 	// a mesh between every node.
 	FullGranularity Granularity = "full"
+	// CrossGranularity indicates that network is encrypted only
+	// between nodes in different locations.
+	CrossGranularity Granularity = "cross"
 	// AutoGranularity can be used with kgctl to obtain
 	// the granularity automatically.
 	AutoGranularity Granularity = "auto"

pkg/mesh/graph.go

@@ -49,17 +49,24 @@ func (t *Topology) Dot() (string, error) {
 	}
 
 	for i, s := range t.segments {
-		if err := g.AddSubGraph("kilo", subGraphName(s.location), nil); err != nil {
+		location := s.location
+		plainConnection := false
+		if s.nodeLocation != "" {
+			location = s.nodeLocation
+			plainConnection = true
+		}
+
+		if err := g.AddSubGraph("kilo", subGraphName(location), nil); err != nil {
 			return "", fmt.Errorf("failed to add subgraph")
 		}
-		if err := g.AddAttr(subGraphName(s.location), string(gographviz.Label), graphEscape(s.location)); err != nil {
+		if err := g.AddAttr(subGraphName(location), string(gographviz.Label), graphEscape(location)); err != nil {
 			return "", fmt.Errorf("failed to add label to subgraph")
 		}
-		if err := g.AddAttr(subGraphName(s.location), string(gographviz.Style), `"dashed,rounded"`); err != nil {
+		if err := g.AddAttr(subGraphName(location), string(gographviz.Style), `"dashed,rounded"`); err != nil {
 			return "", fmt.Errorf("failed to add style to subgraph")
 		}
 		for j := range s.cidrs {
-			if err := g.AddNode(subGraphName(s.location), graphEscape(s.hostnames[j]), nodeAttrs); err != nil {
+			if err := g.AddNode(subGraphName(location), graphEscape(s.hostnames[j]), nodeAttrs); err != nil {
 				return "", fmt.Errorf("failed to add node to subgraph")
 			}
 			var wg net.IP
@@ -75,11 +82,11 @@ func (t *Topology) Dot() (string, error) {
 			if s.privateIPs != nil {
 				priv = s.privateIPs[j]
 			}
-			if err := g.Nodes.Lookup[graphEscape(s.hostnames[j])].Attrs.Add(string(gographviz.Label), nodeLabel(s.location, s.hostnames[j], s.cidrs[j], priv, wg, endpoint)); err != nil {
+			if err := g.Nodes.Lookup[graphEscape(s.hostnames[j])].Attrs.Add(string(gographviz.Label), nodeLabel(location, s.hostnames[j], s.cidrs[j], priv, wg, endpoint)); err != nil {
 				return "", fmt.Errorf("failed to add label to node")
 			}
 		}
-		meshSubGraph(g, g.Relations.SortedChildren(subGraphName(s.location)), s.leader, nil)
+		meshSubGraph(g, g.Relations.SortedChildren(subGraphName(location)), s.leader, plainConnection, nil)
 		leaders[i] = graphEscape(s.hostnames[s.leader])
 	}
 	meshGraph(g, leaders, nil)
@@ -116,15 +123,26 @@ func meshGraph(g *gographviz.Graph, nodes []string, attrs gographviz.Attrs) {
 			if i == j {
 				continue
 			}
+			dsts := g.Edges.SrcToDsts[nodes[i]]
+			if dsts != nil && len(dsts[nodes[j]]) != 0 {
+				// nodes already connected via plain connection
+				continue
+			}
+
 			g.Edges.Add(&gographviz.Edge{Src: nodes[i], Dst: nodes[j], Dir: true, Attrs: attrs})
 		}
 	}
 }
 
-func meshSubGraph(g *gographviz.Graph, nodes []string, leader int, attrs gographviz.Attrs) {
+func meshSubGraph(g *gographviz.Graph, nodes []string, leader int, plainConnection bool, attrs gographviz.Attrs) {
 	if attrs == nil {
 		attrs = make(gographviz.Attrs)
 		attrs[gographviz.Dir] = "both"
+		if plainConnection {
+			attrs[gographviz.Style] = "dotted"
+			attrs[gographviz.ArrowHead] = "none"
+			attrs[gographviz.ArrowTail] = "none"
+		}
 	}
 	for i := range nodes {
 		if i == leader {

pkg/mesh/routes.go

@@ -147,7 +147,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 	}
 	for _, segment := range t.segments {
 		// Add routes for the current segment if local is true.
-		if segment.location == t.location {
+		if (segment.location == t.location) || (t.nodeLocation != "" && segment.nodeLocation == t.nodeLocation) {
 			// If the local node does not have a private IP address,
 			// then skip adding routes, because the node is in its own location.
 			if local && t.privateIP != nil {
@@ -402,9 +402,12 @@ func (t *Topology) Rules(cni, iptablesForwardRule bool) iptables.RuleSet {
 		// Make sure packets to allowed location IPs go through the KILO-NAT chain, so they can be MASQUERADEd,
 		// Otherwise packets to these destinations will reach the destination, but never find their way back.
 		// We only want to NAT in locations of the corresponding allowed location IPs.
+		// Skip the jump when the source is in the same allowed-location CIDR: that traffic is local L2
+		// (e.g. node-to-VIP) and must not be MASQUERADEd, otherwise floating IPs / VIPs whose addresses
+		// have no explicit RETURN rule fall through to MASQUERADE and break etcd / control-plane HA.
 		if t.location == s.location {
 			for _, alip := range s.allowedLocationIPs {
-				rules.AddToPrepend(iptables.NewRule(iptables.GetProtocol(alip.IP), "nat", "POSTROUTING", "-d", alip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"))
+				rules.AddToPrepend(iptables.NewRule(iptables.GetProtocol(alip.IP), "nat", "POSTROUTING", "-d", alip.String(), "!", "-s", alip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"))
 			}
 		}
 	}

pkg/mesh/topology.go

@@ -37,10 +37,12 @@ type Topology struct {
 	// key is the private key of the node creating the topology.
 	key  wgtypes.Key
 	port int
-	// Location is the logical location of the local host.
+	// location is the logical location of the local host.
 	location string
-	segments []*segment
-	peers    []*Peer
+	// nodeLocation is the location annotation of the node. This is set only in cross location topology.
+	nodeLocation string
+	segments     []*segment
+	peers        []*Peer
 
 	// hostname is the hostname of the local host.
 	hostname string
@@ -75,8 +77,10 @@ type segment struct {
 	endpoint            *wireguard.Endpoint
 	key                 wgtypes.Key
 	persistentKeepalive time.Duration
-	// Location is the logical location of this segment.
+	// location is the logical location of this segment.
 	location string
+	// nodeLocation is the node location annotation. This is set only for cross location topology.
+	nodeLocation string
 
 	// cidrs is a slice of subnets of all peers in the segment.
 	cidrs []*net.IPNet
@@ -97,14 +101,34 @@ type segment struct {
 	allowedLocationIPs []net.IPNet
 }
 
+// topoKey is used to group nodes into locations.
+type topoKey struct {
+	location     string
+	nodeLocation string
+}
+
 // NewTopology creates a new Topology struct from a given set of nodes and peers.
 func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port int, key wgtypes.Key, subnet *net.IPNet, serviceCIDRs []*net.IPNet, persistentKeepalive time.Duration, logger log.Logger) (*Topology, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
-	topoMap := make(map[string][]*Node)
+	topoMap := make(map[topoKey][]*Node)
+	var localLocation, localNodeLocation string
+	switch granularity {
+	case LogicalGranularity:
+		localLocation = logicalLocationPrefix + nodes[hostname].Location
+		if nodes[hostname].InternalIP == nil {
+			localLocation = nodeLocationPrefix + hostname
+		}
+	case FullGranularity:
+		localLocation = nodeLocationPrefix + hostname
+	case CrossGranularity:
+		localLocation = nodeLocationPrefix + hostname
+		localNodeLocation = logicalLocationPrefix + nodes[hostname].Location
+	}
+
 	for _, node := range nodes {
-		var location string
+		var location, nodeLocation string
 		switch granularity {
 		case LogicalGranularity:
 			location = logicalLocationPrefix + node.Location
@@ -115,25 +139,20 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			}
 		case FullGranularity:
 			location = nodeLocationPrefix + node.Name
+		case CrossGranularity:
+			location = nodeLocationPrefix + node.Name
+			nodeLocation = logicalLocationPrefix + node.Location
 		}
-		topoMap[location] = append(topoMap[location], node)
-	}
-	var localLocation string
-	switch granularity {
-	case LogicalGranularity:
-		localLocation = logicalLocationPrefix + nodes[hostname].Location
-		if nodes[hostname].InternalIP == nil {
-			localLocation = nodeLocationPrefix + hostname
-		}
-	case FullGranularity:
-		localLocation = nodeLocationPrefix + hostname
+		key := topoKey{location: location, nodeLocation: nodeLocation}
+		topoMap[key] = append(topoMap[key], node)
 	}
 
 	t := Topology{
 		key:                 key,
 		port:                port,
 		hostname:            hostname,
 		location:            localLocation,
+		nodeLocation:        localNodeLocation,
 		persistentKeepalive: persistentKeepalive,
 		privateIP:           nodes[hostname].InternalIP,
 		subnet:              nodes[hostname].Subnet,
@@ -148,7 +167,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			return topoMap[location][i].Name < topoMap[location][j].Name
 		})
 		leader := findLeader(topoMap[location])
-		if location == localLocation && topoMap[location][leader].Name == hostname {
+		if location.nodeLocation != "" || (location.location == localLocation && topoMap[location][leader].Name == hostname) {
 			t.leader = true
 		}
 		var allowedIPs []net.IPNet
@@ -190,7 +209,8 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 			endpoint:            topoMap[location][leader].Endpoint,
 			key:                 topoMap[location][leader].Key,
 			persistentKeepalive: topoMap[location][leader].PersistentKeepalive,
-			location:            location,
+			location:            location.location,
+			nodeLocation:        location.nodeLocation,
 			cidrs:               cidrs,
 			hostnames:           hostnames,
 			leader:              leader,
@@ -235,7 +255,7 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra
 
 		// Now that the topology is ordered, update the discoveredEndpoints map
 		// add new ones by going through the ordered topology: segments, nodes
-		for _, node := range topoMap[segment.location] {
+		for _, node := range topoMap[topoKey{location: segment.location, nodeLocation: segment.nodeLocation}] {
 			for key := range node.DiscoveredEndpoints {
 				if _, ok := t.discoveredEndpoints[key]; !ok {
 					t.discoveredEndpoints[key] = node.DiscoveredEndpoints[key]
@@ -323,7 +343,7 @@ func (t *Topology) Conf() *wireguard.Conf {
 		},
 	}
 	for _, s := range t.segments {
-		if s.location == t.location {
+		if (s.location == t.location) || (t.nodeLocation != "" && t.nodeLocation == s.nodeLocation) {
 			continue
 		}
 		peer := wireguard.Peer{