diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index 6413ba425da..e1176a87e90 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" + "k8s.io/utils/set" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/ovs" @@ -730,13 +731,16 @@ func (c *Controller) setIptables() error { ) } } + _, subnetCidrs, err := c.getDefaultVpcSubnetsCIDR(protocol) if err != nil { klog.Errorf("get subnets failed, %+v", err) return err } + subnetNames := set.New[string]() for name, subnetCidr := range subnetCidrs { + subnetNames.Insert(name) iptablesRules = append(iptablesRules, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(fmt.Sprintf(`-m comment --comment %s,%s -s %s`, util.OvnSubnetGatewayIptables, name, subnetCidr))}, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(fmt.Sprintf(`-m comment --comment %s,%s -d %s`, util.OvnSubnetGatewayIptables, name, subnetCidr))}, @@ -749,27 +753,31 @@ func (c *Controller) setIptables() error { return err } + pattern := fmt.Sprintf(`-m comment --comment "%s,`, util.OvnSubnetGatewayIptables) for _, rule := range rules { - if !strings.Contains(rule, util.OvnSubnetGatewayIptables) { + if !strings.Contains(rule, pattern) { continue } - - var inUse bool - for name := range subnetCidrs { - if slices.Contains(util.DoubleQuotedFields(rule), fmt.Sprintf("%s,%s", util.OvnSubnetGatewayIptables, name)) { - inUse = true - break - } + fields := util.DoubleQuotedFields(rule) + // -A FORWARD -d 10.16.0.0/16 -m comment --comment "ovn-subnet-gateway,ovn-default" + if len(fields) != 8 || fields[6] != "--comment" { + continue + } + commentFields := strings.Split(fields[7], ",") + if len(commentFields) != 2 { + continue + } + if subnetNames.Has(commentFields[1]) { + continue } - if !inUse { - // rule[11:] skip "-A FORWARD " - if err = deleteIptablesRule(ipt, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: util.DoubleQuotedFields(rule[11:])}); err != nil { - klog.Error(err) - return err - } + // use fields[2:] to skip prefix "-A FORWARD" + if err = deleteIptablesRule(ipt, util.IPTableRule{Table: "filter", Chain: "FORWARD", Rule: fields[2:]}); err != nil { + klog.Error(err) + return err } } + var natPreroutingRules, natPostroutingRules, ovnMasqueradeRules, manglePostroutingRules []util.IPTableRule for _, rule := range iptablesRules { if rule.Table == NAT { @@ -1257,95 +1265,73 @@ func (c *Controller) setOvnSubnetGatewayMetric() { } for _, rule := range rules { + if !strings.Contains(rule, util.OvnSubnetGatewayIptables) { + continue + } + items := util.DoubleQuotedFields(rule) - cidr := "" - direction := "" - subnetName := "" - var currentPackets, currentPacketBytes int - if len(items) <= 10 { + if len(items) != 11 { continue } - for _, item := range items { - if strings.Contains(item, util.OvnSubnetGatewayIptables) { - cidr = items[3] - - switch items[2] { - case "-s": - direction = "egress" - case "-d": - direction = "ingress" - default: - break - } - comments := strings.Split(items[7], ",") - if len(comments) != 2 { - break - } - subnetName = comments[1][:len(comments[1])-1] - currentPackets, err = strconv.Atoi(items[9]) - if err != nil { - break - } - currentPacketBytes, err = strconv.Atoi(items[10]) - if err != nil { - break - } - } + comments := strings.Split(items[7], ",") + if len(comments) != 2 || comments[0] != util.OvnSubnetGatewayIptables { + continue + } + subnetName := comments[1] + + var direction string + switch items[2] { + case "-s": + direction = "egress" + case "-d": + direction = "ingress" + default: + continue } + cidr := items[3] proto := util.CheckProtocol(cidr) - - if cidr == "" || direction == "" || subnetName == "" && proto != "" { + if proto == "" { + klog.Errorf("failed to get protocol from cidr %q", cidr) continue } - lastPacketBytes := 0 - lastPackets := 0 - diffPacketBytes := 0 - diffPackets := 0 + currentPackets, err := strconv.Atoi(items[9]) + if err != nil { + klog.Errorf("failed to parse packets %q: %v", items[9], err) + continue + } + currentPacketBytes, err := strconv.Atoi(items[10]) + if err != nil { + klog.Errorf("failed to parse packet bytes %q: %v", items[10], err) + continue + } key := strings.Join([]string{subnetName, direction, proto}, "/") - if ret, ok := c.gwCounters[key]; ok { - lastPackets = ret.Packets - lastPacketBytes = ret.PacketBytes - } else { - c.gwCounters[key] = &util.GwIPtableCounters{ - Packets: lastPackets, - PacketBytes: lastPacketBytes, - } + if c.gwCounters[key] == nil { + c.gwCounters[key] = new(util.GwIPtableCounters) } + lastPackets, lastPacketBytes := c.gwCounters[key].Packets, c.gwCounters[key].PacketBytes + c.gwCounters[key].Packets, c.gwCounters[key].PacketBytes = currentPackets, currentPacketBytes - if lastPacketBytes == 0 && lastPackets == 0 { + if lastPackets == 0 && lastPacketBytes == 0 { // the gwCounters may just initialize don't cal the diff values, // it may loss packets to calculate during a metric period - c.gwCounters[key].Packets = currentPackets - c.gwCounters[key].PacketBytes = currentPacketBytes continue } - - if currentPackets >= lastPackets && currentPacketBytes >= lastPacketBytes { - diffPacketBytes = currentPacketBytes - lastPacketBytes - diffPackets = currentPackets - lastPackets - } else { + if currentPackets < lastPackets || currentPacketBytes < lastPacketBytes { // if currentPacketBytes < lastPacketBytes, the reason is that iptables rule is reset , // it may loss packets to calculate during a metric period - c.gwCounters[key].Packets = currentPackets - c.gwCounters[key].PacketBytes = currentPacketBytes continue } - c.gwCounters[key].Packets = currentPackets - c.gwCounters[key].PacketBytes = currentPacketBytes - + diffPackets := currentPackets - lastPackets + diffPacketBytes := currentPacketBytes - lastPacketBytes klog.V(3).Infof(`hostname %s key %s cidr %s direction %s proto %s has diffPackets %d diffPacketBytes %d currentPackets %d currentPacketBytes %d lastPackets %d lastPacketBytes %d`, hostname, key, cidr, direction, proto, diffPackets, diffPacketBytes, currentPackets, currentPacketBytes, lastPackets, lastPacketBytes) - if diffPackets > 0 { - metricOvnSubnetGatewayPackets.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPackets)) - } - if diffPacketBytes > 0 { - metricOvnSubnetGatewayPacketBytes.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPacketBytes)) - } + metricOvnSubnetGatewayPackets.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPackets)) + metricOvnSubnetGatewayPacketBytes.WithLabelValues(hostname, key, cidr, direction, proto).Add(float64(diffPacketBytes)) } } }