multiple external network dnat bug

[zhaocongqi]

ovn-org/ovn#222

Kube-OVN通过loadbalance实现端口细粒度的dnat
实现方式为：

kubectl ko nbctl lb-add dnat1 192.168.80.129:80 192.50.0.4:80 tcp
kubectl ko nbctl lr-lb-add vpc1 dnat1

上述命令相当于给192.168.80.129创建dnat映射80端口
创建另一个外部网络的dnat后

kubectl ko nbctl lb-add dnat2 192.168.94.129:80 192.100.0.4:80 tcp
kubectl ko nbctl lr-lb-add vpc1 dnat2

192.168.94.0/24为连接vpc1的另一个外部网络，此时仅有1个dnat生效

发现原因如下：dnat至不同外部网络时，都使用同一个distributed gateway port
ovn逻辑流表如下
vpc1 Ingress

table=7 (lr_in_dnat         ), priority=120  , match=(ct.est && !ct.rel && ip4 && reg0 == 192.168.80.129 && tcp && reg9[16..31] == 80 && ct_mark.natted == 1 && is_chassis_resident("cr-vpc1-external")), action=(next;)
  table=7 (lr_in_dnat         ), priority=120  , match=(ct.est && !ct.rel && ip4 && reg0 == 192.168.94.129 && tcp && reg9[16..31] == 80 && ct_mark.natted == 1 && is_chassis_resident("cr-vpc1-external")), action=(next;)
  table=7 (lr_in_dnat         ), priority=120  , match=(ct.new && !ct.rel && ip4 && reg0 == 192.168.80.129 && tcp && reg9[16..31] == 80 && is_chassis_resident("cr-vpc1-external")), action=(ct_lb_mark(backends=192.50.0.4:80);)
  table=7 (lr_in_dnat         ), priority=120  , match=(ct.new && !ct.rel && ip4 && reg0 == 192.168.94.129 && tcp && reg9[16..31] == 80 && is_chassis_resident("cr-vpc1-external")), action=(ct_lb_mark(backends=192.100.0.4:80);)

vpc1 Egress

table=1 (lr_out_undnat      ), priority=120  , match=(ip4 && ((ip4.src == 192.100.0.4 && tcp.src == 80)) && outport == "vpc1-external" && is_chassis_resident("cr-vpc1-external")), action=(ct_dnat_in_czone;)
  table=1 (lr_out_undnat      ), priority=120  , match=(ip4 && ((ip4.src == 192.50.0.4 && tcp.src == 80)) && outport == "vpc1-external" && is_chassis_resident("cr-vpc1-external")), action=(ct_dnat_in_czone;)

loadbalance会调用dnat和undnat，然而loadbalance不支持多个distributed gateway port

static void
build_lrouter_lbs_check(const struct ovn_datapaths *lr_datapaths)
{
    struct ovn_datapath *od;

    HMAP_FOR_EACH (od, key_node, &lr_datapaths->datapaths) {
        ovs_assert(od->nbr);

        if (od->has_lb_vip && od->n_l3dgw_ports > 1
                && !smap_get(&od->nbr->options, "chassis")) {
            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
            VLOG_WARN_RL(&rl, "Load-balancers are configured on logical "
                         "router %s, which has %"PRIuSIZE" distributed "
                         "gateway ports. Load-balancer is not supported "
                         "yet when there is more than one distributed "
                         "gateway port on the router.",
                         od->nbr->name, od->n_l3dgw_ports);
        }
    }
}

然而ovn dnat的逻辑支持根据可达性ip判断distributed gateway port

static int
lrouter_check_nat_entry(struct ovn_datapath *od, const struct nbrec_nat *nat,
                        const struct hmap *lr_ports, ovs_be32 *mask,
                        bool *is_v6, int *cidr_bits, struct eth_addr *mac,
                        bool *distributed, struct ovn_port **nat_l3dgw_port)
{
    for (size_t i = 0; i < od->n_l3dgw_ports; i++) {
               if (find_lrp_member_ip(od->l3dgw_ports[i], nat->external_ip)) {
                   *nat_l3dgw_port = od->l3dgw_ports[i];
                   break;
               }
            }
}

[zhaocongqi]

验证后发现通过nat表项添加dnat规则没有问题
kubectl ko nbctl lr-nat-add vpc1 dnat 192.168.80.129 192.50.0.4
kubectl ko nbctl lr-nat-add vpc1 dnat 192.168.94.129 192.100.0.4

其下发的相关逻辑流表如下：
vpc1 Ingress：
table=7 (lr_in_dnat ), priority=100 , match=(ip && ip4.dst == 192.168.80.129 && inport == "vpc1-external" && is_chassis_resident("cr-vpc1-external")), action=(ct_dnat_in_czone(192.50.0.4);)
table=7 (lr_in_dnat ), priority=100 , match=(ip && ip4.dst == 192.168.94.129 && inport == "vpc1-ext1" && is_chassis_resident("cr-vpc1-ext1")), action=(ct_dnat_in_czone(192.100.0.4);)

vpc1 Egress：
table=1 (lr_out_undnat ), priority=100 , match=(ip && ip4.src == 192.100.0.4 && outport == "vpc1-ext1" && is_chassis_resident("cr-vpc1-ext1")), action=(ct_dnat_in_czone;)
table=1 (lr_out_undnat ), priority=100 , match=(ip && ip4.src == 192.50.0.4 && outport == "vpc1-external" && is_chassis_resident("cr-vpc1-external")), action=(ct_dnat_in_czone;)

但是nat表中dnat无法实现端口细粒度的控制

[zhaocongqi]

执行kubectl ko nbctl lr-lb-add vpc1 dnat1会进入build_lrouter_nat_flows_for_lb这个函数，它在下发match规则时默认使用od->l3dgw_ports[0] 导致所有的 is_chassis_resident 都相同