Skip to content

Commit

Permalink
label node with host subnet selector name to specify specific subnet …
Browse files Browse the repository at this point in the history 
…range

To specify specic subnet range for a specific node, first in the
ovn-config configmap, define multiple subnet cidrs in net_cidr with the
format of <cidr1>,<cidr2>@<selector_name>...; then label the node that that
needs specific subnet range with label
k8s.ovn.org/subnet_selector_name=<selector_name>.

Signed-off-by: Yun Zhou <[email protected]>
  • Loading branch information
@cathy-zhou
cathy-zhou committed Apr 20, 2020
1 parent b5137fe commit 43197e8
Show file tree
Hide file tree
Showing 8 changed files with 183 additions and 77 deletions.
12 changes: 9 additions & 3 deletions go-controller/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ const DefaultEncapPort = 6081

const DefaultAPIServer = "http://localhost:8443"

// DefaultNodeSubnetSelectorName captures default name for the node subnet selector
const DefaultNodeSubnetSelectorName = "default"

// IP address range from which subnet is allocated for per-node join switch
const (
V4JoinSubnet = "100.64.0.0/16"
Expand Down Expand Up @@ -144,9 +147,12 @@ type DefaultConfig struct {
// RawClusterSubnets holds the unparsed cluster subnets. Should only be
// used inside config module.
RawClusterSubnets string `gcfg:"cluster-subnets"`
// ClusterSubnets holds parsed cluster subnet entries and may be used
// ClusterSubnets holds all parsed cluster subnet entries and may be used
// outside the config module.
ClusterSubnets []CIDRNetworkEntry
// ClusterSubnetsBySelector holds all parsed cluster subnet entries keyed by selectorName
// and may be used outside the config module.
ClusterSubnetsBySelector map[string][]CIDRNetworkEntry
}

// LoggingConfig holds logging-related parsed config file parameters and command-line overrides
Expand Down Expand Up @@ -1031,7 +1037,7 @@ func buildHybridOverlayConfig(ctx *cli.Context, cli, file *config, allSubnets *c

if HybridOverlay.Enabled {
var err error
HybridOverlay.ClusterSubnets, err = ParseClusterSubnetEntries(HybridOverlay.RawClusterSubnets)
_, HybridOverlay.ClusterSubnets, err = ParseClusterSubnetEntries(HybridOverlay.RawClusterSubnets)
if err != nil {
return fmt.Errorf("hybrid overlay cluster subnet invalid: %v", err)
}
Expand Down Expand Up @@ -1061,7 +1067,7 @@ func buildDefaultConfig(cli, file *config, allSubnets *configSubnets) error {
}

var err error
Default.ClusterSubnets, err = ParseClusterSubnetEntries(Default.RawClusterSubnets)
Default.ClusterSubnetsBySelector, Default.ClusterSubnets, err = ParseClusterSubnetEntries(Default.RawClusterSubnets)
if err != nil {
return fmt.Errorf("cluster subnet invalid: %v", err)
}
Expand Down
35 changes: 26 additions & 9 deletions go-controller/pkg/config/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,37 @@ func (e CIDRNetworkEntry) HostBits() uint32 {
// ParseClusterSubnetEntries returns the parsed set of CIDRNetworkEntries passed by the user on the command line
// These entries define the clusters network space by specifying a set of CIDR and netmasks the SDN can allocate
// addresses from.
func ParseClusterSubnetEntries(clusterSubnetCmd string) ([]CIDRNetworkEntry, error) {
func ParseClusterSubnetEntries(clusterSubnetCmd string) (map[string][]CIDRNetworkEntry, []CIDRNetworkEntry, error) {
var parsedClusterListMap map[string][]CIDRNetworkEntry
var parsedClusterList []CIDRNetworkEntry
ipv6 := false
clusterEntriesList := strings.Split(clusterSubnetCmd, ",")

for _, clusterEntry := range clusterEntriesList {
parsedClusterListMap = make(map[string][]CIDRNetworkEntry)
for _, clusterEntryWithSelectro := range clusterEntriesList {
var parsedClusterEntry CIDRNetworkEntry

clusterEntryInfo := strings.SplitN(clusterEntryWithSelectro, "@", 2)

// if no selector name is specified, assign the default selector name
selectorName := DefaultNodeSubnetSelectorName
if len(clusterEntryInfo) == 2 {
selectorName = strings.TrimSpace(clusterEntryInfo[1])
if len(selectorName) == 0 {
return nil, nil, fmt.Errorf("invalid selector name %s for %q", selectorName, clusterEntryWithSelectro)
}
}
clusterEntry := clusterEntryInfo[0]
splitClusterEntry := strings.Split(clusterEntry, "/")

if len(splitClusterEntry) < 2 || len(splitClusterEntry) > 3 {
return nil, fmt.Errorf("CIDR %q not properly formatted", clusterEntry)
return nil, nil, fmt.Errorf("CIDR %q not properly formatted", clusterEntry)
}

var err error
_, parsedClusterEntry.CIDR, err = net.ParseCIDR(fmt.Sprintf("%s/%s", splitClusterEntry[0], splitClusterEntry[1]))
if err != nil {
return nil, err
return nil, nil, err
}

if utilnet.IsIPv6(parsedClusterEntry.CIDR.IP) {
Expand All @@ -52,12 +65,12 @@ func ParseClusterSubnetEntries(clusterSubnetCmd string) ([]CIDRNetworkEntry, err
if len(splitClusterEntry) == 3 {
tmp, err := strconv.ParseUint(splitClusterEntry[2], 10, 32)
if err != nil {
return nil, err
return nil, nil, err
}
parsedClusterEntry.HostSubnetLength = uint32(tmp)

if ipv6 && parsedClusterEntry.HostSubnetLength != 64 {
return nil, fmt.Errorf("IPv6 only supports /64 host subnets")
return nil, nil, fmt.Errorf("IPv6 only supports /64 host subnets")
}
} else {
if ipv6 {
Expand All @@ -69,18 +82,22 @@ func ParseClusterSubnetEntries(clusterSubnetCmd string) ([]CIDRNetworkEntry, err
}

if parsedClusterEntry.HostSubnetLength <= uint32(entryMaskLength) {
return nil, fmt.Errorf("cannot use a host subnet length mask shorter than or equal to the cluster subnet mask. "+
return nil, nil, fmt.Errorf("cannot use a host subnet length mask shorter than or equal to the cluster subnet mask. "+
"host subnet length: %d, cluster subnet length: %d", parsedClusterEntry.HostSubnetLength, entryMaskLength)
}

if _, ok := parsedClusterListMap[selectorName]; !ok {
parsedClusterListMap[selectorName] = []CIDRNetworkEntry{}
}
parsedClusterListMap[selectorName] = append(parsedClusterListMap[selectorName], parsedClusterEntry)
parsedClusterList = append(parsedClusterList, parsedClusterEntry)
}

if len(parsedClusterList) == 0 {
return nil, fmt.Errorf("failed to parse any CIDRs from %q", clusterSubnetCmd)
return nil, nil, fmt.Errorf("failed to parse any CIDRs from %q", clusterSubnetCmd)
}

return parsedClusterList, nil
return parsedClusterListMap, parsedClusterList, nil
}

type configSubnetType string
Expand Down
2 changes: 1 addition & 1 deletion go-controller/pkg/config/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func TestParseClusterSubnetEntries(t *testing.T) {

for _, tc := range tests {

parsedList, err := ParseClusterSubnetEntries(tc.cmdLineArg)
_, parsedList, err := ParseClusterSubnetEntries(tc.cmdLineArg)
if err != nil && !tc.expectedErr {
t.Errorf("Test case \"%s\" expected no errors, got %v", tc.name, err)
}
Expand Down
116 changes: 92 additions & 24 deletions go-controller/pkg/ovn/master.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

kapi "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/leaderelection"
Expand All @@ -22,10 +23,13 @@ import (

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/allocator"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
)

const (
// OvnNodeSubnetSelector is the label string representing the selector that node subnet belongs to
OvnNodeSubnetSelector = "k8s.ovn.org/subnet_selector_name"
// OvnServiceIdledAt is a constant string representing the Service annotation key
// whose value indicates the time stamp in RFC3339 format when a Service was idled
OvnServiceIdledAt = "k8s.ovn.org/idled-at"
Expand Down Expand Up @@ -94,6 +98,16 @@ func (oc *Controller) Start(kClient kubernetes.Interface, nodeName string) error
return nil
}

func (oc *Controller) getSubnetSelectorName(node *kapi.Node) string {
for selectorName, subnetSelector := range oc.masterSubnetLabelSelector {
nodeSelector, _ := metav1.LabelSelectorAsSelector(subnetSelector)
if nodeSelector.Matches(labels.Set(node.Labels)) {
return selectorName
}
}
return config.DefaultNodeSubnetSelectorName
}

// StartClusterMaster runs a subnet IPAM and a controller that watches arrival/departure
// of nodes in the cluster
// On an addition to the cluster (node create), a new subnet is created for it that will translate
Expand All @@ -119,18 +133,43 @@ func (oc *Controller) StartClusterMaster(masterNodeName string) error {
klog.Errorf("Error in initializing/fetching subnets: %v", err)
return err
}
for _, clusterEntry := range config.Default.ClusterSubnets {
err := oc.masterSubnetAllocator.AddNetworkRange(clusterEntry.CIDR, clusterEntry.HostBits())
if err != nil {
return err
for selectorName, ClusterEntryList := range config.Default.ClusterSubnetsBySelector {
if selectorName != config.DefaultNodeSubnetSelectorName {
_, ok := oc.masterSubnetLabelSelector[selectorName]
if !ok {
klog.V(3).Infof("CATHY DEBUG create label selector for selector %s", selectorName)
if nodeSelector, err := metav1.ParseToLabelSelector(OvnNodeSubnetSelector + "=" + selectorName); err == nil {
oc.masterSubnetLabelSelector[selectorName] = nodeSelector
} else {
return fmt.Errorf("failed to create label selector for subnet selector %s: %v", selectorName, err)
}
}
}
_, ok := oc.masterSubnetAllocator[selectorName]
if !ok {
klog.V(3).Infof("CATHY DEBUG init subnet allocator for selector %s", selectorName)
oc.masterSubnetAllocator[selectorName] = allocator.NewSubnetAllocator()
}
for _, clusterEntry := range ClusterEntryList {
err := oc.masterSubnetAllocator[selectorName].AddNetworkRange(clusterEntry.CIDR, clusterEntry.HostBits())
if err != nil {
return err
}
}
}
for _, node := range existingNodes.Items {
hostsubnet, _ := util.ParseNodeHostSubnetAnnotation(&node)
selectorName := oc.getSubnetSelectorName(&node)

if hostsubnet != nil {
err := oc.masterSubnetAllocator.MarkAllocatedNetwork(hostsubnet)
subnetAllocator, ok := oc.masterSubnetAllocator[selectorName]
if !ok {
utilruntime.HandleError(fmt.Errorf("node subnet selector name %s is not configured", selectorName))
}
err := subnetAllocator.MarkAllocatedNetwork(hostsubnet)
if err != nil {
utilruntime.HandleError(err)
utilruntime.HandleError(fmt.Errorf("node subnet %s is not in selector %s cidr: %v",
hostsubnet.String(), selectorName, err))
}
}
joinsubnet, _ := util.ParseNodeJoinSubnetAnnotation(&node)
Expand Down Expand Up @@ -433,7 +472,7 @@ func addStaticRouteToHost(node *kapi.Node, nicIPs []*net.IPNet) error {
return nil
}

func (oc *Controller) ensureNodeLogicalNetwork(nodeName string, hostsubnet *net.IPNet) error {
func (oc *Controller) ensureNodeLogicalNetwork(nodeName, selectorName string, hostsubnet *net.IPNet) error {
gwIfAddr := util.GetNodeGatewayIfAddr(hostsubnet)
mgmtIfAddr := util.GetNodeManagementIfAddr(hostsubnet)
hybridOverlayIfAddr := util.GetNodeHybridOverlayIfAddr(hostsubnet)
Expand Down Expand Up @@ -468,6 +507,10 @@ func (oc *Controller) ensureNodeLogicalNetwork(nodeName string, hostsubnet *net.
"other-config:exclude_ips="+excludeIPs,
)
}
if selectorName != config.DefaultNodeSubnetSelectorName {
args = append(args,
[]string{"--", "set", "logical_switch", nodeName, "external-ids:subnet-selector=" + selectorName}...)
}
stdout, stderr, err := util.RunOVNNbctl(args...)
if err != nil {
klog.Errorf("Failed to create a logical switch %v, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
Expand Down Expand Up @@ -602,13 +645,19 @@ func (oc *Controller) addNode(node *kapi.Node) (*net.IPNet, error) {
oc.clearInitialNodeNetworkUnavailableCondition(node, nil)

hostsubnet, _ := util.ParseNodeHostSubnetAnnotation(node)
selectorName := oc.getSubnetSelectorName(node)
if hostsubnet != nil {
// Node already has subnet assigned; ensure its logical network is set up
return hostsubnet, oc.ensureNodeLogicalNetwork(node.Name, hostsubnet)
return hostsubnet, oc.ensureNodeLogicalNetwork(node.Name, selectorName, hostsubnet)
}

subnetAllocator, ok := oc.masterSubnetAllocator[selectorName]
if !ok {
return nil, fmt.Errorf("subnet selector %s for node %s not configured", selectorName, node.Name)
}

// Node doesn't have a subnet assigned; reserve a new one for it
hostsubnets, err := oc.masterSubnetAllocator.AllocateNetworks()
hostsubnets, err := subnetAllocator.AllocateNetworks()
if err != nil {
return nil, fmt.Errorf("Error allocating network for node %s: %v", node.Name, err)
}
Expand All @@ -621,12 +670,12 @@ func (oc *Controller) addNode(node *kapi.Node) (*net.IPNet, error) {
defer func() {
// Release the allocation on error
if err != nil {
_ = oc.masterSubnetAllocator.ReleaseNetwork(hostsubnet)
_ = subnetAllocator.ReleaseNetwork(hostsubnet)
}
}()

// Ensure that the node's logical network has been created
err = oc.ensureNodeLogicalNetwork(node.Name, hostsubnet)
err = oc.ensureNodeLogicalNetwork(node.Name, selectorName, hostsubnet)
if err != nil {
return nil, err
}
Expand All @@ -642,8 +691,12 @@ func (oc *Controller) addNode(node *kapi.Node) (*net.IPNet, error) {
return hostsubnet, nil
}

func (oc *Controller) deleteNodeHostSubnet(nodeName string, subnet *net.IPNet) error {
err := oc.masterSubnetAllocator.ReleaseNetwork(subnet)
func (oc *Controller) deleteNodeHostSubnet(nodeName, selectorName string, subnet *net.IPNet) error {
subnetAllocator, ok := oc.masterSubnetAllocator[selectorName]
if !ok {
return fmt.Errorf("Error subnet selector %s for node %q not configured", selectorName, nodeName)
}
err := subnetAllocator.ReleaseNetwork(subnet)
if err != nil {
return fmt.Errorf("Error deleting subnet %v for node %q: %s", subnet, nodeName, err)
}
Expand All @@ -667,10 +720,10 @@ func (oc *Controller) deleteNodeLogicalNetwork(nodeName string) error {
return nil
}

func (oc *Controller) deleteNode(nodeName string, nodeSubnet, joinSubnet *net.IPNet) error {
func (oc *Controller) deleteNode(nodeName, selectorName string, nodeSubnet, joinSubnet *net.IPNet) error {
// Clean up as much as we can but don't hard error
if nodeSubnet != nil {
if err := oc.deleteNodeHostSubnet(nodeName, nodeSubnet); err != nil {
if nodeSubnet != nil && selectorName != "" {
if err := oc.deleteNodeHostSubnet(nodeName, selectorName, nodeSubnet); err != nil {
klog.Errorf("Error deleting node %s HostSubnet %v: %v", nodeName, nodeSubnet, err)
}
}
Expand Down Expand Up @@ -834,8 +887,8 @@ func (oc *Controller) syncNodes(nodes []interface{}) {
delete(chassisMap, nodeName)
}

nodeSwitches, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading",
"--columns=name,other-config", "find", "logical_switch",
nodeSwitches, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--format=csv",
"--columns=name,other-config,external_ids", "find", "logical_switch",
"other-config:"+subnetAttr+"!=_")
if err != nil {
klog.Errorf("Failed to get node logical switches: stderr: %q, error: %v",
Expand All @@ -844,14 +897,15 @@ func (oc *Controller) syncNodes(nodes []interface{}) {
}

type NodeSubnets struct {
hostSubnet *net.IPNet
joinSubnet *net.IPNet
hostSubnet *net.IPNet
joinSubnet *net.IPNet
selectorName string
}
NodeSubnetsMap := make(map[string]*NodeSubnets)
for _, result := range strings.Split(nodeSwitches, "\n\n") {
for _, result := range strings.Split(nodeSwitches, "\n") {
// Split result into name and other-config
items := strings.Split(result, "\n")
if len(items) != 2 || len(items[0]) == 0 {
items := strings.Split(result, ",")
if len(items) != 3 || len(items[0]) == 0 {
continue
}
isJoinSwitch := false
Expand All @@ -865,6 +919,18 @@ func (oc *Controller) syncNodes(nodes []interface{}) {
continue
}

// get selector name for node logical switch
selectorName := config.DefaultNodeSubnetSelectorName
if !isJoinSwitch {
attrs := strings.Fields(items[2])
for _, attr := range attrs {
if strings.HasPrefix(attr, "subnet-selector=") {
selectorName = strings.TrimPrefix(attr, "subnet-selector=")
break
}
}
}

var subnet *net.IPNet
attrs := strings.Fields(items[1])
for _, attr := range attrs {
Expand All @@ -888,11 +954,13 @@ func (oc *Controller) syncNodes(nodes []interface{}) {
nodeSubnets.joinSubnet = subnet
} else {
nodeSubnets.hostSubnet = subnet
nodeSubnets.selectorName = selectorName
}
}

for nodeName, nodeSubnets := range NodeSubnetsMap {
if err := oc.deleteNode(nodeName, nodeSubnets.hostSubnet, nodeSubnets.joinSubnet); err != nil {
if err := oc.deleteNode(nodeName, nodeSubnets.selectorName, nodeSubnets.hostSubnet,
nodeSubnets.joinSubnet); err != nil {
klog.Error(err)
}
//remove the node from the chassis map so we don't delete it twice
Expand Down
Loading

0 comments on commit 43197e8

Please sign in to comment.