make sure nr mycelium have peers or fail to the public peers

Author: Omarabdul3ziz

pkg/network/mycelium/config.go

@@ -15,19 +15,27 @@ type NodeConfig struct {
 	privateKey x25519.PrivateKey
 }
 
-func (n *NodeConfig) FindPeers(ctx context.Context, filter ...Filter) error {
+func FindPeers(ctx context.Context, filter ...Filter) (Peers, error) {
 	// fetching a peer list goes as this
 	// - Always include the list of peers from
 	peers, err := fetchZosMyList()
 	if err != nil {
-		return errors.Wrap(err, "failed to get zos public peer list")
+		return nil, errors.Wrap(err, "failed to get zos public peer list")
 	}
 
 	peers, err = peers.Ups(filter...)
 	if err != nil {
-		return errors.Wrap(err, "failed to filter out peer list")
+		return nil, errors.Wrap(err, "failed to filter out peer list")
 	}
 
+	return peers, nil
+}
+
+func (n *NodeConfig) AssignPeers(ctx context.Context, filter ...Filter) error {
+	peers, err := FindPeers(ctx, filter...)
+	if err != nil {
+		return err
+	}
 	n.Peers = peers
 	return nil
 }

pkg/network/mycelium/mycelium.go

@@ -131,7 +131,7 @@ func EnsureMycelium(ctx context.Context, privateKey ed25519.PrivateKey, ns Mycel
 	z := zinit.Default()
 
 	cfg := GenerateConfig(privateKey)
-	if err := cfg.FindPeers(ctx, filter); err != nil {
+	if err := cfg.AssignPeers(ctx, filter); err != nil {
 		return nil, err
 	}
 

pkg/network/nr/net_resource.go

@@ -2,6 +2,7 @@ package nr
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
@@ -13,9 +14,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v3"
 	"github.com/threefoldtech/zosbase/pkg/gridtypes/zos"
 	"github.com/threefoldtech/zosbase/pkg/network/ifaceutil"
 	"github.com/threefoldtech/zosbase/pkg/network/macvlan"
+	"github.com/threefoldtech/zosbase/pkg/network/mycelium"
 	"github.com/threefoldtech/zosbase/pkg/network/ndmz"
 	"github.com/threefoldtech/zosbase/pkg/network/options"
 	"github.com/threefoldtech/zosbase/pkg/network/tuntap"
@@ -309,19 +312,10 @@ func (nr *NetResource) SetMycelium() (err error) {
 		"--peers",
 	}
 
-	// set mycelium public addresses as the private peers
-	mycNamespace, mycInterface := ndmz.DmzNamespace, ndmz.DmzPub4
-	if namespace.Exists(publicNamespaceName) {
-		mycNamespace = publicNamespaceName
-		mycInterface = publicInterfaceName
-	}
-
-	ips, err := baseifaceutil.GetIPsForIFace(mycInterface, mycNamespace)
+	hostPeers, err := getMyceliumPeers()
 	if err != nil {
-		return errors.Wrap(err, "failed to get IPs for npub4")
+		return err
 	}
-
-	hostPeers := baseifaceutil.BuildMyceliumPeerURLs(ips)
 	args = append(args, hostPeers...)
 
 	err = zinit.AddService(name, zinit.InitService{
@@ -334,6 +328,46 @@ func (nr *NetResource) SetMycelium() (err error) {
 	return init.Monitor(name)
 }
 
+// will keep retrying to get the host interface ips for a min, eventually will failback to the public peers
+func getMyceliumPeers() ([]string, error) {
+	// set mycelium public addresses as the private peers
+	mycNamespace, mycInterface := ndmz.DmzNamespace, ndmz.DmzPub4
+	if namespace.Exists(publicNamespaceName) {
+		mycNamespace = publicNamespaceName
+		mycInterface = publicInterfaceName
+	}
+
+	var ips []net.IPNet
+	bo := backoff.NewExponentialBackOff()
+	bo.MaxElapsedTime = time.Minute
+	bo.MaxInterval = 10 * time.Second
+
+	err := backoff.Retry(func() error {
+		ips, err := baseifaceutil.GetIPsForIFace(mycInterface, mycNamespace)
+		if err != nil {
+			return backoff.Permanent(errors.Wrap(err, "failed to get IPs"))
+		}
+		if len(ips) == 0 {
+			return fmt.Errorf("no IPs available yet")
+		}
+		return nil
+	}, bo)
+
+	var hostPeers []string
+	if err != nil {
+		log.Warn().Msg("failed to get IPs after 1 minute, falling back to public mycelium peers")
+		peers, err := mycelium.FindPeers(context.Background(), nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to find public mycelium peers")
+		}
+		hostPeers = peers
+	} else {
+		hostPeers = baseifaceutil.BuildMyceliumPeerURLs(ips)
+	}
+
+	return hostPeers, nil
+}
+
 func (nr *NetResource) ensureMyceliumNetwork(keyFile string) error {
 	// applies network configuration for this mycelium instance inside the proper namespace
 	mycelium, err := nr.inspectMycelium(keyFile)