Added cleanup feature for MUs and SCC repositories

cmd/sync.go

@@ -2,12 +2,14 @@ package cmd
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/manifoldco/promptui"
 	"github.com/spf13/cobra"
 	"github.com/uyuni-project/minima/get"
 	yaml "gopkg.in/yaml.v2"
@@ -166,7 +168,9 @@ func syncersFromConfig(configString string) (result []*get.Syncer, err error) {
 		}
 		result = append(result, get.NewSyncer(*repoURL, archs, storage))
 	}
-
+	if cleanup {
+		RemoveOldChannelsFromFileStorage(config)
+	}
 	return
 }
 
@@ -176,3 +180,66 @@ func init() {
 	RootCmd.PersistentFlags().StringVarP(&archs, "arch", "a", "", "flag that specifies covered archs in the given repo")
 	RootCmd.PersistentFlags().BoolVarP(&syncLegacyPackages, "legacypackages", "l", false, "flag that triggers mirroring of i586 pkgs in x86_64 repos")
 }
+
+func RemoveOldChannelsFromFileStorage (config Config) (err error) {
+		// DO CLEANUP - TO BE IMPLEMENTED
+		log.Println("searching for outdated repositories...")
+		//fmt.Printf("List of Repos from config: %s  ---> %s\n", config.SCC.RepoNames, config.HTTP)
+		mappedRepos := make(map[string]bool)
+		var urlink *url.URL
+		for _, elem := range config.HTTP {
+			urlink, err = url.Parse(elem.URL)
+			if err != nil {
+				panic(err)
+			}
+			mappedRepos[filepath.Join(config.Storage.Path, urlink.Path)] = true
+		}
+			//fmt.Printf("MAPPED REPOS: %v\n", mappedRepos)
+			path := config.Storage.Path
+			muChannelList := make(map[string]bool)
+			filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+				if info.IsDir() {
+					if info.Name() == "repodata" {
+						muChannelList[strings.Replace(path, "/repodata", "", 1)] = true
+						return nil
+					}
+					if files, err := ioutil.ReadDir(path); len(files) == 0 && path != config.Storage.Path {
+						if err != nil {
+							log.Fatal(err)
+						}
+						log.Printf("Removing unused empty folders: %s\n", path)
+						os.RemoveAll(path)
+					}
+				}
+				return nil
+			})
+			//fmt.Printf("CHANNEL LIST: %v\n", muChannelList)
+			for ind, _ := range muChannelList {
+				if mappedRepos[ind] {
+					log.Printf("Repo %s is registered...\n", ind)
+				} else {
+					log.Printf("Repo %s is not registered in the yaml file...", ind)
+					if autoApprove {
+						log.Printf("Removing repo %s ...\n", ind)
+						os.RemoveAll(ind)
+					} else {
+						prompt := promptui.Select{
+							Label: fmt.Sprintf("Delete repo: %s ??? [Yes/No]", ind),
+							Items: []string{"Yes", "No"},
+						}
+						_, result, err := prompt.Run()
+						if err != nil {
+							log.Fatalf("Prompt failed %v\n", err)
+						}
+						if result == "Yes" {
+							log.Printf("Removing repo: %s ...\n", ind)
+							os.RemoveAll(ind)
+						} else {
+							log.Printf("Keeping repo: %s ...\n", ind)
+						}
+					}
+				}
+			}
+		log.Println("...done!")
+	return
+}

cmd/updates.go

@@ -50,6 +50,7 @@ to quickly create a Cobra application.`,
 	justSearch bool
 	thisMU string
 	cleanup bool
+	autoApprove bool
 )
 
 func init() {
@@ -58,6 +59,7 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&justSearch, "search", "s", false, "flag that would trigger only looking for updates on OBS")
 	RootCmd.PersistentFlags().StringVarP(&thisMU, "maintupdate", "m", "", "flag that consumes the name of an MU, like 'SUSE:Maintenance:Incident:ReleaseRequest'")
 	RootCmd.PersistentFlags().BoolVarP(&cleanup, "cleanup", "k", false, "flag that triggers cleaning up the storage (from old MU channels)")
+	RootCmd.PersistentFlags().BoolVar(&autoApprove, "auto-approve", false, "triggers automatic deleting of repositories")
 }
 
 func muFindAndSync() {
@@ -74,7 +76,7 @@ func muFindAndSync() {
 		if err != nil {
 			log.Fatalf("There is an error: %v", err)
 		}
-		err = RemoveOldChannels(config, updateList)
+		err = RemoveOldChannels(config, updateList, autoApprove)
 		if err != nil {
 			log.Fatalf("There is an error: %v", err)
 		}
@@ -261,45 +263,85 @@ func GetUpdatesAndChannels(usr, passwd string, justsearch bool) (updlist []Updat
 	return updlist, err
 }
 
-func RemoveOldChannels(config Config, updates []Updates) (err error) {
+func RemoveOldChannels(config Config, updates []Updates, deleteNow bool) (err error) {
 	mappedUpdates := MakeAMap(updates)
+	fmt.Printf("Mapped Updates: %s\n", mappedUpdates)
 	switch config.Storage.Type {
 	case "file":
 		var muChannelList []string
-		err = filepath.Walk(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/"), func(path string, info os.FileInfo, err error) error {
-			if info.IsDir() {
-				muChannelList = append(muChannelList, path)
-			}
-			return nil
-		})
-		if err != nil {
-			return 
+		if _, err := os.Stat(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/")); os.IsNotExist(err) {
+			log.Printf("This path: \"%s\" does not exist! Check your mirror...\n")
+		} else {
+			err = filepath.Walk(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/"), func(path string, info os.FileInfo, err error) error {
+				if info.Name() == "repodata" {
+					muChannelList = append(muChannelList, path)
+				}
+				if info.IsDir() {
+					if files, err := ioutil.ReadDir(path); len(files) == 0  {
+						if err != nil {
+							log.Fatal(err)
+						}
+						log.Printf("Removing unused empty folders: %s\n", path)
+						os.RemoveAll(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/", path))
+					}
+				}
+				return nil
+				})
 		}
-		//templ := regexp.MustCompile(`/\d{5,6}/`)
+			if err != nil {
+				return
+			}
+		//List of repositories that will be deleted: reposToDel
+		var reposToDel map[string]bool
+		fmt.Printf("MuchannelList: %v\n", muChannelList)
 		for _, elem := range muChannelList {
-			if regexp.MustCompile(`/\d{5,6}/`).FindString(elem) != "" {
-				_, exists := mappedUpdates[strings.Replace(regexp.MustCompile(`/\d{5,6}/`).FindString(elem), "/", "", 10)] 
-				if !exists {
-					log.Printf("removing: %s...\n", elem)
-					err = os.RemoveAll(elem)
-					if err != nil {
-						return
+				if regexp.MustCompile(`/\d{5,6}/`).FindString(elem) != "" {
+					_, exists := mappedUpdates[strings.Replace(regexp.MustCompile(`/\d{5,6}/`).FindString(elem), "/", "", 10)]
+					if !exists {
+						if deleteNow {
+							log.Printf("removing: %s...\n", elem)
+							err = os.RemoveAll(elem)
+							if err != nil {
+								return
+							}
+						} else {
+							reposToDel[elem] = true
+						}
 					}
 				}
-			}			
-		}	
+		}
+		log.Println("Mirror will have following changes:")
+		if reposToDel == nil {
+			log.Println("...there is nothing to delete!")
+		}
+		for index, _ := range reposToDel {
+			fmt.Printf("- %s", index)
+		}
+	case "s3":
 	}
 	return
 }
 
 func MakeAMap(updates []Updates) (updatesMap map[string]bool){
 	updatesMap = make(map[string]bool)
 	for _, elem := range updates {
+		if elem.IncidentNumber == "" {
+			elem.IncidentNumber = fmt.Sprintf("%s", elem.Repositories[0])
+		}
 		updatesMap[elem.IncidentNumber] = true
 	}
 	return
 }
 
+func CheckIFMapHasKey(updatesMap map[string]bool, prefix, elem string) (yes bool) {
+	for key, _ := range updatesMap {
+		if strings.Contains(key, strings.Replace(elem, prefix, "", 1)) {
+			yes = true
+		}
+	}
+	return
+}
+
 type Updates struct {
 	IncidentNumber string
 	ReleaseRequest string

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 // indirect
 	github.com/jtolds/gls v4.2.1+incompatible // indirect
+	github.com/manifoldco/promptui v0.8.0
 	github.com/smartystreets/assertions v0.0.0-20170818220048-9c0ea8acbc1d // indirect
 	github.com/smartystreets/goconvey v0.0.0-20170825221426-e5b2b7c91115 // indirect
 	github.com/smartystreets/gunit v0.0.0-20170705041418-d235ada78c07 // indirect

go.sum

@@ -47,6 +47,7 @@ github.com/aws/aws-sdk-go v0.0.0-20170914235556-ad2c5a3ad388 h1:Aa5a+YIfo9fdEe2f
 github.com/aws/aws-sdk-go v0.0.0-20170914235556-ad2c5a3ad388/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -155,12 +156,22 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU=
+github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw=
+github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo=
+github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -300,6 +311,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=