Merge pull request #35 from kthcloud/dev

Update to fetch kubeconfigs locally

Author: saffronjam

.gitignore

@@ -130,4 +130,7 @@ dist
 .pnp.*
 
 
-.idea/
+.idea/
+
+# KubeConfig directory
+kube/

dto/body/capacities.go

@@ -11,12 +11,18 @@ type TimestampedCapacities struct {
 }
 
 type Capacities struct {
-	RAM     RamCapacities     `json:"ram" bson:"ram"`
 	CpuCore CpuCoreCapacities `json:"cpuCore" bson:"cpuCore"`
+	RAM     RamCapacities     `json:"ram" bson:"ram"`
 	GPU     GpuCapacities     `json:"gpu" bson:"gpu"`
 	Hosts   []HostCapacities  `json:"hosts" bson:"hosts"`
 }
 
+type ClusterCapacities struct {
+	Name    string `json:"cluster" bson:"cluster"`
+	RAM     RamCapacities
+	CpuCore CpuCoreCapacities
+}
+
 type HostGpuCapacities struct {
 	Count int `json:"count" bson:"count"`
 }
@@ -31,12 +37,10 @@ type HostCapacities struct {
 }
 
 type RamCapacities struct {
-	Used  int `json:"used" bson:"used"`
 	Total int `json:"total" bson:"total"`
 }
 
 type CpuCoreCapacities struct {
-	Used  int `json:"used" bson:"used"`
 	Total int `json:"total" bson:"total"`
 }
 

models/config.go

@@ -21,11 +21,8 @@ type ConfigType struct {
 	} `yaml:"keycloak"`
 
 	K8s struct {
-		Clusters []struct {
-			Name string `yaml:"name"`
-			URL  string `yaml:"url"`
-		} `yaml:"clusters"`
-		Clients map[string]kubernetes.Clientset
+		ConfigDir string `yaml:"configDir"`
+		Clients   map[string]kubernetes.Clientset
 	} `yaml:"k8s"`
 
 	CS struct {

pkg/config/config.go

@@ -1,19 +1,13 @@
 package config
 
 import (
-	"errors"
 	"fmt"
 	"gopkg.in/yaml.v3"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"log"
 	"os"
-	"regexp"
 	"strings"
 	"sys-api/models"
-	"sys-api/pkg/imp/cloudstack"
-	"sys-api/pkg/repository"
-	"sys-api/pkg/subsystems/cs"
 )
 
 func Setup() error {
@@ -36,153 +30,46 @@ func Setup() error {
 		return makeError(err)
 	}
 
-	csClient := cloudstack.NewAsyncClient(
-		models.Config.CS.URL,
-		models.Config.CS.ApiKey,
-		models.Config.CS.Secret,
-		true,
-	)
-
-	// Load Kubernetes clusters from cloudstack
-	listClusterParams := csClient.Kubernetes.NewListKubernetesClustersParams()
-	listClusterParams.SetListall(true)
-	clusters, err := csClient.Kubernetes.ListKubernetesClusters(listClusterParams)
+	models.Config.K8s.Clients = make(map[string]kubernetes.Clientset)
+
+	// Load config from models.Config.K8s.ConfigDir
+	// Filename without an extension is used as the cluster name
+	files, err := os.ReadDir(models.Config.K8s.ConfigDir)
 	if err != nil {
 		return makeError(err)
 	}
 
-	fetchConfig := func(name string, publicUrl string) string {
-		clusterIdx := -1
-		for idx, cluster := range clusters.KubernetesClusters {
-			if cluster.Name == name {
-				clusterIdx = idx
-				break
-			}
-		}
-
-		if clusterIdx == -1 {
-			fmt.Println("cluster", name, "not found")
-			return ""
-		}
-
-		params := csClient.Kubernetes.NewGetKubernetesClusterConfigParams()
-		params.SetId(clusters.KubernetesClusters[clusterIdx].Id)
-
-		k8sConfig, err := csClient.Kubernetes.GetKubernetesClusterConfig(params)
-		if err != nil {
-			log.Println("failed to fetch config for cluster", name, ". details:", err)
-			return ""
-		}
-
-		// use regex to replace the private ip in config.ConffigData 172.31.1.* with the public ip
-		regex := regexp.MustCompile(`https://172.31.1.[0-9]+:6443`)
-
-		k8sConfig.ClusterConfig.Configdata = regex.ReplaceAllString(k8sConfig.ClusterConfig.Configdata, publicUrl)
-
-		return k8sConfig.ClusterConfig.Configdata
-	}
-
-	models.Config.K8s.Clients = make(map[string]kubernetes.Clientset)
-
-	for _, cluster := range models.Config.K8s.Clusters {
-		// get the public ip of the cluster
-		publicURL := cluster.URL
-
-		// get the config data from cloudstack
-		configData := fetchConfig(cluster.Name, publicURL)
-		if configData == "" {
+	for _, file := range files {
+		if file.IsDir() {
 			continue
 		}
 
-		// create the k8s client
-		client, err := createK8sClient([]byte(configData))
+		clusterName := strings.TrimSuffix(file.Name(), filepath[strings.LastIndex(filepath, "."):])
+		configData, err := os.ReadFile(models.Config.K8s.ConfigDir + "/" + file.Name())
 		if err != nil {
-			fmt.Println(makeError(errors.New("failed to connect to k8s cluster " + cluster.Name + ". details: " + err.Error())))
+			fmt.Println(makeError(err))
 			continue
 		}
 
-		if client == nil {
-			fmt.Println(makeError(errors.New("failed to connect to k8s cluster " + cluster.Name + ", client is nil")))
-			continue
-		}
-
-		models.Config.K8s.Clients[cluster.Name] = *client
-	}
-
-	clusterNames := make([]string, len(models.Config.K8s.Clients))
-	i := 0
-	for name := range models.Config.K8s.Clients {
-		clusterNames[i] = name
-		i++
-	}
-
-	if len(clusterNames) > 0 {
-		fmt.Println("successfully connected to k8s clusters:", strings.Join(clusterNames, ", "))
-	} else {
-		fmt.Println("failed to connect to any k8s clusters")
-	}
-
-	return nil
-}
-
-func SyncCloudStackHosts() error {
-	// Register hosts
-	hosts, err := cs.NewClient(cs.ClientConfig{
-		URL:    models.Config.CS.URL,
-		ApiKey: models.Config.CS.ApiKey,
-		Secret: models.Config.CS.Secret,
-	}).ListHosts()
-
-	if err != nil {
-		return fmt.Errorf("failed to fetch hosts from cloudstack. details: %s", err)
-	}
-
-	if len(hosts) == 0 {
-		return fmt.Errorf("no hosts found in cloudstack. this is likely due to no hosts being available")
-	}
-
-	zones := make(map[string]models.Zone)
-	for _, host := range hosts {
-		convertedName := convertCloudStackZone(host.Zone)
-		if convertedName == nil {
-			log.Printf("zone %s not found in cloudstack zone name conversion map. this is likely a unoffical zone\n", host.Zone)
+		client, err := createK8sClient(configData)
+		if err != nil {
+			fmt.Println(makeError(err))
 			continue
 		}
 
-		newHost := models.NewHost(host.Name, host.DisplayName, *convertedName, host.IP, host.Port, host.Enabled)
-		if err = repository.NewClient().RegisterHost(newHost); err != nil {
-			return err
-		}
+		models.Config.K8s.Clients[clusterName] = *client
 
-		if _, exists := zones[host.Zone]; !exists {
-			zones[host.Zone] = models.Zone{Name: *convertedName}
-		}
+		fmt.Println("Successfully connected to k8s cluster:", clusterName)
 	}
 
-	// Register zones
-	for _, zone := range zones {
-		if err := repository.NewClient().RegisterZone(&zone); err != nil {
-			return err
-		}
+	if len(models.Config.K8s.Clients) == 0 {
+		fmt.Println("No k8s clusters found. Please check your config file and ensure that the k8s config directory is correct")
 	}
 
 	return nil
 }
 
-// convertCloudStackZone converts a cloudstack zone
-// It is a temporary solution since CloudStack does not use the same zone names as other systems
-func convertCloudStackZone(zoneName string) *string {
-	switch zoneName {
-	case "Flemingsberg":
-		s := "se-flem"
-		return &s
-	case "Kista":
-		s := "se-kista"
-		return &s
-	}
-	return nil
-}
-
+// createK8sClient creates a k8s client from the text content of a kubeconfig file
 func createK8sClient(configData []byte) (*kubernetes.Clientset, error) {
 	makeError := func(err error) error {
 		return fmt.Errorf("failed to create k8s client. details: %s", err)

pkg/subsystems/k8s/model.go

@@ -0,0 +1,11 @@
+package k8s
+
+type Node struct {
+	Name string `json:"name"`
+	CPU  struct {
+		Total int `json:"total"`
+	} `json:"cpu"`
+	RAM struct {
+		Total int `json:"total"`
+	} `json:"ram"`
+}

pkg/subsystems/k8s/nodes.go

@@ -0,0 +1,32 @@
+package k8s
+
+import (
+	"context"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (c *Client) GetNodes() ([]Node, error) {
+	nodes, err := c.K8sClient.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	var ret []Node
+	for _, node := range nodes.Items {
+		ret = append(ret, Node{
+			Name: node.Name,
+			CPU: struct {
+				Total int `json:"total"`
+			}{
+				Total: int(node.Status.Capacity.Cpu().MilliValue()) / 1000,
+			},
+			RAM: struct {
+				Total int `json:"total"`
+			}{
+				Total: int(float64(node.Status.Capacity.Memory().Value() / 1024 / 1024 / 1024)),
+			},
+		})
+	}
+
+	return ret, nil
+}