use cluster scripts (#442)

* use cluster scripts

Get rid of kubevirtci included in the project and ugly Makefile scripts.
Scripts introduced in this PR are abstracting interaction with the cluster
and are used with other kubevirtci projects as well.

Signed-off-by: Petr Horacek <[email protected]>

* fix kubevirtci.sh

Signed-off-by: Quique Llorente <[email protected]>

* Use k8s-1.17 provider by default

Signed-off-by: Quique Llorente <[email protected]>

* Fix docs

Signed-off-by: Quique Llorente <[email protected]>

* Cleanup Makefile

Signed-off-by: Quique Llorente <[email protected]>

* Use cluster/ tools instead of kubevirtci/cluster-up/ ones

Signed-off-by: Quique Llorente <[email protected]>

Co-authored-by: Félix Enrique Llorente Pastora <[email protected]>

.gitignore

@@ -3,7 +3,7 @@ build/_output
 build/_test
 
 # Cloned kubevirtci
-kubevirtci
+_kubevirtci
 
 # Junit
 *junit*.xml

Makefile

@@ -46,20 +46,10 @@ GO := $(GOBIN)/go
 
 LOCAL_REGISTRY ?= registry:5000
 
-CLUSTER_DIR ?= kubevirtci/cluster-up/
-KUBECONFIG ?= $(CURDIR)/kubevirtci/_ci-configs/$(KUBEVIRT_PROVIDER)/.kubeconfig
-export KUBECTL ?= $(CLUSTER_DIR)/kubectl.sh
-CLUSTER_UP ?= $(CLUSTER_DIR)/up.sh
-CLUSTER_DOWN ?= $(CLUSTER_DIR)/down.sh
-CLI ?= $(CLUSTER_DIR)/cli.sh
-export SSH ?= $(CLUSTER_DIR)/ssh.sh
-
-install_kubevirtci := hack/install-kubevirtci.sh
 local_handler_manifest = build/_output/handler.local.yaml
 versioned_operator_manifest = build/_output/versioned/operator.yaml
 description = build/_output/description
 
-resources = deploy/namespace.yaml deploy/service_account.yaml deploy/role.yaml deploy/role_binding.yaml
 all: check handler
 
 check: vet whitespace-check gofmt-check
@@ -113,75 +103,31 @@ test/unit: $(GINKGO)
 	INTERFACES_FILTER="" NODE_NAME=node01 $(GINKGO) $(unit_test_args) $(WHAT)
 
 test/e2e: $(OPERATOR_SDK)
-	# We have to unset mod=vendor here since operator-sdk is already
-	# building with it, and go tool fail if it's specified twice
 	mkdir -p test_logs/e2e
 	unset GOFLAGS && $(OPERATOR_SDK) test local ./test/e2e \
-		--kubeconfig $(KUBECONFIG) \
+		--kubeconfig $(shell ./cluster/kubeconfig.sh) \
 		--namespace nmstate \
 		--no-setup \
 		--go-test-flags "$(e2e_test_args)"
 
-$(local_handler_manifest): deploy/operator.yaml
-	mkdir -p $(dir $@)
-	sed "s#REPLACE_IMAGE#$(LOCAL_REGISTRY)/$(HANDLER_IMAGE_FULL_NAME)#" \
-		deploy/operator.yaml > $@
-
 
 $(versioned_operator_manifest): HANDLER_IMAGE_SUFFIX = :$(shell hack/version.sh)
 $(versioned_operator_manifest): version/version.go
 	mkdir -p $(dir $@)
 	sed "s#REPLACE_IMAGE#$(HANDLER_IMAGE)#" \
 		deploy/operator.yaml > $@
 
-$(CLUSTER_DIR)/%: $(install_kubevirtci)
-	$(install_kubevirtci)
-
-cluster-prepare:
-	hack/install-nm.sh
-	hack/flush-secondary-nics.sh
-
-provider-up: $(CLUSTER_UP)
-	$(CLUSTER_UP)
-
-cluster-up: provider-up cluster-prepare
-
-cluster-down: $(CLUSTER_DOWN)
-	$(CLUSTER_DOWN)
-
-cluster-clean: $(KUBECTL)
-	$(KUBECTL) delete --ignore-not-found -f build/_output/
-	$(KUBECTL) delete --ignore-not-found -f deploy/
-	$(KUBECTL) delete --ignore-not-found -f deploy/crds/nmstate.io_nodenetworkstates_crd.yaml
-	$(KUBECTL) delete --ignore-not-found -f deploy/crds/nmstate.io_nodenetworkconfigurationpolicies_crd.yaml
-	$(KUBECTL) delete --ignore-not-found -f deploy/crds/nmstate.io_nodenetworkconfigurationenactments_crd.yaml
-	if [[ "$$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \
-		$(KUBECTL) delete --ignore-not-found -f deploy/openshift/; \
-	fi
-
-cluster-sync-resources: $(KUBECTL)
-	if [[ "$$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \
-		while ! $(KUBECTL) get securitycontextconstraints; do sleep 1; done; \
-	fi
-	for resource in $(resources); do \
-		$(KUBECTL) apply -f $$resource || exit 1; \
-	done
-	if [[ "$$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \
-		$(KUBECTL) apply -f deploy/openshift/; \
-	fi
-
-cluster-sync-handler: cluster-sync-resources $(local_handler_manifest)
-	if [[ "$$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \
-		IMAGE_REGISTRY=localhost:$$($(CLI) ports --container-name=cluster registry | tr -d '\r') \
-				   make push-handler;  \
-	else \
-		IMAGE_REGISTRY=localhost:$$($(CLI) ports registry | tr -d '\r') \
-				   make push-handler; \
-	fi
-	local_handler_manifest=$(local_handler_manifest) ./hack/cluster-sync-handler.sh
-
-
-cluster-sync: cluster-sync-handler
+cluster-up:
+	./cluster/up.sh
+
+cluster-down:
+	./cluster/down.sh
+
+cluster-clean:
+	./cluster/clean.sh
+
+cluster-sync:
+	./cluster/sync.sh
 
 $(description): version/description
 	mkdir -p $(dir $@)

cluster/clean.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ex
+
+echo 'Cleaning up ...'
+
+./cluster/kubectl.sh delete --ignore-not-found -f deploy/
+./cluster/kubectl.sh delete --ignore-not-found -f deploy/crds/nmstate.io_nodenetworkconfigurationenactments_crd.yaml
+./cluster/kubectl.sh delete --ignore-not-found -f deploy/crds/nmstate.io_nodenetworkconfigurationpolicies_crd.yaml
+./cluster/kubectl.sh delete --ignore-not-found -f deploy/crds/nmstate.io_nodenetworkstates_crd.yaml
+
+if [[ "$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$ ]]; then
+    ./cluster/kubectl.sh delete --ignore-not-found -f deploy/openshift/
+fi

cluster/cli.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source ./cluster/kubevirtci.sh
+kubevirtci::install
+
+$(kubevirtci::path)/cluster-up/cli.sh "$@"

cluster/down.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+source ./cluster/kubevirtci.sh
+kubevirtci::install
+
+$(kubevirtci::path)/cluster-up/down.sh

cluster/kubeconfig.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+source ./cluster/kubevirtci.sh
+kubevirtci::kubeconfig

cluster/kubectl.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source ./cluster/kubevirtci.sh
+kubevirtci::install
+
+$(kubevirtci::path)/cluster-up/kubectl.sh "$@"

cluster/kubevirtci.sh

@@ -0,0 +1,38 @@
+export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-'k8s-1.17'}
+
+KUBEVIRTCI_VERSION='95096c8189c8b620ddc1310e12388df2190e1cc8'
+KUBEVIRTCI_REPO='https://github.com/kubevirt/kubevirtci.git'
+KUBEVIRTCI_PATH="${PWD}/_kubevirtci"
+
+function kubevirtci::_get_repo() {
+    git --git-dir ${KUBEVIRTCI_PATH}/.git remote get-url origin
+}
+
+function kubevirtci::_get_version() {
+    git --git-dir ${KUBEVIRTCI_PATH}/.git log --format="%H" -n 1
+}
+
+function kubevirtci::install() {
+    # Remove cloned kubevirtci repository if it does not match the requested one
+    if [ -d ${KUBEVIRTCI_PATH} ]; then
+        if [ $(kubevirtci::_get_repo) != ${KUBEVIRTCI_REPO} -o $(kubevirtci::_get_version) != ${KUBEVIRTCI_VERSION} ]; then
+            rm -rf ${KUBEVIRTCI_PATH}
+        fi
+    fi
+
+    if [ ! -d ${KUBEVIRTCI_PATH} ]; then
+        git clone ${KUBEVIRTCI_REPO} ${KUBEVIRTCI_PATH}
+        (
+            cd ${KUBEVIRTCI_PATH}
+            git checkout ${KUBEVIRTCI_VERSION}
+        )
+    fi
+}
+
+function kubevirtci::path() {
+    echo -n ${KUBEVIRTCI_PATH}
+}
+
+function kubevirtci::kubeconfig() {
+    echo -n ${KUBEVIRTCI_PATH}/_ci-configs/${KUBEVIRT_PROVIDER}/.kubeconfig
+}

cluster/sync.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -ex
+
+function getDesiredNumberScheduled {
+    echo $(./cluster/kubectl.sh get daemonset -n nmstate $1 -o=jsonpath='{.status.desiredNumberScheduled}')
+}
+
+function getNumberAvailable {
+    numberAvailable=$(./cluster/kubectl.sh get daemonset -n nmstate $1 -o=jsonpath='{.status.numberAvailable}')
+    echo ${numberAvailable:-0}
+}
+
+function consistently {
+    cmd=$@
+    retries=3
+    interval=1
+    cnt=1
+    while [[ $cnt -le $retries ]]; do
+        $cmd
+        sleep $interval
+        cnt=$(($cnt + 1))
+    done
+}
+
+function isOk {
+    desiredNumberScheduled=$(getDesiredNumberScheduled $1)
+    numberAvailable=$(getNumberAvailable $1)
+
+    if [ "$desiredNumberScheduled" == "$numberAvailable" ]; then
+        echo "$1 DS is ready"
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Cleanup previous deployment, if there is any
+make cluster-clean
+
+# Fetch registry port that can be used to upload images to the local kubevirtci cluster
+registry_port=$(./cluster/cli.sh ports registry | tr -d '\r')
+if [[ "${KUBEVIRT_PROVIDER}" =~ ^(okd|ocp)-.*$ ]]; then \
+		registry=localhost:$(./cluster/cli.sh ports --container-name=cluster registry | tr -d '\r')
+else
+    registry=localhost:$(./cluster/cli.sh ports registry | tr -d '\r')
+fi
+
+# Build new handler image from local sources and push it to the kubevirtci cluster
+IMAGE_REGISTRY=${registry} make push-handler
+
+# Deploy all needed manifests
+./cluster/kubectl.sh apply -f deploy/namespace.yaml
+./cluster/kubectl.sh apply -f deploy/service_account.yaml
+./cluster/kubectl.sh apply -f deploy/role.yaml
+./cluster/kubectl.sh apply -f deploy/role_binding.yaml
+./cluster/kubectl.sh apply -f deploy/crds/nmstate.io_nodenetworkstates_crd.yaml
+./cluster/kubectl.sh apply -f deploy/crds/nmstate.io_nodenetworkconfigurationpolicies_crd.yaml
+./cluster/kubectl.sh apply -f deploy/crds/nmstate.io_nodenetworkconfigurationenactments_crd.yaml
+if [[ "$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$ ]]; then
+		./cluster/kubectl.sh apply -f deploy/openshift/
+fi
+sed \
+    -e "s#--v=production#--v=debug#" \
+    -e "s#REPLACE_IMAGE#registry:5000/nmstate/kubernetes-nmstate-handler#" \
+    deploy/operator.yaml | ./cluster/kubectl.sh create -f -
+
+# Wait until the handler becomes ready on all nodes
+for i in {300..0}; do
+    # We have to re-check desired number, sometimes takes some time to be filled in
+    if consistently isOk nmstate-handler && consistently isOk nmstate-handler-worker; then
+       break
+    fi
+
+    if [ $i -eq 0 ]; then
+        echo "nmstate-handler or nmstate-handler-worker DS haven't turned ready within the given timeout"
+    exit 1
+    fi
+
+    sleep 1
+done

cluster/up.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+source ./cluster/kubevirtci.sh
+kubevirtci::install
+
+$(kubevirtci::path)/cluster-up/up.sh
+
+if [[ "$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \
+		while ! $(KUBECTL) get securitycontextconstraints; do sleep 1; done; \
+fi
+
+if [[ "$KUBEVIRT_PROVIDER" =~ k8s- ]]; then
+    echo 'Install NetworkManager on nodes'
+    for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do
+        ./cluster/cli.sh ssh ${node} sudo -- yum install -y yum-plugin-copr
+        ./cluster/cli.sh ssh ${node} sudo -- yum copr enable -y networkmanager/NetworkManager-1.22
+        ./cluster/cli.sh ssh ${node} sudo -- yum install -y NetworkManager
+        ./cluster/cli.sh ssh ${node} sudo -- systemctl daemon-reload
+        ./cluster/cli.sh ssh ${node} sudo -- systemctl restart NetworkManager
+        echo "Check NetworkManager is working fine on node $node"
+        ./cluster/cli.sh ssh ${node} -- nmcli device show > /dev/null
+    done
+fi
+
+for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do
+    for nic in $FIRST_SECONDARY_NIC $SECOND_SECONDARY_NIC; do
+	      uuid=$(./cluster/cli.sh ssh $node -- nmcli --fields=device,uuid  c show  |grep $nic|awk '{print $2}')
+	      if [ ! -z "$uuid" ]; then
+        	  echo "$node: Flushing nic $nic"
+        	  ./cluster/cli.sh ssh $node -- sudo nmcli con del $uuid
+	      fi
+    done
+done
+

docs/deployment-local-cluster.md

@@ -35,22 +35,22 @@ cluster.
 make cluster-sync
 ```
 
-You can ssh into the created nodes using `kubevirtci/cluster-up/ssh.sh`.
+You can ssh into the created nodes using `cluster/cli.sh ssh`.
 
 ```shell
-kubevirtci/cluster-up/ssh.sh node01
+cluster/cli.sh ssh node01
 ```
 
-Finally, you can access Kubernetes API using `kubevirtci/cluster-up/kubectl.sh`.
+Finally, you can access Kubernetes API using `cluster/kubectl.sh`.
 
 ```shell
-kubevirtci/cluster-up/kubectl.sh get nodes
+cluster/kubectl.sh get nodes
 ```
 
 If you want to use `kubectl` to access the cluster, start a proxy.
 
 ```shell
-kubevirtci/cluster-up/kubectl.sh proxy --port=8080 --disable-filter=true &
+cluster/kubectl.sh proxy --port=8080 --disable-filter=true &
 ```
 
 You can stop here and play with the cluster on your own or continue with the

docs/user-guide-policy-configure-linux-bond-with-vlans.md

@@ -16,7 +16,7 @@ All you have to do in order to create the bond and the sub vlan interface on all
 to apply the following policy:
 
 ```yaml
-cat <<EOF | ./kubevirtci/cluster-up/kubectl.sh create -f -
+cat <<EOF | ./cluster/kubectl.sh create -f -
 apiVersion: nmstate.io/v1alpha1
 kind: NodeNetworkConfigurationPolicy
 metadata:

docs/user-guide-policy-configure-linux-bond.md

@@ -16,7 +16,7 @@ All you have to do in order to create the bond on all nodes across cluster is
 to apply the following policy:
 
 ```yaml
-cat <<EOF | ./kubevirtci/cluster-up/kubectl.sh create -f -
+cat <<EOF | ./cluster/kubectl.sh create -f -
 apiVersion: nmstate.io/v1alpha1
 kind: NodeNetworkConfigurationPolicy
 metadata: