Gather e2e tests logs in parallel and improve some logs (#373)

* Run logs gathering in parallel Signed-off-by: Quique Llorente <[email protected]> * Convert e2e test reporter to ginkgo custom reporter Signed-off-by: Quique Llorente <[email protected]> * Be quiet at reporter cmd runs Signed-off-by: Quique Llorente <[email protected]> * Dump pod logs on failure to GinkgoWriter It helps with prow jobs, since you navigate them using the junit report Signed-off-by: Quique Llorente <[email protected]> * Fix automation scripts typo Signed-off-by: Quique Llorente <[email protected]> * Dump only non passed test log files Signed-off-by: Quique Llorente <[email protected]> * Reuse banner format Signed-off-by: Quique Llorente <[email protected]> * Move knmstate reporter to new package Signed-off-by: Quique Llorente <[email protected]> * Don't fail if there is no logs Signed-off-by: Quique Llorente <[email protected]>
nmstate · Feb 3, 2020 · aaf2d9c · aaf2d9c
1 parent 16984e3
commit aaf2d9c
Show file tree

Hide file tree

Showing 12 changed files with 409 additions and 245 deletions.
diff --git a/automation/check-patch.e2e-k8s.sh b/automation/check-patch.e2e-k8s.sh
@@ -10,7 +10,8 @@
 teardown() {
     make cluster-down
     cp $(find . -name "*junit*.xml") $ARTIFACTS
-    [ -d ${E2E_LOGS} ] && cp ${E2E_LOGS}/*.log ${ARTIFACTS}
+    # Don't fail if there is no logs
+    cp ${E2E_LOGS}/*.log ${ARTIFACTS} || true
 }
 
 main() {
@@ -26,7 +27,7 @@ main() {
     make cluster-up
     trap teardown EXIT SIGINT SIGTERM SIGSTOP
     make cluster-sync
-    make E2E_TEST_ARG="-ginkgo.noColor" test/e2e
+    make E2E_TEST_ARGS="-ginkgo.noColor" test/e2e
 }
 
 [[ "${BASH_SOURCE[0]}" == "$0" ]] && main "$@"
diff --git a/automation/check-patch.e2e-ocp.sh b/automation/check-patch.e2e-ocp.sh
@@ -10,7 +10,8 @@
 teardown() {
     make cluster-down
     cp $(find . -name "*junit*.xml") $ARTIFACTS
-    [ -d ${E2E_LOGS} ] && cp ${E2E_LOGS}/*.log ${ARTIFACTS}
+    # Don't fail if there is no logs
+    cp ${E2E_LOGS}/*.log ${ARTIFACTS} || true
 }
 
 main() {
@@ -26,7 +27,7 @@ main() {
     make cluster-up
     trap teardown EXIT SIGINT SIGTERM SIGSTOP
     make cluster-sync
-    make E2E_TEST_ARG="-ginkgo.noColor" test/e2e
+    make E2E_TEST_ARGS="-ginkgo.noColor" test/e2e
 }
 
 [[ "${BASH_SOURCE[0]}" == "$0" ]] && main "$@"
diff --git a/automation/check-patch.e2e-okd.sh b/automation/check-patch.e2e-okd.sh
@@ -13,7 +13,8 @@ exit 0
 teardown() {
     make cluster-down
     cp $(find . -name "*junit*.xml") $ARTIFACTS
-    [ -d ${E2E_LOGS} ] && cp ${E2E_LOGS}/*.log ${ARTIFACTS}
+    # Don't fail if there is no logs
+    cp ${E2E_LOGS}/*.log ${ARTIFACTS} || true
 }
 
 main() {
@@ -29,7 +30,7 @@ main() {
     make cluster-up
     trap teardown EXIT SIGINT SIGTERM SIGSTOP
     make cluster-sync
-    make E2E_TEST_ARG="-ginkgo.noColor" test/e2e
+    make E2E_TEST_ARGS="-ginkgo.noColor" test/e2e
 }
 
 [[ "${BASH_SOURCE[0]}" == "$0" ]] && main "$@"
diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go
@@ -12,7 +12,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/onsi/ginkgo/reporters"
+	ginkgoreporters "github.com/onsi/ginkgo/reporters"
 
 	corev1 "k8s.io/api/core/v1"
 	dynclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -21,12 +21,12 @@ import (
 
 	apis "github.com/nmstate/kubernetes-nmstate/pkg/apis"
 	nmstatev1alpha1 "github.com/nmstate/kubernetes-nmstate/pkg/apis/nmstate/v1alpha1"
+	knmstatereporter "github.com/nmstate/kubernetes-nmstate/test/e2e/reporter"
 )
 
 var (
 	f                    = framework.Global
 	t                    *testing.T
-	namespace            string
 	nodes                []string
 	startTime            time.Time
 	bond1                string
@@ -36,7 +36,6 @@ var (
 	secondSecondaryNic   = flag.String("secondSecondaryNic", "eth2", "Second secondary network interface name")
 	nodesInterfacesState = make(map[string][]byte)
 	interfacesToIgnore   = []string{"flannel.1", "dummy0"}
-	reporter             = NewKubernetesNMStateReporter("test_logs/e2e")
 )
 
 var _ = BeforeSuite(func() {
@@ -45,14 +44,7 @@ var _ = BeforeSuite(func() {
 	err := framework.AddToFrameworkScheme(apis.AddToScheme, nodeNetworkStateList)
 	Expect(err).ToNot(HaveOccurred())
 
-	By("Getting node list from cluster")
-	nodeList := corev1.NodeList{}
-	err = framework.Global.Client.List(context.TODO(), &nodeList, &dynclient.ListOptions{})
-	Expect(err).ToNot(HaveOccurred())
-	reporter.BeforeSuiteDidRun()
-	for _, node := range nodeList.Items {
-		nodes = append(nodes, node.Name)
-	}
+	prepare(t)
 })
 
 func TestMain(m *testing.M) {
@@ -62,23 +54,29 @@ func TestMain(m *testing.M) {
 func TestE2E(tapi *testing.T) {
 	t = tapi
 	RegisterFailHandler(Fail)
-	junitReporter := reporters.NewJUnitReporter("junit.functest.xml")
-	RunSpecsWithDefaultAndCustomReporters(t, "E2E Test Suite", []Reporter{junitReporter})
+
+	By("Getting node list from cluster")
+	nodeList := corev1.NodeList{}
+	err := framework.Global.Client.List(context.TODO(), &nodeList, &dynclient.ListOptions{})
+	Expect(err).ToNot(HaveOccurred())
+	for _, node := range nodeList.Items {
+		nodes = append(nodes, node.Name)
+	}
+	knmstateReporter := knmstatereporter.New("test_logs/e2e", framework.Global.Namespace, nodes)
+	junitReporter := ginkgoreporters.NewJUnitReporter("junit.functest.xml")
+	RunSpecsWithDefaultAndCustomReporters(t, "E2E Test Suite", []Reporter{junitReporter, knmstateReporter})
 
 }
 
 var _ = BeforeEach(func() {
 	bond1 = nextBond()
 	bridge1 = nextBridge()
-	_, namespace = prepare(t)
 	startTime = time.Now()
 	By("Getting nodes initial state")
 	for _, node := range nodes {
 		nodeState := nodeInterfacesState(node, interfacesToIgnore)
 		nodesInterfacesState[node] = nodeState
 	}
-	reporter.dumpStateBeforeEach(getTestName())
-
 })
 
 var _ = AfterEach(func() {
@@ -87,7 +85,6 @@ var _ = AfterEach(func() {
 		nodeState := nodeInterfacesState(node, interfacesToIgnore)
 		Expect(nodesInterfacesState[node]).Should(MatchJSON(nodeState))
 	}
-	reporter.dumpStateAfterEach(getTestName(), namespace, startTime)
 })
 
 func getMaxFailsFromEnv() int {

diff --git a/test/e2e/nnce_conditions_test.go b/test/e2e/nnce_conditions_test.go
@@ -10,6 +10,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	nmstatev1alpha1 "github.com/nmstate/kubernetes-nmstate/pkg/apis/nmstate/v1alpha1"
+	runner "github.com/nmstate/kubernetes-nmstate/test/e2e/runner"
 )
 
 func invalidConfig(bridgeName string) nmstatev1alpha1.State {
@@ -24,13 +25,13 @@ var _ = Describe("EnactmentCondition", func() {
 	Context("when applying valid config", func() {
 		BeforeEach(func() {
 			By("Add some sleep time to vlan-filtering")
-			runAtPods("cp", "/usr/local/bin/vlan-filtering", "/usr/local/bin/vlan-filtering.bak")
-			runAtPods("sed", "-i", "$ a\\sleep 5", "/usr/local/bin/vlan-filtering")
+			runner.RunAtPods("cp", "/usr/local/bin/vlan-filtering", "/usr/local/bin/vlan-filtering.bak")
+			runner.RunAtPods("sed", "-i", "$ a\\sleep 5", "/usr/local/bin/vlan-filtering")
 			updateDesiredState(linuxBrUp(bridge1))
 		})
 		AfterEach(func() {
 			By("Restore original vlan-filtering")
-			runAtPods("mv", "/usr/local/bin/vlan-filtering.bak", "/usr/local/bin/vlan-filtering")
+			runner.RunAtPods("mv", "/usr/local/bin/vlan-filtering.bak", "/usr/local/bin/vlan-filtering")
 
 			By("Remove the bridge")
 			updateDesiredState(linuxBrAbsent(bridge1))

diff --git a/test/e2e/reporter.go b/test/e2e/reporter.go