Merge pull request #225 from hyperhq/integration-test

Optimize integration test

Author: Jimmy-Xu

hack/generate-hyper-conf-qa.sh

@@ -19,6 +19,10 @@ else
   echo "========== Task: test BRANCH ${BRANCH} =========="
 fi
 
+if [[ "${ACCESS_KEY}" == "" ]] || [[ "${SECRET_KEY}" == "" ]];then
+  echo "Error: Please set ACCESS_KEY and SECRET_KEY"
+  exit 1
+fi
 
 if [[ "$@" != "./build.sh" ]];then
     #ensure config for hyper cli
@@ -88,7 +92,7 @@ fi
 #execute command
 if [[ $# -ne 0 ]];then
     echo "========== Test Cmd: $@ =========="
-    cd /go/src/github.com/hyperhq/hypercli/integration-cli && eval $@
+    cd /go/src/github.com/hyperhq/hypercli/integration-cli && $@
     if [[ "$@" == "./build.sh" ]];then
     #show make result
         if [[ $? -eq 0 ]];then

integration-cli/README.md

@@ -345,10 +345,17 @@ $ docker run -it --rm \
 $ docker run -it --rm \
     -e ACCESS_KEY=${ACCESS_KEY} \
     -e SECRET_KEY=${SECRET_KEY} \
-		-e BRANCH="#221" \
+    -e BRANCH="#221" \
     hyperhq/hypercli-auto-test:qa go test -check.f TestCli -timeout 180m
 
 
+//test `specified case name`
+$ docker run -it --rm \
+    -e ACCESS_KEY="${ACCESS_KEY}" \
+    -e SECRET_KEY="${SECRET_KEY}" \
+    hyperhq/hypercli-auto-test:qa go test -check.f 'TestCliInfo|TestCliFip' -timeout 180m
+
+
 //test `specified branch` with `packet` apirouter
 $ docker run -it --rm \
     -e ACCESS_KEY=${ACCESS_KEY} \

integration-cli/check_test.go

@@ -35,13 +35,23 @@ func (s *DockerSuite) TearDownTest(c *check.C) {
 	deleteAllImages()
 	deleteAllSnapshots()
 	deleteAllVolumes()
+	deleteAllFips()
 	//deleteAllNetworks()
 }
 
 func init() {
 	check.Suite(&DockerRegistrySuite{
 		ds: &DockerSuite{},
 	})
+
+	fmt.Printf("INFO: clear resources(containers, images, snapshots, volumes, fips)\n")
+	deleteAllContainers()
+	deleteAllImages()
+	deleteAllSnapshots()
+	deleteAllVolumes()
+	deleteAllFips()
+
+	fmt.Println("INFO: finish init")
 }
 
 type DockerRegistrySuite struct {

integration-cli/docker_utils.go

@@ -114,7 +114,6 @@ func init() {
 	if os.Getenv("DOCKER_HOST") != "" {
 		flag_host = "--host=" + os.Getenv("DOCKER_HOST")
 	}
-	fmt.Println("finish init")
 }
 
 // Daemon represents a Docker daemon for the testing framework.
@@ -845,6 +844,44 @@ func deleteAllImages() error {
 	return nil
 }
 
+func deleteAllFips() error {
+	fips, err := getAllFips()
+	if err != nil {
+		return err
+	}
+
+	for _, v := range strings.Split(fips,"\n") {
+		if v == "" {
+			continue
+		}
+		fip := strings.Trim(v, " ")
+		releaseFipCmd := exec.Command(dockerBinary, flag_host, "fip", "release", fip)
+		out, exitCode, err := runCommandWithOutput(releaseFipCmd)
+		if exitCode != 0 && err == nil {
+			err = fmt.Errorf("failed to release fip: %v\n", out)
+			return err
+		}
+	}
+	return nil
+}
+
+func getAllFips() (string, error) {
+	//load via pipe
+	getFipsCmd := exec.Command(dockerBinary, flag_host, "fip", "ls")
+	grepCmd := exec.Command("grep", "-v", "^Floating IP")
+
+	getFipOut, err := getFipsCmd.StdoutPipe()
+	getFipsCmd.Start()
+
+	grepCmd.Stdin = getFipOut
+	output, err := grepCmd.Output()
+
+	if err != nil {
+		err = fmt.Errorf("failed to get a list of fips: %v\n", output)
+	}
+	return string(output), err
+}
+
 //status only support : created, restarting, running, exited (https://github.com/getdvm/hyper-api-router/blob/master/pkg/apiserver/router/local/container.go#L204)
 /*
 func getPausedContainers() (string, error) {

integration-cli/hyper_cli_links_test.go

@@ -192,6 +192,7 @@ func (s *DockerSuite) TestCliLinksEnvs(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	printTestCaseName()
 	defer printTestDuration(time.Now())
+	pullImageIfNotExist("busybox")
 	dockerCmd(c, "run", "-d", "-e", "e1=", "-e", "e2=v2", "-e", "e3=v3=v3", "--name=first", "busybox", "top")
 	out, _ := dockerCmd(c, "run", "--name=second", "--link=first:first", "busybox", "env")
 	/* FIXME

integration-cli/hyper_cli_logs_test.go

@@ -69,9 +69,11 @@ func (s *DockerSuite) TestCliLogsTimestamps(c *check.C) {
 	testLen := 100
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo =; done;", testLen))
+	time.Sleep(5 * time.Second)
 
 	id := strings.TrimSpace(out)
 	dockerCmd(c, "stop", id)
+	time.Sleep(5 * time.Second)
 
 	out, _ = dockerCmd(c, "logs", "-t", id)
 
@@ -99,9 +101,11 @@ func (s *DockerSuite) TestCliLogsSeparateStderr(c *check.C) {
 	pullImageIfNotExist("busybox")
 	msg := "stderr_log"
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("echo %s 1>&2", msg))
+	time.Sleep(5 * time.Second)
 
 	id := strings.TrimSpace(out)
 	dockerCmd(c, "stop", id)
+	time.Sleep(5 * time.Second)
 
 	stdout, stderr, _ := dockerCmdWithStdoutStderr(c, "logs", id)
 
@@ -120,9 +124,11 @@ func (s *DockerSuite) TestCliLogsStderrInStdout(c *check.C) {
 	pullImageIfNotExist("busybox")
 	msg := "stderr_log"
 	out, _ := dockerCmd(c, "run", "-d", "-t", "busybox", "sh", "-c", fmt.Sprintf("echo %s 1>&2", msg))
+	time.Sleep(5 * time.Second)
 
 	id := strings.TrimSpace(out)
 	dockerCmd(c, "stop", id)
+	time.Sleep(5 * time.Second)
 
 	stdout, stderr, _ := dockerCmdWithStdoutStderr(c, "logs", id)
 	c.Assert(stderr, checker.Equals, "")
@@ -139,9 +145,11 @@ func (s *DockerSuite) TestCliLogsTail(c *check.C) {
 	testLen := 100
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo =; done;", testLen))
+	time.Sleep(5 * time.Second)
 
 	id := strings.TrimSpace(out)
 	dockerCmd(c, "stop", id)
+	time.Sleep(5 * time.Second)
 
 	out, _ = dockerCmd(c, "logs", "--tail", "5", id)
 
@@ -169,9 +177,11 @@ func (s *DockerSuite) TestCliLogsFollowStopped(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "echo", "hello")
+	time.Sleep(5 * time.Second)
 
 	id := strings.TrimSpace(out)
 	dockerCmd(c, "stop", id)
+	time.Sleep(5 * time.Second)
 
 	logsCmd := exec.Command(dockerBinary, "logs", "-f", id)
 	c.Assert(logsCmd.Start(), checker.IsNil)
@@ -198,7 +208,8 @@ func (s *DockerSuite) TestCliLogsSince(c *check.C) {
 	pullImageIfNotExist("busybox")
 	name := "testlogssince"
 	dockerCmd(c, "run", "--name="+name, "-d", "busybox", "/bin/sh", "-c", "for i in $(seq 1 30); do sleep 2; echo log$i; done")
-	time.Sleep(5*time.Second)
+	time.Sleep(10*time.Second)
+
 	out, _ := dockerCmd(c, "logs", "-t", name)
 
 	log2Line := strings.Split(strings.Split(out, "\n")[1], " ")
@@ -238,17 +249,19 @@ func (s *DockerSuite) TestCliLogsSinceFutureFollow(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", `for i in $(seq 1 50); do date +%s; sleep 1; done`)
+	time.Sleep(5 * time.Second)
 	id := strings.TrimSpace(out)
 
 	now := daemonTime(c).Unix()
 	since := now - 5
+
 	out, _ = dockerCmd(c, "logs", "-f", fmt.Sprintf("--since=%v", since), id)
 	lines := strings.Split(strings.TrimSpace(out), "\n")
 	c.Assert(lines, checker.Not(checker.HasLen), 0)
 	for _, v := range lines {
 		ts, err := strconv.ParseInt(v, 10, 64)
 		c.Assert(err, checker.IsNil, check.Commentf("cannot parse timestamp output from log: '%v'\nout=%s", v, out))
-		c.Assert(ts >= since, checker.Equals, true, check.Commentf("earlier log found. since=%v logdate=%v", since, ts))
+		c.Assert( (ts+1) >= since, checker.Equals, true, check.Commentf("earlier log found. since=%v logdate=%v", since, ts))
 	}
 }
 
@@ -260,13 +273,14 @@ func (s *DockerSuite) TestCliLogsFollowSlowStdoutConsumer(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", `usleep 600000;yes X | head -c 200000`)
-
+	time.Sleep(5 * time.Second)
 	id := strings.TrimSpace(out)
 
 	stopSlowRead := make(chan bool)
 
 	go func() {
 		exec.Command(dockerBinary, "stop", id).Run()
+		time.Sleep(5 * time.Second)
 		stopSlowRead <- true
 	}()
 

integration-cli/hyper_cli_ps_test.go

@@ -117,8 +117,6 @@ func (s *DockerSuite) TestCliPsListContainersBase(c *check.C) {
 }
 
 func assertContainerList(out string, expected []string) bool {
-	printTestCaseName()
-	defer printTestDuration(time.Now())
 
 	lines := strings.Split(strings.Trim(out, "\n "), "\n")
 	if len(lines)-1 != len(expected) {

integration-cli/hyper_cli_restart_test.go

@@ -16,14 +16,15 @@ func (s *DockerSuite) TestCliRestartStoppedContainer(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "echo", "foobar")
-
+	time.Sleep(5 * time.Second)
 	cleanedContainerID := strings.TrimSpace(out)
 	c.Assert(waitExited(cleanedContainerID, 30*time.Second), checker.IsNil)
 
 	out, _ = dockerCmd(c, "logs", cleanedContainerID)
 	c.Assert(out, checker.Equals, "foobar\n")
 
 	dockerCmd(c, "restart", cleanedContainerID)
+	time.Sleep(5 * time.Second)
 
 	out, _ = dockerCmd(c, "logs", cleanedContainerID)
 	c.Assert(out, checker.Equals, "foobar\n")
@@ -35,7 +36,7 @@ func (s *DockerSuite) TestCliRestartRunningContainer(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", "echo foobar && sleep 30 && echo 'should not print this'")
-
+	time.Sleep(5 * time.Second)
 	cleanedContainerID := strings.TrimSpace(out)
 
 	c.Assert(waitRun(cleanedContainerID), checker.IsNil)
@@ -44,6 +45,7 @@ func (s *DockerSuite) TestCliRestartRunningContainer(c *check.C) {
 	c.Assert(out, checker.Equals, "foobar\n")
 
 	dockerCmd(c, "restart", "-t", "1", cleanedContainerID)
+	time.Sleep(5 * time.Second)
 
 	out, _ = dockerCmd(c, "logs", cleanedContainerID)
 

integration-cli/hyper_cli_run_test.go

@@ -127,11 +127,14 @@ func (s *DockerSuite) TestCliRunDetachedContainerIDPrinting(c *check.C) {
 	defer printTestDuration(time.Now())
 	pullImageIfNotExist("busybox")
 	out, _ := dockerCmd(c, "run", "-d", "busybox", "true")
+	time.Sleep(5 * time.Second)
 
 	out = strings.TrimSpace(out)
 	dockerCmd(c, "stop", out)
+	time.Sleep(5 * time.Second)
 
 	rmOut, _ := dockerCmd(c, "rm", out)
+	time.Sleep(5 * time.Second)
 
 	rmOut = strings.TrimSpace(rmOut)
 	if rmOut != out {
@@ -995,6 +998,7 @@ func (s *DockerSuite) TestCliRunUnknownCommand(c *check.C) {
 
 	cID := strings.TrimSpace(out)
 	_, _, err := dockerCmdWithError("start", cID)
+	time.Sleep(5 * time.Second)
 
 	// Windows and Linux are different here by architectural design. Linux will
 	// fail to start the container, so an error is expected. Windows will
@@ -1106,23 +1110,22 @@ func (s *DockerSuite) TestCliRunRestartMaxRetries(c *check.C) {
 	printTestCaseName()
 	defer printTestDuration(time.Now())
 	pullImageIfNotExist("busybox")
-	out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:3", "busybox", "sh", "-c", "sleep 15; false")
-	timeout := 60 * time.Second
-	if daemonPlatform == "windows" {
-		timeout = 45 * time.Second
-	}
+
+	RETRY_COUNT := "3"
+	out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:" + RETRY_COUNT, "busybox", "sh", "-c", "sleep 35; false")
+	timeout := 120 * time.Second
 
 	time.Sleep(timeout)
 	id := strings.TrimSpace(string(out))
 
 	count := inspectField(c, id, "RestartCount")
-	if count != "3" {
-		c.Fatalf("Container was restarted %s times, expected %d", count, 3)
+	if count != RETRY_COUNT {
+		c.Fatalf("Container was restarted %s times, expected %d", count, RETRY_COUNT)
 	}
 
 	MaximumRetryCount := inspectField(c, id, "HostConfig.RestartPolicy.MaximumRetryCount")
-	if MaximumRetryCount != "3" {
-		c.Fatalf("Container Maximum Retry Count is %s, expected %s", MaximumRetryCount, "3")
+	if MaximumRetryCount != RETRY_COUNT {
+		c.Fatalf("Container Maximum Retry Count is %s, expected %s", MaximumRetryCount, RETRY_COUNT)
 	}
 }