Merge pull request #36 from uc-cdis/fix/retries

fix(retries): retry limited times

Author: giangbui

handlers/handler.go

@@ -13,6 +13,10 @@ import (
 	"github.com/golang/glog"
 )
 
+const (
+	MAX_RETRIES int = 3
+)
+
 type SQSHandler struct {
 	QueueURL      string
 	Start         bool
@@ -145,9 +149,11 @@ func (handler *SQSHandler) StartMonitoringProcess() {
 					if k8sJob.Status == "Unknown" || k8sJob.Status == "Running" || k8sJob.Status == "Completed" {
 						jobInfo.Status = k8sJob.Status
 					} else if k8sJob.Status == "Failed" {
-						glog.Errorf("The k8s job %s failed. Detail %s", jobInfo.Name, err)
-						//glog.Errorf("The k8s job %s failed. Detail %s. Resend the message to the queue", jobInfo.Name, err)
-						//handler.ResendSQSMessage(handler.QueueURL, jobInfo.SQSMessage)
+						if jobInfo.Retries < MAX_RETRIES {
+							glog.Errorf("The k8s job %s failed. Detail %s. Resend the message to the queue", jobInfo.Name, err)
+							handler.ResendSQSMessage(handler.QueueURL, jobInfo.SQSMessage)
+							jobInfo.Retries += 1
+						}
 					}
 				}
 				nextMonitoredJobs = append(nextMonitoredJobs, jobInfo)

handlers/jobs.go

@@ -31,6 +31,7 @@ type JobInfo struct {
 	Name       string `json:"name"`
 	Status     string `json:"status"`
 	URL        string `json:"url"`
+	Retries    int    `json:"retries"`
 	SQSMessage *sqs.Message
 }
 
@@ -327,6 +328,7 @@ func CreateK8sJob(inputURL string, jobConfig JobConfig) (*JobInfo, error) {
 	ji.Name = newJob.Name
 	ji.UID = string(newJob.GetUID())
 	ji.URL = inputURL
+	ji.Retries = 0
 	ji.Status = jobStatusToString(&newJob.Status)
 	return &ji, nil
 }