Merge pull request #28 from dclaisse/master

HTTP mirroring: add an option to follow HTTP redirections

Author: dclaisse

README.md

@@ -128,6 +128,7 @@ or
 {
   "type": "sink.http",
   "config": {
+    "follow_redirects": true,
     "timeout": "1s",
     "target_url": "{req.meta.target.string}"
   }
@@ -136,11 +137,12 @@ or
 
 
 
-| Param        | Value                                                       |
-| ------------ | ----------------------------------------------------------- |
-| `target_url` | URL to send the requests to. The path in the URL is ignored |
-| `timeout`    | Requests timeout. Ex: `1s`, `200ms`, `1m30s`                |
-| `parallel`   | How many requests to send in parallel. Default: 10          |
+| Param              | Value                                                       |
+| ------------------ | ----------------------------------------------------------- |
+| `target_url`       | URL to send the requests to. The path in the URL is ignored |
+| `timeout`          | Requests timeout. Ex: `1s`, `200ms`, `1m30s`                |
+| `parallel`         | How many requests to send in parallel. Default: 10          |
+| `follow_redirects` | Follow HTTP redirections. Default: False                    |
 
 #### sink.file
 

mirror/modules/sink/http.go

@@ -42,9 +42,10 @@ func init() {
 }
 
 type HTTPConfig struct {
-	TargetURL *expr.StringExpr `json:"target_url,omitempty"`
-	Timeout   string           `json:"timeout,omitempty"`
-	Parallel  int              `json:"parallel"`
+	FollowRedirects bool             `json:"follow_redirects,omitempty"`
+	TargetURL       *expr.StringExpr `json:"target_url,omitempty"`
+	Timeout         string           `json:"timeout,omitempty"`
+	Parallel        int              `json:"parallel"`
 }
 
 type HTTP struct {
@@ -77,17 +78,22 @@ func NewHTTP(ctx *mirror.ModuleContext, cfg []byte) (mirror.Module, error) {
 		maxWorkers = c.Parallel
 	}
 
+	httpClient := &http.Client{
+		Timeout: timeout,
+	}
+
+	if !c.FollowRedirects {
+		httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+
 	mod := &HTTP{
-		ctx:   ctx,
-		cfg:   c,
-		out:   make(chan mirror.Request),
-		tasks: make(chan mirror.Request),
-		client: &http.Client{
-			CheckRedirect: func(req *http.Request, via []*http.Request) error {
-				return http.ErrUseLastResponse
-			},
-			Timeout: timeout,
-		},
+		ctx:        ctx,
+		cfg:        c,
+		out:        make(chan mirror.Request),
+		tasks:      make(chan mirror.Request),
+		client:     httpClient,
 		maxWorkers: maxWorkers,
 	}
 	return mod, nil