feat: RequestSetting增加设置代理、跳过tls证书校验功能

Author: CC11001100

README.md

@@ -1,12 +1,12 @@
-# HTTP请求工具 
-
+# HTTP请求工具
 
 ```bash
 go get -u github.com/crawler-go-go-go/go-requests
 ```
 
-# TODO 
-- [x] 方便的做响应格式处理 
-- [ ] 兼容HTTP2 
+# TODO
+
+- [x] 方便的做响应格式处理
+- [ ] 兼容HTTP2
 - [ ] 兼容JA3指纹 
 

http.go

@@ -10,19 +10,7 @@ import (
 // DefaultMaxTryTimes 默认情况下的最大重试次数
 const DefaultMaxTryTimes = 3
 
-type RequestSetting func(client *http.Client, request *http.Request) error
-
-const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
-
-func DefaultUserAgentRequestSetting() RequestSetting {
-	return func(client *http.Client, httpRequest *http.Request) error {
-		httpRequest.Header.Set("User-Agent", DefaultUserAgent)
-		return nil
-	}
-}
-
-// ------------------------------------------------- --------------------------------------------------------------------
-
+// SendRequest 底层API，不建议直接调用
 func SendRequest[Request any, Response any](ctx context.Context, options *Options[Request, Response]) (Response, error) {
 
 	// TODO set default params
@@ -32,7 +20,7 @@ func SendRequest[Request any, Response any](ctx context.Context, options *Option
 	for tryTimes := 0; tryTimes < options.MaxTryTimes; tryTimes++ {
 		var client http.Client
 
-		// 忽略证书验证 
+		// 忽略证书验证
 		client.Transport = &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		}

request_setting.go

@@ -0,0 +1,68 @@
+package requests
+
+import (
+	"net/http"
+	"net/url"
+)
+
+type RequestSetting func(client *http.Client, request *http.Request) error
+
+const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
+
+// DefaultRequestSetting 每个请求默认添加的设置
+func DefaultRequestSetting() RequestSetting {
+	return func(client *http.Client, httpRequest *http.Request) error {
+		settings := []RequestSetting{
+			RequestSettingUserAgent(),
+			RequestSettingSkipTlsVerify(),
+		}
+		for _, setting := range settings {
+			err := setting(client, httpRequest)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// RequestSettingUserAgent 设置User-Agent，如果不传递的则使用默认的User-Agent
+func RequestSettingUserAgent(userAgent ...string) RequestSetting {
+	if len(userAgent) == 0 {
+		userAgent = append(userAgent, DefaultUserAgent)
+	}
+	return func(client *http.Client, httpRequest *http.Request) error {
+		httpRequest.Header.Set("User-Agent", userAgent[0])
+		return nil
+	}
+}
+
+// RequestSettingProxy 配置代理IP
+func RequestSettingProxy(proxy string) RequestSetting {
+	parse, err := url.Parse(proxy)
+	return func(client *http.Client, httpRequest *http.Request) error {
+		if err != nil {
+			return err
+		}
+		transport, ok := client.Transport.(*http.Transport)
+		if !ok {
+			transport = &http.Transport{}
+		}
+		transport.Proxy = http.ProxyURL(parse)
+		client.Transport = transport
+		return nil
+	}
+}
+
+// RequestSettingSkipTlsVerify 跳过https证书验证
+func RequestSettingSkipTlsVerify() RequestSetting {
+	return func(client *http.Client, httpRequest *http.Request) error {
+		transport, ok := client.Transport.(*http.Transport)
+		if !ok {
+			transport = &http.Transport{}
+		}
+		transport.TLSClientConfig.InsecureSkipVerify = true
+		client.Transport = transport
+		return nil
+	}
+}