init

Author: CC11001100

.gitignore

@@ -0,0 +1 @@
+.idea

README.md

@@ -0,0 +1,7 @@
+# HTTP请求工具 
+
+
+# TODO 
+- [x] 方便的做响应格式处理 
+- [ ] 兼容HTTP2 
+- [ ] 兼容JA3指纹 

api.go

@@ -0,0 +1,92 @@
+package requests
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+)
+
+// ------------------------------------------------- --------------------------------------------------------------------
+
+// GetYaml 响应内容是YAML格式的
+func GetYaml[Response any](ctx context.Context, targetUrl string, options ...*Options[any, Response]) (Response, error) {
+
+	if len(options) == 0 {
+		options = append(options, NewOptions[any, Response](targetUrl, YamlResponseHandler[Response]()))
+	}
+
+	options[0] = options[0].WithTargetURL(targetUrl).WithYamlResponseHandler()
+
+	return SendRequest[any, Response](ctx, options[0])
+}
+
+// ------------------------------------------------- --------------------------------------------------------------------
+
+// GetJson 响应内容是JSON格式的
+func GetJson[Response any](ctx context.Context, targetUrl string, options ...*Options[any, Response]) (Response, error) {
+
+	if len(options) == 0 {
+		options = append(options, NewOptions[any, Response](targetUrl, JsonResponseHandler[Response]()))
+	}
+
+	options[0] = options[0].WithTargetURL(targetUrl).
+		WithJsonResponseHandler().
+		AppendRequestSetting(func(httpRequest *http.Request) error {
+			httpRequest.Header.Set("Content-Type", "application/json")
+			return nil
+		})
+
+	return SendRequest[any, Response](ctx, options[0])
+}
+
+func PostJson[Request any, Response any](ctx context.Context, targetUrl string, request Request, options ...*Options[Request, Response]) (Response, error) {
+
+	if len(options) == 0 {
+		options = append(options, NewOptions[Request, Response](targetUrl, JsonResponseHandler[Response]()))
+	}
+
+	marshal, err := json.Marshal(request)
+	if err != nil {
+		var zero Response
+		return zero, fmt.Errorf("PostJson json marshal request error: %s, typer = %s", err.Error(), reflect.TypeOf(request).String())
+	}
+
+	options[0] = options[0].WithTargetURL(targetUrl).
+		WithJsonResponseHandler().
+		AppendRequestSetting(func(httpRequest *http.Request) error {
+			httpRequest.Header.Set("Content-Type", "application/json")
+			return nil
+		}).
+		WithBody(marshal)
+
+	return SendRequest[Request, Response](ctx, options[0])
+}
+
+// ------------------------------------------------- --------------------------------------------------------------------
+
+func GetBytes(ctx context.Context, targetUrl string, options ...*Options[any, []byte]) ([]byte, error) {
+
+	if len(options) == 0 {
+		options = append(options, NewOptions[any, []byte](targetUrl, BytesResponseHandler()))
+	}
+
+	options[0] = options[0].WithTargetURL(targetUrl).
+		WithResponseHandler(BytesResponseHandler())
+
+	return SendRequest[any, []byte](ctx, options[0])
+}
+
+// ------------------------------------------------- --------------------------------------------------------------------
+
+func GetString(ctx context.Context, targetUrl string, options ...*Options[any, []byte]) (string, error) {
+	responseBytes, err := GetBytes(ctx, targetUrl, options...)
+	if err != nil {
+		return "", err
+	}
+	return string(responseBytes), nil
+}
+
+// ------------------------------------------------- --------------------------------------------------------------------
+

example/request-text/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+
+}

go.mod

@@ -0,0 +1,5 @@
+module github.com/crawler-go-go-go/go-requests
+
+go 1.18
+
+require gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

http.go

@@ -0,0 +1,64 @@
+package requests
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+)
+
+// DefaultMaxTryTimes 默认情况下的最大重试次数
+const DefaultMaxTryTimes = 3
+
+type RequestSetting func(httpRequest *http.Request) error
+
+const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
+
+func DefaultUserAgentRequestSetting() RequestSetting {
+	return func(httpRequest *http.Request) error {
+		httpRequest.Header.Set("User-Agent", DefaultUserAgent)
+		return nil
+	}
+}
+
+// ------------------------------------------------- --------------------------------------------------------------------
+
+func SendRequest[Request any, Response any](ctx context.Context, options *Options[Request, Response]) (Response, error) {
+
+	// TODO set default params
+
+	var lastErr error
+	for tryTimes := 0; tryTimes < options.MaxTryTimes; tryTimes++ {
+		var client http.Client
+		httpRequest, err := http.NewRequest(options.Method, options.TargetURL, bytes.NewReader(options.Body))
+		if err != nil {
+			lastErr = err
+			continue
+		}
+
+		httpRequest = httpRequest.WithContext(ctx)
+
+		for _, requestSettingFunc := range options.RequestSettingSlice {
+			if err := requestSettingFunc(httpRequest); err != nil {
+				lastErr = err
+				continue
+			}
+		}
+
+		httpResponse, err := client.Do(httpRequest)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		defer httpResponse.Body.Close()
+
+		response, err := options.ResponseHandler(httpResponse)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		return response, nil
+	}
+
+	var zero Response
+	return zero, lastErr
+}

options.go

@@ -0,0 +1,75 @@
+package requests
+
+import "net/http"
+
+const DefaultMethod = http.MethodGet
+
+type Options[Request any, Response any] struct {
+	MaxTryTimes         int
+	TargetURL           string
+	Method              string
+	Body                []byte
+	RequestSettingSlice []RequestSetting
+	ResponseHandler     ResponseHandler[Response]
+}
+
+func NewOptions[Request any, Response any](targetUrl string, responseHandler ResponseHandler[Response]) *Options[Request, Response] {
+	return &Options[Request, Response]{
+		MaxTryTimes:         DefaultMaxTryTimes,
+		TargetURL:           targetUrl,
+		Method:              DefaultMethod,
+		Body:                []byte{},
+		RequestSettingSlice: nil,
+		ResponseHandler:     responseHandler,
+	}
+}
+
+func (x *Options[Request, Response]) WithMaxTryTimes(maxTryTimes int) *Options[Request, Response] {
+	x.MaxTryTimes = maxTryTimes
+	return x
+}
+
+func (x *Options[Request, Response]) WithTargetURL(targetURL string) *Options[Request, Response] {
+	x.TargetURL = targetURL
+	return x
+}
+
+func (x *Options[Request, Response]) WithMethod(method string) *Options[Request, Response] {
+	x.Method = method
+	return x
+}
+
+func (x *Options[Request, Response]) WithBody(body []byte) *Options[Request, Response] {
+	if x.Method == DefaultMethod {
+		x.Method = http.MethodPost
+	}
+	x.Body = body
+	return x
+}
+
+func (x *Options[Request, Response]) WithRequestSettingSlice(requestSettingSlice []RequestSetting) *Options[Request, Response] {
+	x.RequestSettingSlice = requestSettingSlice
+	return x
+}
+
+func (x *Options[Request, Response]) AppendRequestSetting(requestSetting RequestSetting) *Options[Request, Response] {
+	x.RequestSettingSlice = append(x.RequestSettingSlice, requestSetting)
+	return x
+}
+
+func (x *Options[Request, Response]) WithResponseHandler(responseHandler ResponseHandler[Response]) *Options[Request, Response] {
+	x.ResponseHandler = responseHandler
+	return x
+}
+
+func (x *Options[Request, Response]) WithYamlResponseHandler() *Options[Request, Response] {
+	x.ResponseHandler = YamlResponseHandler[Response]()
+	return x
+}
+
+func (x *Options[Request, Response]) WithJsonResponseHandler() *Options[Request, Response] {
+	x.ResponseHandler = JsonResponseHandler[Response]()
+	return x
+}
+
+

response_handler.go

@@ -0,0 +1,79 @@
+package requests
+
+import (
+	"encoding/json"
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io"
+	"net/http"
+	"reflect"
+)
+
+type ResponseHandler[Response any] func(httpResponse *http.Response) (Response, error)
+
+func BytesResponseHandler(readResponseOnStatusCodeIn ...int) ResponseHandler[[]byte] {
+
+	// By default, the response body is read only when the status code is 200
+	if len(readResponseOnStatusCodeIn) == 0 {
+		readResponseOnStatusCodeIn = append(readResponseOnStatusCodeIn, http.StatusOK)
+	}
+
+	return func(httpResponse *http.Response) ([]byte, error) {
+		for _, status := range readResponseOnStatusCodeIn {
+			if status == httpResponse.StatusCode {
+				responseBodyBytes, err := io.ReadAll(httpResponse.Body)
+				if err != nil {
+					return nil, fmt.Errorf("response statuc code: %d, read body error: %s", httpResponse.StatusCode, err.Error())
+				}
+				return responseBodyBytes, nil
+			}
+		}
+		return nil, fmt.Errorf("response status code: %d", httpResponse.StatusCode)
+	}
+}
+
+func StringResponseHandler(readResponseOnStatusCodeIn ...int) ResponseHandler[string] {
+	return func(httpResponse *http.Response) (string, error) {
+		responseBytes, err := BytesResponseHandler(readResponseOnStatusCodeIn...)(httpResponse)
+		if err != nil {
+			return "", err
+		}
+		return string(responseBytes), nil
+	}
+}
+
+func YamlResponseHandler[Response any](readResponseOnStatusCodeIn ...int) ResponseHandler[Response] {
+	return func(httpResponse *http.Response) (Response, error) {
+
+		var r Response
+
+		responseBytes, err := BytesResponseHandler(readResponseOnStatusCodeIn...)(httpResponse)
+		if err != nil {
+			return r, err
+		}
+
+		err = yaml.Unmarshal(responseBytes, &r)
+		if err != nil {
+			return r, fmt.Errorf("response body yaml unmarshal error: %s, type: %s, response body: %s", err.Error(), reflect.TypeOf(r).String(), string(responseBytes))
+		}
+		return r, nil
+	}
+}
+
+func JsonResponseHandler[Response any](readResponseOnStatusCodeIn ...int) ResponseHandler[Response] {
+	return func(httpResponse *http.Response) (Response, error) {
+
+		var r Response
+
+		responseBytes, err := BytesResponseHandler(readResponseOnStatusCodeIn...)(httpResponse)
+		if err != nil {
+			return r, err
+		}
+
+		err = json.Unmarshal(responseBytes, &r)
+		if err != nil {
+			return r, fmt.Errorf("response body json unmarshal error: %s, type: %s, response body: %s", err.Error(), reflect.TypeOf(r).String(), string(responseBytes))
+		}
+		return r, nil
+	}
+}