Test ref dir

code/go/0chain.net/blobbercore/challenge/protocol.go

@@ -1,6 +1,8 @@
 package challenge
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"errors"
@@ -221,14 +223,17 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
 		return err
 	}
 	allocMu.RUnlock()
-
-	postDataBytes, err := json.Marshal(postData)
-	logging.Logger.Info("[challenge]post: ", zap.Any("challenge_id", cr.ChallengeID), zap.Any("post_data_len", len(postDataBytes)/(1024*1024)))
+	validationTicketPayload := new(bytes.Buffer)
+	gw := gzip.NewWriter(validationTicketPayload)
+	err = json.NewEncoder(gw).Encode(postData)
 	if err != nil {
-		logging.Logger.Error("[db]form: " + err.Error())
+		logging.Logger.Error("json encoding failed: " + err.Error())
 		cr.CancelChallenge(ctx, err)
 		return err
 	}
+	gw.Close() //nolint:errcheck
+	logging.Logger.Info("[challenge]post: ", zap.Any("challenge_id", cr.ChallengeID), zap.Any("post_data_len", validationTicketPayload.Len()/(1024*1024)))
+
 	responses := make(map[string]ValidationTicket)
 	if cr.ValidationTickets == nil {
 		cr.ValidationTickets = make([]*ValidationTicket, len(cr.Validators))
@@ -264,7 +269,7 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error {
 		go func(url, validatorID string, i int) {
 			defer swg.Done()
 
-			resp, err := util.SendPostRequest(url, postDataBytes, nil)
+			resp, err := util.SendPostRequest(url, validationTicketPayload.Bytes(), nil)
 			if err != nil {
 				numFailed++
 				logging.Logger.Error("[challenge]post: ", zap.Any("error", err.Error()))

code/go/0chain.net/blobbercore/config/config.go

@@ -52,7 +52,7 @@ func SetupDefaultConfig() {
 	viper.SetDefault("finalize_allocations_interval", time.Duration(-1))
 
 	viper.SetDefault("max_dirs_files", 50000)
-	viper.SetDefault("max_objects_dir", 1000)
+	viper.SetDefault("max_objects_dir", 10000)
 }
 
 /*SetupConfig - setup the configuration system */

code/go/0chain.net/blobbercore/handler/handler.go

@@ -200,7 +200,7 @@ func setupHandlers(r *mux.Router) {
 		RateLimitByGeneralRL(common.ToJSONResponse(WithReadOnlyConnection(FileStatsHandler)))) // TODO: add swagger
 
 	r.HandleFunc("/v1/file/referencepath/{allocation}",
-		RateLimitByObjectRL(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) // TODO: add handler
+		RateLimitByObjectRL(common.ToGzipJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) // TODO: add handler
 
 	r.HandleFunc("/v1/file/latestwritemarker/{allocation}",
 		RateLimitByObjectRL(common.ToJSONResponse(WithReadOnlyConnection(WriteMarkerHandler))))

code/go/0chain.net/blobbercore/reference/referencepath.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
@@ -126,7 +127,7 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths,
 	var refs []Ref
 	t := datastore.GetStore().GetTransaction(ctx)
 	db := t.DB
-
+	now := time.Now()
 	pathsAdded := make(map[string]bool)
 	var shouldOr bool
 	for _, path := range paths {
@@ -178,7 +179,7 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths,
 		refMap[refs[i].Path] = &refs[i]
 
 	}
-
+	elapsedRef := time.Since(now)
 	for _, path := range objTreePath {
 		ref, err := GetObjectTree(ctx, allocationID, path)
 		if err != nil {
@@ -196,6 +197,8 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths,
 			refMap[ref.Path].childrenLoaded = true
 		}
 	}
+	elapsedObjectTree := time.Since(now) - elapsedRef
+	logging.Logger.Info("getReferencePathFromPaths", zap.Duration("total_time", time.Since(now)), zap.Duration("elapsed_ref", elapsedRef), zap.Duration("elapsed_obj_tree", elapsedObjectTree))
 
 	return &refs[0], nil
 }

code/go/0chain.net/core/common/handler.go

@@ -2,12 +2,15 @@ package common
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
+
+	"github.com/pierrec/lz4/v4"
 )
 
 const (
@@ -60,6 +63,50 @@ func Respond(w http.ResponseWriter, data interface{}, err error) {
 	}
 }
 
+func RespondGzip(w http.ResponseWriter, data any, err error) {
+	w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all.
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	if err != nil {
+		data := make(map[string]interface{}, 2)
+		data["error"] = err.Error()
+		if cerr, ok := err.(*Error); ok {
+			data["code"] = cerr.Code
+		}
+		buf := bytes.NewBuffer(nil)
+		json.NewEncoder(buf).Encode(data) //nolint:errcheck // checked in previous step
+		w.WriteHeader(400)
+		fmt.Fprintln(w, buf.String())
+	} else if data != nil {
+		w.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(w)
+		defer gw.Close()
+		json.NewEncoder(gw).Encode(data) //nolint:errcheck // checked in previous step
+	}
+}
+
+func RespondLz4(w http.ResponseWriter, data any, err error) {
+	w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all.
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	if err != nil {
+		data := make(map[string]interface{}, 2)
+		data["error"] = err.Error()
+		if cerr, ok := err.(*Error); ok {
+			data["code"] = cerr.Code
+		}
+		buf := bytes.NewBuffer(nil)
+		json.NewEncoder(buf).Encode(data) //nolint:errcheck // checked in previous step
+		w.WriteHeader(400)
+		fmt.Fprintln(w, buf.String())
+	} else if data != nil {
+		w.Header().Set("Content-Encoding", "lz4")
+		lw := lz4.NewWriter(w)
+		defer lw.Close()
+		json.NewEncoder(lw).Encode(data) //nolint:errcheck // checked in previous step
+	}
+}
+
 var domainRE = regexp.MustCompile(`^(?:https?:\/\/)?(?:[^@\/\n]+@)?(?:www\.)?([^:\/\n]+)`) //nolint:unused,deadcode,varcheck // might be used later?
 
 func ToByteStream(handler JSONResponderF) ReqRespHandlerf {
@@ -119,6 +166,24 @@ func ToJSONResponse(handler JSONResponderF) ReqRespHandlerf {
 	}
 }
 
+/*ToGzipJSONResponse - An adapter that takes a handler of the form
+* func AHandler(r *http.Request) (interface{}, error)
+* which takes a request object, processes and returns an object or an error
+* and converts into a standard request/response handler
+ */
+func ToGzipJSONResponse(handler JSONResponderF) ReqRespHandlerf {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Access-Control-Allow-Origin", "*") // CORS for all.
+		if r.Method == "OPTIONS" {
+			SetupCORSResponse(w, r)
+			return
+		}
+		ctx := r.Context()
+		data, err := handler(ctx, r)
+		RespondGzip(w, data, err)
+	}
+}
+
 /*ToJSONReqResponse - An adapter that takes a handler of the form
 * func AHandler(json map[string]interface{}) (interface{}, error)
 * which takes a parsed json map from the request, processes and returns an object or an error

code/go/0chain.net/core/util/http.go

@@ -23,7 +23,8 @@ const SLEEP_BETWEEN_RETRIES = 5
 func NewHTTPRequest(method, url string, data []byte) (*http.Request, context.Context, context.CancelFunc, error) {
 	requestHash := encryption.Hash(data)
 	req, err := http.NewRequest(method, url, bytes.NewBuffer(data))
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Content-Encoding", "gzip")
 	req.Header.Set("Access-Control-Allow-Origin", "*")
 	req.Header.Set("X-App-Client-ID", node.Self.ID)
 	req.Header.Set("X-App-Client-Key", node.Self.PublicKey)
@@ -32,16 +33,6 @@ func NewHTTPRequest(method, url string, data []byte) (*http.Request, context.Con
 	return req, ctx, cncl, err
 }
 
-func SendMultiPostRequest(urls []string, data []byte) {
-	wg := sync.WaitGroup{}
-	wg.Add(len(urls))
-
-	for _, url := range urls {
-		go SendPostRequest(url, data, &wg) //nolint:errcheck // goroutines
-	}
-	wg.Wait()
-}
-
 func SendPostRequest(postURL string, data []byte, wg *sync.WaitGroup) (body []byte, err error) {
 	if wg != nil {
 		defer wg.Done()
@@ -63,7 +54,6 @@ func SendPostRequest(postURL string, data []byte, wg *sync.WaitGroup) (body []by
 
 		req, ctx, cncl, err = NewHTTPRequest(http.MethodPost, u.String(), data)
 		defer cncl()
-
 		resp, err = http.DefaultClient.Do(req.WithContext(ctx))
 		if err == nil {
 			if resp.StatusCode >= 200 && resp.StatusCode <= 299 {

code/go/0chain.net/validatorcore/storage/challenge_handler.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"compress/gzip"
 	"context"
 	"encoding/hex"
 	"encoding/json"
@@ -66,9 +67,14 @@ func NewChallengeRequest(r *http.Request) (*ChallengeRequest, string, error) {
 	requestHash := r.Header.Get("X-App-Request-Hash")
 	h := sha3.New256()
 	tReader := io.TeeReader(r.Body, h)
+	var requestBody io.Reader
+	if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
+		requestBody, _ = gzip.NewReader(tReader)
+	} else {
+		requestBody = tReader
+	}
 	var challengeRequest ChallengeRequest
-	decoder := json.NewDecoder(tReader)
-	err := decoder.Decode(&challengeRequest)
+	err := json.NewDecoder(requestBody).Decode(&challengeRequest)
 	if err != nil {
 		logging.Logger.Error("Error decoding the input to validator")
 		return nil, "", common.NewError("input_decode_error", "Error in decoding the input."+err.Error())

go.mod

@@ -69,6 +69,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect
 	github.com/aws/smithy-go v1.19.0 // indirect
 	github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.51.0 // indirect
 )

go.sum

@@ -678,6 +678,8 @@ github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 h1:6ob53CVz+ja2i7e
 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=