From 8c8745a4727c57a5897b5b660aea616d7d7c67e4 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Fri, 16 Jul 2021 22:22:50 +0545 Subject: [PATCH 1/7] WIP: Object tree pagination --- .../0chain.net/blobbercore/handler/handler.go | 14 +++++ .../blobbercore/handler/storage_handler.go | 59 ++++++++++++++++++- .../blobbercore/reference/referencepath.go | 25 ++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 45b7163fb..0b2d744b3 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -239,6 +239,20 @@ func ObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error return response, nil } +func PaginatedObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error) { + if r.Method == "POST" { + return nil, common.NewError("invalid method", "Invalid method used. Use GET instead") + } + ctx = setupHandlerContext(ctx, r) + + response, err := storageHandler.GetPaginatedObjectTree(ctx, r) + if err != nil { + return nil, err + } + + return response, nil +} + func RenameHandler(ctx context.Context, r *http.Request) (interface{}, error) { ctx = setupHandlerContext(ctx, r) response, err := storageHandler.RenameObject(ctx, r) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 002f3ad3c..56c322012 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -26,6 +26,7 @@ const ( DOWNLOAD_CONTENT_FULL = "full" DOWNLOAD_CONTENT_THUMB = "thumbnail" + PAGE_LIMIT = 10 //TODO make it to exat limit; for eg: 1MB ) type StorageHandler struct{} @@ -631,7 +632,13 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( if len(path) == 0 { return nil, common.NewError("invalid_parameters", "Invalid path") } - + // bOffset := r.FormValue("bOffset") + // bLimit := r.FormValue("bLimit") + // if len(bOffset) == 0{ + // breadthOffset := OFFSET + // }else{ + // breadthOffset := int(bOffset) + // } rootRef, err := reference.GetObjectTree(ctx, allocationID, path) if err != nil { return nil, err @@ -640,6 +647,8 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( refPath := &reference.ReferencePath{Ref: rootRef} refsToProcess := make([]*reference.ReferencePath, 0) refsToProcess = append(refsToProcess, refPath) + //It seems it can be controlled from this for loop + //SQL query needs to be changed as it can be heavy result for len(refsToProcess) > 0 { refToProcess := refsToProcess[0] refToProcess.Meta = refToProcess.Ref.GetListingData(ctx) @@ -671,6 +680,54 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( return &refPathResult, nil } +func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.Request) (*blobberhttp.ReferencePathResult, error) { + allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) + allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) + + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) + } + + clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string) + valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey) + if !valid || err != nil { + return nil, common.NewError("invalid_signature", "Invalid signature") + } + + allocationID := allocationObj.ID + clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string) + if len(clientID) == 0 || allocationObj.OwnerID != clientID { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") + } + path := r.FormValue("path") + if len(path) == 0 { + return nil, common.NewError("invalid_parameters", "Invalid path") + } + + pageStr := r.FormValue("page") + var page int + if len(pageStr) == 0 { + page = 1 + } else { + o, err := strconv.Atoi(pageStr) + if err != nil { + return nil, common.NewError("invalid_parameters", "Invalid page value type") + } + if o < 0 { + page = 1 + } else { + page = o + } + } + + refs, err := reference.GetPaginatedObjectTree(ctx, allocationID, path, page) // Also return total pages + if err != nil { + return nil, err + } + // Refs will be returned as it is and object tree will be build in gosdk + return nil, nil +} + func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) (interface{}, error) { if r.Method != "POST" { return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead") diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 4103d8dc0..3d0e7b27b 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -8,6 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" ) +const PAGE_SIZE = 100 + type ReferencePath struct { Meta map[string]interface{} `json:"meta_data"` List []*ReferencePath `json:"list,omitempty"` @@ -99,3 +101,26 @@ func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, } return &refs[0], nil } + +func GetPaginatedObjectTree(ctx context.Context, allocationID string, path string, page int) (*[]Ref, error) { + var refs []Ref + path = filepath.Clean(path) + db := datastore.GetStore().GetTransaction(ctx) + offset := (page - 1) * PAGE_SIZE + db = db.Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + // Select * from reference_objects where allocation_id = {allocatioid} AND (path=path OR path LIKE {path}%) + // db = db.Where("allocation_id = ? AND (path = ? OR path LIKE ?)", allocationID, path, path) + // db = db.Where(Ref{Path: path, AllocationID: allocationID}) + // db = db.Or("path LIKE ? AND allocation_id = ?", (path + "/%"), allocationID) + db = db.Order("level, lookup_hash") + db = db.Offset(offset).Limit(PAGE_SIZE) + err := db.Find(&refs).Error + if err != nil { + return nil, err + } + + if len(refs) == 0 { + return nil, common.NewError("invalid_parameters", "Invalid path. Could not find object tree") + } + return &refs, nil +} From 0f78ff001465a30260f31fddaaa6f3093b6957ae Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Sun, 18 Jul 2021 16:22:58 +0545 Subject: [PATCH 2/7] WIP: Update code --- .../blobbercore/blobberhttp/response.go | 7 +++ .../0chain.net/blobbercore/handler/handler.go | 2 +- .../blobbercore/handler/storage_handler.go | 34 +++++++---- .../0chain.net/blobbercore/reference/ref.go | 60 +++++++++---------- .../blobbercore/reference/referencepath.go | 25 +++++--- 5 files changed, 76 insertions(+), 52 deletions(-) diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index e09b4430a..d774e1c27 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -34,6 +34,13 @@ type ReferencePathResult struct { LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` } +type ObjectTreeResult struct { + Page int64 `json:"page"` + TotalPages int64 `json:"total_pages"` + Refs *[]reference.Ref `json:"refs"` + LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` +} + type ObjectPathResult struct { *reference.ObjectPath LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 0b2d744b3..7a167aeda 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -48,7 +48,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler)))) r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler)))) - + r.HandleFunc("/v1/file/pg_objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(PaginatedObjectTreeHandler)))) //admin related r.HandleFunc("/_debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines))) r.HandleFunc("/_config", common.UserRateLimit(common.ToJSONResponse(GetConfig))) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 56c322012..4ada2108f 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -632,13 +632,7 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( if len(path) == 0 { return nil, common.NewError("invalid_parameters", "Invalid path") } - // bOffset := r.FormValue("bOffset") - // bLimit := r.FormValue("bLimit") - // if len(bOffset) == 0{ - // breadthOffset := OFFSET - // }else{ - // breadthOffset := int(bOffset) - // } + rootRef, err := reference.GetObjectTree(ctx, allocationID, path) if err != nil { return nil, err @@ -647,8 +641,7 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( refPath := &reference.ReferencePath{Ref: rootRef} refsToProcess := make([]*reference.ReferencePath, 0) refsToProcess = append(refsToProcess, refPath) - //It seems it can be controlled from this for loop - //SQL query needs to be changed as it can be heavy result + for len(refsToProcess) > 0 { refToProcess := refsToProcess[0] refToProcess.Meta = refToProcess.Ref.GetListingData(ctx) @@ -680,7 +673,7 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( return &refPathResult, nil } -func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.Request) (*blobberhttp.ReferencePathResult, error) { +func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.Request) (*blobberhttp.ObjectTreeResult, error) { allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) @@ -720,12 +713,29 @@ func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.R } } - refs, err := reference.GetPaginatedObjectTree(ctx, allocationID, path, page) // Also return total pages + refs, totalPages, err := reference.GetPaginatedObjectTree(ctx, allocationID, path, page) // Also return total pages if err != nil { return nil, err } + var latestWM *writemarker.WriteMarkerEntity + if len(allocationObj.AllocationRoot) == 0 { + latestWM = nil + } else { + latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) + if err != nil { + return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) + } + } + + var oTreeResult blobberhttp.ObjectTreeResult + oTreeResult.Refs = refs + oTreeResult.Page = int64(page) + oTreeResult.TotalPages = totalPages + if latestWM != nil { + oTreeResult.LatestWM = &latestWM.WM + } // Refs will be returned as it is and object tree will be build in gosdk - return nil, nil + return &oTreeResult, nil } func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 6d7b3813e..06a5abc4d 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -56,40 +56,40 @@ func (a *Attributes) Validate() (err error) { } type Ref struct { - ID int64 `gorm:"column:id;primary_key"` - Type string `gorm:"column:type" dirlist:"type" filelist:"type"` - AllocationID string `gorm:"column:allocation_id"` - LookupHash string `gorm:"column:lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"` - Name string `gorm:"column:name" dirlist:"name" filelist:"name"` - Path string `gorm:"column:path" dirlist:"path" filelist:"path"` - Hash string `gorm:"column:hash" dirlist:"hash" filelist:"hash"` - NumBlocks int64 `gorm:"column:num_of_blocks" dirlist:"num_of_blocks" filelist:"num_of_blocks"` - PathHash string `gorm:"column:path_hash" dirlist:"path_hash" filelist:"path_hash"` - ParentPath string `gorm:"column:parent_path"` - PathLevel int `gorm:"column:level"` - CustomMeta string `gorm:"column:custom_meta" filelist:"custom_meta"` - ContentHash string `gorm:"column:content_hash" filelist:"content_hash"` - Size int64 `gorm:"column:size" dirlist:"size" filelist:"size"` - MerkleRoot string `gorm:"column:merkle_root" filelist:"merkle_root"` - ActualFileSize int64 `gorm:"column:actual_file_size" filelist:"actual_file_size"` - ActualFileHash string `gorm:"column:actual_file_hash" filelist:"actual_file_hash"` - MimeType string `gorm:"column:mimetype" filelist:"mimetype"` - WriteMarker string `gorm:"column:write_marker"` - ThumbnailSize int64 `gorm:"column:thumbnail_size" filelist:"thumbnail_size"` - ThumbnailHash string `gorm:"column:thumbnail_hash" filelist:"thumbnail_hash"` - ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" filelist:"actual_thumbnail_size"` - ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" filelist:"actual_thumbnail_hash"` - EncryptedKey string `gorm:"column:encrypted_key" filelist:"encrypted_key"` - Attributes datatypes.JSON `gorm:"column:attributes" filelist:"attributes"` + ID int64 `gorm:"column:id;primary_key" json:"id"` + Type string `gorm:"column:type" dirlist:"type" filelist:"type" json:"type"` + AllocationID string `gorm:"column:allocation_id" json:"allocation_id"` + LookupHash string `gorm:"column:lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash" json:"lookup_hash"` + Name string `gorm:"column:name" dirlist:"name" filelist:"name" json:"name"` + Path string `gorm:"column:path" dirlist:"path" filelist:"path" json:"path"` + Hash string `gorm:"column:hash" dirlist:"hash" filelist:"hash" json:"hash"` + NumBlocks int64 `gorm:"column:num_of_blocks" dirlist:"num_of_blocks" filelist:"num_of_blocks" json:"num_blocks"` + PathHash string `gorm:"column:path_hash" dirlist:"path_hash" filelist:"path_hash" json:"path_hash"` + ParentPath string `gorm:"column:parent_path" json:"parent_path"` + PathLevel int `gorm:"column:level" json:"level"` + CustomMeta string `gorm:"column:custom_meta" filelist:"custom_meta" json:"custom_meta"` + ContentHash string `gorm:"column:content_hash" filelist:"content_hash" json:"content_hash"` + Size int64 `gorm:"column:size" dirlist:"size" filelist:"size" json:"size"` + MerkleRoot string `gorm:"column:merkle_root" filelist:"merkle_root" json:"merkle_root"` + ActualFileSize int64 `gorm:"column:actual_file_size" filelist:"actual_file_size" json:"actual_file_size"` + ActualFileHash string `gorm:"column:actual_file_hash" filelist:"actual_file_hash" json:"actual_file_hash"` + MimeType string `gorm:"column:mimetype" filelist:"mimetype" json:"mimetype"` + WriteMarker string `gorm:"column:write_marker" json:"write_marker"` + ThumbnailSize int64 `gorm:"column:thumbnail_size" filelist:"thumbnail_size" json:"thumbnail_size"` + ThumbnailHash string `gorm:"column:thumbnail_hash" filelist:"thumbnail_hash" json:"thumbnail_hash"` + ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" filelist:"actual_thumbnail_size" json:"actual_thumbnail_size"` + ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" filelist:"actual_thumbnail_hash" json:"actual_thumbnail_hash"` + EncryptedKey string `gorm:"column:encrypted_key" filelist:"encrypted_key" json:"encrypted_key"` + Attributes datatypes.JSON `gorm:"column:attributes" filelist:"attributes" json:"attributes"` Children []*Ref `gorm:"-"` childrenLoaded bool - OnCloud bool `gorm:"column:on_cloud" filelist:"on_cloud"` - CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns"` - CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at"` - UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at"` + OnCloud bool `gorm:"column:on_cloud" filelist:"on_cloud" json:"on_cloud"` + CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns" json:""` + CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at" json:"updated_at"` - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index" json:"-"` // soft deletion } func (Ref) TableName() string { diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 3d0e7b27b..5adcaf584 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -2,13 +2,14 @@ package reference import ( "context" + "math" "path/filepath" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -const PAGE_SIZE = 100 +const PAGE_SIZE = 5 type ReferencePath struct { Meta map[string]interface{} `json:"meta_data"` @@ -102,25 +103,31 @@ func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, return &refs[0], nil } -func GetPaginatedObjectTree(ctx context.Context, allocationID string, path string, page int) (*[]Ref, error) { +func GetPaginatedObjectTree(ctx context.Context, allocationID string, path string, page int) (*[]Ref, int64, error) { var refs []Ref + var totalRows int64 + var totalPages int64 path = filepath.Clean(path) db := datastore.GetStore().GetTransaction(ctx) offset := (page - 1) * PAGE_SIZE - db = db.Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) // Select * from reference_objects where allocation_id = {allocatioid} AND (path=path OR path LIKE {path}%) - // db = db.Where("allocation_id = ? AND (path = ? OR path LIKE ?)", allocationID, path, path) - // db = db.Where(Ref{Path: path, AllocationID: allocationID}) - // db = db.Or("path LIKE ? AND allocation_id = ?", (path + "/%"), allocationID) + db = db.Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + // db = db.Where("deleted_at = null") db = db.Order("level, lookup_hash") db = db.Offset(offset).Limit(PAGE_SIZE) + err := db.Find(&refs).Error + if err != nil { - return nil, err + return nil, 0, err } + tx := datastore.GetStore().GetTransaction(ctx) + tx = tx.Model(&Ref{}).Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + tx.Count(&totalRows) + totalPages = int64(math.Ceil(float64(totalRows) / PAGE_SIZE)) if len(refs) == 0 { - return nil, common.NewError("invalid_parameters", "Invalid path. Could not find object tree") + return nil, 0, common.NewError("invalid_parameters", "Invalid path. Could not find object tree") } - return &refs, nil + return &refs, totalPages, nil } From 26a6e08571b853a0991163d9043d07ea9c612630 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Wed, 21 Jul 2021 14:32:55 +0545 Subject: [PATCH 3/7] WIP: Refactor with index creation --- .../blobbercore/blobberhttp/response.go | 9 ++++--- .../blobbercore/handler/storage_handler.go | 13 ++++++++- .../blobbercore/reference/referencepath.go | 27 +++++++++---------- sql/create-path-index.sql | 1 + 4 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 sql/create-path-index.sql diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index d774e1c27..5794c8a03 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -35,10 +35,11 @@ type ReferencePathResult struct { } type ObjectTreeResult struct { - Page int64 `json:"page"` - TotalPages int64 `json:"total_pages"` - Refs *[]reference.Ref `json:"refs"` - LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` + Page int64 `json:"page"` + TotalPages int64 `json:"total_pages"` + NewOffsetPath string `json:"offsetPath"` + Refs *[]reference.Ref `json:"refs"` + LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` } type ObjectPathResult struct { diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 4ada2108f..da3cd45eb 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -713,7 +713,17 @@ func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.R } } - refs, totalPages, err := reference.GetPaginatedObjectTree(ctx, allocationID, path, page) // Also return total pages + offsetPath := r.FormValue("offsetPath") + // pathLevelStr := r.FormValue("pathLevel") + // var pathLevel int + // if len(pageStr) > 0 { + // p, err := strconv.Atoi(pathLevelStr) + // if err != nil || p < 0 { + // return nil, common.NewError("invalid_parameters", "Invalid level value type") + // } + // } + + refs, totalPages, newOffsetPath, err := reference.GetPaginatedObjectTree(ctx, allocationID, path, page, offsetPath) // Also return total pages if err != nil { return nil, err } @@ -731,6 +741,7 @@ func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.R oTreeResult.Refs = refs oTreeResult.Page = int64(page) oTreeResult.TotalPages = totalPages + oTreeResult.NewOffsetPath = newOffsetPath if latestWM != nil { oTreeResult.LatestWM = &latestWM.WM } diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 5adcaf584..0f6776941 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -103,31 +103,28 @@ func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, return &refs[0], nil } -func GetPaginatedObjectTree(ctx context.Context, allocationID string, path string, page int) (*[]Ref, int64, error) { - var refs []Ref +//This function retrieves refrence_objects tables rows with pagination. Check for issue https://github.com/0chain/gosdk/issues/117 +//Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful +func GetPaginatedObjectTree(ctx context.Context, allocationID string, path string, page int, offsetPath string) (refs *[]Ref, totalPages int64, newOffsetPath string, err error) { var totalRows int64 - var totalPages int64 + var pRefs []Ref path = filepath.Clean(path) db := datastore.GetStore().GetTransaction(ctx) - offset := (page - 1) * PAGE_SIZE // Select * from reference_objects where allocation_id = {allocatioid} AND (path=path OR path LIKE {path}%) - db = db.Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) - // db = db.Where("deleted_at = null") - db = db.Order("level, lookup_hash") - db = db.Offset(offset).Limit(PAGE_SIZE) + db = db.Where(Ref{AllocationID: allocationID}).Where("path>'?'", offsetPath).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + db = db.Order("path") + db = db.Limit(PAGE_SIZE) - err := db.Find(&refs).Error + err = db.Find(&pRefs).Error if err != nil { - return nil, 0, err + return } - + refs = &pRefs + newOffsetPath = pRefs[len(pRefs)-1].Path tx := datastore.GetStore().GetTransaction(ctx) tx = tx.Model(&Ref{}).Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) tx.Count(&totalRows) totalPages = int64(math.Ceil(float64(totalRows) / PAGE_SIZE)) - if len(refs) == 0 { - return nil, 0, common.NewError("invalid_parameters", "Invalid path. Could not find object tree") - } - return &refs, totalPages, nil + return } diff --git a/sql/create-path-index.sql b/sql/create-path-index.sql new file mode 100644 index 000000000..a01e1ff50 --- /dev/null +++ b/sql/create-path-index.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX path_idx ON public.reference_objects USING btree (path) \ No newline at end of file From 1ebd977fe1f3ef844f7b3062391647090a50ec1e Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Wed, 28 Jul 2021 19:08:44 +0545 Subject: [PATCH 4/7] WIP: Refactor to refs pagination --- .../blobbercore/blobberhttp/response.go | 8 +- .../0chain.net/blobbercore/handler/handler.go | 6 +- .../0chain.net/blobbercore/handler/helper.go | 12 ++ .../blobbercore/handler/storage_handler.go | 97 +++++++--- .../0chain.net/blobbercore/reference/ref.go | 64 +++---- .../blobbercore/reference/referencepath.go | 177 ++++++++++++++++-- sql/create-indexes.sql | 8 + sql/create-path-index.sql | 1 - 8 files changed, 290 insertions(+), 83 deletions(-) create mode 100644 sql/create-indexes.sql delete mode 100644 sql/create-path-index.sql diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index 5794c8a03..0a8c0bd06 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -34,10 +34,10 @@ type ReferencePathResult struct { LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` } -type ObjectTreeResult struct { - Page int64 `json:"page"` - TotalPages int64 `json:"total_pages"` - NewOffsetPath string `json:"offsetPath"` +type RefResult struct { + TotalPages int `json:"total_pages"` + NewOffsetPath string `json:"offsetPath,omitempty"` + NewOffsetDate string `json:"offsetDate,omitempty"` Refs *[]reference.Ref `json:"refs"` LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 7a167aeda..ccc921977 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -48,7 +48,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler)))) r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler)))) - r.HandleFunc("/v1/file/pg_objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(PaginatedObjectTreeHandler)))) + r.HandleFunc("/v1/file/refs/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(RefsHandler)))) //admin related r.HandleFunc("/_debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines))) r.HandleFunc("/_config", common.UserRateLimit(common.ToJSONResponse(GetConfig))) @@ -239,13 +239,13 @@ func ObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error return response, nil } -func PaginatedObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error) { +func RefsHandler(ctx context.Context, r *http.Request) (interface{}, error) { if r.Method == "POST" { return nil, common.NewError("invalid method", "Invalid method used. Use GET instead") } ctx = setupHandlerContext(ctx, r) - response, err := storageHandler.GetPaginatedObjectTree(ctx, r) + response, err := storageHandler.GetRefs(ctx, r) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/helper.go b/code/go/0chain.net/blobbercore/handler/helper.go index c0b32be6b..add9ea100 100644 --- a/code/go/0chain.net/blobbercore/handler/helper.go +++ b/code/go/0chain.net/blobbercore/handler/helper.go @@ -2,8 +2,10 @@ package handler import ( "context" + "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" + "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/gorilla/mux" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "google.golang.org/grpc" @@ -18,3 +20,13 @@ func registerGRPCServices(r *mux.Router, server *grpc.Server) { r.PathPrefix("/").Handler(grpcGatewayHandler) } + +func checkValidDate(s string) error { + if s != "" { + _, err := time.Parse("2006-01-02 15:04:05.999999999", s) + if err != nil { + return common.NewError("invalid_parameters", err.Error()) + } + } + return nil +} diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index da3cd45eb..a94300d05 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -26,7 +26,10 @@ const ( DOWNLOAD_CONTENT_FULL = "full" DOWNLOAD_CONTENT_THUMB = "thumbnail" - PAGE_LIMIT = 10 //TODO make it to exat limit; for eg: 1MB + PAGE_LIMIT = 100 //100 rows will make upto 100 KB + UPDATED = "updated" + DELETED = "deleted" + REGULAR = "regular" ) type StorageHandler struct{} @@ -673,7 +676,9 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) ( return &refPathResult, nil } -func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.Request) (*blobberhttp.ObjectTreeResult, error) { +//Retrieves file refs. One can use three types to refer to regular, updated and deleted. Regular type gives all undeleted rows. +//Updated gives rows that is updated compared to the date given. And deleted gives deleted refs compared to the date given. +func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobberhttp.RefResult, error) { allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string) allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false) @@ -697,33 +702,67 @@ func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.R return nil, common.NewError("invalid_parameters", "Invalid path") } - pageStr := r.FormValue("page") - var page int - if len(pageStr) == 0 { - page = 1 + pageLimitStr := r.FormValue("pageLimit") + var pageLimit int + if len(pageLimitStr) == 0 { + pageLimit = PAGE_LIMIT } else { - o, err := strconv.Atoi(pageStr) + o, err := strconv.Atoi(pageLimitStr) if err != nil { - return nil, common.NewError("invalid_parameters", "Invalid page value type") + return nil, common.NewError("invalid_parameters", "Invalid page limit value type") } - if o < 0 { - page = 1 + if o <= 0 { + return nil, common.NewError("invalid_parameters", "Zero/Negative page limit value is not allowed") + } else if o > PAGE_LIMIT { + pageLimit = PAGE_LIMIT } else { - page = o + pageLimit = o } } - offsetPath := r.FormValue("offsetPath") - // pathLevelStr := r.FormValue("pathLevel") - // var pathLevel int - // if len(pageStr) > 0 { - // p, err := strconv.Atoi(pathLevelStr) - // if err != nil || p < 0 { - // return nil, common.NewError("invalid_parameters", "Invalid level value type") - // } - // } + offsetDate := r.FormValue("offsetDate") + updatedDate := r.FormValue("updatedDate") + err = checkValidDate(offsetDate) + if err != nil { + return nil, err + } + err = checkValidDate(updatedDate) + if err != nil { + return nil, err + } + fileType := r.FormValue("fileType") + levelStr := r.FormValue("level") + var level int + if len(levelStr) != 0 { + level, err = strconv.Atoi(levelStr) + if err != nil { + return nil, common.NewError("invalid_parameters", err.Error()) + } + if level < 0 { + return nil, common.NewError("invalid_parameters", "Negative level value is not allowed") + } + } + + refType := r.FormValue("refType") + var refs *[]reference.Ref + var totalPages int + var newOffsetPath string + var newOffsetDate string + + switch { + case refType == REGULAR: + refs, totalPages, newOffsetPath, err = reference.GetRefs(ctx, allocationID, path, offsetPath, fileType, level, pageLimit) + + case refType == UPDATED: + refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs(ctx, allocationID, path, offsetPath, fileType, updatedDate, offsetDate, level, pageLimit) + + case refType == DELETED: + refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetDeletedRefs(ctx, allocationID, updatedDate, offsetPath, offsetDate, pageLimit) + + default: + return nil, common.NewError("invalid_parameters", "refType param should have value regular/updated/deleted") + } - refs, totalPages, newOffsetPath, err := reference.GetPaginatedObjectTree(ctx, allocationID, path, page, offsetPath) // Also return total pages if err != nil { return nil, err } @@ -737,16 +776,16 @@ func (fsh *StorageHandler) GetPaginatedObjectTree(ctx context.Context, r *http.R } } - var oTreeResult blobberhttp.ObjectTreeResult - oTreeResult.Refs = refs - oTreeResult.Page = int64(page) - oTreeResult.TotalPages = totalPages - oTreeResult.NewOffsetPath = newOffsetPath + var refResult blobberhttp.RefResult + refResult.Refs = refs + refResult.TotalPages = totalPages + refResult.NewOffsetPath = newOffsetPath + refResult.NewOffsetDate = newOffsetDate if latestWM != nil { - oTreeResult.LatestWM = &latestWM.WM + refResult.LatestWM = &latestWM.WM } - // Refs will be returned as it is and object tree will be build in gosdk - return &oTreeResult, nil + // Refs will be returned as it is and object tree will be build in client side + return &refResult, nil } func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 06a5abc4d..061a5a25d 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -56,38 +56,38 @@ func (a *Attributes) Validate() (err error) { } type Ref struct { - ID int64 `gorm:"column:id;primary_key" json:"id"` - Type string `gorm:"column:type" dirlist:"type" filelist:"type" json:"type"` - AllocationID string `gorm:"column:allocation_id" json:"allocation_id"` - LookupHash string `gorm:"column:lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash" json:"lookup_hash"` - Name string `gorm:"column:name" dirlist:"name" filelist:"name" json:"name"` - Path string `gorm:"column:path" dirlist:"path" filelist:"path" json:"path"` - Hash string `gorm:"column:hash" dirlist:"hash" filelist:"hash" json:"hash"` - NumBlocks int64 `gorm:"column:num_of_blocks" dirlist:"num_of_blocks" filelist:"num_of_blocks" json:"num_blocks"` - PathHash string `gorm:"column:path_hash" dirlist:"path_hash" filelist:"path_hash" json:"path_hash"` - ParentPath string `gorm:"column:parent_path" json:"parent_path"` - PathLevel int `gorm:"column:level" json:"level"` - CustomMeta string `gorm:"column:custom_meta" filelist:"custom_meta" json:"custom_meta"` - ContentHash string `gorm:"column:content_hash" filelist:"content_hash" json:"content_hash"` - Size int64 `gorm:"column:size" dirlist:"size" filelist:"size" json:"size"` - MerkleRoot string `gorm:"column:merkle_root" filelist:"merkle_root" json:"merkle_root"` - ActualFileSize int64 `gorm:"column:actual_file_size" filelist:"actual_file_size" json:"actual_file_size"` - ActualFileHash string `gorm:"column:actual_file_hash" filelist:"actual_file_hash" json:"actual_file_hash"` - MimeType string `gorm:"column:mimetype" filelist:"mimetype" json:"mimetype"` - WriteMarker string `gorm:"column:write_marker" json:"write_marker"` - ThumbnailSize int64 `gorm:"column:thumbnail_size" filelist:"thumbnail_size" json:"thumbnail_size"` - ThumbnailHash string `gorm:"column:thumbnail_hash" filelist:"thumbnail_hash" json:"thumbnail_hash"` - ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" filelist:"actual_thumbnail_size" json:"actual_thumbnail_size"` - ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" filelist:"actual_thumbnail_hash" json:"actual_thumbnail_hash"` - EncryptedKey string `gorm:"column:encrypted_key" filelist:"encrypted_key" json:"encrypted_key"` - Attributes datatypes.JSON `gorm:"column:attributes" filelist:"attributes" json:"attributes"` - Children []*Ref `gorm:"-"` - childrenLoaded bool - - OnCloud bool `gorm:"column:on_cloud" filelist:"on_cloud" json:"on_cloud"` - CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns" json:""` - CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at" json:"created_at"` - UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at" json:"updated_at"` + ID int64 `gorm:"column:id;primary_key" json:"id,omitempty"` + Type string `gorm:"column:type" dirlist:"type" filelist:"type" json:"type,omitempty"` + AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` + LookupHash string `gorm:"column:lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash" json:"lookup_hash,omitempty"` + Name string `gorm:"column:name" dirlist:"name" filelist:"name" json:"name,omitempty"` + Path string `gorm:"column:path" dirlist:"path" filelist:"path" json:"path,omitempty"` + Hash string `gorm:"column:hash" dirlist:"hash" filelist:"hash" json:"hash,omitempty"` + NumBlocks int64 `gorm:"column:num_of_blocks" dirlist:"num_of_blocks" filelist:"num_of_blocks" json:"num_blocks,omitempty"` + PathHash string `gorm:"column:path_hash" dirlist:"path_hash" filelist:"path_hash" json:"path_hash,omitempty"` + ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"` + PathLevel int `gorm:"column:level" json:"level,omitempty"` + CustomMeta string `gorm:"column:custom_meta" filelist:"custom_meta" json:"custom_meta,omitempty"` + ContentHash string `gorm:"column:content_hash" filelist:"content_hash" json:"content_hash,omitempty"` + Size int64 `gorm:"column:size" dirlist:"size" filelist:"size" json:"size,omitempty"` + MerkleRoot string `gorm:"column:merkle_root" filelist:"merkle_root" json:"merkle_root,omitempty"` + ActualFileSize int64 `gorm:"column:actual_file_size" filelist:"actual_file_size" json:"actual_file_size,omitempty"` + ActualFileHash string `gorm:"column:actual_file_hash" filelist:"actual_file_hash" json:"actual_file_hash,omitempty"` + MimeType string `gorm:"column:mimetype" filelist:"mimetype" json:"mimetype,omitempty"` + WriteMarker string `gorm:"column:write_marker" json:"write_marker,omitempty"` + ThumbnailSize int64 `gorm:"column:thumbnail_size" filelist:"thumbnail_size" json:"thumbnail_size,omitempty"` + ThumbnailHash string `gorm:"column:thumbnail_hash" filelist:"thumbnail_hash" json:"thumbnail_hash,omitempty"` + ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" filelist:"actual_thumbnail_size" json:"actual_thumbnail_size,omitempty"` + ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" filelist:"actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"` + EncryptedKey string `gorm:"column:encrypted_key" filelist:"encrypted_key" json:"encrypted_key,omitempty"` + Attributes datatypes.JSON `gorm:"column:attributes" filelist:"attributes" json:"attributes,omitempty"` + Children []*Ref `gorm:"-" json:"-"` + childrenLoaded bool `json:"-"` + + OnCloud bool `gorm:"column:on_cloud" filelist:"on_cloud" json:"on_cloud,omitempty"` + CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns" json:"-"` + CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at" json:"created_at,omitempty"` + UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at" json:"updated_at,omitempty"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index" json:"-"` // soft deletion } diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 0f6776941..d84bfdb45 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -4,13 +4,13 @@ import ( "context" "math" "path/filepath" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "gorm.io/gorm" ) -const PAGE_SIZE = 5 - type ReferencePath struct { Meta map[string]interface{} `json:"meta_data"` List []*ReferencePath `json:"list,omitempty"` @@ -105,26 +105,175 @@ func GetObjectTree(ctx context.Context, allocationID string, path string) (*Ref, //This function retrieves refrence_objects tables rows with pagination. Check for issue https://github.com/0chain/gosdk/issues/117 //Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful -func GetPaginatedObjectTree(ctx context.Context, allocationID string, path string, page int, offsetPath string) (refs *[]Ref, totalPages int64, newOffsetPath string, err error) { +func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int) (refs *[]Ref, totalPages int, newOffsetPath string, err error) { var totalRows int64 var pRefs []Ref path = filepath.Clean(path) - db := datastore.GetStore().GetTransaction(ctx) - // Select * from reference_objects where allocation_id = {allocatioid} AND (path=path OR path LIKE {path}%) - db = db.Where(Ref{AllocationID: allocationID}).Where("path>'?'", offsetPath).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) - db = db.Order("path") - db = db.Limit(PAGE_SIZE) - err = db.Find(&pRefs).Error + db := datastore.GetStore().GetDB() + db1 := db.Session(&gorm.Session{}) + db2 := db.Session(&gorm.Session{}) + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + db1 = db1.Model(&Ref{}).Where("allocation_id = ?", allocationID). + Where(db1.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + if _type != "" { + db1 = db1.Where("type = ?", _type) + } + if level != 0 { + db1 = db1.Where("level >= ?", level) + } + db1 = db1.Count(&totalRows) + + db1 = db1.Where("path > ?", offsetPath) + + db1 = db1.Order("path") + err = db1.Limit(pageLimit).Find(&pRefs).Error + wg.Done() + }() + + go func() { + db2 = db2.Model(&Ref{}).Where("allocation_id = ?", allocationID). + Where(db2.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + if _type != "" { + db2 = db2.Where("type = ?", _type) + } + if level != 0 { + db2 = db2.Where("level >= ?", level) + } + db2.Count(&totalRows) + wg.Done() + }() + wg.Wait() + if err != nil { + return + } + + refs = &pRefs + if len(pRefs) > 0 { + newOffsetPath = pRefs[len(pRefs)-1].Path + + } + totalPages = int(math.Ceil(float64(totalRows) / float64(pageLimit))) + return +} + +//Retrieves updated refs compared to some update_at value. Useful to localCache +func GetUpdatedRefs(ctx context.Context, allocationID, path, offsetPath, _type, updatedDate, offsetDate string, level, pageLimit int) (refs *[]Ref, totalPages int, newOffsetPath, newOffsetDate string, err error) { + var totalRows int64 + var pRefs []Ref + db := datastore.GetStore().GetDB() + db1 := db.Session(&gorm.Session{}) //TODO Might need to use transaction from db1/db2 to avoid injection attack + db2 := db.Session(&gorm.Session{}) + + wg := sync.WaitGroup{} + wg.Add(2) + + go func() { + db1 = db1.Model(&Ref{}).Where("allocation_id = ?", allocationID). + Where(db1.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + if _type != "" { + db1 = db1.Where("type = ?", _type) + } + if level != 0 { + db1 = db1.Where("level >= ?", level) + } + if updatedDate != "" { + db1 = db1.Where("updated_at > ?", updatedDate) + } + + if offsetDate != "" { + db1 = db1.Where("(updated_at, path) > (?, ?)", offsetDate, offsetPath) + } + db1 = db1.Order("updated_at, path") + db1 = db1.Limit(pageLimit) + err = db1.Find(&pRefs).Error + wg.Done() + }() + go func() { + db2 = db2.Model(&Ref{}).Where("allocation_id = ?", allocationID). + Where(db2.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) + if _type != "" { + db2 = db2.Where("type > ?", level) + } + if level != 0 { + db2 = db2.Where("level >= ?", level) + } + if updatedDate != "" { + db2 = db2.Where("updated_at > ?", updatedDate) + } + db2 = db2.Count(&totalRows) + wg.Done() + }() + wg.Wait() if err != nil { return } + + if len(pRefs) != 0 { + lastIdx := len(pRefs) - 1 + newOffsetDate = pRefs[lastIdx].UpdatedAt.String() + newOffsetPath = pRefs[lastIdx].Path + } + refs = &pRefs + totalPages = int(math.Ceil(float64(totalRows) / float64(pageLimit))) + return +} + +//Retrieves deleted refs compared to some update_at value. Useful for localCache. +func GetDeletedRefs(ctx context.Context, allocationID, updatedDate, offsetPath, offsetDate string, pageLimit int) (refs *[]Ref, totalPages int, newOffsetPath, newOffsetDate string, err error) { + var totalRows int64 + var pRefs []Ref + db := datastore.GetStore().GetDB() + + db1 := db.Session(&gorm.Session{NewDB: true}) + db2 := db.Session(&gorm.Session{NewDB: true}) + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + db1 = db1.Model(&Ref{}).Unscoped(). + Select("path", "path_hash", "deleted_at", "updated_at"). + Where("allocation_id = ?", allocationID) + + if updatedDate == "" { + db1 = db1.Where("deleted_at IS NOT null") + } else { + db1 = db1.Where("deleted_at > ?", updatedDate) + } + + if offsetDate != "" { + db1 = db1.Where("(updated_at, path) > (?, ?)", offsetDate, offsetPath) + } + + err = db1.Order("updated_at, path").Limit(pageLimit).Find(&pRefs).Error + wg.Done() + }() + + go func() { + + db2 = db2.Model(&Ref{}).Unscoped().Where("allocation_id = ?", allocationID) + + if updatedDate == "" { + db2 = db2.Where("deleted_at IS NOT null") + } else { + db2 = db2.Where("deleted_at > ?", updatedDate) + } + + db2 = db2.Count(&totalRows) + wg.Done() + }() + wg.Wait() + if len(pRefs) != 0 { + lastIdx := len(pRefs) - 1 + newOffsetDate = pRefs[lastIdx].DeletedAt.Time.String() + newOffsetPath = pRefs[lastIdx].Path + + } refs = &pRefs - newOffsetPath = pRefs[len(pRefs)-1].Path - tx := datastore.GetStore().GetTransaction(ctx) - tx = tx.Model(&Ref{}).Where(Ref{AllocationID: allocationID}).Where(db.Where("path = ?", path).Or("path LIKE ?", (path + "%"))) - tx.Count(&totalRows) - totalPages = int64(math.Ceil(float64(totalRows) / PAGE_SIZE)) + totalPages = int(math.Ceil(float64(totalRows) / float64(pageLimit))) return } diff --git a/sql/create-indexes.sql b/sql/create-indexes.sql new file mode 100644 index 000000000..f0d57dbe6 --- /dev/null +++ b/sql/create-indexes.sql @@ -0,0 +1,8 @@ +\connect blobber_meta; +BEGIN; +DROP INDEX IF EXISTS path_idx; +DROP INDEX IF EXISTS update_idx; +-- Create index on path column; It cannot be Unique index because of soft delete by gorm +CREATE INDEX path_idx ON reference_objects (path); +CREATE INDEX update_idx ON reference_objects (updated_at); +COMMIT; \ No newline at end of file diff --git a/sql/create-path-index.sql b/sql/create-path-index.sql deleted file mode 100644 index a01e1ff50..000000000 --- a/sql/create-path-index.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE UNIQUE INDEX path_idx ON public.reference_objects USING btree (path) \ No newline at end of file From 0badb03384d2e2c8b128bc29038ae9467e009cc5 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Wed, 28 Jul 2021 19:33:05 +0545 Subject: [PATCH 5/7] Fix: lint error NewDB --- code/go/0chain.net/blobbercore/reference/referencepath.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index d84bfdb45..7283ced2e 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -229,8 +229,8 @@ func GetDeletedRefs(ctx context.Context, allocationID, updatedDate, offsetPath, var pRefs []Ref db := datastore.GetStore().GetDB() - db1 := db.Session(&gorm.Session{NewDB: true}) - db2 := db.Session(&gorm.Session{NewDB: true}) + db1 := db.Session(&gorm.Session{}) + db2 := db.Session(&gorm.Session{}) wg := sync.WaitGroup{} wg.Add(2) From 6cee08262d7e2041c2b209ba3f72654cc9c80082 Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Thu, 29 Jul 2021 17:21:10 +0545 Subject: [PATCH 6/7] Review: Changes as per review suggestion. --- .../0chain.net/blobbercore/handler/handler.go | 5 +--- .../handler/object_operation_handler.go | 8 +++--- .../blobbercore/handler/storage_handler.go | 25 ++++++++----------- .../0chain.net/blobbercore/reference/ref.go | 2 +- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index ccc921977..f1d42e9db 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -48,7 +48,7 @@ func SetupHandlers(r *mux.Router) { r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler)))) r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))) r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler)))) - r.HandleFunc("/v1/file/refs/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(RefsHandler)))) + r.HandleFunc("/v1/file/refs/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(RefsHandler)))).Methods("GET") //admin related r.HandleFunc("/_debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines))) r.HandleFunc("/_config", common.UserRateLimit(common.ToJSONResponse(GetConfig))) @@ -240,9 +240,6 @@ func ObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error } func RefsHandler(ctx context.Context, r *http.Request) (interface{}, error) { - if r.Method == "POST" { - return nil, common.NewError("invalid method", "Invalid method used. Use GET instead") - } ctx = setupHandlerContext(ctx, r) response, err := storageHandler.GetRefs(ctx, r) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 3f3357d0b..f9236c2b9 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -202,7 +202,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( } // get and parse file params - if err = r.ParseMultipartForm(FORM_FILE_PARSE_MAX_MEMORY); nil != err { + if err = r.ParseMultipartForm(FormFileParseMaxMemory); nil != err { Logger.Info("download_file - request_parse_error", zap.Error(err)) return nil, common.NewErrorf("download_file", "request_parse_error: %v", err) @@ -362,7 +362,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) ( downloadMode = r.FormValue("content") respData []byte ) - if len(downloadMode) > 0 && downloadMode == DOWNLOAD_CONTENT_THUMB { + if len(downloadMode) > 0 && downloadMode == DownloadContentThumb { var fileData = &filestore.FileInputData{} fileData.Name = fileref.Name fileData.Path = fileref.Path @@ -470,7 +470,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - if err = r.ParseMultipartForm(FORM_FILE_PARSE_MAX_MEMORY); nil != err { + if err = r.ParseMultipartForm(FormFileParseMaxMemory); nil != err { Logger.Info("Error Parsing the request", zap.Any("error", err)) return nil, common.NewError("request_parse_error", err.Error()) } @@ -940,7 +940,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") } - if err := r.ParseMultipartForm(FORM_FILE_PARSE_MAX_MEMORY); err != nil { + if err := r.ParseMultipartForm(FormFileParseMaxMemory); err != nil { Logger.Info("Error Parsing the request", zap.Any("error", err)) return nil, common.NewError("request_parse_error", err.Error()) } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index a94300d05..e1a541b9d 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -22,14 +22,11 @@ import ( ) const ( - FORM_FILE_PARSE_MAX_MEMORY = 10 * 1024 * 1024 - - DOWNLOAD_CONTENT_FULL = "full" - DOWNLOAD_CONTENT_THUMB = "thumbnail" - PAGE_LIMIT = 100 //100 rows will make upto 100 KB - UPDATED = "updated" - DELETED = "deleted" - REGULAR = "regular" + FormFileParseMaxMemory = 10 * 1024 * 1024 + + DownloadCcontentFull = "full" + DownloadContentThumb = "thumbnail" + PageLimit = 100 //100 rows will make upto 100 KB ) type StorageHandler struct{} @@ -705,7 +702,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb pageLimitStr := r.FormValue("pageLimit") var pageLimit int if len(pageLimitStr) == 0 { - pageLimit = PAGE_LIMIT + pageLimit = PageLimit } else { o, err := strconv.Atoi(pageLimitStr) if err != nil { @@ -713,8 +710,8 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb } if o <= 0 { return nil, common.NewError("invalid_parameters", "Zero/Negative page limit value is not allowed") - } else if o > PAGE_LIMIT { - pageLimit = PAGE_LIMIT + } else if o > PageLimit { + pageLimit = PageLimit } else { pageLimit = o } @@ -750,13 +747,13 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb var newOffsetDate string switch { - case refType == REGULAR: + case refType == "regular": refs, totalPages, newOffsetPath, err = reference.GetRefs(ctx, allocationID, path, offsetPath, fileType, level, pageLimit) - case refType == UPDATED: + case refType == "updated": refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs(ctx, allocationID, path, offsetPath, fileType, updatedDate, offsetDate, level, pageLimit) - case refType == DELETED: + case refType == "deleted": refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetDeletedRefs(ctx, allocationID, updatedDate, offsetPath, offsetDate, pageLimit) default: diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 061a5a25d..ecacdb741 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -89,7 +89,7 @@ type Ref struct { CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at" json:"created_at,omitempty"` UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at" json:"updated_at,omitempty"` - DeletedAt gorm.DeletedAt `gorm:"column:deleted_at;index" json:"-"` // soft deletion + DeletedAt gorm.DeletedAt `gorm:"column:deleted_at" json:"-"` // soft deletion } func (Ref) TableName() string { From 937c1964c55fa45ff0c12f91ed8b651be012577b Mon Sep 17 00:00:00 2001 From: Laxmi Prasad Oli Date: Sun, 1 Aug 2021 10:44:26 +0545 Subject: [PATCH 7/7] Fix: Remove unrequired function --- code/go/0chain.net/blobbercore/handler/helper.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/helper.go b/code/go/0chain.net/blobbercore/handler/helper.go index add9ea100..0a5414a49 100644 --- a/code/go/0chain.net/blobbercore/handler/helper.go +++ b/code/go/0chain.net/blobbercore/handler/helper.go @@ -1,26 +1,11 @@ package handler import ( - "context" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/gorilla/mux" - "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - "google.golang.org/grpc" ) -func registerGRPCServices(r *mux.Router, server *grpc.Server) { - blobberService := newGRPCBlobberService() - grpcGatewayHandler := runtime.NewServeMux() - - blobbergrpc.RegisterBlobberServer(server, blobberService) - _ = blobbergrpc.RegisterBlobberHandlerServer(context.Background(), grpcGatewayHandler, blobberService) - r.PathPrefix("/").Handler(grpcGatewayHandler) - -} - func checkValidDate(s string) error { if s != "" { _, err := time.Parse("2006-01-02 15:04:05.999999999", s)