Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

File References Pagination #273

Merged
merged 10 commits into from
Aug 1, 2021
Merged

File References Pagination #273

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
8c8745a
WIP: Object tree pagination
lpoli Jul 16, 2021
0f78ff0
WIP: Update code
lpoli Jul 18, 2021
26a6e08
WIP: Refactor with index creation
lpoli Jul 21, 2021
1ebd977
WIP: Refactor to refs pagination
lpoli Jul 28, 2021
0badb03
Fix: lint error NewDB
lpoli Jul 28, 2021
6cee082
Review: Changes as per review suggestion.
lpoli Jul 29, 2021
2199d14
Merge branch 'master' into lp-pagination
lpoli Jul 29, 2021
adab8a6
Merge branch 'master' of https://github.com/0chain/blobber into lp-pa…
lpoli Jul 29, 2021
21d3af8
Merge branch 'master' into lp-pagination
lpoli Aug 1, 2021
937c196
Fix: Remove unrequired function
lpoli Aug 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions code/go/0chain.net/blobbercore/blobberhttp/response.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ type ReferencePathResult struct {
LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
}

type RefResult struct {
TotalPages int `json:"total_pages"`
NewOffsetPath string `json:"offsetPath,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could you add some documentation about what is NewOffsetPath/NewoffsetDate? it is confused there without any comment.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

total_pages is snake_case, and offsetPath is lower_case. should we use snake_case only?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and the field name is NewOffsetPath, but it is offsetPath in json. why they are different?

NewOffsetDate string `json:"offsetDate,omitempty"`
Refs *[]reference.Ref `json:"refs"`
LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
}

type ObjectPathResult struct {
*reference.ObjectPath
LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
Expand Down
13 changes: 12 additions & 1 deletion code/go/0chain.net/blobbercore/handler/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func SetupHandlers(r *mux.Router) {
r.HandleFunc("/v1/file/objectpath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectPathHandler))))
r.HandleFunc("/v1/file/referencepath/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler))))
r.HandleFunc("/v1/file/objecttree/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(ObjectTreeHandler))))

r.HandleFunc("/v1/file/refs/{allocation}", common.UserRateLimit(common.ToJSONResponse(WithReadOnlyConnection(RefsHandler)))).Methods("GET")
//admin related
r.HandleFunc("/_debug", common.UserRateLimit(common.ToJSONResponse(DumpGoRoutines)))
r.HandleFunc("/_config", common.UserRateLimit(common.ToJSONResponse(GetConfig)))
Expand Down Expand Up @@ -252,6 +252,17 @@ func ObjectTreeHandler(ctx context.Context, r *http.Request) (interface{}, error
return response, nil
}

func RefsHandler(ctx context.Context, r *http.Request) (interface{}, error) {
ctx = setupHandlerContext(ctx, r)

response, err := storageHandler.GetRefs(ctx, r)
if err != nil {
return nil, err
}

return response, nil
}

func RenameHandler(ctx context.Context, r *http.Request) (interface{}, error) {
ctx = setupHandlerContext(ctx, r)
response, err := storageHandler.RenameObject(ctx, r)
Expand Down
32 changes: 32 additions & 0 deletions code/go/0chain.net/blobbercore/handler/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package handler

import (
"context"
"time"

"github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobbergrpc"
"github.com/0chain/blobber/code/go/0chain.net/core/common"
"github.com/gorilla/mux"
"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
"google.golang.org/grpc"
)

func registerGRPCServices(r *mux.Router, server *grpc.Server) {
blobberService := newGRPCBlobberService()
grpcGatewayHandler := runtime.NewServeMux()

blobbergrpc.RegisterBlobberServer(server, blobberService)
_ = blobbergrpc.RegisterBlobberHandlerServer(context.Background(), grpcGatewayHandler, blobberService)
r.PathPrefix("/").Handler(grpcGatewayHandler)

}

func checkValidDate(s string) error {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about Timezone? it will be an issue if client send date with local time zone.

if s != "" {
_, err := time.Parse("2006-01-02 15:04:05.999999999", s)
if err != nil {
return common.NewError("invalid_parameters", err.Error())
}
}
return nil
}
8 changes: 4 additions & 4 deletions code/go/0chain.net/blobbercore/handler/object_operation_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ func (fsh *StorageHandler) DownloadFile(
}

// get and parse file params
if err = r.ParseMultipartForm(FORM_FILE_PARSE_MAX_MEMORY); nil != err {
if err = r.ParseMultipartForm(FormFileParseMaxMemory); nil != err {
Logger.Info("download_file - request_parse_error", zap.Error(err))
return nil, common.NewErrorf("download_file",
"request_parse_error: %v", err)
Expand Down Expand Up @@ -394,7 +394,7 @@ func (fsh *StorageHandler) DownloadFile(
downloadMode = r.FormValue("content")
respData []byte
)
if len(downloadMode) > 0 && downloadMode == DOWNLOAD_CONTENT_THUMB {
if len(downloadMode) > 0 && downloadMode == DownloadContentThumb {
var fileData = &filestore.FileInputData{}
fileData.Name = fileref.Name
fileData.Path = fileref.Path
Expand Down Expand Up @@ -561,7 +561,7 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation")
}

if err = r.ParseMultipartForm(FORM_FILE_PARSE_MAX_MEMORY); nil != err {
if err = r.ParseMultipartForm(FormFileParseMaxMemory); nil != err {
Logger.Info("Error Parsing the request", zap.Any("error", err))
return nil, common.NewError("request_parse_error", err.Error())
}
Expand Down Expand Up @@ -1122,7 +1122,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation")
}

if err := r.ParseMultipartForm(FORM_FILE_PARSE_MAX_MEMORY); err != nil {
if err := r.ParseMultipartForm(FormFileParseMaxMemory); err != nil {
Logger.Info("Error Parsing the request", zap.Any("error", err))
return nil, common.NewError("request_parse_error", err.Error())
}
Expand Down
120 changes: 117 additions & 3 deletions code/go/0chain.net/blobbercore/handler/storage_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@ import (
)

const (
FORM_FILE_PARSE_MAX_MEMORY = 10 * 1024 * 1024
FormFileParseMaxMemory = 10 * 1024 * 1024

DOWNLOAD_CONTENT_FULL = "full"
DOWNLOAD_CONTENT_THUMB = "thumbnail"
DownloadCcontentFull = "full"
DownloadContentThumb = "thumbnail"
PageLimit = 100 //100 rows will make upto 100 KB
)

type StorageHandler struct{}
Expand Down Expand Up @@ -636,6 +637,7 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) (
refPath := &reference.ReferencePath{Ref: rootRef}
refsToProcess := make([]*reference.ReferencePath, 0)
refsToProcess = append(refsToProcess, refPath)

for len(refsToProcess) > 0 {
refToProcess := refsToProcess[0]
refToProcess.Meta = refToProcess.Ref.GetListingData(ctx)
Expand Down Expand Up @@ -667,6 +669,118 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) (
return &refPathResult, nil
}

//Retrieves file refs. One can use three types to refer to regular, updated and deleted. Regular type gives all undeleted rows.
//Updated gives rows that is updated compared to the date given. And deleted gives deleted refs compared to the date given.
func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobberhttp.RefResult, error) {
allocationTx := ctx.Value(constants.ALLOCATION_CONTEXT_KEY).(string)
allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false)

if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error())
}

clientSign, _ := ctx.Value(constants.CLIENT_SIGNATURE_HEADER_KEY).(string)
valid, err := verifySignatureFromRequest(allocationTx, clientSign, allocationObj.OwnerPublicKey)
if !valid || err != nil {
return nil, common.NewError("invalid_signature", "Invalid signature")
}

allocationID := allocationObj.ID
clientID := ctx.Value(constants.CLIENT_CONTEXT_KEY).(string)
if len(clientID) == 0 || allocationObj.OwnerID != clientID {
return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation")
}
path := r.FormValue("path")
if len(path) == 0 {
return nil, common.NewError("invalid_parameters", "Invalid path")
}

pageLimitStr := r.FormValue("pageLimit")
var pageLimit int
if len(pageLimitStr) == 0 {
pageLimit = PageLimit
} else {
o, err := strconv.Atoi(pageLimitStr)
if err != nil {
return nil, common.NewError("invalid_parameters", "Invalid page limit value type")
}
if o <= 0 {
return nil, common.NewError("invalid_parameters", "Zero/Negative page limit value is not allowed")
} else if o > PageLimit {
pageLimit = PageLimit
} else {
pageLimit = o
}
}
offsetPath := r.FormValue("offsetPath")
offsetDate := r.FormValue("offsetDate")
updatedDate := r.FormValue("updatedDate")
err = checkValidDate(offsetDate)
if err != nil {
return nil, err
}
err = checkValidDate(updatedDate)
if err != nil {
return nil, err
}
fileType := r.FormValue("fileType")
levelStr := r.FormValue("level")
var level int
if len(levelStr) != 0 {
level, err = strconv.Atoi(levelStr)
if err != nil {
return nil, common.NewError("invalid_parameters", err.Error())
}
if level < 0 {
return nil, common.NewError("invalid_parameters", "Negative level value is not allowed")
}
}

refType := r.FormValue("refType")
var refs *[]reference.Ref
var totalPages int
var newOffsetPath string
var newOffsetDate string

switch {
case refType == "regular":
refs, totalPages, newOffsetPath, err = reference.GetRefs(ctx, allocationID, path, offsetPath, fileType, level, pageLimit)

case refType == "updated":
refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs(ctx, allocationID, path, offsetPath, fileType, updatedDate, offsetDate, level, pageLimit)

case refType == "deleted":
refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetDeletedRefs(ctx, allocationID, updatedDate, offsetPath, offsetDate, pageLimit)

default:
return nil, common.NewError("invalid_parameters", "refType param should have value regular/updated/deleted")
}

if err != nil {
return nil, err
}
var latestWM *writemarker.WriteMarkerEntity
if len(allocationObj.AllocationRoot) == 0 {
latestWM = nil
} else {
latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot)
if err != nil {
return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error())
}
}

var refResult blobberhttp.RefResult
refResult.Refs = refs
refResult.TotalPages = totalPages
refResult.NewOffsetPath = newOffsetPath
refResult.NewOffsetDate = newOffsetDate
if latestWM != nil {
refResult.LatestWM = &latestWM.WM
}
// Refs will be returned as it is and object tree will be build in client side
return &refResult, nil
}

func (fsh *StorageHandler) CalculateHash(ctx context.Context, r *http.Request) (interface{}, error) {
if r.Method != "POST" {
return nil, common.NewError("invalid_method", "Invalid method used. Use POST instead")
Expand Down
68 changes: 34 additions & 34 deletions code/go/0chain.net/blobbercore/reference/ref.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,40 +56,40 @@ func (a *Attributes) Validate() (err error) {
}

type Ref struct {
ID int64 `gorm:"column:id;primary_key"`
Type string `gorm:"column:type" dirlist:"type" filelist:"type"`
AllocationID string `gorm:"column:allocation_id"`
LookupHash string `gorm:"column:lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"`
Name string `gorm:"column:name" dirlist:"name" filelist:"name"`
Path string `gorm:"column:path" dirlist:"path" filelist:"path"`
Hash string `gorm:"column:hash" dirlist:"hash" filelist:"hash"`
NumBlocks int64 `gorm:"column:num_of_blocks" dirlist:"num_of_blocks" filelist:"num_of_blocks"`
PathHash string `gorm:"column:path_hash" dirlist:"path_hash" filelist:"path_hash"`
ParentPath string `gorm:"column:parent_path"`
PathLevel int `gorm:"column:level"`
CustomMeta string `gorm:"column:custom_meta" filelist:"custom_meta"`
ContentHash string `gorm:"column:content_hash" filelist:"content_hash"`
Size int64 `gorm:"column:size" dirlist:"size" filelist:"size"`
MerkleRoot string `gorm:"column:merkle_root" filelist:"merkle_root"`
ActualFileSize int64 `gorm:"column:actual_file_size" filelist:"actual_file_size"`
ActualFileHash string `gorm:"column:actual_file_hash" filelist:"actual_file_hash"`
MimeType string `gorm:"column:mimetype" filelist:"mimetype"`
WriteMarker string `gorm:"column:write_marker"`
ThumbnailSize int64 `gorm:"column:thumbnail_size" filelist:"thumbnail_size"`
ThumbnailHash string `gorm:"column:thumbnail_hash" filelist:"thumbnail_hash"`
ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" filelist:"actual_thumbnail_size"`
ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" filelist:"actual_thumbnail_hash"`
EncryptedKey string `gorm:"column:encrypted_key" filelist:"encrypted_key"`
Attributes datatypes.JSON `gorm:"column:attributes" filelist:"attributes"`
Children []*Ref `gorm:"-"`
childrenLoaded bool

OnCloud bool `gorm:"column:on_cloud" filelist:"on_cloud"`
CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns"`
CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at"`

DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion
ID int64 `gorm:"column:id;primary_key" json:"id,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why add json annotations explicitly? Gorm handles the json naming imo.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It gives flexibility and also I am not sure about Gorm handling json naming. It supports json dataTypes but it always fills struct fields with its value and its easy to use it with json tag.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

json column field is used when we unmarshal an interface to the following dataType, but since we already have gorm annotation, gorm has default serialising strategy as snake_case hence I said this change is redundant.

Type string `gorm:"column:type" dirlist:"type" filelist:"type" json:"type,omitempty"`
AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"`
LookupHash string `gorm:"column:lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash" json:"lookup_hash,omitempty"`
Name string `gorm:"column:name" dirlist:"name" filelist:"name" json:"name,omitempty"`
Path string `gorm:"column:path" dirlist:"path" filelist:"path" json:"path,omitempty"`
Hash string `gorm:"column:hash" dirlist:"hash" filelist:"hash" json:"hash,omitempty"`
NumBlocks int64 `gorm:"column:num_of_blocks" dirlist:"num_of_blocks" filelist:"num_of_blocks" json:"num_blocks,omitempty"`
PathHash string `gorm:"column:path_hash" dirlist:"path_hash" filelist:"path_hash" json:"path_hash,omitempty"`
ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"`
PathLevel int `gorm:"column:level" json:"level,omitempty"`
CustomMeta string `gorm:"column:custom_meta" filelist:"custom_meta" json:"custom_meta,omitempty"`
ContentHash string `gorm:"column:content_hash" filelist:"content_hash" json:"content_hash,omitempty"`
Size int64 `gorm:"column:size" dirlist:"size" filelist:"size" json:"size,omitempty"`
MerkleRoot string `gorm:"column:merkle_root" filelist:"merkle_root" json:"merkle_root,omitempty"`
ActualFileSize int64 `gorm:"column:actual_file_size" filelist:"actual_file_size" json:"actual_file_size,omitempty"`
ActualFileHash string `gorm:"column:actual_file_hash" filelist:"actual_file_hash" json:"actual_file_hash,omitempty"`
MimeType string `gorm:"column:mimetype" filelist:"mimetype" json:"mimetype,omitempty"`
WriteMarker string `gorm:"column:write_marker" json:"write_marker,omitempty"`
ThumbnailSize int64 `gorm:"column:thumbnail_size" filelist:"thumbnail_size" json:"thumbnail_size,omitempty"`
ThumbnailHash string `gorm:"column:thumbnail_hash" filelist:"thumbnail_hash" json:"thumbnail_hash,omitempty"`
ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" filelist:"actual_thumbnail_size" json:"actual_thumbnail_size,omitempty"`
ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" filelist:"actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"`
EncryptedKey string `gorm:"column:encrypted_key" filelist:"encrypted_key" json:"encrypted_key,omitempty"`
Attributes datatypes.JSON `gorm:"column:attributes" filelist:"attributes" json:"attributes,omitempty"`
Children []*Ref `gorm:"-" json:"-"`
childrenLoaded bool `json:"-"`

OnCloud bool `gorm:"column:on_cloud" filelist:"on_cloud" json:"on_cloud,omitempty"`
CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns" json:"-"`
CreatedAt time.Time `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at" json:"created_at,omitempty"`
UpdatedAt time.Time `gorm:"column:updated_at" dirlist:"updated_at" filelist:"updated_at" json:"updated_at,omitempty"`

DeletedAt gorm.DeletedAt `gorm:"column:deleted_at" json:"-"` // soft deletion
}

func (Ref) TableName() string {
Expand Down
Loading