Feature/list recent files (#755)

* Remove whopays attribute * Remove dot logger * Fix hashes of test * Remove redundant payer id field * Remove authticket association with readmarker * Fix conflicts * Remove proto file * Update go modules * Generate/Update grpc code * Go mod tidy * Remove payer in test insertion * Make reader pay for read instead of owner * Go mod tidy * Add separate handler for directory creation * Separate directory creation handling * Replace time.Time with common.Timestamp in refs Remove deleted_at column Update tests Optimize *change.go files * Optimize upload file changer * Remove unrequired files * Remove dot import and fmt prints * Fix imports * Update timestamp * Remove calculate hash feature * Fix gosimple issue * Return nil for root path * Reduce slice size * Fix rename issues * Fix timestamp issue and missing db save * Update timestamp for delete operation * Fix tests * Use condition * Check error * Fix root row retrieval issue * Remove deleted columns usages * Add log * Use deleted_at functionality to delete files Use children loaded condition * Fix missing level and ordering issue * Fix missing level issue * Add path hash * Undo path hash removal * Uncomment workers * Uncomment workers and add loop label * Fix unit tests * Merge stashed changes * Remove unrequired argument * Complete function * Use goroutines to execute two queries * Add endpoint * Add from date parameter to filter refs * Merge staging into feature/list-recent-files * Remove same file with two names * Remove fmt print * Remove fmt print * Rename URI * Remove total pages count in response * Use appropriate constants Co-authored-by: peterlimg <54137706+peterlimg@users.noreply.github.com> Co-authored-by: peterlimg <peterlimg@protonmail.com> Co-authored-by: Lz <imlangzi@qq.com>

Author: 3 people

code/go/0chain.net/blobbercore/allocation/copyfilechange.go

@@ -28,7 +28,7 @@ func (rf *CopyFileChange) DeleteTempFile() error {
 func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationChange,
 allocationRoot string, ts common.Timestamp) (*reference.Ref, error) {
 
-totalRefs, err := reference.CountRefs(ctx, rf.AllocationID)
+totalRefs, err := reference.CountRefs(rf.AllocationID)
 if err != nil {
 return nil, err
 }

code/go/0chain.net/blobbercore/allocation/file_changer_upload.go

@@ -24,7 +24,7 @@ type UploadFileChanger struct {
 func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *AllocationChange,
 allocationRoot string, ts common.Timestamp) (*reference.Ref, error) {
 
-totalRefs, err := reference.CountRefs(ctx, nf.AllocationID)
+totalRefs, err := reference.CountRefs(nf.AllocationID)
 if err != nil {
 return nil, err
 }

code/go/0chain.net/blobbercore/allocation/newdirchange.go

@@ -23,7 +23,7 @@ type NewDir struct {
 func (nf *NewDir) ApplyChange(ctx context.Context, change *AllocationChange,
 allocationRoot string, ts common.Timestamp) (*reference.Ref, error) {
 
-totalRefs, err := reference.CountRefs(ctx, nf.AllocationID)
+totalRefs, err := reference.CountRefs(nf.AllocationID)
 if err != nil {
 return nil, err
 }

code/go/0chain.net/blobbercore/blobberhttp/response.go

@@ -42,6 +42,10 @@ type RefResult struct {
 LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`
 }
 
+type RecentRefResult struct {
+Offset int `json:"offset"`
+Refs []*reference.PaginatedRef `json:"refs"`
+}
 type ObjectPathResult struct {
 *reference.ObjectPath
 LatestWM *writemarker.WriteMarker `json:"latest_write_marker"`

code/go/0chain.net/blobbercore/handler/handler.go

@@ -61,6 +61,7 @@ func SetupHandlers(r *mux.Router) {
 r.HandleFunc("/v1/file/referencepath/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(ReferencePathHandler)))
 r.HandleFunc("/v1/file/objecttree/{allocation}", common.ToStatusCode(WithStatusReadOnlyConnection(ObjectTreeHandler))).Methods(http.MethodGet, http.MethodOptions)
 r.HandleFunc("/v1/file/refs/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(RefsHandler))).Methods(http.MethodGet, http.MethodOptions)
+r.HandleFunc("/v1/file/refs/recent/{allocation}", common.ToJSONResponse(WithReadOnlyConnection(RecentRefsRequestHandler))).Methods(http.MethodGet, http.MethodOptions)
 //admin related
 r.HandleFunc("/_debug", common.ToJSONResponse(DumpGoRoutines))
 r.HandleFunc("/_config", common.ToJSONResponse(GetConfig))
@@ -240,6 +241,15 @@ func RefsHandler(ctx context.Context, r *http.Request) (interface{}, error) {
 return response, nil
 }
 
+func RecentRefsRequestHandler(ctx context.Context, r *http.Request) (interface{}, error) {
+ctx = setupHandlerContext(ctx, r)
+response, err := storageHandler.GetRecentlyAddedRefs(ctx, r)
+if err != nil {
+return nil, err
+}
+return response, nil
+}
+
 func RenameHandler(ctx context.Context, r *http.Request) (interface{}, error) {
 
 ctx = setupHandlerContext(ctx, r)

code/go/0chain.net/blobbercore/handler/storage_handler.go

@@ -30,7 +30,8 @@ const (
 OffsetDateLayout = "2006-01-02T15:04:05.99999Z07:00"
 DownloadContentFull = "full"
 DownloadContentThumb = "thumbnail"
-PageLimit = 100 //100 rows will make up to 100 KB
+MaxPageLimit = 100 //100 rows will make up to 100 KB
+DefaultPageLimit = 20
 )
 
 type StorageHandler struct{}
@@ -782,6 +783,88 @@ func (fsh *StorageHandler) GetObjectTree(ctx context.Context, r *http.Request) (
 return &refPathResult, nil
 }
 
+func (fsh *StorageHandler) GetRecentlyAddedRefs(ctx context.Context, r *http.Request) (*blobberhttp.RecentRefResult, error) {
+allocationTx := ctx.Value(constants.ContextKeyAllocation).(string)
+allocationObj, err := fsh.verifyAllocation(ctx, allocationTx, false)
+
+if err != nil {
+return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error())
+}
+
+clientID := ctx.Value(constants.ContextKeyClient).(string)
+if clientID == "" {
+return nil, common.NewError("invalid_operation", "Client id is required")
+}
+
+publicKey, _ := ctx.Value(constants.ContextKeyClientKey).(string)
+if publicKey == "" {
+if clientID == allocationObj.OwnerID {
+publicKey = allocationObj.OwnerPublicKey
+} else {
+return nil, common.NewError("empty_public_key", "public key is required")
+}
+}
+
+clientSign := ctx.Value(constants.ContextKeyClientSignatureHeaderKey).(string)
+
+valid, err := verifySignatureFromRequest(allocationTx, clientSign, publicKey)
+if !valid || err != nil {
+return nil, common.NewError("invalid_signature", "Invalid signature")
+}
+
+allocationID := allocationObj.ID
+
+var offset, pageLimit int
+offsetStr := r.FormValue("offset")
+
+if offsetStr != "" {
+offset, err = strconv.Atoi(offsetStr)
+if err != nil {
+return nil, common.NewError("invalid_parameters", "Invalid offset value "+err.Error())
+}
+}
+
+pageLimitStr := r.FormValue("page_limit")
+if pageLimitStr != "" {
+pageLimit, err = strconv.Atoi(pageLimitStr)
+if err != nil {
+return nil, common.NewError("invalid_parameters", "Invalid page limit value. Got Error "+err.Error())
+}
+if pageLimit < 0 {
+return nil, common.NewError("invalid_parameters", "Zero/Negative page limit value is not allowed")
+}
+
+if pageLimit > MaxPageLimit {
+pageLimit = MaxPageLimit
+}
+
+} else {
+pageLimit = DefaultPageLimit
+}
+
+var fromDate int
+fromDateStr := r.FormValue("from_date")
+if fromDateStr != "" {
+fromDate, err = strconv.Atoi(fromDateStr)
+if err != nil {
+return nil, common.NewError("invalid_parameters", "Invalid from date value. Got error "+err.Error())
+}
+if fromDate < 0 {
+return nil, common.NewError("invalid_parameters", "Negative from date value is not allowed")
+}
+}
+
+refs, offset, err := reference.GetRecentlyCreatedRefs(ctx, allocationID, pageLimit, offset, fromDate)
+if err != nil {
+return nil, err
+}
+
+return &blobberhttp.RecentRefResult{
+Refs: refs,
+Offset: offset,
+}, nil
+}
+
 //Retrieves file refs. One can use three types to refer to regular, updated and deleted. Regular type gives all undeleted rows.
 //Updated gives rows that is updated compared to the date given. And deleted gives deleted refs compared to the date given.
 //Updated date time format should be as declared in above constant; OffsetDateLayout
@@ -886,16 +969,16 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb
 pageLimitStr := r.FormValue("pageLimit")
 var pageLimit int
 if pageLimitStr == "" {
-pageLimit = PageLimit
+pageLimit = DefaultPageLimit
 } else {
 o, err := strconv.Atoi(pageLimitStr)
 if err != nil {
 return nil, common.NewError("invalid_parameters", "Invalid page limit value type")
 }
 if o <= 0 {
 return nil, common.NewError("invalid_parameters", "Zero/Negative page limit value is not allowed")
-} else if o > PageLimit {
-pageLimit = PageLimit
+} else if o > MaxPageLimit {
+pageLimit = MaxPageLimit
 } else {
 pageLimit = o
 }

code/go/0chain.net/blobbercore/reference/ref.go

@@ -57,8 +57,8 @@ type Ref struct {
 OnCloud bool `gorm:"column:on_cloud;default:false" filelist:"on_cloud"`
 
 CommitMetaTxns []CommitMetaTxn `gorm:"foreignkey:ref_id" filelist:"commit_meta_txns"`
-CreatedAt common.Timestamp `gorm:"column:created_at" dirlist:"created_at" filelist:"created_at"`
-UpdatedAt common.Timestamp `gorm:"column:updated_at;index:idx_updated_at;" dirlist:"updated_at" filelist:"updated_at"`
+CreatedAt common.Timestamp `gorm:"column:created_at;index:idx_created_at,sort:desc" dirlist:"created_at" filelist:"created_at"`
+UpdatedAt common.Timestamp `gorm:"column:updated_at;index:idx_updated_at,sort:desc;" dirlist:"updated_at" filelist:"updated_at"`
 
 DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion
 

code/go/0chain.net/blobbercore/reference/referencepath.go

@@ -294,7 +294,36 @@ func GetUpdatedRefs(ctx context.Context, allocationID, path, offsetPath, _type,
 return
 }
 
-func CountRefs(ctx context.Context, allocationID string) (int64, error) {
+// GetRecentlyCreatedRefs will return recently created refs with pagination. As opposed to getting
+// refs ordered by path in ascending order, this will return paths in decending order for same timestamp.
+// So if a file is created with path "/a/b/c/d/e/f.txt" and if "/a" didn't exist previously then
+// creation date for "/a", "/a/b", "/a/b/c", "/a/b/c/d", "/a/b/c/d/e" and "/a/b/c/d/e/f.txt" will be the same.
+// The refs returned will be in "/a/b/c/d/e/f.txt", "/a/b/c/d/e", ... order.
+//
+// pageLimit --> maximum number of refs to return
+// fromDate --> timestamp to begin searching refs from i.e. refs created date greater than fromDate
+// newOffset --> offset to use for subsequent request
+func GetRecentlyCreatedRefs(
+// Note: Above mentioned function will only be feasible after splitting reference_objects table.
+// Since current limit is 50,000 files per allocation, Using common offset method should not be a big
+// deal
+ctx context.Context,
+allocID string,
+pageLimit, offset, fromDate int,
+) (refs []*PaginatedRef, newOffset int, err error) {
+
+db := datastore.GetStore().GetTransaction(ctx)
+err = db.Model(&Ref{}).Where("allocation_id=? AND created_at > ?",
+allocID, fromDate).
+Order("created_at desc, path asc").
+Offset(offset).
+Limit(pageLimit).Find(&refs).Error
+
+newOffset = offset + len(refs)
+return
+}
+
+func CountRefs(allocationID string) (int64, error) {
 var totalRows int64
 
 db := datastore.GetStore().GetDB()