Fix/limit thumbnail size (#678)

* Optimize delete * Limit thumbnail size to 1MB * Set hashToBeComputed field for affected directories * Fix delete * Fix hash calculation issue * Fix sql query with sequential rather than goroutine * Add db constraint * Fix hash calculation issue for delete operation * Skip go routine for count not 0 * Uncapatalize error string

Author: lpoli

code/go/0chain.net/blobbercore/allocation/deletefilechange.go

@@ -3,13 +3,14 @@ package allocation
 import (
 "context"
 "encoding/json"
-"os"
+"fmt"
+"sync"
 
 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
-"github.com/0chain/blobber/code/go/0chain.net/core/common"
-. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+"github.com/0chain/blobber/code/go/0chain.net/core/logging"
+"gorm.io/gorm"
 
 "go.uber.org/zap"
 )
@@ -21,11 +22,10 @@ type DeleteFileChange struct {
 Path string `json:"path"`
 Size int64 `json:"size"`
 Hash string `json:"hash"`
-ContentHash map[string]bool
 }
 
 func (nf *DeleteFileChange) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) {
-rootRef, contentHash, err := reference.DeleteObject(ctx, nf.AllocationID, nf.Path)
+rootRef, err := reference.DeleteObject(ctx, nf.AllocationID, nf.Path)
 if err != nil {
 return nil, err
 }
@@ -34,8 +34,6 @@ func (nf *DeleteFileChange) ApplyChange(ctx context.Context, change *AllocationC
 return nil, err
 }
 
-nf.ContentHash = contentHash
-
 return nil, nil
 }
 
@@ -58,20 +56,67 @@ func (nf *DeleteFileChange) DeleteTempFile() error {
 
 func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error {
 db := datastore.GetStore().GetTransaction(ctx)
-var errFileWasDeleted error
-for contenthash := range nf.ContentHash {
-var count int64
-err := db.Table((&reference.Ref{}).TableName()).Where(db.Where(&reference.Ref{ThumbnailHash: contenthash}).Or(&reference.Ref{ContentHash: contenthash})).Where("deleted_at IS null").Where(&reference.Ref{AllocationID: nf.AllocationID}).Count(&count).Error
-if err == nil && count == 0 {
-Logger.Info("Deleting content file", zap.String("content_hash", contenthash))
-if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil {
-if os.IsNotExist(err) {
-errFileWasDeleted = common.ErrFileWasDeleted
+type Result struct {
+ContentHash string
+ThumbnailHash string
+}
+
+limitCh := make(chan struct{}, 10)
+wg := &sync.WaitGroup{}
+var results []Result
+err := db.Model(&reference.Ref{}).Unscoped().
+Select("content_hash", "thumbnail_hash").
+Where("allocation_id=? AND path LIKE ? AND type=? AND deleted_at is not NULL",
+nf.AllocationID, nf.Path+"%", reference.FILE).
+FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
+
+for _, res := range results {
+var count int64
+tx.Model(&reference.Ref{}).
+Where("allocation_id=? AND content_hash=?", nf.AllocationID, res.ContentHash).
+Count(&count)
+
+if count != 0 {
 continue
 }
-Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err))
+
+limitCh <- struct{}{}
+wg.Add(1)
+
+go func(res Result, count int64) {
+defer func() {
+<-limitCh
+wg.Done()
+}()
+
+err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ContentHash)
+if err != nil {
+logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
+zap.String("content_hash", res.ContentHash))
+}
+
+if res.ThumbnailHash != "" {
+err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ThumbnailHash)
+if err != nil {
+logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail: %s", err.Error()),
+zap.String("thumbnail", res.ThumbnailHash))
+}
+}
+
+}(res, count)
+
 }
-}
+return nil
+}).Error
+
+wg.Wait()
+
+if err != nil {
+return err
 }
-return errFileWasDeleted
+
+return db.Model(&reference.Ref{}).Unscoped().
+Delete(&reference.Ref{},
+"allocation_id = ? AND path LIKE ? AND deleted_at IS NOT NULL",
+nf.AllocationID, nf.Path+"%").Error
 }

code/go/0chain.net/blobbercore/handler/file_command_update.go

@@ -3,6 +3,7 @@ package handler
 import (
 "context"
 "encoding/json"
+"fmt"
 "net/http"
 
 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
@@ -11,12 +12,15 @@ import (
 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
 "github.com/0chain/blobber/code/go/0chain.net/core/common"
 "github.com/0chain/blobber/code/go/0chain.net/core/logging"
-"github.com/0chain/gosdk/constants"
-sdkConstants "github.com/0chain/gosdk/constants"
+sdkConst "github.com/0chain/gosdk/constants"
 "github.com/0chain/gosdk/zboxcore/fileref"
 "go.uber.org/zap"
 )
 
+const (
+UpdateMeta = "updatedMeta"
+)
+
 // UpdateFileCommand command for updating file
 type UpdateFileCommand struct {
 existingFileRef *reference.Ref
@@ -26,11 +30,11 @@ type UpdateFileCommand struct {
 
 // IsValidated validate request.
 func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, clientID string) error {
-uploadMetaString := req.FormValue("uploadMeta")
+uploadMetaString := req.FormValue(UploadMeta)
 
 if uploadMetaString == "" {
 // backward compatibility for old update request
-uploadMetaString = req.FormValue("updatedMeta")
+uploadMetaString = req.FormValue(UpdateMeta)
 }
 
 err := json.Unmarshal([]byte(uploadMetaString), &cmd.fileChanger)
@@ -54,6 +58,14 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request
 return common.NewError("invalid_operation", "Operation needs to be performed by the owner, collaborator or the payer of the allocation")
 }
 
+_, thumbHeader, _ := req.FormFile(UploadThumbnailFile)
+if thumbHeader != nil {
+if thumbHeader.Size > MaxThumbnailSize {
+return common.NewError("max_thumbnail_size",
+fmt.Sprintf("thumbnail size %d should not be greater than %d", thumbHeader.Size, MaxThumbnailSize))
+}
+}
+
 return nil
 }
 
@@ -63,7 +75,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ
 
 result.Filename = cmd.fileChanger.Filename
 
-origfile, _, err := req.FormFile("uploadFile")
+origfile, _, err := req.FormFile(UploadFile)
 if err != nil {
 return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error())
 }
@@ -109,7 +121,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ
 cmd.allocationChange = &allocation.AllocationChange{}
 cmd.allocationChange.ConnectionID = connectionObj.ID
 cmd.allocationChange.Size = allocationSize - cmd.existingFileRef.Size
-cmd.allocationChange.Operation = sdkConstants.FileOperationUpdate
+cmd.allocationChange.Operation = sdkConst.FileOperationUpdate
 
 if cmd.fileChanger.IsFinal {
 connectionObj.Size = allocationSize - cmd.existingFileRef.Size
@@ -122,7 +134,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ
 
 // ProcessThumbnail flush thumbnail file to FileStorage if it has.
 func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error {
-thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile")
+thumbfile, thumbHeader, _ := req.FormFile(UploadThumbnailFile)
 
 if thumbHeader != nil {
 defer thumbfile.Close()
@@ -145,7 +157,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Re
 
 func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationChangeCollector) {
 for _, c := range connectionObj.Changes {
-if c.Operation != constants.FileOperationUpdate {
+if c.Operation != sdkConst.FileOperationUpdate {
 continue
 }
 
@@ -168,7 +180,7 @@ func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationC
 // UpdateChange add UpdateFileChanger in db
 func (cmd *UpdateFileCommand) UpdateChange(ctx context.Context, connectionObj *allocation.AllocationChangeCollector) error {
 for _, c := range connectionObj.Changes {
-if c.Operation != constants.FileOperationUpdate {
+if c.Operation != sdkConst.FileOperationUpdate {
 continue
 }
 

code/go/0chain.net/blobbercore/handler/file_command_upload.go

@@ -19,6 +19,13 @@ import (
 "go.uber.org/zap"
 )
 
+const (
+MaxThumbnailSize = MB
+UploadMeta = "uploadMeta"
+UploadFile = "uploadFile"
+UploadThumbnailFile = "uploadThumbnailFile"
+)
+
 // UploadFileCommand command for resuming file
 type UploadFileCommand struct {
 allocationChange *allocation.AllocationChange
@@ -33,7 +40,7 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
 
 fileChanger := &allocation.UploadFileChanger{}
 
-uploadMetaString := req.FormValue("uploadMeta")
+uploadMetaString := req.FormValue(UploadMeta)
 err := json.Unmarshal([]byte(uploadMetaString), fileChanger)
 if err != nil {
 return common.NewError("invalid_parameters",
@@ -64,6 +71,14 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
 return err
 }
 
+_, thumbHeader, _ := req.FormFile(UploadThumbnailFile)
+if thumbHeader != nil {
+if thumbHeader.Size > MaxThumbnailSize {
+return common.NewError("max_thumbnail_size",
+fmt.Sprintf("thumbnail size %d should not be greater than %d", thumbHeader.Size, MaxThumbnailSize))
+}
+}
+
 if fileChanger.ChunkSize <= 0 {
 fileChanger.ChunkSize = fileref.CHUNK_SIZE
 }
@@ -77,7 +92,7 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
 func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) (blobberhttp.UploadResult, error) {
 result := blobberhttp.UploadResult{}
 
-origfile, _, err := req.FormFile("uploadFile")
+origfile, _, err := req.FormFile(UploadFile)
 if err != nil {
 return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error())
 }
@@ -137,7 +152,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ
 
 // ProcessThumbnail flush thumbnail file to FileStorage if it has.
 func (cmd *UploadFileCommand) ProcessThumbnail(ctx context.Context, req *http.Request, allocationObj *allocation.Allocation, connectionObj *allocation.AllocationChangeCollector) error {
-thumbfile, thumbHeader, _ := req.FormFile("uploadThumbnailFile")
+thumbfile, thumbHeader, _ := req.FormFile(UploadThumbnailFile)
 
 if thumbHeader != nil {
 defer thumbfile.Close()

code/go/0chain.net/blobbercore/reference/object.go

@@ -3,77 +3,48 @@ package reference
 import (
 "context"
 "path/filepath"
-"strings"
 
 "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 "github.com/0chain/blobber/code/go/0chain.net/core/common"
-_ "gorm.io/gorm"
-"gorm.io/gorm/clause"
 )
 
-// DeleteObject delete object from tree, and return tree root and deleted content hash list
-func DeleteObject(ctx context.Context, allocationID, path string) (*Ref, map[string]bool, error) {
-rootRef, err := GetObjectTree(ctx, allocationID, "/")
-if err != nil {
-return nil, nil, err
-}
-
-rootRef.HashToBeComputed = true
-db := datastore.GetStore().
-GetTransaction(ctx)
+func DeleteObject(ctx context.Context, allocationID, objPath string) (*Ref, error) {
 
-var deletedObjects []*Ref
-txDelete := db.Clauses(clause.Returning{Columns: []clause.Column{{Name: "content_hash"}, {Name: "thumbnail_hash"}, {Name: "type"}}})
+db := datastore.GetStore().GetTransaction(ctx)
+err := db.Delete(&Ref{}, "allocation_id=? AND path LIKE ? AND path != ?",
+allocationID, objPath+"%", "/").Error
 
-path = filepath.Join("/", path)
-txDelete = txDelete.Where("allocation_id = ? and deleted_at IS NULL and (path LIKE ? or path = ?) and path != ? ", allocationID, path+"%", path, "/")
-
-err = txDelete.Delete(&deletedObjects).Error
 if err != nil {
-return nil, nil, common.NewError("bad_db_operation", err.Error())
+return nil, err
 }
 
-deletedFiles := make(map[string]bool)
-for _, it := range deletedObjects {
-if it.Type == FILE {
-deletedFiles[it.ContentHash] = true
-if it.ThumbnailHash != "" {
-deletedFiles[it.ThumbnailHash] = true
-}
-}
-}
-
-// remove deleted object from tree
-if path == "/" {
-rootRef.Children = nil
-return rootRef, deletedFiles, nil
+parentPath := filepath.Dir(objPath)
+rootRef, err := GetReferencePath(ctx, allocationID, parentPath)
+if err != nil {
+return nil, err
 }
 
-path = strings.TrimSuffix(path, "/")
-tSubDirs := GetSubDirsFromPath(path)
+subDirs := GetSubDirsFromPath(parentPath)
 dirRef := rootRef
-treelevel := 0
-for treelevel < len(tSubDirs)-1 {
-found := false
-for _, child := range dirRef.Children {
-if child.Name == tSubDirs[treelevel] && child.Type == DIRECTORY {
-dirRef = child
-dirRef.HashToBeComputed = true
+
+for _, subDir := range subDirs {
+var found bool
+for _, ref := range dirRef.Children {
+if ref.Name == subDir && ref.Type == DIRECTORY {
+ref.HashToBeComputed = true
+ref.childrenLoaded = true
 found = true
+dirRef = ref
 break
 }
 }
+
 if !found {
-return nil, nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber")
+return nil, common.NewError("invalid_reference_path", "Reference path has invalid references")
 }
-treelevel++
 }
 
-for i, child := range dirRef.Children {
-if child.Path == path {
-dirRef.RemoveChild(i)
-return rootRef, deletedFiles, nil
-}
-}
-return nil, nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber")
+rootRef.HashToBeComputed = true
+rootRef.childrenLoaded = true
+return rootRef, nil
 }

code/go/0chain.net/blobbercore/stats/filestats.go

@@ -12,7 +12,7 @@ import (
 type FileStats struct {
 ID int64 `gorm:"column:id;primaryKey" json:"-"`
 RefID int64 `gorm:"column:ref_id;unique" json:"-"`
-Ref reference.Ref `gorm:"foreignKey:RefID"`
+Ref reference.Ref `gorm:"foreignKey:RefID;constraint:OnDelete:CASCADE"`
 NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"`
 NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"`
 SuccessChallenges int64 `gorm:"column:num_of_challenges" json:"num_of_challenges"`
@@ -75,3 +75,8 @@ func GetFileStats(ctx context.Context, refID int64) (*FileStats, error) {
 }
 return stats, err
 }
+
+func DeleteFileStats(ctx context.Context, refID int64) error {
+db := datastore.GetStore().GetTransaction(ctx)
+return db.Unscoped().Delete(&FileStats{}, "ref_id=?", refID).Error
+}