Fix/challenge for bbr (#686)

Author: cnlangzi

.gitignore

@@ -1,3 +1,4 @@
+/data
 coverage.txt
 .DS_Store
 .idea/

code/go/0chain.net/blobbercore/allocation/protocol.go

@@ -23,7 +23,7 @@ func GetAllocationByID(ctx context.Context, allocID string) (a *Allocation, err
 
 a = new(Allocation)
 err = tx.Model(&Allocation{}).
-Where(&Allocation{ID: allocID}).
+Where("id=?", allocID).
 First(a).Error
 return
 }

code/go/0chain.net/blobbercore/automigration/automigration.go

@@ -129,6 +129,7 @@ func grantPrivileges(db *gorm.DB) error {
 }
 
 func migrateSchema(db *gorm.DB) error {
+// !FIXME : tables can't be dropped as default on blobber startup. Because blobber will lose all data if it restarts. It might be a critical bug on production.
 tablesToKeep := make(map[string]struct{})
 
 for _, tb := range config.Configuration.DBTablesToKeep {

code/go/0chain.net/blobbercore/challenge/entity.go

@@ -112,12 +112,15 @@ func marshalField(obj interface{}, dest *datatypes.JSON) error {
 }
 
 func unMarshalField(stringObj datatypes.JSON, dest interface{}) error {
-retBytes, err := stringObj.Value()
+if len(stringObj) == 0 {
+return nil
+}
+retBytes, err := stringObj.MarshalJSON()
 if err != nil {
 return err
 }
 if retBytes != nil {
-return json.Unmarshal(retBytes.([]byte), dest)
+return json.Unmarshal(retBytes, dest)
 }
 return nil
 }
@@ -151,20 +154,16 @@ func (cr *ChallengeEntity) SaveWith(db *gorm.DB) error {
 
 func (cr *ChallengeEntity) UnmarshalFields() error {
 var err error
-
-cr.Validators = make([]ValidationNode, 0)
 err = unMarshalField(cr.ValidatorsString, &cr.Validators)
 if err != nil {
 return err
 }
 
-cr.LastCommitTxnIDs = make([]string, 0)
 err = unMarshalField(cr.LastCommitTxnList, &cr.LastCommitTxnIDs)
 if err != nil {
 return err
 }
 
-cr.ValidationTickets = make([]*ValidationTicket, 0)
 err = unMarshalField(cr.ValidationTicketsString, &cr.ValidationTickets)
 if err != nil {
 return err

code/go/0chain.net/blobbercore/filestore/storage.go

@@ -385,32 +385,49 @@ func (fs *FileStore) GetBlocksMerkleTreeForChallenge(allocID string,
 return nil, nil, common.NewError("stat_error", err.Error())
 }
 
-totalChunks := int(math.Ceil(float64(fi.Size()) / float64(fileData.ChunkSize)))
+numChunks := int(math.Ceil(float64(fi.Size()) / float64(fileData.ChunkSize)))
 
 fixedMT := util.NewFixedMerkleTree(int(fileData.ChunkSize))
+merkleChunkSize := int(fileData.ChunkSize) / 1024
+if merkleChunkSize == 0 {
+merkleChunkSize = 1
+}
 
 bytesBuf := bytes.NewBuffer(make([]byte, 0))
-for i := 0; i < totalChunks; i++ {
-n, err := io.CopyN(bytesBuf, file, fileData.ChunkSize)
-if err != nil {
-if n == 0 && errors.Is(err, io.EOF) {
-break
+for chunkIndex := 0; chunkIndex < numChunks; chunkIndex++ {
+written, err := io.CopyN(bytesBuf, file, fileData.ChunkSize)
+
+if written > 0 {
+dataBytes := bytesBuf.Bytes()
+
+errWrite := fixedMT.Write(dataBytes, chunkIndex)
+if errWrite != nil {
+return nil, nil, common.NewError("hash_error", errWrite.Error())
 }
-return nil, nil, err
-}
 
-dataBytes := bytesBuf.Bytes()
-err = fixedMT.Write(dataBytes, i)
-if err != nil {
-return nil, nil, common.NewError("buffer_write_error", err.Error())
-}
+offset := 0
+
+for i := 0; i < len(dataBytes); i += merkleChunkSize {
+end := i + merkleChunkSize
+if end > len(dataBytes) {
+end = len(dataBytes)
+}
 
-startInd := blockoffset * MerkleChunkSize
-endInd := startInd + MerkleChunkSize
-returnBytes = append(returnBytes, dataBytes[startInd:endInd]...)
+if offset == blockoffset {
+returnBytes = append(returnBytes, dataBytes[i:end]...)
+}
 
-bytesBuf.Reset()
+offset++
+if offset >= 1024 {
+offset = 1
+}
+}
+bytesBuf.Reset()
+}
 
+if err != nil && err == io.EOF {
+break
+}
 }
 
 return returnBytes, fixedMT.GetMerkleTree(), nil

code/go/0chain.net/blobbercore/handler/protocol.go

@@ -17,7 +17,6 @@ import (
 "github.com/0chain/blobber/code/go/0chain.net/core/transaction"
 "github.com/0chain/blobber/code/go/0chain.net/core/util"
 
-"github.com/0chain/gosdk/zboxcore/sdk"
 "github.com/0chain/gosdk/zcncore"
 "go.uber.org/zap"
 )
@@ -97,14 +96,14 @@ func RegisterBlobber(ctx context.Context) error {
 return err
 }
 
-if t, err := TransactionVerify(txnHash); err != nil {
+t, err := TransactionVerify(txnHash)
+if err != nil {
 logging.Logger.Error("Failed to verify blobber add/update transaction", zap.Any("err", err), zap.String("txn.Hash", txnHash))
-} else {
-logging.Logger.Info("Verified blobber add/update transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput))
+return err
 }
 
-return err
-
+logging.Logger.Info("Verified blobber add/update transaction", zap.String("txn_hash", t.Hash), zap.Any("txn_output", t.TransactionOutput))
+return nil
 }
 
 return SendHealthCheck()
@@ -160,7 +159,7 @@ func sendSmartContractBlobberAdd(ctx context.Context) (string, error) {
 // UpdateBlobberOnChain updates latest changes in blobber's settings, capacity,etc.
 func UpdateBlobberOnChain(ctx context.Context) error {
 
-_, err := sdk.GetBlobber(node.Self.ID)
+_, err := zcn.GetBlobber(node.Self.ID)
 if err != nil { // blobber is not registered yet
 logging.Logger.Warn("failed to get blobber from blockchain", zap.Error(err))
 return err

dev.local/cli.sh

@@ -83,8 +83,11 @@ change_zcn() {
 
 install_debuggger() {
  [ -d ../.vscode ] || mkdir -p ../.vscode
- sed "s/Hostname/$hostname/g" launch.json > ../.vscode/launch.json
- echo "debugbbers are installed"
+ sed "s/Hostname/$hostname/g" launch.json > ./launch.1.json
+ base=$(echo "$root" | sed 's/\//\\\//g')
+ sed "s/root/$base/g" ./launch.1.json > ../.vscode/launch.json
+ rm -rf ./launch.1.json
+ echo "debugbbers are installed" 
 }
 
 cleanAll() {
@@ -200,7 +203,7 @@ start_blobber () {
  keys_file="../docker.local/keys_config/b0bnode${i}_keys.txt"
  minio_file="../docker.local/keys_config/minio_config.txt"
  config_dir="./data/blobber$i/config"
- files_dir="./data/blobber$i/files"
+ files_dir="${root}/data/blobber$i/files"
  log_dir="./data/blobber$i/log"
  db_dir="./data/blobber$i/data"
 

dev.local/launch.json

@@ -17,7 +17,7 @@
  "--keys_file", "../../../../docker.local/keys_config/b0bnode1_keys.txt",
  "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt",
 
- "--files_dir","../../../../dev.local/data/blobber/files",
+ "--files_dir","root/data/blobber/files",
  "--log_dir" ,"../../../../dev.local/data/blobber/log",
  "--db_dir", "../../../../dev.local/data/blobber/data", 
  "--config_dir", "../../../../dev.local/data/blobber/config"
@@ -36,7 +36,7 @@
  "--keys_file", "../../../../docker.local/keys_config/b0bnode1_keys.txt",
  "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt",
 
- "--files_dir","../../../../dev.local/data/blobber1/files",
+ "--files_dir","root/data/blobber1/files",
  "--log_dir" ,"../../../../dev.local/data/blobber1/log",
  "--db_dir", "../../../../dev.local/data/blobber1/data", 
  "--config_dir", "../../../../dev.local/data/blobber1/config"
@@ -55,7 +55,7 @@
  "--keys_file", "../../../../docker.local/keys_config/b0bnode2_keys.txt",
  "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt",
 
- "--files_dir","../../../../dev.local/data/blobber2/files",
+ "--files_dir","root/data/blobber2/files",
  "--log_dir" ,"../../../../dev.local/data/blobber2/log",
  "--db_dir", "../../../../dev.local/data/blobber2/data", 
  "--config_dir", "../../../../dev.local/data/blobber2/config"
@@ -74,7 +74,7 @@
  "--keys_file", "../../../../docker.local/keys_config/b0bnode3_keys.txt",
  "--minio_file" ,"../../../../docker.local/keys_config/minio_config.txt",
 
- "--files_dir","../../../../dev.local/data/blobber3/files",
+ "--files_dir","root/data/blobber3/files",
  "--log_dir" ,"../../../../dev.local/data/blobber3/log",
  "--db_dir", "../../../../dev.local/data/blobber3/data", 
  "--config_dir", "../../../../dev.local/data/blobber3/config"