Skip to content

Only write the modified tablets to file system. #304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 9, 2018
Merged

Only write the modified tablets to file system. #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e00bc8f
Only write the modified tablets to file system.
jetfuel Mar 9, 2018
ced1ac7
Merge branch 'develop' into onlyWriteModifiedTablets
jetfuel Mar 9, 2018
87f0fb3
Persist the data in the destructor
jetfuel Mar 9, 2018
d55f049
Merge branch 'onlyWriteModifiedTablets' of github.com:jetfuel/VisualD…
jetfuel Mar 9, 2018
0b6a6b0
Use -1 for the unknown.
jetfuel Mar 9, 2018
d086de4
Merge branch 'develop' into onlyWriteModifiedTablets
jetfuel Mar 9, 2018
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 34 additions & 3 deletions visualdl/storage/storage.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ Storage::Storage() {
data_ = std::make_shared<storage::Storage>();
tablets_ = std::make_shared<std::map<std::string, storage::Tablet>>();
modes_ = std::make_shared<std::set<std::string>>();
modified_tablet_set_ = std::unordered_set<std::string>();
time_t t;
time(&t);
data_->set_timestamp(t);
Expand All @@ -31,6 +32,10 @@ Storage::Storage(const Storage& other)
dir_ = other.dir_;
}

Storage::~Storage() {
PersistToDisk();
}

void Storage::AddMode(const std::string& x) {
// avoid duplicate modes.
if (modes_->count(x) != 0) return;
Expand All @@ -43,11 +48,20 @@ Tablet Storage::AddTablet(const std::string& x) {
CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed";
(*tablets_)[x] = storage::Tablet();
AddTag(x);
MarkTabletModified(x);
// WRITE_GUARD
PersistToDisk();
return Tablet(&(*tablets_)[x], this);
}

void Storage::SetDir(const std::string& dir) {
*dir_ = dir;
}

std::string Storage::dir() const {
return *dir_;
}

void Storage::PersistToDisk() { PersistToDisk(*dir_); }

void Storage::PersistToDisk(const std::string& dir) {
Expand All @@ -56,12 +70,29 @@ void Storage::PersistToDisk(const std::string& dir) {

fs::SerializeToFile(*data_, meta_path(dir));
for (auto tag : data_->tags()) {
auto it = tablets_->find(tag);
CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
fs::SerializeToFile(it->second, tablet_path(dir, tag));
if (modified_tablet_set_.count(tag) > 0){
auto it = tablets_->find(tag);
CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
fs::SerializeToFile(it->second, tablet_path(dir, tag));
}
}
modified_tablet_set_.clear();
}

Storage* Storage::parent() {
return this;
}

void Storage::MarkTabletModified(const std::string tag) {
modified_tablet_set_.insert(tag);
}

void Storage::AddTag(const std::string& x) {
*data_->add_tags() = x;
WRITE_GUARD
}

// StorageReader
std::vector<std::string> StorageReader::all_tags() {
storage::Storage storage;
Reload(storage);
Expand Down
16 changes: 8 additions & 8 deletions visualdl/storage/storage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ struct Storage {

Storage();
Storage(const Storage& other);

~Storage();
// Add a mode. Mode is similar to TB's FileWriter, It can be "train" or "test"
// or something else.
void AddMode(const std::string& x);
Expand All @@ -67,8 +67,8 @@ struct Storage {
Tablet AddTablet(const std::string& x);

// Set storage's directory.
void SetDir(const std::string& dir) { *dir_ = dir; }
std::string dir() const { return *dir_; }
void SetDir(const std::string& dir);
std::string dir() const;

// Save content in memory to `dir_`.
void PersistToDisk();
Expand All @@ -77,20 +77,20 @@ struct Storage {
void PersistToDisk(const std::string& dir);

// A trick help to retrieve the storage's `SimpleSyncMeta`.
Storage* parent() { return this; }
Storage* parent();

void MarkTabletModified(const std::string tag);

protected:
// Add a tag which content is `x`.
void AddTag(const std::string& x) {
*data_->add_tags() = x;
WRITE_GUARD
}
void AddTag(const std::string& x);

private:
std::shared_ptr<std::string> dir_;
std::shared_ptr<std::map<std::string, storage::Tablet>> tablets_;
std::shared_ptr<storage::Storage> data_;
std::shared_ptr<std::set<std::string>> modes_;
std::unordered_set<std::string> modified_tablet_set_;
};

// Storage reader, each method will trigger a reading from disk.
Expand Down
16 changes: 16 additions & 0 deletions visualdl/storage/tablet.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,25 @@ See the License for the specific language governing permissions and
limitations under the License. */

#include "visualdl/storage/tablet.h"
#include "visualdl/storage/storage.h"

namespace visualdl {

void Tablet::SetTag(const std::string& mode, const std::string& tag) {
auto internal_tag = mode + "/" + tag;
string::TagEncode(internal_tag);
internal_encoded_tag_ = internal_tag;
data_->set_tag(internal_tag);
WRITE_GUARD
}

Record Tablet::AddRecord() {
parent()->MarkTabletModified(internal_encoded_tag_);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MarkTableModified use a set and that takes an overhead if AddRecord is triggered frequently(e.g. in a scalar), and theoretically, it should be placed in WRITE_GUARD so that modifying a record will also mark the tablet "modified". But currently, this implementation is ok.

There is a trick that can be faster, assigning each tablet an internal member id_ (continuously ascending from zero), and make modified_tablet_set_ a vector, the marking logic like this

vector<bool> modified_tablet_set_; // each time add a tablet will append a true Record Tablet::AddRecord() { parent()->MarkTabletModified(id_); // ... } inline void Storage::MarkTabletModified(int tagid) { modified_tablet_set_[tagid] = true; } void Storage::PersistToDisk(const std::string& dir) ... // clear states
IncTotalRecords();
WRITE_GUARD
return Record(data_->add_records(), parent());
}

TabletReader Tablet::reader() { return TabletReader(*data_); }

} // namespace visualdl
20 changes: 6 additions & 14 deletions visualdl/storage/tablet.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ struct TabletReader;
* Tablet is a helper for operations on storage::Tablet.
*/
struct Tablet {
enum Type { kScalar = 0, kHistogram = 1, kImage = 2 };
enum Type { kScalar = 0, kHistogram = 1, kImage = 2, kUnknown = -1};

DECL_GUARD(Tablet);

Tablet(storage::Tablet* x, Storage* parent) : data_(x), x_(parent) {}
Tablet(storage::Tablet* x, Storage* parent) : data_(x), x_(parent), internal_encoded_tag_("") {}

static Type type(const std::string& name) {
if (name == "scalar") {
Expand All @@ -46,6 +46,7 @@ struct Tablet {
return kImage;
}
LOG(ERROR) << "unknown component: " << name;
return kUnknown;
}

// write operations.
Expand All @@ -59,18 +60,8 @@ struct Tablet {
WRITE_GUARD
}

void SetTag(const std::string& mode, const std::string& tag) {
auto internal_tag = mode + "/" + tag;
string::TagEncode(internal_tag);
data_->set_tag(internal_tag);
WRITE_GUARD
}

Record AddRecord() {
IncTotalRecords();
WRITE_GUARD
return Record(data_->add_records(), parent());
}
void SetTag(const std::string& mode, const std::string& tag);
Record AddRecord();

template <typename T>
Entry MutableMeta() {
Expand Down Expand Up @@ -102,6 +93,7 @@ struct Tablet {
private:
Storage* x_;
storage::Tablet* data_{nullptr};
std::string internal_encoded_tag_;
};

/*
Expand Down