feature/refactor storage code

visualdl/storage/storage.cc

@@ -0,0 +1,83 @@
+#include "visualdl/storage/storage.h"
+
+namespace visualdl {
+
+Storage::Storage() {
+ dir_ = std::make_shared<std::string>();
+ data_ = std::make_shared<storage::Storage>();
+ tablets_ = std::make_shared<std::map<std::string, storage::Tablet>>();
+ modes_ = std::make_shared<std::set<std::string>>();
+ time_t t;
+ time(&t);
+ data_->set_timestamp(t);
+}
+
+Storage::Storage(const Storage& other)
+ : data_(other.data_), tablets_(other.tablets_), modes_(other.modes_) {
+ dir_ = other.dir_;
+}
+
+void Storage::AddMode(const std::string& x) {
+ // avoid duplicate modes.
+ if (modes_->count(x) != 0) return;
+ *data_->add_modes() = x;
+ modes_->insert(x);
+ WRITE_GUARD
+}
+
+Tablet Storage::AddTablet(const std::string& x) {
+ CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed";
+ (*tablets_)[x] = storage::Tablet();
+ AddTag(x);
+ LOG(INFO) << "really add tag " << x;
+ // WRITE_GUARD
+ PersistToDisk();
+ return Tablet(&(*tablets_)[x], this);
+}
+
+void Storage::PersistToDisk() { PersistToDisk(*dir_); }
+
+void Storage::PersistToDisk(const std::string& dir) {
+ CHECK(!dir.empty()) << "dir should be set.";
+ fs::TryRecurMkdir(dir);
+
+ fs::SerializeToFile(*data_, meta_path(dir));
+ for (auto tag : data_->tags()) {
+ auto it = tablets_->find(tag);
+ CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
+ fs::SerializeToFile(it->second, tablet_path(dir, tag));
+ }
+}
+
+std::vector<std::string> StorageReader::all_tags() {
+ storage::Storage storage;
+ Reload(storage);
+ return std::vector<std::string>(storage.tags().begin(), storage.tags().end());
+}
+
+std::vector<std::string> StorageReader::tags(Tablet::Type component) {
+ auto tags = all_tags();
+ auto it =
+ std::remove_if(tags.begin(), tags.end(), [&](const std::string& tag) {
+ auto tb = tablet(tag);
+ return tb.type() != component;
+ });
+ tags.resize(it - tags.begin());
+ return tags;
+}
+
+std::vector<std::string> StorageReader::modes() {
+ storage::Storage storage;
+ Reload(storage);
+ return std::vector<std::string>(storage.modes().begin(),
+ storage.modes().end());
+}
+
+TabletReader StorageReader::tablet(const std::string& tag) const {
+ auto path = tablet_path(dir_, tag);
+ storage::Tablet tablet;
+ fs::DeSerializeFromFile(&tablet, path);
+ return TabletReader(tablet);
+}
+
+} // namespace visualdl

visualdl/storage/storage.h

@@ -20,10 +20,13 @@ static std::string meta_path(const std::string& dir) {
  return dir + "/" + meta_file_name;
 }
 static std::string tablet_path(const std::string& dir, const std::string& tag) {
- CHECK(!dir.empty()) << "dir should be set first";
+ CHECK(!dir.empty()) << "dir is empty";
  return dir + "/" + tag;
 }
 
+// A simple logic to sync storage between memory and disk. Each writing
+// operation will trigger an `Inc`, and check whether `ToSync`, if true, write
+// memory to disk.
 struct SimpleSyncMeta {
  void Inc() { counter++; }
 
@@ -33,78 +36,37 @@ struct SimpleSyncMeta {
  int cycle;
 };
 
-/*
- * Helper for operations on storage::Storage.
- */
+// Helper class for operations on storage::Storage.
 struct Storage {
  DECL_GUARD(Storage)
 
  mutable SimpleSyncMeta meta;
 
- Storage() {
- dir_ = std::make_shared<std::string>();
- data_ = std::make_shared<storage::Storage>();
- tablets_ = std::make_shared<std::map<std::string, storage::Tablet>>();
- modes_ = std::make_shared<std::set<std::string>>();
- time_t t;
- time(&t);
- data_->set_timestamp(t);
- }
- Storage(const Storage& other)
- : data_(other.data_), tablets_(other.tablets_), modes_(other.modes_) {
- dir_ = other.dir_;
- }
+ Storage();
+ Storage(const Storage& other);
 
- // write operations
- void AddMode(const std::string& x) {
- // avoid duplicate modes.
- if (modes_->count(x) != 0) return;
- *data_->add_modes() = x;
- modes_->insert(x);
- WRITE_GUARD
- }
+ // Add a mode. Mode is similar to TB's FileWriter, It can be "train" or "test"
+ // or something else.
+ void AddMode(const std::string& x);
 
- Tablet AddTablet(const std::string& x) {
- CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed";
- (*tablets_)[x] = storage::Tablet();
- AddTag(x);
- LOG(INFO) << "really add tag " << x;
- // WRITE_GUARD
- PersistToDisk();
- return Tablet(&(*tablets_)[x], this);
- }
+ // Add a tablet which tag is `x`.
+ Tablet AddTablet(const std::string& x);
 
+ // Set storage's directory.
  void SetDir(const std::string& dir) { *dir_ = dir; }
  std::string dir() const { return *dir_; }
- void PersistToDisk() {
- CHECK(!dir_->empty()) << "dir should be set.";
- fs::TryRecurMkdir(*dir_);
-
- fs::SerializeToFile(*data_, meta_path(*dir_));
- for (auto tag : data_->tags()) {
- auto it = tablets_->find(tag);
- CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
- fs::SerializeToFile(it->second, tablet_path(*dir_, tag));
- }
- }
- /*
- * Save memory to disk.
- */
- void PersistToDisk(const std::string& dir) {
- CHECK(!dir.empty()) << "dir should be set.";
- fs::TryRecurMkdir(dir);
-
- fs::SerializeToFile(*data_, meta_path(dir));
- for (auto tag : data_->tags()) {
- auto it = tablets_->find(tag);
- CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
- fs::SerializeToFile(it->second, tablet_path(dir, tag));
- }
- }
 
+ // Save content in memory to `dir_`.
+ void PersistToDisk();
+
+ // Save content in memory to `dir`.
+ void PersistToDisk(const std::string& dir);
+
+ // A trick help to retrieve the storage's `SimpleSyncMeta`.
  Storage* parent() { return this; }
 
 protected:
+ // Add a tag which content is `x`.
  void AddTag(const std::string& x) {
  *data_->add_tags() = x;
  WRITE_GUARD
@@ -117,44 +79,24 @@ struct Storage {
  std::shared_ptr<std::set<std::string>> modes_;
 };
 
-/*
- * Storage reader, each interface will trigger a read.
- */
+// Storage reader, each method will trigger a reading from disk.
 struct StorageReader {
  StorageReader(const std::string& dir) : dir_(dir) {}
 
- // read operations
- std::vector<std::string> all_tags() {
- storage::Storage storage;
- Reload(storage);
- return std::vector<std::string>(storage.tags().begin(),
- storage.tags().end());
- }
- std::vector<std::string> tags(Tablet::Type component) {
- auto tags = all_tags();
- auto it =
- std::remove_if(tags.begin(), tags.end(), [&](const std::string& tag) {
- auto tb = tablet(tag);
- return tb.type() != component;
- });
- tags.resize(it - tags.begin());
- return tags;
- }
- std::vector<std::string> modes() {
- storage::Storage storage;
- Reload(storage);
- return std::vector<std::string>(storage.modes().begin(),
- storage.modes().end());
- }
+ // Get all tags of the storage.
+ std::vector<std::string> all_tags();
 
- TabletReader tablet(const std::string& tag) const {
- auto path = tablet_path(dir_, tag);
- storage::Tablet tablet;
- fs::DeSerializeFromFile(&tablet, path);
- return TabletReader(tablet);
- }
+ // Get all the tags of the tablet of the kind of `component`.
+ std::vector<std::string> tags(Tablet::Type component);
+
+ // Get all the modes of the storage.
+ std::vector<std::string> modes();
+
+ // Get a tablet whose tag is `tag`.
+ TabletReader tablet(const std::string& tag) const;
 
 protected:
+ // Load meta from disk to memory.
  void Reload(storage::Storage& storage) {
  const std::string path = meta_path(dir_);
  fs::DeSerializeFromFile(&storage, path);