5.6.36-82.0

Author: cvicentiu

storage/tokudb/CMakeLists.txt

@@ -1,4 +1,4 @@
-SET(TOKUDB_VERSION 5.6.35-80.0)
+SET(TOKUDB_VERSION 5.6.36-82.0)
 # PerconaFT only supports x86-64 and cmake-2.8.9+
 IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND
  NOT CMAKE_VERSION VERSION_LESS "2.8.9")
@@ -111,7 +111,8 @@ SET(TOKUDB_SOURCES
  tokudb_background.cc
  tokudb_information_schema.cc
  tokudb_sysvars.cc
- tokudb_thread.cc)
+ tokudb_thread.cc
+ tokudb_dir_cmd.cc)
 MYSQL_ADD_PLUGIN(tokudb ${TOKUDB_SOURCES} STORAGE_ENGINE MODULE_ONLY
  LINK_LIBRARIES tokufractaltree_static tokuportability_static ${ZLIB_LIBRARY} stdc++)
 SET(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} -flto -fuse-linker-plugin")

storage/tokudb/PerconaFT/buildheader/make_tdb.cc

@@ -425,6 +425,9 @@ static void print_db_env_struct (void) {
  "bool (*set_dir_per_db)(DB_ENV *, bool new_val)",
  "bool (*get_dir_per_db)(DB_ENV *)",
  "const char *(*get_data_dir)(DB_ENV *env)",
+ "int (*dirtool_attach)(DB_ENV *, DB_TXN *, const char *, const char *)",
+ "int (*dirtool_detach)(DB_ENV *, DB_TXN *, const char *)",
+ "int (*dirtool_move)(DB_ENV *, DB_TXN *, const char *, const char *)",
  NULL};
 
  sort_and_dump_fields("db_env", true, extra);

storage/tokudb/PerconaFT/ft/ft-ops.cc

@@ -70,6 +70,8 @@ const char *toku_copyright_string = "Copyright (c) 2006, 2015, Percona and/or it
 #include "util/status.h"
 #include "util/context.h"
 
+#include <functional>
+
 // Include ydb_lib.cc here so that its constructor/destructor gets put into
 // ydb.o, to make sure they don't get erased at link time (when linking to
 // a static libtokufractaltree.a that was compiled with gcc). See #5094.
@@ -1314,6 +1316,159 @@ static const char *env_get_data_dir(DB_ENV *env) {
  return env->i->real_data_dir;
 }
 
+static int env_dirtool_attach(DB_ENV *env,
+ DB_TXN *txn,
+ const char *dname,
+ const char *iname) {
+ int r;
+ DBT dname_dbt;
+ DBT iname_dbt;
+
+ HANDLE_PANICKED_ENV(env);
+ if (!env_opened(env)) {
+ return EINVAL;
+ }
+ HANDLE_READ_ONLY_TXN(txn);
+ toku_fill_dbt(&dname_dbt, dname, strlen(dname) + 1);
+ toku_fill_dbt(&iname_dbt, iname, strlen(iname) + 1);
+
+ r = toku_db_put(env->i->directory,
+ txn,
+ &dname_dbt,
+ &iname_dbt,
+ 0,
+ true);
+ return r;
+}
+
+static int env_dirtool_detach(DB_ENV *env,
+ DB_TXN *txn,
+ const char *dname) {
+ int r;
+ DBT dname_dbt;
+ DBT old_iname_dbt;
+
+ HANDLE_PANICKED_ENV(env);
+ if (!env_opened(env)) {
+ return EINVAL;
+ }
+ HANDLE_READ_ONLY_TXN(txn);
+
+ toku_fill_dbt(&dname_dbt, dname, strlen(dname) + 1);
+ toku_init_dbt_flags(&old_iname_dbt, DB_DBT_REALLOC);
+
+ r = toku_db_get(env->i->directory,
+ txn,
+ &dname_dbt,
+ &old_iname_dbt,
+ DB_SERIALIZABLE); // allocates memory for iname
+ if (r == DB_NOTFOUND)
+ return EEXIST;
+ toku_free(old_iname_dbt.data);
+
+ r = toku_db_del(env->i->directory, txn, &dname_dbt, DB_DELETE_ANY, true);
+
+ return r;
+}
+
+static int env_dirtool_move(DB_ENV *env,
+ DB_TXN *txn,
+ const char *old_dname,
+ const char *new_dname) {
+ int r;
+ DBT old_dname_dbt;
+ DBT new_dname_dbt;
+ DBT iname_dbt;
+
+ HANDLE_PANICKED_ENV(env);
+ if (!env_opened(env)) {
+ return EINVAL;
+ }
+ HANDLE_READ_ONLY_TXN(txn);
+
+ toku_fill_dbt(&old_dname_dbt, old_dname, strlen(old_dname) + 1);
+ toku_fill_dbt(&new_dname_dbt, new_dname, strlen(new_dname) + 1);
+ toku_init_dbt_flags(&iname_dbt, DB_DBT_REALLOC);
+
+ r = toku_db_get(env->i->directory,
+ txn,
+ &old_dname_dbt,
+ &iname_dbt,
+ DB_SERIALIZABLE); // allocates memory for iname
+ if (r == DB_NOTFOUND)
+ return EEXIST;
+
+ r = toku_db_del(
+ env->i->directory, txn, &old_dname_dbt, DB_DELETE_ANY, true);
+ if (r != 0)
+ goto exit;
+
+ r = toku_db_put(
+ env->i->directory, txn, &new_dname_dbt, &iname_dbt, 0, true);
+
+exit:
+ toku_free(iname_dbt.data);
+ return r;
+}
+
+static int locked_env_op(DB_ENV *env,
+ DB_TXN *txn,
+ std::function<int(DB_TXN *)> f) {
+ int ret, r;
+ HANDLE_READ_ONLY_TXN(txn);
+ HANDLE_ILLEGAL_WORKING_PARENT_TXN(env, txn);
+
+ DB_TXN *child_txn = NULL;
+ int using_txns = env->i->open_flags & DB_INIT_TXN;
+ if (using_txns) {
+ ret = toku_txn_begin(env, txn, &child_txn, 0);
+ lazy_assert_zero(ret);
+ }
+
+ // cannot begin a checkpoint
+ toku_multi_operation_client_lock();
+ r = f(child_txn);
+ toku_multi_operation_client_unlock();
+
+ if (using_txns) {
+ if (r == 0) {
+ ret = locked_txn_commit(child_txn, 0);
+ lazy_assert_zero(ret);
+ } else {
+ ret = locked_txn_abort(child_txn);
+ lazy_assert_zero(ret);
+ }
+ }
+ return r;
+
+}
+
+static int locked_env_dirtool_attach(DB_ENV *env,
+ DB_TXN *txn,
+ const char *dname,
+ const char *iname) {
+ auto f = std::bind(
+ env_dirtool_attach, env, std::placeholders::_1, dname, iname);
+ return locked_env_op(env, txn, f);
+}
+
+static int locked_env_dirtool_detach(DB_ENV *env,
+ DB_TXN *txn,
+ const char *dname) {
+ auto f = std::bind(
+ env_dirtool_detach, env, std::placeholders::_1, dname);
+ return locked_env_op(env, txn, f);
+}
+
+static int locked_env_dirtool_move(DB_ENV *env,
+ DB_TXN *txn,
+ const char *old_dname,
+ const char *new_dname) {
+ auto f = std::bind(
+ env_dirtool_move, env, std::placeholders::_1, old_dname, new_dname);
+ return locked_env_op(env, txn, f);
+}
+
 static int env_dbremove(DB_ENV * env, DB_TXN *txn, const char *fname, const char *dbname, uint32_t flags);
 
 static int
@@ -2646,6 +2801,9 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) {
 #define SENV(name) result->name = locked_env_ ## name
  SENV(dbremove);
  SENV(dbrename);
+ SENV(dirtool_attach);
+ SENV(dirtool_detach);
+ SENV(dirtool_move);
  //SENV(set_noticecall);
 #undef SENV
 #define USENV(name) result->name = env_ ## name
@@ -2975,8 +3133,10 @@ env_dbremove(DB_ENV * env, DB_TXN *txn, const char *fname, const char *dbname, u
  if (txn && r) {
  if (r == EMFILE || r == ENFILE)
  r = toku_ydb_do_error(env, r, "toku dbremove failed because open file limit reached\n");
- else
+ else if (r != ENOENT)
  r = toku_ydb_do_error(env, r, "toku dbremove failed\n");
+ else
+ r = 0;
  goto exit;
  }
  if (txn) {

storage/tokudb/PerconaFT/src/ydb.cc

@@ -5254,17 +5254,17 @@ int ha_tokudb::fill_range_query_buf(
  DEBUG_SYNC(ha_thd(), "tokudb_icp_asc_scan_out_of_range");
  goto cleanup;
  } else if (result == ICP_NO_MATCH) {
- // if we are performing a DESC ICP scan and have no end_range
- // to compare to stop using ICP filtering as there isn't much more
- // that we can do without going through contortions with remembering
- // and comparing key parts.
+ // Optimizer change for MyRocks also benefits us here in TokuDB as
+ // opt_range.cc QUICK_SELECT::get_next now sets end_range during
+ // descending scan. We should not ever hit this condition, but
+ // leaving this code in to prevent any possibility of a descending
+ // scan to the beginning of an index and catch any possibility
+ // in debug builds with an assertion
+ assert_debug(!(!end_range && direction < 0));
  if (!end_range &&
  direction < 0) {
-
  cancel_pushed_idx_cond();
- DEBUG_SYNC(ha_thd(), "tokudb_icp_desc_scan_invalidate");
  }
-
  error = TOKUDB_CURSOR_CONTINUE;
  goto cleanup;
  }
@@ -6122,7 +6122,6 @@ int ha_tokudb::info(uint flag) {
  stats.records = share->row_count() + share->rows_from_locked_table;
  stats.deleted = 0;
  if (!(flag & HA_STATUS_NO_LOCK)) {
- uint64_t num_rows = 0;
 
  error = txn_begin(db_env, NULL, &txn, DB_READ_UNCOMMITTED, ha_thd());
  if (error) {
@@ -6132,20 +6131,13 @@ int ha_tokudb::info(uint flag) {
  // we should always have a primary key
  assert_always(share->file != NULL);
 
- error = estimate_num_rows(share->file, &num_rows, txn);
- if (error == 0) {
- share->set_row_count(num_rows, false);
- stats.records = num_rows;
- } else {
- goto cleanup;
- }
-
  DB_BTREE_STAT64 dict_stats;
  error = share->file->stat64(share->file, txn, &dict_stats);
  if (error) {
  goto cleanup;
  }
-
+ share->set_row_count(dict_stats.bt_ndata, false);
+ stats.records = dict_stats.bt_ndata;
  stats.create_time = dict_stats.bt_create_time_sec;
  stats.update_time = dict_stats.bt_modify_time_sec;
  stats.check_time = dict_stats.bt_verify_time_sec;
@@ -7841,7 +7833,7 @@ ha_rows ha_tokudb::records_in_range(uint keynr, key_range* start_key, key_range*
  // As a result, equal may be 0 and greater may actually be equal+greater
  // So, we call key_range64 on the key, and the key that is after it.
  if (!start_key && !end_key) {
- error = estimate_num_rows(kfile, &rows, transaction);
+ error = estimate_num_rows(share->file, &rows, transaction);
  if (error) {
  ret_val = HA_TOKUDB_RANGE_COUNT;
  goto cleanup;

storage/tokudb/ha_tokudb.cc

@@ -0,0 +1,70 @@
+SET GLOBAL tokudb_dir_per_db=ON;
+CREATE PROCEDURE create_table()
+BEGIN
+CREATE TABLE test.t1 (
+a INT
+) ENGINE = TokuDB
+PARTITION BY RANGE (a)
+(PARTITION p100 VALUES LESS THAN (100) ENGINE = TokuDB,
+PARTITION p_to_del VALUES LESS THAN (200) ENGINE = TokuDB,
+PARTITION p300 VALUES LESS THAN (300) ENGINE = TokuDB,
+PARTITION p400 VALUES LESS THAN (400) ENGINE = TokuDB
+);
+END|
+### Create partitioned table
+CALL create_table();
+## Looking for *.tokudb files in data_dir
+## Looking for *.tokudb files in data_dir/test
+t1_P_p100_main_id.tokudb
+t1_P_p100_status_id.tokudb
+t1_P_p300_main_id.tokudb
+t1_P_p300_status_id.tokudb
+t1_P_p400_main_id.tokudb
+t1_P_p400_status_id.tokudb
+t1_P_p_to_del_main_id.tokudb
+t1_P_p_to_del_status_id.tokudb
+### Stop server
+### Remove 'main' file of one of the partitions
+### Start server
+### Make sure 'main' partition file is deleted
+## Looking for *.tokudb files in data_dir
+## Looking for *.tokudb files in data_dir/test
+t1_P_p100_main_id.tokudb
+t1_P_p100_status_id.tokudb
+t1_P_p300_main_id.tokudb
+t1_P_p300_status_id.tokudb
+t1_P_p400_main_id.tokudb
+t1_P_p400_status_id.tokudb
+t1_P_p_to_del_status_id.tokudb
+### Make sure the table still exists
+SHOW TABLES;
+Tables_in_test
+t1
+### Drop table
+DROP TABLE t1;
+### Make sure the table is dropped
+SHOW TABLES;
+Tables_in_test
+### Check what files still exist after DROP TABLE
+## Looking for *.tokudb files in data_dir
+## Looking for *.tokudb files in data_dir/test
+### Remove the rest of the files
+### Make sure there are no tokudb files
+## Looking for *.tokudb files in data_dir
+## Looking for *.tokudb files in data_dir/test
+### Create the same table once more
+CALL create_table();
+## Looking for *.tokudb files in data_dir
+## Looking for *.tokudb files in data_dir/test
+t1_P_p100_main_id.tokudb
+t1_P_p100_status_id.tokudb
+t1_P_p300_main_id.tokudb
+t1_P_p300_status_id.tokudb
+t1_P_p400_main_id.tokudb
+t1_P_p400_status_id.tokudb
+t1_P_p_to_del_main_id.tokudb
+t1_P_p_to_del_status_id.tokudb
+### Restore state
+DROP TABLE t1;
+DROP PROCEDURE create_table;
+SET GLOBAL tokudb_dir_per_db=default;