Merge branch 'merge/merge-tokudb-5.6' into 10.0

5.6.31-77.0

Author: vuvova

storage/tokudb/CMakeLists.txt

@@ -1,4 +1,4 @@
-SET(TOKUDB_VERSION 5.6.30-76.3)
+SET(TOKUDB_VERSION 5.6.31-77.0)
 # PerconaFT only supports x86-64 and cmake-2.8.9+
 IF(CMAKE_VERSION VERSION_LESS "2.8.9")
  MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB")
@@ -82,10 +82,6 @@ IF(DEFINED TOKUDB_NOPATCH_CONFIG)
  ADD_DEFINITIONS("-DTOKUDB_NOPATCH_CONFIG=${TOKUDB_NOPATCH_CONFIG}")
 ENDIF()
 
-IF(DEFINED TOKUDB_CHECK_JEMALLOC)
- ADD_DEFINITIONS("-DTOKUDB_CHECK_JEMALLOC=${TOKUDB_CHECK_JEMALLOC}")
-ENDIF()
-
 macro(set_cflags_if_supported)
  foreach(flag ${ARGN})
  string(REGEX REPLACE "-" "_" temp_flag ${flag})

storage/tokudb/PerconaFT/buildheader/make_tdb.cc

@@ -420,6 +420,8 @@ static void print_db_env_struct (void) {
  "int (*set_client_pool_threads)(DB_ENV *, uint32_t)",
  "int (*set_cachetable_pool_threads)(DB_ENV *, uint32_t)",
  "int (*set_checkpoint_pool_threads)(DB_ENV *, uint32_t)",
+ "void (*set_check_thp)(DB_ENV *, bool new_val)",
+ "bool (*get_check_thp)(DB_ENV *)",
  NULL};
 
  sort_and_dump_fields("db_env", true, extra);

storage/tokudb/PerconaFT/src/ydb-internal.h

@@ -132,6 +132,7 @@ struct __toku_db_env_internal {
  int datadir_lockfd;
  int logdir_lockfd;
  int tmpdir_lockfd;
+ bool check_thp; // if set check if transparent huge pages are disables
  uint64_t (*get_loader_memory_size_callback)(void);
  uint64_t default_lock_timeout_msec;
  uint64_t (*get_lock_timeout_callback)(uint64_t default_lock_timeout_msec);

storage/tokudb/PerconaFT/src/ydb.cc

@@ -623,56 +623,80 @@ ydb_recover_log_exists(DB_ENV *env) {
 }
 
 // Validate that all required files are present, no side effects.
-// Return 0 if all is well, ENOENT if some files are present but at least one is missing, 
+// Return 0 if all is well, ENOENT if some files are present but at least one is
+// missing,
 // other non-zero value if some other error occurs.
 // Set *valid_newenv if creating a new environment (all files missing).
-// (Note, if special dictionaries exist, then they were created transactionally and log should exist.)
-static int 
-validate_env(DB_ENV * env, bool * valid_newenv, bool need_rollback_cachefile) {
+// (Note, if special dictionaries exist, then they were created transactionally
+// and log should exist.)
+static int validate_env(DB_ENV *env,
+ bool *valid_newenv,
+ bool need_rollback_cachefile) {
  int r;
- bool expect_newenv = false;  // set true if we expect to create a new env
+ bool expect_newenv = false; // set true if we expect to create a new env
  toku_struct_stat buf;
- char* path = NULL;
+ char *path = NULL;
 
  // Test for persistent environment
- path = toku_construct_full_name(2, env->i->dir, toku_product_name_strings.environmentdictionary);
+ path = toku_construct_full_name(
+ 2, env->i->dir, toku_product_name_strings.environmentdictionary);
  assert(path);
  r = toku_stat(path, &buf);
  if (r == 0) {
  expect_newenv = false; // persistent info exists
- }
- else {
+ } else {
  int stat_errno = get_error_errno();
  if (stat_errno == ENOENT) {
  expect_newenv = true;
  r = 0;
- }
- else {
- r = toku_ydb_do_error(env, stat_errno, "Unable to access persistent environment\n");
+ } else {
+ r = toku_ydb_do_error(
+ env,
+ stat_errno,
+ "Unable to access persistent environment [%s] in [%s]\n",
+ toku_product_name_strings.environmentdictionary,
+ env->i->dir);
  assert(r);
  }
  }
  toku_free(path);
 
  // Test for existence of rollback cachefile if it is expected to exist
  if (r == 0 && need_rollback_cachefile) {
- path = toku_construct_full_name(2, env->i->dir, toku_product_name_strings.rollback_cachefile);
+ path = toku_construct_full_name(
+ 2, env->i->dir, toku_product_name_strings.rollback_cachefile);
  assert(path);
  r = toku_stat(path, &buf);
- if (r == 0) { 
- if (expect_newenv) // rollback cachefile exists, but persistent env is missing
- r = toku_ydb_do_error(env, ENOENT, "Persistent environment is missing\n");
- }
- else {
+ if (r == 0) {
+ if (expect_newenv) // rollback cachefile exists, but persistent env
+ // is missing
+ r = toku_ydb_do_error(
+ env,
+ ENOENT,
+ "Persistent environment is missing while looking for "
+ "rollback cachefile [%s] in [%s]\n",
+ toku_product_name_strings.rollback_cachefile, env->i->dir);
+ } else {
  int stat_errno = get_error_errno();
  if (stat_errno == ENOENT) {
- if (!expect_newenv) // rollback cachefile is missing but persistent env exists
- r = toku_ydb_do_error(env, ENOENT, "rollback cachefile directory is missing\n");
- else 
- r = 0; // both rollback cachefile and persistent env are missing
- }
- else {
- r = toku_ydb_do_error(env, stat_errno, "Unable to access rollback cachefile\n");
+ if (!expect_newenv) // rollback cachefile is missing but
+ // persistent env exists
+ r = toku_ydb_do_error(
+ env,
+ ENOENT,
+ "rollback cachefile [%s] is missing from [%s]\n",
+ toku_product_name_strings.rollback_cachefile,
+ env->i->dir);
+ else
+ r = 0; // both rollback cachefile and persistent env are
+ // missing
+ } else {
+ r = toku_ydb_do_error(
+ env,
+ stat_errno,
+ "Unable to access rollback cachefile [%s] in [%s]\n",
+ toku_product_name_strings.rollback_cachefile,
+ env->i->dir);
  assert(r);
  }
  }
@@ -681,23 +705,41 @@ validate_env(DB_ENV * env, bool * valid_newenv, bool need_rollback_cachefile) {
 
  // Test for fileops directory
  if (r == 0) {
- path = toku_construct_full_name(2, env->i->dir, toku_product_name_strings.fileopsdirectory);
+ path = toku_construct_full_name(
+ 2, env->i->dir, toku_product_name_strings.fileopsdirectory);
  assert(path);
  r = toku_stat(path, &buf);
- if (r == 0) { 
- if (expect_newenv) // fileops directory exists, but persistent env is missing
- r = toku_ydb_do_error(env, ENOENT, "Persistent environment is missing\n");
- }
- else {
+ if (r == 0) {
+ if (expect_newenv) // fileops directory exists, but persistent env
+ // is missing
+ r = toku_ydb_do_error(
+ env,
+ ENOENT,
+ "Persistent environment is missing while looking for "
+ "fileops directory [%s] in [%s]\n",
+ toku_product_name_strings.fileopsdirectory,
+ env->i->dir);
+ } else {
  int stat_errno = get_error_errno();
  if (stat_errno == ENOENT) {
- if (!expect_newenv) // fileops directory is missing but persistent env exists
- r = toku_ydb_do_error(env, ENOENT, "Fileops directory is missing\n");
- else 
- r = 0; // both fileops directory and persistent env are missing
- }
- else {
- r = toku_ydb_do_error(env, stat_errno, "Unable to access fileops directory\n");
+ if (!expect_newenv) // fileops directory is missing but
+ // persistent env exists
+ r = toku_ydb_do_error(
+ env,
+ ENOENT,
+ "Fileops directory [%s] is missing from [%s]\n",
+ toku_product_name_strings.fileopsdirectory,
+ env->i->dir);
+ else
+ r = 0; // both fileops directory and persistent env are
+ // missing
+ } else {
+ r = toku_ydb_do_error(
+ env,
+ stat_errno,
+ "Unable to access fileops directory [%s] in [%s]\n",
+ toku_product_name_strings.fileopsdirectory,
+ env->i->dir);
  assert(r);
  }
  }
@@ -709,16 +751,26 @@ validate_env(DB_ENV * env, bool * valid_newenv, bool need_rollback_cachefile) {
  // if using transactions, test for existence of log
  r = ydb_recover_log_exists(env); // return 0 or ENOENT
  if (expect_newenv && (r != ENOENT))
- r = toku_ydb_do_error(env, ENOENT, "Persistent environment information is missing (but log exists)\n");
+ r = toku_ydb_do_error(env,
+ ENOENT,
+ "Persistent environment information is "
+ "missing (but log exists) while looking for "
+ "recovery log files in [%s]\n",
+ env->i->real_log_dir);
  else if (!expect_newenv && r == ENOENT)
- r = toku_ydb_do_error(env, ENOENT, "Recovery log is missing (persistent environment information is present)\n");
+ r = toku_ydb_do_error(env,
+ ENOENT,
+ "Recovery log is missing (persistent "
+ "environment information is present) while "
+ "looking for recovery log files in [%s]\n",
+ env->i->real_log_dir);
  else
  r = 0;
  }
 
  if (r == 0)
  *valid_newenv = expect_newenv;
- else 
+ else
  *valid_newenv = false;
  return r;
 }
@@ -768,7 +820,7 @@ env_open(DB_ENV * env, const char *home, uint32_t flags, int mode) {
  goto cleanup;
  }
 
- if (toku_os_huge_pages_enabled()) {
+ if (env->get_check_thp(env) && toku_os_huge_pages_enabled()) {
  r = toku_ydb_do_error(env, TOKUDB_HUGE_PAGES_ENABLED,
  "Huge pages are enabled, disable them before continuing\n");
  goto cleanup;
@@ -1234,6 +1286,18 @@ env_set_checkpoint_pool_threads(DB_ENV * env, uint32_t threads) {
  return 0;
 }
 
+static void
+env_set_check_thp(DB_ENV * env, bool new_val) {
+ assert(env);
+ env->i->check_thp = new_val;
+}
+
+static bool
+env_get_check_thp(DB_ENV * env) {
+ assert(env);
+ return env->i->check_thp;
+}
+
 static int env_dbremove(DB_ENV * env, DB_TXN *txn, const char *fname, const char *dbname, uint32_t flags);
 
 static int
@@ -2634,6 +2698,8 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) {
  USENV(get_loader_memory_size);
  USENV(set_killed_callback);
  USENV(do_backtrace);
+ USENV(set_check_thp);
+ USENV(get_check_thp);
 #undef USENV
 
  // unlocked methods
@@ -2659,6 +2725,8 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) {
  env_fs_init(result);
  env_fsync_log_init(result);
 
+ result->i->check_thp = true;
+
  result->i->bt_compare = toku_builtin_compare_fun;
 
  r = toku_logger_create(&result->i->logger);

storage/tokudb/hatoku_defines.h

@@ -69,10 +69,6 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #pragma interface /* gcc class implementation */
 #endif
 
-#if !defined(TOKUDB_CHECK_JEMALLOC)
-#define TOKUDB_CHECK_JEMALLOC 1
-#endif
-
 #if 100000 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099
 // mariadb 10.0
 #define TOKU_USE_DB_TYPE_TOKUDB 1

storage/tokudb/hatoku_hton.cc

@@ -266,7 +266,6 @@ static int tokudb_init_func(void *p) {
  db_env = NULL;
  tokudb_hton = (handlerton *) p;
 
-#if TOKUDB_CHECK_JEMALLOC
  if (tokudb::sysvars::check_jemalloc) {
  typedef int (*mallctl_type)(
  const char*,
@@ -293,7 +292,6 @@ static int tokudb_init_func(void *p) {
  goto error;
  }
  }
-#endif
 
  r = tokudb_set_product_name();
  if (r) {
@@ -538,6 +536,8 @@ static int tokudb_init_func(void *p) {
  db_env,
  tokudb_get_loader_memory_size_callback);
 
+ db_env->set_check_thp(db_env, tokudb::sysvars::check_jemalloc);
+
  r = db_env->open(
  db_env,
  tokudb_home,

storage/tokudb/mysql-test/rpl/t/suite.opt

@@ -1 +1 @@
-$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0
+$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 --loose-tokudb-cache-size=512M --loose-tokudb-block-size=1M

storage/tokudb/mysql-test/tokudb/bulk-fetch-gen.py

@@ -71,6 +71,7 @@ def sqlgen_range_query_full(table):
 ]
 
 # Code generation stats here
+print "source include/have_tokudb.inc;"
 print "# Tokutek"
 print "# Test that bulk fetch works with various table types"
 print ""

storage/tokudb/mysql-test/tokudb/locks-blocking-row-locks-testgen.py

@@ -56,6 +56,7 @@ def mysqlgen_cleanup():
 timeouts = [0, 500]
 
 # Here's where all the magic happens
+print "source include/have_tokudb.inc;"
 print "# Tokutek"
 print "# Blocking row lock tests;"
 print "# Generated by %s on %s;" % (__file__, datetime.date.today())

storage/tokudb/mysql-test/tokudb/replace-ignore-gen.py

@@ -23,6 +23,7 @@ def sqlgen_explain_and_do(query):
 def sqlgen_drop_table():
  print "drop table t;"
 
+print "source include/have_tokudb.inc;"
 print "# Tokutek"
 print "# Test that replace into and insert ignore insertions "
 print "# work under various index schemas. "