Merge 10.2 into 10.3

Author: dr-m

extra/mariabackup/backup_copy.cc

@@ -58,6 +58,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "backup_mysql.h"
 #include <btr0btr.h>
 
+#define ROCKSDB_BACKUP_DIR "#rocksdb"
+
 /* list of files to sync for --rsync mode */
 static std::set<std::string> rsync_list;
 /* locations of tablespaces read from .isl files */
@@ -66,6 +68,21 @@ static std::map<std::string, std::string> tablespace_locations;
 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
 bool binlog_locked;
 
+static void rocksdb_create_checkpoint();
+static bool has_rocksdb_plugin();
+static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
+static void rocksdb_backup_checkpoint();
+static void rocksdb_copy_back();
+
+static bool is_abs_path(const char *path)
+{
+#ifdef _WIN32
+return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
+#else
+return path[0] == '/';
+#endif
+}
+
 /************************************************************************
 Struct represents file or directory. */
 struct datadir_node_t {
@@ -1138,7 +1155,8 @@ bool
 copy_or_move_file(const char *src_file_path,
  const char *dst_file_path,
  const char *dst_dir,
- uint thread_n)
+ uint thread_n,
+ bool copy = xtrabackup_copy_back)
 {
 ds_ctxt_t *datasink = ds_data;/* copy to datadir by default */
 char filedir[FN_REFLEN];
@@ -1186,7 +1204,7 @@ copy_or_move_file(const char *src_file_path,
 free(link_filepath);
 }
 
-ret = (xtrabackup_copy_back ?
+ret = (copy ?
 copy_file(datasink, src_file_path, dst_file_path, thread_n) :
 move_file(datasink, src_file_path, dst_file_path,
  dst_dir, thread_n));
@@ -1371,6 +1389,10 @@ bool backup_start()
 return false;
 }
 
+if (has_rocksdb_plugin()) {
+rocksdb_create_checkpoint();
+}
+
 // There is no need to stop slave thread before coping non-Innodb data when
 // --no-lock option is used because --no-lock option requires that no DDL or
 // DML to non-transaction tables can occur.
@@ -1456,6 +1478,10 @@ bool backup_finish()
 }
 }
 
+if (has_rocksdb_plugin()) {
+rocksdb_backup_checkpoint();
+}
+
 msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir);
 if (mysql_binlog_position != NULL) {
 msg("MySQL binlog position: %s\n", mysql_binlog_position);
@@ -1771,6 +1797,16 @@ copy_back()
 int i_tmp;
 bool is_ibdata_file;
 
+if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
+#ifdef _WIN32
+|| strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
+#endif
+)
+{
+// copied at later step
+continue;
+}
+
 /* create empty directories */
 if (node.is_empty_dir) {
 char path[FN_REFLEN];
@@ -1855,6 +1891,8 @@ copy_back()
 }
 }
 
+rocksdb_copy_back();
+
 cleanup:
 if (it != NULL) {
 datadir_iter_free(it);
@@ -2031,3 +2069,234 @@ static bool backup_files_from_datadir(const char *dir_path)
 os_file_closedir(dir);
 return ret;
 }
+
+
+static int rocksdb_remove_checkpoint_directory()
+{
+xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
+return 0;
+}
+
+static bool has_rocksdb_plugin()
+{
+static bool first_time = true;
+static bool has_plugin= false;
+if (!first_time || !xb_backup_rocksdb)
+return has_plugin;
+
+const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
+MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
+MYSQL_ROW row = mysql_fetch_row(result);
+if (row)
+has_plugin = !strcmp(row[0], "1");
+mysql_free_result(result);
+first_time = false;
+return has_plugin;
+}
+
+static char *trim_trailing_dir_sep(char *path)
+{
+size_t path_len = strlen(path);
+while (path_len)
+{
+char c = path[path_len - 1];
+if (c == '/' IF_WIN(|| c == '\\', ))
+path_len--;
+else
+break;
+}
+path[path_len] = 0;
+return path;
+}
+
+/*
+Create a file hardlink.
+@return true on success, false on error.
+*/
+static bool make_hardlink(const char *from_path, const char *to_path)
+{
+DBUG_EXECUTE_IF("no_hardlinks", return false;);
+char to_path_full[FN_REFLEN];
+if (!is_abs_path(to_path))
+{
+fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
+}
+else
+{
+strncpy(to_path_full, to_path, sizeof(to_path_full));
+}
+#ifdef _WIN32
+return CreateHardLink(to_path_full, from_path, NULL);
+#else
+return !link(from_path, to_path_full);
+#endif
+}
+
+/*
+ Copies or moves a directory (non-recursively so far).
+ Helper function used to backup rocksdb checkpoint, or copy-back the
+ rocksdb files.
+
+ Has optimization that allows to use hardlinks when possible
+ (source and destination are directories on the same device)
+*/
+static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
+{
+datadir_node_t node;
+datadir_node_init(&node);
+datadir_iter_t *it = datadir_iter_new(from, false);
+
+while (datadir_iter_next(it, &node))
+{
+char to_path[FN_REFLEN];
+const char *from_path = node.filepath;
+snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
+bool rc = false;
+if (do_copy && allow_hardlinks)
+{
+rc = make_hardlink(from_path, to_path);
+if (rc)
+{
+msg_ts("[%02u] Creating hardlink from %s to %s\n",
+1, from_path, to_path);
+}
+else
+{
+allow_hardlinks = false;
+}
+}
+
+if (!rc) 
+{
+rc = (do_copy ?
+copy_file(ds_data, from_path, to_path, 1) :
+move_file(ds_data, from_path, node.filepath_rel,
+to, 1));
+}
+if (!rc)
+exit(EXIT_FAILURE);
+}
+datadir_iter_free(it);
+datadir_node_free(&node);
+
+}
+
+/*
+ Obtain user level lock , to protect the checkpoint directory of the server
+ from being user/overwritten by different backup processes, if backups are
+ running in parallel.
+ 
+ This lock will be acquired before rocksdb checkpoint is created, held
+ while all files from it are being copied to their final backup destination,
+ and finally released after the checkpoint is removed.
+*/
+static void rocksdb_lock_checkpoint()
+{
+msg_ts("Obtaining rocksdb checkpoint lock.\n");
+MYSQL_RES *res =
+xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);
+
+MYSQL_ROW r = mysql_fetch_row(res);
+if (r && r[0] && strcmp(r[0], "1"))
+{
+msg_ts("Could not obtain rocksdb checkpont lock\n");
+exit(EXIT_FAILURE);
+}
+}
+
+static void rocksdb_unlock_checkpoint()
+{
+xb_mysql_query(mysql_connection, 
+"SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
+}
+
+
+/*
+ Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
+ directory.
+ A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
+ acquired be this function.
+*/
+#define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
+static char rocksdb_checkpoint_dir[FN_REFLEN];
+
+static void rocksdb_create_checkpoint()
+{
+MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
+MYSQL_ROW row = mysql_fetch_row(result);
+
+DBUG_ASSERT(row && row[0] && row[1]);
+
+char *rocksdbdir = row[0];
+char *datadir = row[1];
+
+if (is_abs_path(rocksdbdir))
+{
+snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
+"%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
+}
+else 
+{
+snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
+"%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
+trim_dotslash(rocksdbdir));
+}
+mysql_free_result(result);
+
+#ifdef _WIN32
+for (char *p = rocksdb_checkpoint_dir; *p; p++)
+if (*p == '\\') *p = '/';
+#endif
+
+rocksdb_lock_checkpoint();
+
+if (!access(rocksdb_checkpoint_dir, 0))
+{
+msg_ts("Removing rocksdb checkpoint from previous backup attempt.\n");
+rocksdb_remove_checkpoint_directory();
+}
+
+char query[FN_REFLEN + 32];
+snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
+xb_mysql_query(mysql_connection, query, false, true);
+}
+
+/*
+ Copy files from rocksdb temporary checkpoint to final destination.
+ remove temp.checkpoint directory (in server's datadir)
+ and release user level lock acquired inside rocksdb_create_checkpoint().
+*/
+static void rocksdb_backup_checkpoint()
+{
+msg_ts("Backing up rocksdb files.\n");
+char rocksdb_backup_dir[FN_REFLEN];
+snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
+bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
+if (backup_to_directory) 
+{
+if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
+msg_ts("Can't create rocksdb backup directory %s\n", rocksdb_backup_dir);
+exit(EXIT_FAILURE);
+}
+}
+copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
+rocksdb_remove_checkpoint_directory();
+rocksdb_unlock_checkpoint();
+}
+
+/*
+ Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
+*/
+static void rocksdb_copy_back() {
+if (access(ROCKSDB_BACKUP_DIR, 0))
+return;
+char rocksdb_home_dir[FN_REFLEN];
+ if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
+strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof(rocksdb_home_dir));
+} else {
+ snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home, 
+xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
+}
+mkdirp(rocksdb_home_dir, 0777, MYF(0));
+copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
+}

extra/mariabackup/xtrabackup.cc

@@ -146,6 +146,8 @@ char *xtrabackup_tmpdir;
 char *xtrabackup_tables;
 char *xtrabackup_tables_file;
 char *xtrabackup_tables_exclude;
+char *xb_rocksdb_datadir;
+my_bool xb_backup_rocksdb = 1;
 
 typedef std::list<regex_t> regex_list_t;
 static regex_list_t regex_include_list;
@@ -683,7 +685,9 @@ enum options_xtrabackup
  OPT_XTRA_TABLES_EXCLUDE,
  OPT_XTRA_DATABASES_EXCLUDE,
  OPT_PROTOCOL,
- OPT_LOCK_DDL_PER_TABLE
+ OPT_LOCK_DDL_PER_TABLE,
+ OPT_ROCKSDB_DATADIR,
+ OPT_BACKUP_ROCKSDB
 };
 
 struct my_option xb_client_options[] =
@@ -1182,7 +1186,7 @@ struct my_option xb_server_options[] =
  "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
  "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
  &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
- REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0},
+ REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},
 
  {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
  "Directory where undo tablespace files live, this path can be absolute.",
@@ -1227,6 +1231,17 @@ struct my_option xb_server_options[] =
  (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
  GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
+ {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
+ "This option is only used with --copy-back or --move-back option",
+ &xb_rocksdb_datadir, &xb_rocksdb_datadir,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+ { "backup-rocksdb", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
+ "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
+ &xb_backup_rocksdb, &xb_backup_rocksdb,
+ 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
+
+
  { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 

extra/mariabackup/xtrabackup.h

@@ -44,6 +44,9 @@ extern char	*xtrabackup_incremental_basedir;
 extern char*innobase_data_home_dir;
 extern char*innobase_buffer_pool_filename;
 extern char*xb_plugin_dir;
+extern char*xb_rocksdb_datadir;
+extern my_boolxb_backup_rocksdb;
+
 extern uintopt_protocol;
 extern ds_ctxt_t*ds_meta;
 extern ds_ctxt_t*ds_data;