MDEV-23580: WSREP_SST: [ERROR] rsync daemon port has been taken

This commit contains a large set of further bug fixes and improvements to SST scripts for Galera, continuing the work that was started in MDEV-24962 to make SST scripts work smoothly in different network configurations (especially using ipv6) and with different environment settings: 1) The ipv6 addresses were incorrectly handled in the SST script for rsync (incorrect address substitution for establishing a connection, incorrect address substitution for bind, and so on); 2) Checking the locality of the ip-address in SST scripts did not support ipv6 addresses (such as "[::1]"), which were falsely identified as non-local ip, which further did not allow running two SSTs on different local addresses on the same machine. On the other hand, this bug masked some other errors (related to handling ipv6 addresses); 3) The code for checking the locality of the ip address was different in the SST scripts for rsync and for mysqldump, with individual flaws. This code is now made common and moved to wsrep_sst_common; 4) Waiting for the start of the transport channel (socat, nc, rsync, stunnel) in the wait_for_listen() and check_pid_and_port() functions did not process ipv6 addresses correctly in all cases (not for all branches); 5) Waiting for the start of the transport channel (socat, nc, rsync, stunnel) in the wait_for_listen() and check_pid_and_port() functions for some code branches could give a false positive result due to the textual match of prefixes in the port number and/or PID of the process; 6) Waiting for the start of the transport channel (socat, nc, rsync, stunnel) was supported through different utilities in SST scripts for mariabackup and for rsync, and with various minor flaws in the code. Now the code is still different in these scripts, but it supports a common set of utilities (lsof, ss, sockstat) and is synchronized across patterns that used to check the output of these utilities; 7) In SST via mariabackup, the signal about readiness to receive data is sometimes sent too early - immediately after listen(), and not after accept() (which are called by socat or netcat utility). 8) Checking availability of the some options of some utilities was done using the grep pattern, which easily gives false positives; 9) Common name (CN) for local addresses, if not explicitly specified, is now always replaced to "localhost" to avoid the need to generate many separate certificates for local addresses of one machine and not to depend on which the local address is currently used in test (ipv4 or ipv6, etc.); 10) In tests galera_sst_mariabackup_encrypt_with_key_server and galera_sst_rsync_encrypt_with_key_server the correct certificate is selected to avoid commonname (CN) mismatch problems; 11) Further refactoring to protect against spaces in file names. 12) Further general refactoring to eliminate bash-specific constructs or to improve code readability; 13) The code for setting options for the nc (netcat) utility was different in different scripts for SST - now it is made identical. 14) Fixed long-time broken encryption via xbcrypt in combination with mariabackup and added support for key-based encryption via openssl utility, which is now enabled by default for encrypt=1 mode (this default mode can be changed using a new configuration file option "encypt-format=openssl|xbcrypt", which can be placed in the [mysqld], [sst] or in the [xtrabackup] section) - this change will allow us to use and to test the encypt=1 encryption without installing non-standard third-party utilities.

Author: sysprg

mysql-test/suite/galera/r/galera_sst_mariabackup_encrypt_with_key-openssl.result

@@ -0,0 +1,3 @@
+SELECT 1;
+1
+1

mysql-test/suite/galera/t/galera_sst_mariabackup_encrypt_with_key-openssl.cnf

@@ -0,0 +1,13 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=ON
+
+[sst]
+encrypt-format=openssl
+encrypt=1
+encrypt-algo=aes-256-ctr
+encrypt-key=4FA92C5873672E20FB163A0BCB2BB4A4
+transferfmt=@ENV.MTR_GALERA_TFMT

mysql-test/suite/galera/t/galera_sst_mariabackup_encrypt_with_key-openssl.test

@@ -0,0 +1,12 @@
+#
+# This test checks that encryption with key using openssl with options
+# passed to mariabackup via the my.cnf file
+#
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+SELECT 1;
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc

mysql-test/suite/galera/t/galera_sst_mariabackup_encrypt_with_key_server.cnf

@@ -5,8 +5,8 @@ wsrep_sst_method=mariabackup
 wsrep_sst_auth="root:"
 wsrep_debug=1
 
-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem
-ssl-key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem
+ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem
+ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem
 ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem
 
 [sst]

mysql-test/suite/galera/t/galera_sst_mariabackup_encrypt_with_key_server.test

@@ -18,7 +18,7 @@ SELECT 1;
 
 # Confirm that transfer was SSL-encrypted
 --let $assert_text = Using openssl based encryption with socat
---let $assert_select = Using openssl based encryption with socat: with key and c
+--let $assert_select = Using openssl based encryption with socat: with key and crt
 --let $assert_count = 1
 --let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
 --let $assert_only_after = CURRENT_TEST

mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.cnf

@@ -2,8 +2,8 @@
 
 [mysqld]
 wsrep_sst_method=rsync
-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem
-ssl-key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem
+ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem
+ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem
 ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem
 
 [sst]

scripts/wsrep_sst_common.sh

@@ -51,7 +51,7 @@ case "$1" in
  #
  # Break address string into host:port/path parts
  #
- case "${WSREP_SST_OPT_ADDR}" in
+ case "$WSREP_SST_OPT_ADDR" in
  \[*)
  # IPv6
  # Remove the starting and ending square brackets, if present:
@@ -81,18 +81,18 @@ case "$1" in
  # up to "/" (if present):
  WSREP_SST_OPT_ADDR_PORT="${remain%%/*}"
  # If the "/" character is present, then the path is not empty:
- if [ "${remain#*/}" != "${remain}" ]; then
+ if [ "${remain#*/}" != "$remain" ]; then
  # This operation removes everything up to the "/" character,
  # effectively removing the port number from the string:
  readonly WSREP_SST_OPT_PATH="${remain#*/}"
  else
  readonly WSREP_SST_OPT_PATH=""
  fi
  # The rest of the string is the same as the path (for now):
- remain="${WSREP_SST_OPT_PATH}"
+ remain="$WSREP_SST_OPT_PATH"
  # If there is one more "/" in the string, then everything before
  # it will be the module name, otherwise the module name is empty:
- if [ "${remain%%/*}" != "${remain}" ]; then
+ if [ "${remain%%/*}" != "$remain" ]; then
  # This operation removes the tail after the very first
  # occurrence of the "/" character (inclusively):
  readonly WSREP_SST_OPT_MODULE="${remain%%/*}"
@@ -103,7 +103,7 @@ case "$1" in
  remain="${WSREP_SST_OPT_PATH#*/}"
  # If the rest of the string does not match the original, then there
  # was something else besides the module name:
- if [ "$remain" != "${WSREP_SST_OPT_PATH}" ]; then
+ if [ "$remain" != "$WSREP_SST_OPT_PATH" ]; then
  # Extract the part that matches the LSN by removing all
  # characters starting from the very first "/":
  readonly WSREP_SST_OPT_LSN="${remain%%/*}"
@@ -113,7 +113,7 @@ case "$1" in
  # If the remainder does not match the original string,
  # then there is something else (the version number in
  # our case):
- if [ "$remain" != "${WSREP_SST_OPT_LSN}" ]; then
+ if [ "$remain" != "$WSREP_SST_OPT_LSN" ]; then
  # Let's extract the version number by removing the tail
  # after the very first occurence of the "/" character
  # (inclusively):
@@ -535,7 +535,8 @@ readonly WSREP_SST_OPT_ADDR_PORT
 
 # try to use my_print_defaults, mysql and mysqldump that come with the sources
 # (for MTR suite)
-SCRIPTS_DIR="$(cd $(dirname "$0"); pwd -P)"
+script_binary=$(dirname "$0")
+SCRIPTS_DIR=$(cd "$script_binary"; pwd -P)
 EXTRA_DIR="$SCRIPTS_DIR/../extra"
 CLIENT_DIR="$SCRIPTS_DIR/../client"
 
@@ -581,30 +582,45 @@ readonly MY_PRINT_DEFAULTS="$MY_PRINT_DEFAULTS $WSREP_SST_OPT_CONF"
 #
 parse_cnf()
 {
- local group="$1"
+ local groups="$1"
  local var="$2"
  local reval=""
 
- # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin)
- # then search for needed variable
- # finally get the variable value (if variables has been specified multiple time use the last value only)
-
- if [ "$group" = '--mysqld' -o \
- "$group" = 'mysqld' ]; then
- if [ -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
- reval=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk 'BEGIN {OFS=FS="="} {sub(/^--loose/,"-",$0); gsub(/_/,"-",$1); if ($1=="--'"$var"'") lastval=substr($0,length($1)+2)} END {print lastval}')
- fi
- fi
-
- if [ -z "$reval" ]; then
- reval=$($MY_PRINT_DEFAULTS "$group" | awk 'BEGIN {OFS=FS="="} {sub(/^--loose/,"-",$0); gsub(/_/,"-",$1); if ($1=="--'"$var"'") lastval=substr($0,length($1)+2)} END {print lastval}')
- fi
+ # normalize the variable names specified in the .cnf file
+ # (user can use '_' or '-', for example, log-bin or log_bin),
+ # then search for the last instance of the desired variable
+ # and finally get the value of that variable (if the variable
+ # was specified several times - we use only its last instance):
+
+ local pattern='BEGIN {OFS=FS="="} {sub(/^--loose/,"-",$0); gsub(/_/,"-",$1); if ($1=="--'"$var"'") lastval=substr($0,length($1)+2)} END {print lastval}'
+
+ while [ -n "$groups" ]; do
+ # Remove the largest suffix starting with the '|' character:
+ local group="${groups%%\|*}"
+ # Remove the remainder (the group name) from the rest
+ # of the groups list (as if it were a prefix):
+ groups="${groups#$group}"
+ groups="${groups#\|}"
+ # if the group name is the same as the "[--]mysqld", then
+ # try to use it together with the group suffix:
+ if [ "${group#--}" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
+ reval=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
+ if [ -n "$reval" ]; then
+ break
+ fi
+ fi
+ # Let's try to use the group name as it is:
+ reval=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern")
+ if [ -n "$reval" ]; then
+ break
+ fi
+ done
 
- # use default if we haven't found a value
+ # use default if we haven't found a value:
  if [ -z "$reval" ]; then
  [ -n "${3:-}" ] && reval="$3"
  fi
- echo $reval
+ echo "$reval"
 }
 
 #
@@ -615,18 +631,37 @@ parse_cnf()
 #
 in_config()
 {
- local group="$1"
+ local groups="$1"
  local var="$2"
  local found=0
- if [ "$group" = '--mysqld' -o \
- "$group" = 'mysqld' ]; then
- if [ -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
- found=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk 'BEGIN {OFS=FS="="; found=0} {sub(/^--loose/,"-",$0); gsub(/_/,"-",$1); if ($1=="--'"$var"'") found=1} END {print found}')
- fi
- fi
- if [ $found -eq 0 ]; then
- found=$($MY_PRINT_DEFAULTS "$group" | awk 'BEGIN {OFS=FS="="; found=0} {sub(/^--loose/,"-",$0); gsub(/_/,"-",$1); if ($1=="--'"$var"'") found=1} END {print found}')
- fi
+
+ # normalize the variable names specified in the .cnf file
+ # (user can use '_' or '-', for example, log-bin or log_bin),
+ # then search for the last instance(s) of the desired variable:
+
+ local pattern='BEGIN {OFS=FS="="; found=0} {sub(/^--loose/,"-",$0); gsub(/_/,"-",$1); if ($1=="--'"$var"'") found=1} END {print found}'
+
+ while [ -n "$groups" ]; do
+ # Remove the largest suffix starting with the '|' character:
+ local group="${groups%%\|*}"
+ # Remove the remainder (the group name) from the rest
+ # of the groups list (as if it were a prefix):
+ groups="${groups#$group}"
+ groups="${groups#\|}"
+ # if the group name is the same as the "[--]mysqld", then
+ # try to use it together with the group suffix:
+ if [ "${group#--}" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
+ found=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
+ if [ $found -ne 0 ]; then
+ break
+ fi
+ fi
+ # Let's try to use the group name as it is:
+ found=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern")
+ if [ $found -ne 0 ]; then
+ break
+ fi
+ done
  echo $found
 }
 
@@ -747,7 +782,7 @@ wsrep_check_programs()
 
  while [ $# -gt 0 ]
  do
- wsrep_check_program $1 || ret=$?
+ wsrep_check_program "$1" || ret=$?
  shift
  done
 
@@ -793,3 +828,52 @@ wsrep_gen_secret()
  $RANDOM $RANDOM $RANDOM $RANDOM
  fi
 }
+
+is_local_ip()
+{
+ [ "$1" = '127.0.0.1' ] && return 0
+ [ "$1" = '127.0.0.2' ] && return 0
+ [ "$1" = 'localhost' ] && return 0
+ [ "$1" = '[::1]' ] && return 0
+ [ "$1" = "$(hostname -s)" ] && return 0
+ [ "$1" = "$(hostname -f)" ] && return 0
+ [ "$1" = "$(hostname -d)" ] && return 0
+
+ local ip_util="$(command -v ip)"
+ if [ -x "$ip_util" ]; then
+ # ip address show ouput format is " inet[6] <address>/<mask>":
+ "$ip_util" address show \
+ | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \
+ | grep -F " $1/" >/dev/null && return 0
+ else
+ local ifconfig_util="$(command -v ifconfig)"
+ if [ -x "$ifconfig_util" ]; then
+ # ifconfig output format is " inet[6] <address> ...":
+ "$ifconfig_util" \
+ | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \
+ | grep -F " $1 " >/dev/null && return 0
+ fi
+ fi
+
+ return 1
+}
+
+check_sockets_utils()
+{
+ lsof_available=0
+ sockstat_available=0
+ ss_available=0
+
+ [ -x "$(command -v lsof)" ] && lsof_available=1
+ [ -x "$(command -v sockstat)" ] && sockstat_available=1
+ [ -x "$(command -v ss)" ] && ss_available=1
+
+ if [ $lsof_available -eq 0 -a \
+ $sockstat_available -eq 0 -a \
+ $ss_available -eq 0 ]
+ then
+ wsrep_log_error "Neither lsof tool, nor ss or sockstat was found in " \
+ "the PATH! Make sure you have it installed."
+ exit 2 # ENOENT
+ fi
+}