Merge branch '10.9' into 10.10

Author: sanja-byelkin

.gitignore

@@ -7,6 +7,7 @@
 *.rpm
 .*.swp
 *.ninja
+.ccls-cache/
 .ninja_*
 *.mri
 *.mri.tpl

.gitlab-ci.yml

@@ -27,6 +27,7 @@ stages:
  - build
  - test
  - Salsa-CI
+ - sast
 
 default:
  # Base image for builds and tests unless otherwise defined
@@ -206,7 +207,7 @@ fedora-sanitizer:
  - builddir/_CPack_Packages/Linux/RPM/SPECS/
  parallel:
  matrix:
- - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES, -DWITH_MSAN=YES]
+ - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES]
 
 centos8:
  stage: build
@@ -298,6 +299,7 @@ centos7:
  main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
  main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
  main.mysql_upgrade_noengine : upgrade output order does not match the expected
+ main.func_math : MDEV-20966 - Wrong error code
  " > skiplist
  - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
 
@@ -370,22 +372,6 @@ mysql-test-run-ubsan:
  junit:
  - junit.xml
 
-mysql-test-run-msan:
- stage: test
- variables:
- RESTART_POLICY: "--force-restart"
- dependencies:
- - "fedora-sanitizer: [-DWITH_MSAN=YES]"
- needs:
- - "fedora-sanitizer: [-DWITH_MSAN=YES]"
- <<: *mysql-test-run-def
- allow_failure: true
- artifacts:
- when: always # Also show results when tests fail
- reports:
- junit:
- - junit.xml
-
 rpmlint:
  stage: test
  dependencies:
@@ -440,52 +426,70 @@ fedora install:
  - installed-database.sql
  - upgraded-database.sql
 
-fedora upgrade:
- stage: test
- dependencies:
-  - fedora
- needs:
- - fedora
+cppcheck: 
+ stage: sast
+ needs: []
+ variables:
+  GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
  script:
- - dnf install -y mariadb-server
- # Fedora does not support running services in Docker (like Debian packages do) so start it manually
- - /usr/libexec/mariadb-check-socket
- - /usr/libexec/mariadb-prepare-db-dir
- - sudo -u mysql /usr/libexec/mariadbd --basedir=/usr & sleep 10
- # Dump database contents in installed state
- - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql
- - /usr/libexec/mariadb-check-upgrade
- # Dump database contents in upgraded state
- - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql
- - mariadb --skip-column-names -e "SELECT @@version, @@version_comment" # Show version
- # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work
- # so do this manual step to remove conflicts until packaging is fixed
- - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config
- - rm -f rpm/*debuginfo* # Not relevant in this test
- - yum install -y rpm/*.rpm
- # nothing provides galera-4 on Fedora, so this step fails if built with wsrep
- - mysql -e "SHUTDOWN;"
- - /usr/bin/mariadb-install-db # This step should not do anything on upgrades, just exit
- - sudo -u mysql /usr/sbin/mariadbd & sleep 10
- # Dump database contents in installed state
- - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true
- # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577)
- # @TODO: Since we did a manual start, we also need to run upgrade manually
- - /usr/bin/mariadb-upgrade
- # Dump database contents in upgraded state
- - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql
- - |
- mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
- grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
- - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
- - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+ - yum install -y cppcheck diffutils
+ # --template: use a single-line template
+ # --force: check large directories without warning
+ # -i<directory>: ignore this directory when scanning
+ # -j: run multiple cppcheck threads
+ # Use newline to escape colon in yaml
+ - >
+ cppcheck --template="{file}:{line}: {severity}: {message}" --force
+ client dbug extra include libmariadb libmysqld libservices mysql-test mysys mysys_ssl pcre plugin
+ strings tests unittest vio wsrep-lib sql sql-common storage
+ -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore
+ --output-file=cppcheck.txt -j $(nproc)
+ # Parallel jobs may output findings in an nondeterministic order. Sort to match ignorelist.
+ - cat cppcheck.txt | sort > cppcheck_sorted.txt
+ # Remove line numbers for diff
+ - sed 's/:[^:]*:/:/' cppcheck_sorted.txt > cppcheck_sorted_no_line_numbers.txt
+ # Only print new issues not found in ignore list
+ - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed)."
+ - diff --changed-group-format='%>' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt || true
+ - echo "Problems found by cppcheck that were not in ignore list."
+ - diff --changed-group-format='%<' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt > lines_not_ignored.txt || true
+ - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
  artifacts:
+ when: always
  paths:
- - old-installed-database.sql
- - old-upgraded-database.sql
- - new-installed-database.sql
- - new-upgraded-database.sql
+ - cppcheck_sorted.txt
 
+flawfinder: 
+ stage: sast
+ needs: []
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y python3 python3-pip jq diffutils git
+ - pip install flawfinder
+ - flawfinder --falsepositive --quiet --html . > flawfinder-all-vulnerabilities.html
+ - cat flawfinder-all-vulnerabilities.html | grep "Hits ="
+ - flawfinder --falsepositive --quiet --minlevel=5 --sarif . > flawfinder-output.json
+ # FlawFinder's --sarif output will display all vulnerabilities despite having --minlevel=5 specified.
+ # Therefore, we postprocess the results with jq and filter out findings where the vulnerability level is less than 5.
+ # Also in the SARIF output format, the vulnerabilities are ranked as 0.2/0.4/0.6/0.8/1.0 which correspond to the --minlevel=1/2/3/4/5 of FlawFinder.
+ # Additionally, we sort the results because individual findings are consistent across different runs, but their ordering may not be.
+ # Vulnerabilities can also be ignored in-line (/* Flawfinder: ignore */), but this option was chosen as to not clutter the codebase.
+ - jq 'del(.runs[] | .tool | .driver | .rules) | del(.runs[] | .results[] | select(.rank < 1)) | del(.runs[] | .results[] | .locations[] | .physicalLocation | .region | .startLine) | .runs[0].results|=sort_by(.fingerprints)' flawfinder-output.json > flawfinder-min-level5.json
+ # Diff against known vulnerabilities, but ignore the line number.
+ - echo "Problems found in ignore list that were not discovered by flawfinder (may have been fixed)."
+ - diff --changed-group-format='%>' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json || true
+ - echo "Problems found by flawfinder that were not in ignore list."
+ - diff --changed-group-format='%<' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json > lines_not_ignored.txt || true
+ - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
+ artifacts:
+ when: always
+ paths:
+ - flawfinder-all-vulnerabilities.html
+ - flawfinder-min-level5.json
+ 
 mini-benchmark:
  stage: test
  dependencies:

CMakeLists.txt

@@ -27,8 +27,6 @@ IF(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
  "None" "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 ENDIF()
 
-PROJECT(MySQL)
-
 # Remove the following comment if you don't want to have striped binaries
 # in RPM's:
 
@@ -39,6 +37,8 @@ FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0075 CMP0069 CMP0135)
  ENDIF()
 ENDFOREACH()
 
+PROJECT(MySQL)
+
 MESSAGE(STATUS "Running cmake version ${CMAKE_VERSION}")
 
 SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}

client/mysql.cc

@@ -2122,7 +2122,7 @@ static int get_options(int argc, char **argv)
  current_db= my_strdup(PSI_NOT_INSTRUMENTED, *argv, MYF(MY_WME));
  }
  if (tty_password)
- opt_password= get_tty_password(NullS);
+ opt_password= my_get_tty_password(NullS);
  if (debug_info_flag)
  my_end_arg= MY_CHECK_ERROR | MY_GIVE_INFO;
  if (debug_check_flag)
@@ -4826,7 +4826,7 @@ char *mysql_authentication_dialog_ask(MYSQL *mysql, int type,
 
  if (type == 2) /* password */
  {
- s= get_tty_password("");
+ s= my_get_tty_password("");
  strnmov(buf, s, buf_len);
  buf[buf_len-1]= 0;
  my_free(s);

client/mysql_upgrade.c

@@ -1451,7 +1451,7 @@ int main(int argc, char **argv)
 
  if (tty_password)
  {
- opt_password= get_tty_password(NullS);
+ opt_password= my_get_tty_password(NullS);
  /* add password to defaults file */
  add_one_option_cnf_file(&ds_args, "password", opt_password);
  }

client/mysqladmin.cc

@@ -369,7 +369,7 @@ int main(int argc,char *argv[])
  }
  commands = temp_argv;
  if (tty_password)
- opt_password = get_tty_password(NullS);
+ opt_password = my_get_tty_password(NullS);
 
  (void) signal(SIGINT,endprog);/* Here if abort */
  (void) signal(SIGTERM,endprog);/* Here if abort */
@@ -1100,8 +1100,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
  else if (argc == 1)
  {
  /* prompt for password */
- typed_password= get_tty_password("New password: ");
- verified= get_tty_password("Confirm new password: ");
+ typed_password= my_get_tty_password("New password: ");
+ verified= my_get_tty_password("Confirm new password: ");
  if (strcmp(typed_password, verified) != 0)
  {
  my_printf_error(0,"Passwords don't match",MYF(ME_BELL));

client/mysqlbinlog.cc

@@ -2545,7 +2545,7 @@ get_one_option(const struct my_option *opt, const char *argument,
  break;
  }
  if (tty_password)
- pass= get_tty_password(NullS);
+ pass= my_get_tty_password(NullS);
 
  return 0;
 }

client/mysqlcheck.c

@@ -489,7 +489,7 @@ static int get_options(int *argc, char ***argv)
  DBUG_RETURN(1);
  }
  if (tty_password)
- opt_password = get_tty_password(NullS);
+ opt_password = my_get_tty_password(NullS);
  if (debug_info_flag)
  my_end_arg= MY_CHECK_ERROR | MY_GIVE_INFO;
  if (debug_check_flag)

client/mysqldump.c

@@ -1323,7 +1323,7 @@ static int get_options(int *argc, char ***argv)
  return EX_USAGE;
  }
  if (tty_password)
- opt_password=get_tty_password(NullS);
+ opt_password=my_get_tty_password(NullS);
  return(0);
 } /* get_options */
 
@@ -2818,11 +2818,7 @@ static uint dump_routines_for_db(char *db)
  routine_type[i], routine_name);
 
  if (mysql_query_with_error_report(mysql, &routine_res, query_buff))
- {
- mysql_free_result(routine_list_res);
- routine_list_res= 0;
- DBUG_RETURN(1);
- }
+ continue;
 
  while ((row= mysql_fetch_row(routine_res)))
  {

client/mysqlimport.c

@@ -326,7 +326,7 @@ static int get_options(int *argc, char ***argv)
  current_db= *((*argv)++);
  (*argc)--;
  if (tty_password)
- opt_password=get_tty_password(NullS);
+ opt_password=my_get_tty_password(NullS);
  return(0);
 }