MDEV-12420 max_recursive_iterations did not prevent a stack-overflow and segfault

post-review fixes * move pcre-specific variable out of mysys * don't use current_thd * move a commonly used macro to my_sys.h * remove new sysvar

Author: vuvova

include/my_sys.h

@@ -227,9 +227,6 @@ extern void (*fatal_error_handler_hook)(uint my_err, const char *str,
  myf MyFlags);
 extern uint my_file_limit;
 extern ulonglong my_thread_stack_size;
-#ifndef EMBEDDED_LIBRARY
-extern ulonglong my_pcre_frame_size;
-#endif
 extern int sf_leaking_memory; /* set to 1 to disable memleak detection */
 
 extern void (*proc_info_hook)(void *, const PSI_stage_info *, PSI_stage_info *,
@@ -909,6 +906,12 @@ extern ulonglong my_getcputime(void);
 #define hrtime_sec_part(X) ((ulong)((X).val % HRTIME_RESOLUTION))
 #define my_time(X) hrtime_to_time(my_hrtime())
 
+#if STACK_DIRECTION < 0
+#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END))
+#else
+#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR))
+#endif
+
 #ifdef HAVE_SYS_MMAN_H
 #include <sys/mman.h>
 

mysys/lf_alloc-pin.c

@@ -328,12 +328,6 @@ static int match_pins(LF_PINS *el, void *addr)
  return 0;
 }
 
-#if STACK_DIRECTION < 0
-#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END))
-#else
-#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR))
-#endif
-
 #define next_node(P, X) (*((uchar * volatile *)(((uchar *)(X)) + (P)->free_ptr_offset)))
 #define anext_node(X) next_node(&allocator->pinbox, (X))
 

mysys/my_init.c

@@ -45,8 +45,6 @@ my_bool my_init_done= 0;
 uintmysys_usage_id= 0; /* Incremented for each my_init() */
 
 ulonglong my_thread_stack_size= (sizeof(void*) <= 4)? 65536: ((256-16)*1024);
-/* http://pcre.org/original/doc/html/pcrestack.html - replaced by init_pcre value */
-ulonglong my_pcre_frame_size= 640 + 16;
 
 static ulong atoi_octal(const char *str)
 {

sql/item_cmpfunc.cc

@@ -5104,6 +5104,15 @@ int Regexp_processor_pcre::default_regex_flags()
  return default_regex_flags_pcre(current_thd);
 }
 
+void Regexp_processor_pcre::set_recursion_limit(THD *thd)
+{
+ long stack_used;
+ DBUG_ASSERT(thd == current_thd);
+ stack_used= available_stack_size(thd->thread_stack, &stack_used);
+ m_pcre_extra.match_limit_recursion=
+ (my_thread_stack_size - stack_used)/my_pcre_frame_size;
+}
+
 
 /**
  Convert string to lib_charset, if needed.

sql/item_cmpfunc.h

@@ -25,7 +25,6 @@
 
 #include "thr_malloc.h" /* sql_calloc */
 #include "item_func.h" /* Item_int_func, Item_bool_func */
-long check_stack_available(long margin, uchar *dummy);
 #define PCRE_STATIC 1 /* Important on Windows */
 #include "pcre.h" /* pcre header file */
 
@@ -1577,15 +1576,11 @@ class Regexp_processor_pcre
  m_library_charset(&my_charset_utf8_general_ci),
  m_subpatterns_needed(0)
  {
-#ifndef EMBEDDED_LIBRARY
- uchar dummy;
- m_pcre_extra.flags= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
- m_pcre_extra.match_limit_recursion= check_stack_available(100, &dummy) / my_pcre_frame_size;
-#else
- m_pcre_extra.flags= 0L;
-#endif
+ m_pcre_extra.flags= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+ m_pcre_extra.match_limit_recursion= 100L;
  }
  int default_regex_flags();
+ void set_recursion_limit(THD *);
  void init(CHARSET_INFO *data_charset, int extra_flags, uint nsubpatterns)
  {
  m_library_flags= default_regex_flags() | extra_flags |

sql/mysqld.cc

@@ -3497,16 +3497,16 @@ static void init_libstrings()
 #endif
 }
 
+ulonglong my_pcre_frame_size;
 
 static void init_pcre()
 {
  pcre_malloc= pcre_stack_malloc= my_str_malloc_mysqld;
  pcre_free= pcre_stack_free= my_str_free_mysqld;
 #ifndef EMBEDDED_LIBRARY
  pcre_stack_guard= check_enough_stack_size_slow;
- /* my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) + 16;
- http://pcre.org/original/doc/html/pcrestack.html has reason for + 16 
- my_pcre_frame_size= -pcre_match(NULL, NULL, NULL, 0, NULL, NULL, 0) + 16; */
+ /* See http://pcre.org/original/doc/html/pcrestack.html */
+ my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) + 16;
 #endif
 }
 

sql/mysqld.h

@@ -488,6 +488,8 @@ extern pthread_t signal_thread;
 extern struct st_VioSSLFd * ssl_acceptor_fd;
 #endif /* HAVE_OPENSSL */
 
+extern ulonglong my_pcre_frame_size;
+
 /*
  The following variables were under INNODB_COMPABILITY_HOOKS
  */

sql/sql_parse.cc

@@ -6163,12 +6163,6 @@ bool check_fk_parent_table_access(THD *thd,
 ****************************************************************************/
 
 
-#if STACK_DIRECTION < 0
-#define used_stack(A,B) (long) (A - B)
-#else
-#define used_stack(A,B) (long) (B - A)
-#endif
-
 #ifndef DBUG_OFF
 long max_stack_used;
 #endif
@@ -6185,7 +6179,7 @@ bool check_stack_overrun(THD *thd, long margin,
 {
  long stack_used;
  DBUG_ASSERT(thd == current_thd);
- if ((stack_used=used_stack(thd->thread_stack,(char*) &stack_used)) >=
+ if ((stack_used= available_stack_size(thd->thread_stack, &stack_used)) >=
  (long) (my_thread_stack_size - margin))
  {
  thd->is_fatal_error= 1;
@@ -6208,14 +6202,6 @@ bool check_stack_overrun(THD *thd, long margin,
  return 0;
 }
 
-long check_stack_available(long margin,
- uchar *buf __attribute__((unused)))
-{
- long stack_top;
- DBUG_ASSERT(current_thd);
- return my_thread_stack_size - margin \
- - used_stack(current_thd->thread_stack,(char*) &stack_top);
-}
 
 #define MY_YACC_INIT 1000// Start with big alloc
 #define MY_YACC_MAX 32000// Because of 'short'

sql/sql_parse.h

@@ -134,7 +134,6 @@ bool check_simple_select();
 Item *normalize_cond(Item *cond);
 Item *negate_expression(THD *thd, Item *expr);
 bool check_stack_overrun(THD *thd, long margin, uchar *dummy);
-long check_stack_available(long margin, uchar *dummy);
 
 /* Variables */
 

sql/sys_vars.cc

@@ -2493,14 +2493,6 @@ static Sys_var_ulonglong Sys_thread_stack(
  VALID_RANGE(128*1024, ULONGLONG_MAX), DEFAULT(DEFAULT_THREAD_STACK),
  BLOCK_SIZE(1024));
 
-#ifndef EMBEDDED_LIBRARY
-static Sys_var_ulonglong Sys_my_pcre_frame_size(
- "pcre_frame_size", "Frame size for pcre_recursion",
- GLOBAL_VAR(my_pcre_frame_size), NO_CMD_LINE,
- VALID_RANGE(500,1024), DEFAULT(640 + 16), 1, NO_MUTEX_GUARD,
- NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(0));
-#endif
-
 static Sys_var_charptr Sys_tmpdir(
  "tmpdir", "Path for temporary files. Several paths may "
  "be specified, separated by a "