Bug #17878183 NDB CONTINUOUSLY REBOOTING DUE TO FILE NOT FOUND (DBLQH: CAUSED BY ERROR 2341)

Fix for this bug. Root cause : - DICT overload resulting in GETTABINFOREF - Backup block failing to complete LCP_PREPARE due to GETTABINFOREF - LQH ignoring failure to start LCP - Node restart finding LCP file missing, and cannot handle. Fixes 1. LQH no longer ignores failure to start LCP 2. DICT no longer REFs internal requests This patch adds 1. A new testcase (testNodeRestart -nGetTabInfoOverload) 2. A new queueing mechanism inside DICT, separating internal + external requests 3. A new SegmentList utils class for kernel code Merge to 7.2+ requires some modifications : - Integrate with more recent fixes there. Applied to 7.1 as 'backport' is requested there.

Author: frazerclement

storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2003, 2014, Oracle and/or its affiliates. All rights reserved.
+ Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -445,6 +445,10 @@ void Dbdict::execCONTINUEB(Signal* signal)
  jam();
  trans_commit_wait_gci(signal);
  break;
+ case ZNEXT_GET_TAB_REQ:
+ jam();
+ startNextGetTabInfoReq(signal);
+ break;
  default :
  ndbrequire(false);
  break;
@@ -486,7 +490,7 @@ void Dbdict::packTableIntoPages(Signal* signal)
  jam();
  sendGET_TABINFOREF(signal, &req_copy,
  GetTabInfoRef::TableNotDefined, __LINE__);
- initRetrieveRecord(0, 0, 0);
+ initRetrieveRecord(signal, 0, 0);
  return;
  }
 
@@ -506,7 +510,7 @@ void Dbdict::packTableIntoPages(Signal* signal)
  // cannot see another uncommitted trans
  sendGET_TABINFOREF(signal, &req_copy,
  (GetTabInfoRef::ErrorCode)err, __LINE__);
- initRetrieveRecord(0, 0, 0);
+ initRetrieveRecord(signal, 0, 0);
  return;
  }
  }
@@ -534,7 +538,7 @@ void Dbdict::packTableIntoPages(Signal* signal)
  jam();
  sendGET_TABINFOREF(signal, &req_copy,
  GetTabInfoRef::TableNotDefined, __LINE__);
- initRetrieveRecord(0, 0, 0);
+ initRetrieveRecord(signal, 0, 0);
  return;
  }
 
@@ -551,7 +555,7 @@ void Dbdict::packTableIntoPages(Signal* signal)
  Uint32 dstRef = c_retrieveRecord.blockRef;
  sendSignal(dstRef, GSN_GET_TABINFOREF, signal,
  GetTabInfoRef::SignalLength, JBB);
- initRetrieveRecord(0,0,0);
+ initRetrieveRecord(signal,0,0);
  return;
  }
  break;
@@ -1977,6 +1981,7 @@ Dbdict::Dbdict(Block_context& ctx):
  c_attributeRecordHash(c_attributeRecordPool),
  c_obj_name_hash(c_obj_pool),
  c_obj_id_hash(c_obj_pool),
+ c_gettabinforeq_q(*this),
  c_schemaOpHash(c_schemaOpPool),
  c_schemaTransHash(c_schemaTransPool),
  c_schemaTransList(c_schemaTransPool),
@@ -2308,14 +2313,34 @@ void Dbdict::initWriteSchemaRecord()
 
 void Dbdict::initRetrieveRecord(Signal* signal, Uint32 i, Uint32 returnCode)
 {
- c_retrieveRecord.busyState = false;
+ jam();
  c_retrieveRecord.blockRef = 0;
  c_retrieveRecord.m_senderData = RNIL;
  c_retrieveRecord.tableId = RNIL;
  c_retrieveRecord.currentSent = 0;
  c_retrieveRecord.retrievedNoOfPages = 0;
  c_retrieveRecord.retrievedNoOfWords = 0;
  c_retrieveRecord.m_useLongSig = false;
+
+ if (!c_gettabinforeq_q.isEmpty())
+ {
+ jam();
+ ndbrequire(signal != NULL);
+
+ /* Take a real-time break now, CONTINUEB will
+ * start processing the next request.
+ * busyState = true will maintain fairness
+ */
+ signal->theData[0] = ZNEXT_GET_TAB_REQ;
+ sendSignal(reference(), GSN_CONTINUEB, signal,
+ 1, JBB);
+ }
+ else
+ {
+ /* Done */
+ c_retrieveRecord.busyState = false;
+ }
+
 }//initRetrieveRecord()
 
 void Dbdict::initSchemaRecord()
@@ -2823,6 +2848,9 @@ void Dbdict::execREAD_CONFIG_REQ(Signal* signal)
  bat[1].bits.q = ZLOG_SIZE_OF_PAGES_IN_WORDS; // 2**13 = 8192 elements
  bat[1].bits.v = 5; // 32 bits per element
 
+ // Initialize Segment Sub pool in GetTabInfoReq queue */
+ ndbrequire(c_gettabinforeq_q.init(jamBuffer()));
+
  initCommonData();
  initRecords();
 
@@ -10033,60 +10061,52 @@ void Dbdict::execGET_TABINFOREQ(Signal* signal)
  }
 
  GetTabInfoReq * const req = (GetTabInfoReq *)&signal->theData[0];
- SectionHandle handle(this, signal);
 
- /**
- * If I get a GET_TABINFO_REQ from myself
- * it's is a one from the time queue
- */
- bool fromTimeQueue = (signal->senderBlockRef() == reference());
-
- if (ERROR_INSERTED(6215) && fromTimeQueue == false)
+ if (ERROR_INSERTED(6215) &&
+ (signal->senderBlockRef() != reference()))
  {
  jam();
  // API tries 100 times and (80/100)^100 is quite small..
  if (rand() % 100 >= 20)
  {
  jam();
+ SectionHandle handle(this, signal);
  releaseSections(handle);
  sendGET_TABINFOREF(signal, req, GetTabInfoRef::Busy, __LINE__);
  return;
  }
  // no CLEAR_ERROR_INSERT_VALUE
  }
 
- if (c_retrieveRecord.busyState && fromTimeQueue == true) {
- jam();
-
- sendSignalWithDelay(reference(), GSN_GET_TABINFOREQ, signal, 30,
-signal->length(),
-&handle);
- return;
- }//if
-
- const Uint32 MAX_WAITERS = 5;
-
- if(c_retrieveRecord.busyState && fromTimeQueue == false)
+ if (c_retrieveRecord.busyState)
  {
  jam();
- if(c_retrieveRecord.noOfWaiters < MAX_WAITERS){
+ NodeInfo sendersNI = getNodeInfo(refToNode(req->senderRef));
+ bool internalReq = (sendersNI.m_type == NodeInfo::DB);
+
+ /* Queue request
+ * Will be processed later when current requests + queue are completed
+ */
+ if (!c_gettabinforeq_q.tryEnqReq(internalReq,
+ signal))
+ {
  jam();
- c_retrieveRecord.noOfWaiters++;
+ /**
+ * Enqueue failure resulting in Busy signal only allowed for
+ * external requests
+ */
+ ndbrequire(!internalReq);
 
- sendSignalWithDelay(reference(), GSN_GET_TABINFOREQ, signal, 30,
- signal->length(),
- &handle);
- return;
+ SectionHandle handle(this, signal);
+ releaseSections(handle);
+
+ sendGET_TABINFOREF(signal, req, GetTabInfoRef::Busy, __LINE__);
  }
- releaseSections(handle);
- sendGET_TABINFOREF(signal, req, GetTabInfoRef::Busy, __LINE__);
+
  return;
  }
 
- if(fromTimeQueue){
- jam();
- c_retrieveRecord.noOfWaiters--;
- }
+ SectionHandle handle(this, signal);
 
  const bool useLongSig = (req->requestType & GetTabInfoReq::LongSignalConf);
  const bool byName = (req->requestType & GetTabInfoReq::RequestByName);
@@ -29799,3 +29819,30 @@ Dbdict::send_event(Signal* signal,
  signal->theData[4] = refToNode(trans_ptr.p->m_clientRef);
  sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 5, JBB);
 }
+
+void
+Dbdict::startNextGetTabInfoReq(Signal* signal)
+{
+ jam();
+
+ /* Retrieve record should be in busy state to block
+ * queue-jumpers. We can clear it now as we will
+ * execute direct from the head of the queue(s)
+ */
+ ndbrequire(c_retrieveRecord.busyState);
+ ndbrequire(!c_gettabinforeq_q.isEmpty());
+
+ /* Directly start next queued request
+ * Prefer internalQueue, but give externalQueue entries
+ * a proportional share to avoid starvation.
+ */
+ ndbrequire(c_gettabinforeq_q.deqReq(signal));
+
+ c_retrieveRecord.busyState = false;
+
+ /* Execute...using EXECUTE_DIRECT to get signal trace */
+ EXECUTE_DIRECT(number(),
+ GSN_GET_TABINFOREQ,
+ signal,
+ GetTabInfoReq::SignalLength);
+}