Merge perch.ndb.mysql.com:/home/jonas/src/50-work

into  perch.ndb.mysql.com:/home/jonas/src/mysql-5.0-ndb

ndb/include/kernel/signaldata/DumpStateOrd.hpp

@@ -68,6 +68,7 @@ public:
     // 100-105 TUP and ACC  
     // 200-240 UTIL
     // 300-305 TRIX
+    QmgrErr935 = 935,
     NdbfsDumpFileStat = 400,
     NdbfsDumpAllFiles = 401,
     NdbfsDumpOpenFiles = 402,

ndb/src/kernel/blocks/ERROR_codes.txt

@@ -21,6 +21,9 @@ Crash president when he starts to run in ArbitState 1-9.
 
 910: Crash new president after node crash
 
+935 : Crash master on node failure (delayed) 
+      and skip sending GSN_COMMIT_FAILREQ to specified node
+
 ERROR CODES FOR TESTING NODE FAILURE, GLOBAL CHECKPOINT HANDLING:
 -----------------------------------------------------------------
 

ndb/src/kernel/blocks/dbdih/DbdihMain.cpp

@@ -3555,7 +3555,6 @@ void Dbdih::endTakeOver(Uint32 takeOverPtrI)
   takeOverPtr.i = takeOverPtrI;
   ptrCheckGuard(takeOverPtr, MAX_NDB_NODES, takeOverRecord);
 
-  releaseTakeOver(takeOverPtrI);
   if ((takeOverPtr.p->toMasterStatus != TakeOverRecord::IDLE) &&
       (takeOverPtr.p->toMasterStatus != TakeOverRecord::TO_WAIT_START_TAKE_OVER)) {
     jam();
@@ -3569,6 +3568,7 @@ void Dbdih::endTakeOver(Uint32 takeOverPtrI)
   }//if
   setAllowNodeStart(takeOverPtr.p->toStartingNode, true);
   initTakeOver(takeOverPtr);
+  releaseTakeOver(takeOverPtrI);
 }//Dbdih::endTakeOver()
 
 void Dbdih::releaseTakeOver(Uint32 takeOverPtrI)
@@ -4710,6 +4710,7 @@ void Dbdih::handleTakeOverNewMaster(Signal* signal, Uint32 takeOverPtrI)
       break;
     }
     ndbrequire(ok);
+    endTakeOver(takeOverPtr.i);
   }//if
 }//Dbdih::handleTakeOverNewMaster()
 

ndb/src/kernel/blocks/qmgr/Qmgr.hpp

@@ -426,6 +426,10 @@ private:
   
   StopReq c_stopReq;
   bool check_multi_node_shutdown(Signal* signal);
+
+#ifdef ERROR_INSERT
+  Uint32 c_error_insert_extra;
+#endif
 };
 
 #endif

ndb/src/kernel/blocks/qmgr/QmgrMain.cpp

@@ -2895,6 +2895,17 @@ void Qmgr::failReportLab(Signal* signal, Uint16 aFailedNode,
     systemErrorLab(signal, __LINE__);
     return;
   }//if
+
+  if (getNodeState().startLevel < NodeState::SL_STARTED)
+  {
+    jam();
+    CRASH_INSERTION(932);
+    char buf[100];
+    BaseString::snprintf(buf, 100, "Node failure during restart");
+    progError(__LINE__, NDBD_EXIT_SR_OTHERNODEFAILED, buf);
+    ndbrequire(false);
+  }
+
   TnoFailedNodes = cnoFailedNodes;
   failReport(signal, failedNodePtr.i, (UintR)ZTRUE, aFailCause);
   if (cpresident == getOwnNodeId()) {
@@ -2981,6 +2992,16 @@ void Qmgr::execPREP_FAILREQ(Signal* signal)
     return;
   }//if
 
+  if (getNodeState().startLevel < NodeState::SL_STARTED)
+  {
+    jam();
+    CRASH_INSERTION(932);
+    char buf[100];
+    BaseString::snprintf(buf, 100, "Node failure during restart");
+    progError(__LINE__, NDBD_EXIT_SR_OTHERNODEFAILED, buf);
+    ndbrequire(false);
+  }
+
   guard0 = cnoPrepFailedNodes - 1;
   arrGuard(guard0, MAX_NDB_NODES);
   for (Tindex = 0; Tindex <= guard0; Tindex++) {
@@ -3158,6 +3179,18 @@ Qmgr::sendCommitFailReq(Signal* signal)
   for (nodePtr.i = 1; nodePtr.i < MAX_NDB_NODES; nodePtr.i++) {
     jam();
     ptrAss(nodePtr, nodeRec);
+
+#ifdef ERROR_INSERT    
+    if (ERROR_INSERTED(935) && nodePtr.i == c_error_insert_extra)
+    {
+      ndbout_c("skipping node %d", c_error_insert_extra);
+      CLEAR_ERROR_INSERT_VALUE;
+      signal->theData[0] = 9999;
+      sendSignalWithDelay(CMVMI_REF, GSN_NDB_TAMPER, signal, 1000, 1);
+      continue;
+    }
+#endif
+
     if (nodePtr.p->phase == ZRUNNING) {
       jam();
       nodePtr.p->sendCommitFailReqStatus = Q_ACTIVE;
@@ -3228,6 +3261,33 @@ void Qmgr::execPREP_FAILREF(Signal* signal)
   return;
 }//Qmgr::execPREP_FAILREF()
 
+static
+Uint32
+clear_nodes(Uint32 dstcnt, Uint16 dst[], Uint32 srccnt, const Uint16 src[])
+{
+  if (srccnt == 0)
+    return dstcnt;
+  
+  Uint32 pos = 0;
+  for (Uint32 i = 0; i<dstcnt; i++)
+  {
+    Uint32 node = dst[i];
+    for (Uint32 j = 0; j<srccnt; j++)
+    {
+      if (node == dst[j])
+      {
+	node = RNIL;
+	break;
+      }
+    }
+    if (node != RNIL)
+    {
+      dst[pos++] = node;
+    }
+  }
+  return pos;
+}
+
 /*---------------------------------------------------------------------------*/
 /*    THE PRESIDENT IS NOW COMMITTING THE PREVIOUSLY PREPARED NODE FAILURE.  */
 /*---------------------------------------------------------------------------*/
@@ -3315,19 +3375,18 @@ void Qmgr::execCOMMIT_FAILREQ(Signal* signal)
 		   NodeFailRep::SignalLength, JBB);
       }//if
     }//for
-    if (cpresident != getOwnNodeId()) {
-      jam();
-      cnoFailedNodes = cnoCommitFailedNodes - cnoFailedNodes;
-      if (cnoFailedNodes > 0) {
-        jam();
-        guard0 = cnoFailedNodes - 1;
-        arrGuard(guard0 + cnoCommitFailedNodes, MAX_NDB_NODES);
-        for (Tj = 0; Tj <= guard0; Tj++) {
-          jam();
-          cfailedNodes[Tj] = cfailedNodes[Tj + cnoCommitFailedNodes];
-        }//for
-      }//if
-    }//if
+
+    /**
+     * Remove committed nodes from failed/prepared
+     */
+    cnoFailedNodes = clear_nodes(cnoFailedNodes, 
+				 cfailedNodes, 
+				 cnoCommitFailedNodes, 
+				 ccommitFailedNodes);
+    cnoPrepFailedNodes = clear_nodes(cnoPrepFailedNodes, 
+				     cprepFailedNodes,
+				     cnoCommitFailedNodes,
+				     ccommitFailedNodes);
     cnoCommitFailedNodes = 0;
   }//if
   /**-----------------------------------------------------------------------
@@ -4706,6 +4765,14 @@ Qmgr::execDUMP_STATE_ORD(Signal* signal)
   default:
     ;
   }//switch
+
+#ifdef ERROR_INSERT
+  if (signal->theData[0] == 935 && signal->getLength() == 2)
+  {
+    SET_ERROR_INSERT_VALUE(935);
+    c_error_insert_extra = signal->theData[1];
+  }
+#endif
 }//Qmgr::execDUMP_STATE_ORD()
 
 void Qmgr::execSET_VAR_REQ(Signal* signal) 

ndb/test/ndbapi/testNodeRestart.cpp

@@ -955,6 +955,96 @@ int runBug24717(NDBT_Context* ctx, NDBT_Step* step){
   return NDBT_OK;
 }
 
+int runBug25364(NDBT_Context* ctx, NDBT_Step* step){
+  int result = NDBT_OK;
+  NdbRestarter restarter;
+  Ndb* pNdb = GETNDB(step);
+  int loops = ctx->getNumLoops();
+  
+  if (restarter.getNumDbNodes() < 4)
+    return NDBT_OK;
+
+  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
+
+  for (; loops; loops --)
+  {
+    int master = restarter.getMasterNodeId();
+    int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
+    int second = restarter.getRandomNodeSameNodeGroup(victim, rand());
+    
+    int dump[] = { 935, victim } ;
+    if (restarter.dumpStateOneNode(master, dump, 2))
+      return NDBT_FAILED;
+  
+    if (restarter.dumpStateOneNode(master, val2, 2))
+      return NDBT_FAILED;
+  
+    if (restarter.restartOneDbNode(second, false, true, true))
+      return NDBT_FAILED;
+
+    int nodes[2] = { master, second };
+    if (restarter.waitNodesNoStart(nodes, 2))
+      return NDBT_FAILED;
+
+    restarter.startNodes(nodes, 2);
+
+    if (restarter.waitNodesStarted(nodes, 2))
+      return NDBT_FAILED;
+  }
+  
+  return NDBT_OK;
+}
+
+int runBug25554(NDBT_Context* ctx, NDBT_Step* step){
+  
+  int result = NDBT_OK;
+  int loops = ctx->getNumLoops();
+  int records = ctx->getNumRecords();
+  NdbRestarter restarter;
+  
+  if (restarter.getNumDbNodes() < 4)
+    return NDBT_OK;
+
+  for (int i = 0; i<loops; i++)
+  {
+    int master = restarter.getMasterNodeId();
+    int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
+    restarter.restartOneDbNode(node1, false, true, true);
+
+    int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
+  
+    if (restarter.dumpStateOneNode(master, val2, 2))
+      return NDBT_FAILED;
+
+    if (restarter.insertErrorInNode(master, 7141))
+      return NDBT_FAILED;
+
+    if (restarter.waitNodesNoStart(&node1, 1))
+      return NDBT_FAILED;
+
+    if (restarter.dumpStateOneNode(node1, val2, 2))
+      return NDBT_FAILED;
+
+    if (restarter.insertErrorInNode(node1, 932))
+      return NDBT_FAILED;
+
+    if (restarter.startNodes(&node1, 1))
+      return NDBT_FAILED;
+
+    int nodes[] = { master, node1 };
+    if (restarter.waitNodesNoStart(nodes, 2))
+      return NDBT_FAILED;
+
+    if (restarter.startNodes(nodes, 2))
+      return NDBT_FAILED;
+
+    if (restarter.waitClusterStarted())
+      return NDBT_FAILED;
+  }    
+
+  return NDBT_OK;
+}
+
 
 NDBT_TESTSUITE(testNodeRestart);
 TESTCASE("NoLoad", 
@@ -1271,6 +1361,12 @@ TESTCASE("Bug20185",
 TESTCASE("Bug24717", ""){
   INITIALIZER(runBug24717);
 }
+TESTCASE("Bug25364", ""){
+  INITIALIZER(runBug25364);
+}
+TESTCASE("Bug25554", ""){
+  INITIALIZER(runBug25554);
+}
 NDBT_TESTSUITE_END(testNodeRestart);
 
 int main(int argc, const char** argv){

ndb/test/run-test/daily-basic-tests.txt

@@ -469,6 +469,14 @@ max-time: 1000
 cmd: testNodeRestart
 args: -n Bug24717 T1
 
+max-time: 1000
+cmd: testNodeRestart
+args: -n Bug25364 T1
+
+max-time: 1000
+cmd: testNodeRestart
+args: -n Bug25554 T1
+
 # OLD FLEX
 max-time: 500
 cmd: flexBench