ndb -

  refactor testSRBank and create new class NdbMixRestarter


storage/ndb/test/ndbapi/testSRBank.cpp:
  refactor testSRBank and create new class NdbMixRestarter
storage/ndb/test/src/Makefile.am:
  refactor testSRBank and create new class NdbMixRestarter
storage/ndb/test/src/NDBT_Test.cpp:
  fix
storage/ndb/test/include/NdbMixRestarter.hpp:
  New BitKeeper file ``storage/ndb/test/include/NdbMixRestarter.hpp''
storage/ndb/test/src/NdbMixRestarter.cpp:
  New BitKeeper file ``storage/ndb/test/src/NdbMixRestarter.cpp''

storage/ndb/test/include/NdbMixRestarter.hpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef NDBT_MIX_RESTARTER_HPP
+#define NDBT_MIX_RESTARTER_HPP
+
+#include <mgmapi.h>
+#include <Vector.hpp>
+#include <BaseString.hpp>
+#include "NdbRestarter.hpp"
+#include "NDBT_Test.hpp"
+
+#define NMR_SR                       "SR"
+#define NMR_SR_THREADS               "SR_ThreadCount"
+#define NMR_SR_THREADS_STOPPED       "SR_ThreadsStoppedCount"
+#define NMR_SR_VALIDATE_THREADS      "SR_ValidateThreadCount"
+#define NMR_SR_VALIDATE_THREADS_DONE "SR_ValidateThreadsDoneCount"
+
+class NdbMixRestarter : public NdbRestarter 
+{
+public:
+  enum RestartTypeMask 
+  {
+    RTM_RestartCluster     = 0x01,
+    RTM_RestartNode        = 0x02,
+    RTM_RestartNodeInitial = 0x04,
+    RTM_StopNode           = 0x08,
+    RTM_StopNodeInitial    = 0x10,
+    RTM_StartNode          = 0x20,
+
+    RTM_COUNT = 6,
+
+    RTM_ALL = 0xFF,
+    RTM_SR  = RTM_RestartCluster,
+    RTM_NR  = 0x2 | 0x4 | 0x8 | 0x10 | 0x20
+  };
+  
+  enum SR_State {
+    SR_RUNNING    = 0,
+    SR_STOPPING   = 1,
+    SR_STOPPED    = 2,
+    SR_VALIDATING = 3
+  };
+  
+  NdbMixRestarter(const char* _addr = 0);
+  ~NdbMixRestarter();
+
+  void setRestartTypeMask(Uint32 mask);
+  int runUntilStopped(NDBT_Context* ctx, NDBT_Step* step, Uint32 freq);
+  int runPeriod(NDBT_Context* ctx, NDBT_Step* step, Uint32 time, Uint32 freq);
+  
+  int init(NDBT_Context* ctx, NDBT_Step* step);
+  int dostep(NDBT_Context* ctx, NDBT_Step* step);
+  int finish(NDBT_Context* ctx, NDBT_Step* step);
+
+private:
+  Uint32 m_mask;
+  Vector<ndb_mgm_node_state> m_nodes;
+  int restart_cluster(NDBT_Context* ctx, NDBT_Step* step, bool abort = true);
+};
+
+#endif

storage/ndb/test/ndbapi/testSRBank.cpp

@@ -20,9 +20,16 @@
 #include <NdbBackup.hpp>
 
 #include "bank/Bank.hpp"
+#include <NdbMixRestarter.hpp>
 
 bool disk = false;
 
+#define CHECK(b) if (!(b)) { \
+  g_err << "ERR: "<< step->getName() \
+         << " failed on line " << __LINE__ << endl; \
+  result = NDBT_FAILED; \
+  continue; } 
+
 int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){
   Bank bank(ctx->m_cluster_connection);
   int overWriteExisting = true;
@@ -37,21 +44,25 @@ int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){
  * SR 1 - shutdown in progress
  * SR 2 - restart in progress
  */
-int runBankTimer(NDBT_Context* ctx, NDBT_Step* step){
+int 
+runBankTimer(NDBT_Context* ctx, NDBT_Step* step){
   int wait = 5; // Max seconds between each "day"
   int yield = 1; // Loops before bank returns 
-
-  ctx->incProperty("ThreadCount");
+  
+  ctx->incProperty(NMR_SR_THREADS);
   while (!ctx->isTestStopped()) 
   {
     Bank bank(ctx->m_cluster_connection);
-    while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1)
+    while(!ctx->isTestStopped() && 
+          ctx->getProperty(NMR_SR) <= NdbMixRestarter::SR_STOPPING)
+    {
       if(bank.performIncreaseTime(wait, yield) == NDBT_FAILED)
 	break;
-
+    }
+    
     ndbout_c("runBankTimer is stopped");
-    ctx->incProperty("ThreadStopped");
-    if(ctx->getPropertyWait("SR", (Uint32)0))
+    ctx->incProperty(NMR_SR_THREADS_STOPPED);
+    if(ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
       break;
   }
   return NDBT_OK;
@@ -61,17 +72,18 @@ int runBankTransactions(NDBT_Context* ctx, NDBT_Step* step){
   int wait = 0; // Max ms between each transaction
   int yield = 1; // Loops before bank returns 
 
-  ctx->incProperty("ThreadCount");
+  ctx->incProperty(NMR_SR_THREADS);
   while (!ctx->isTestStopped()) 
   {
     Bank bank(ctx->m_cluster_connection);
-    while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1)
+    while(!ctx->isTestStopped() && 
+          ctx->getProperty(NMR_SR) <= NdbMixRestarter::SR_STOPPING)
       if(bank.performTransactions(0, 1) == NDBT_FAILED)
 	break;
     
     ndbout_c("runBankTransactions is stopped");
-    ctx->incProperty("ThreadStopped");
-    if(ctx->getPropertyWait("SR", (Uint32)0))
+    ctx->incProperty(NMR_SR_THREADS_STOPPED);
+    if(ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
       break;
   }
   return NDBT_OK;
@@ -80,278 +92,108 @@ int runBankTransactions(NDBT_Context* ctx, NDBT_Step* step){
 int runBankGL(NDBT_Context* ctx, NDBT_Step* step){
   int yield = 1; // Loops before bank returns 
   int result = NDBT_OK;
-
-  ctx->incProperty("ThreadCount");
+  
+  ctx->incProperty(NMR_SR_THREADS);
   while (ctx->isTestStopped() == false) 
   {
     Bank bank(ctx->m_cluster_connection);
-    while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1)
+    while(!ctx->isTestStopped() && 
+          ctx->getProperty(NMR_SR) <= NdbMixRestarter::SR_STOPPING)
       if (bank.performMakeGLs(yield) != NDBT_OK)
       {
-	if(ctx->getProperty("SR") != 0)
+	if(ctx->getProperty(NMR_SR) != NdbMixRestarter::SR_RUNNING)
 	  break;
 	ndbout << "bank.performMakeGLs FAILED" << endl;
+        abort();
 	return NDBT_FAILED;
       }
     
     ndbout_c("runBankGL is stopped");
-    ctx->incProperty("ThreadStopped");
-    if(ctx->getPropertyWait("SR", (Uint32)0))
+    ctx->incProperty(NMR_SR_THREADS_STOPPED);
+    if(ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
       break;
   }
   return NDBT_OK;
 }
 
-int runBankSum(NDBT_Context* ctx, NDBT_Step* step){
-  Bank bank(ctx->m_cluster_connection);
-  int wait = 2000; // Max ms between each sum of accounts
-  int yield = 1; // Loops before bank returns 
-  int result = NDBT_OK;
-
-  while (ctx->isTestStopped() == false) {
-    if (bank.performSumAccounts(wait, yield) != NDBT_OK){
-      ndbout << "bank.performSumAccounts FAILED" << endl;
-      result = NDBT_FAILED;
-    }
-  }
-  return result ;
-}
-
-#define CHECK(b) if (!(b)) { \
-  g_err << "ERR: "<< step->getName() \
-         << " failed on line " << __LINE__ << endl; \
-  result = NDBT_FAILED; \
-  continue; } 
-
-static
-int
-restart_cluster(NDBT_Context* ctx, NDBT_Step* step, NdbRestarter& restarter)
+int 
+runBankSrValidator(NDBT_Context* ctx, NDBT_Step* step)
 {
-  bool abort = true;
-  int timeout = 180;
-  int result = NDBT_OK;
 
-  do 
+  ctx->incProperty(NMR_SR_VALIDATE_THREADS);
+
+  while(!ctx->isTestStopped())
   {
-    ndbout << " -- Shutting down " << endl;
-    ctx->setProperty("SR", 1);
-    CHECK(restarter.restartAll(false, true, abort) == 0);
-    ctx->setProperty("SR", 2);
-    CHECK(restarter.waitClusterNoStart(timeout) == 0);
+    if (ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_VALIDATING))
+      break;
     
-    Uint32 cnt = ctx->getProperty("ThreadCount");
-    Uint32 curr= ctx->getProperty("ThreadStopped");
-    while(curr != cnt && !ctx->isTestStopped())
+    int wait = 0;
+    int yield = 1;
+    Bank bank(ctx->m_cluster_connection);
+    if (bank.performSumAccounts(wait, yield) != 0)
     {
-      ndbout_c("%d %d", curr, cnt);
-      NdbSleep_MilliSleep(100);
-      curr= ctx->getProperty("ThreadStopped");
+      ndbout << "bank.performSumAccounts FAILED" << endl;
+      abort();
+      return NDBT_FAILED;
     }
     
-    ctx->setProperty("ThreadStopped", (Uint32)0);
-    CHECK(restarter.startAll() == 0);
-    CHECK(restarter.waitClusterStarted(timeout) == 0);
-    
-    ndbout << " -- Validating starts " << endl;
+    if (bank.performValidateAllGLs() != 0)
     {
-      int wait = 0;
-      int yield = 1;
-      Bank bank(ctx->m_cluster_connection);
-      if (bank.performSumAccounts(wait, yield) != 0)
-      {
-	ndbout << "bank.performSumAccounts FAILED" << endl;
-	return NDBT_FAILED;
-      }
-      
-      if (bank.performValidateAllGLs() != 0)
-      {
-	ndbout << "bank.performValidateAllGLs FAILED" << endl;
-	return NDBT_FAILED;
-      }
+      ndbout << "bank.performValidateAllGLs FAILED" << endl;
+      abort();
+      return NDBT_FAILED;
     }
     
-    ndbout << " -- Validating complete " << endl;
-  } while(0);
-  ctx->setProperty("SR", (Uint32)0);
-  ctx->broadcast();
-  return result;
-}
-
-static
-ndb_mgm_node_state*
-select_node_to_stop(Vector<ndb_mgm_node_state>& nodes)
-{
-  Uint32 i, j;
-  Vector<ndb_mgm_node_state*> alive_nodes;
-  for(i = 0; i<nodes.size(); i++)
-  {
-    ndb_mgm_node_state* node = &nodes[i];
-    if (node->node_status == NDB_MGM_NODE_STATUS_STARTED)
-      alive_nodes.push_back(node);
-  }
-
-  Vector<ndb_mgm_node_state*> victims;
-  // Remove those with one in node group
-  for(i = 0; i<alive_nodes.size(); i++)
-  {
-    int group = alive_nodes[i]->node_group;
-    for(j = 0; j<alive_nodes.size(); j++) 
-    {
-      if (i != j && alive_nodes[j]->node_group == group)
-      {
-	victims.push_back(alive_nodes[i]);
-	break;
-      }
-    }
-  }
-
-  if (victims.size())
-  {
-    int victim = rand() % victims.size();
-    return victims[victim];
-  }
-  else
-  {
-    return 0;
+    ctx->incProperty(NMR_SR_VALIDATE_THREADS_DONE);
+    
+    if (ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
+      break;
   }
+  
+  return NDBT_OK;
 }
 
-static
-ndb_mgm_node_state*
-select_node_to_start(Vector<ndb_mgm_node_state>& nodes)
-{
-  Uint32 i, j;
-  Vector<ndb_mgm_node_state*> victims;
-  for(i = 0; i<nodes.size(); i++)
-  {
-    ndb_mgm_node_state* node = &nodes[i];
-    if (node->node_status == NDB_MGM_NODE_STATUS_NOT_STARTED)
-      victims.push_back(node);
-  }
+#if 0
+int runBankSum(NDBT_Context* ctx, NDBT_Step* step){
+  Bank bank(ctx->m_cluster_connection);
+  int wait = 2000; // Max ms between each sum of accounts
+  int yield = 1; // Loops before bank returns 
+  int result = NDBT_OK;
 
-  if (victims.size())
-  {
-    int victim = rand() % victims.size();
-    return victims[victim];
-  }
-  else
+  while (ctx->isTestStopped() == false) 
   {
-    return 0;
+    if (bank.performSumAccounts(wait, yield) != NDBT_OK){
+      ndbout << "bank.performSumAccounts FAILED" << endl;
+      result = NDBT_FAILED;
+    }
   }
+  return result ;
 }
+#endif
 
-enum Action {
-  AA_RestartCluster = 0x1,
-  AA_RestartNode    = 0x2,
-  AA_StopNode       = 0x4,
-  AA_StartNode      = 0x8,
-  AA_COUNT = 4
-};
 
 int
 runMixRestart(NDBT_Context* ctx, NDBT_Step* step)
 {
   int result = NDBT_OK;
+  NdbMixRestarter res;
   int runtime = ctx->getNumLoops();
   int sleeptime = ctx->getNumRecords();
-  NdbRestarter restarter;
-  int timeout = 180;
-  Uint32 type = ctx->getProperty("Type", ~(Uint32)0);
-  
-  restarter.waitClusterStarted();
-  Vector<ndb_mgm_node_state> nodes;
-  nodes = restarter.ndbNodes;
-#if 0
-  for (Uint32 i = 0; i<restarter.ndbNodes.size(); i++)
-    nodes.push_back(restarter.ndbNodes[i]);
-#endif  
+  Uint32 mask = ctx->getProperty("Type", ~(Uint32)0);
+  res.setRestartTypeMask(mask);
 
-  
-  Uint32 now;
-  const Uint32 stop = time(0)+ runtime;
-  while(!ctx->isTestStopped() && ((now= time(0)) < stop) && result == NDBT_OK)
+  if (res.runPeriod(ctx, step, runtime, sleeptime))
   {
-    ndbout << " -- Sleep " << sleeptime << "s " << endl;
-    int cnt = sleeptime;
-    while (cnt-- && !ctx->isTestStopped())
-      NdbSleep_SecSleep(1);
-    if (ctx->isTestStopped())
-      return NDBT_FAILED;
-    
-    ndb_mgm_node_state* node = 0;
-    int action;
-loop:
-    while(((action = (1 << (rand() % AA_COUNT))) & type) == 0);
-    switch(action){
-    case AA_RestartCluster:
-      if (restart_cluster(ctx, step, restarter))
-	return NDBT_FAILED;
-      for (Uint32 i = 0; i<nodes.size(); i++)
-	nodes[i].node_status = NDB_MGM_NODE_STATUS_STARTED;
-      break;
-    case AA_RestartNode:
-    case AA_StopNode:
-    {
-      if ((node = select_node_to_stop(nodes)) == 0)
-	goto loop;
-      
-      if (action == AA_RestartNode)
-	g_err << "Restarting " << node->node_id << endl;
-      else
-	g_err << "Stopping " << node->node_id << endl;
-
-      if (restarter.restartOneDbNode(node->node_id, false, true, true))
-	return NDBT_FAILED;
-      
-      if (restarter.waitNodesNoStart(&node->node_id, 1))
-	return NDBT_FAILED;
-      
-      node->node_status = NDB_MGM_NODE_STATUS_NOT_STARTED;
-      
-      if (action == AA_StopNode)
-	break;
-      else
-	goto start;
-    }
-    case AA_StartNode:
-      if ((node = select_node_to_start(nodes)) == 0)
-	goto loop;
-  start:
-      g_err << "Starting " << node->node_id << endl;
-      if (restarter.startNodes(&node->node_id, 1))
-	return NDBT_FAILED;
-      if (restarter.waitNodesStarted(&node->node_id, 1))
-	return NDBT_FAILED;
-      
-      node->node_status = NDB_MGM_NODE_STATUS_STARTED;      
-      break;
-    }
-  }
-
-  Vector<int> not_started;
-  {
-    ndb_mgm_node_state* node = 0;
-    while((node = select_node_to_start(nodes)))
-    {
-      not_started.push_back(node->node_id);
-      node->node_status = NDB_MGM_NODE_STATUS_STARTED;
-    }
-  }
-  
-  if (not_started.size())
-  {
-    g_err << "Starting stopped nodes " << endl;
-    if (restarter.startNodes(not_started.getBase(), not_started.size()))
-      return NDBT_FAILED;
-    if (restarter.waitClusterStarted())
-      return NDBT_FAILED;
+    abort();
+    return NDBT_FAILED;
   }
 
   ctx->stopTest();
   return NDBT_OK;
 }
 
-int runDropBank(NDBT_Context* ctx, NDBT_Step* step){
+int 
+runDropBank(NDBT_Context* ctx, NDBT_Step* step){
   Bank bank(ctx->m_cluster_connection);
   if (bank.dropBank() != NDBT_OK)
     return NDBT_FAILED;
@@ -367,7 +209,7 @@ TESTCASE("SR",
 	 "3.  Restart ndb and verify consistency\n"
 	 "4.  Drop bank\n")
 {
-  TC_PROPERTY("Type", AA_RestartCluster);
+  TC_PROPERTY("Type", NdbMixRestarter::RTM_SR);
   INITIALIZER(runCreateBank);
   STEP(runBankTimer);
   STEP(runBankTransactions);
@@ -381,6 +223,7 @@ TESTCASE("SR",
   STEP(runBankTransactions);
   STEP(runBankTransactions);
   STEP(runBankGL);
+  STEP(runBankSrValidator);
   STEP(runMixRestart);
 }
 TESTCASE("NR", 
@@ -390,7 +233,7 @@ TESTCASE("NR",
 	 "3.  Restart ndb and verify consistency\n"
 	 "4.  Drop bank\n")
 {
-  TC_PROPERTY("Type", AA_RestartNode | AA_StopNode | AA_StartNode);
+  TC_PROPERTY("Type", NdbMixRestarter::RTM_NR);
   INITIALIZER(runCreateBank);
   STEP(runBankTimer);
   STEP(runBankTransactions);
@@ -414,7 +257,7 @@ TESTCASE("Mix",
 	 "3.  Restart ndb and verify consistency\n"
 	 "4.  Drop bank\n")
 {
-  TC_PROPERTY("Type", ~0);
+  TC_PROPERTY("Type", NdbMixRestarter::RTM_ALL);
   INITIALIZER(runCreateBank);
   STEP(runBankTimer);
   STEP(runBankTransactions);
@@ -429,6 +272,7 @@ TESTCASE("Mix",
   STEP(runBankTransactions);
   STEP(runBankGL);
   STEP(runMixRestart);
+  STEP(runBankSrValidator);
   FINALIZER(runDropBank);
 }
 NDBT_TESTSUITE_END(testSRBank);

storage/ndb/test/src/Makefile.am

@@ -24,7 +24,7 @@ libNDBT_a_SOURCES = \
 	NdbRestarter.cpp NdbRestarts.cpp NDBT_Output.cpp \
 	NdbBackup.cpp  NdbConfig.cpp NdbGrep.cpp NDBT_Table.cpp \
 	NdbSchemaCon.cpp NdbSchemaOp.cpp getarg.c \
-	CpcClient.cpp
+	CpcClient.cpp NdbMixRestarter.cpp
 
 INCLUDES_LOC = -I$(top_srcdir)/storage/ndb/src/common/mgmcommon -I$(top_srcdir)/storage/ndb/include/mgmcommon -I$(top_srcdir)/storage/ndb/include/kernel -I$(top_srcdir)/storage/ndb/src/mgmapi
 

storage/ndb/test/src/NDBT_Test.cpp

@@ -146,8 +146,10 @@ void  NDBT_Context::setProperty(const char* _name, Uint32 _val){
   NdbMutex_Lock(propertyMutexPtr);
   const bool b = props.put(_name, _val, true);
   assert(b == true);
+  NdbCondition_Broadcast(propertyCondPtr);
   NdbMutex_Unlock(propertyMutexPtr);
 }
+
 void
 NDBT_Context::decProperty(const char * name){
   NdbMutex_Lock(propertyMutexPtr);
@@ -159,6 +161,7 @@ NDBT_Context::decProperty(const char * name){
   NdbCondition_Broadcast(propertyCondPtr);
   NdbMutex_Unlock(propertyMutexPtr);
 }
+
 void
 NDBT_Context::incProperty(const char * name){
   NdbMutex_Lock(propertyMutexPtr);
@@ -173,6 +176,7 @@ void  NDBT_Context::setProperty(const char* _name, const char* _val){
   NdbMutex_Lock(propertyMutexPtr);
   const bool b = props.put(_name, _val);
   assert(b == true);
+  NdbCondition_Broadcast(propertyCondPtr);
   NdbMutex_Unlock(propertyMutexPtr);
 }
 

storage/ndb/test/src/NdbMixRestarter.cpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "NdbMixRestarter.hpp"
+
+NdbMixRestarter::NdbMixRestarter(const char* _addr) :
+  NdbRestarter(_addr),
+  m_mask(~(Uint32)0)
+{
+}
+
+NdbMixRestarter::~NdbMixRestarter()
+{
+  
+}
+
+#define CHECK(b) if (!(b)) { \
+  ndbout << "ERR: "<< step->getName() \
+         << " failed on line " << __LINE__ << endl; \
+  result = NDBT_FAILED; \
+  continue; } 
+
+int
+NdbMixRestarter::restart_cluster(NDBT_Context* ctx, 
+                                 NDBT_Step* step,
+                                 bool stopabort)
+{
+  int timeout = 180;
+  int result = NDBT_OK;
+
+  do 
+  {
+    ctx->setProperty(NMR_SR_THREADS_STOPPED, (Uint32)0);
+    ctx->setProperty(NMR_SR_VALIDATE_THREADS_DONE, (Uint32)0);
+    
+    ndbout << " -- Shutting down " << endl;
+    ctx->setProperty(NMR_SR, NdbMixRestarter::SR_STOPPING);
+    CHECK(restartAll(false, true, stopabort) == 0);
+    ctx->setProperty(NMR_SR, NdbMixRestarter::SR_STOPPED);
+    CHECK(waitClusterNoStart(timeout) == 0);
+    
+    Uint32 cnt = ctx->getProperty(NMR_SR_THREADS);
+    Uint32 curr= ctx->getProperty(NMR_SR_THREADS_STOPPED);
+    while(curr != cnt && !ctx->isTestStopped())
+    {
+      if (curr > cnt)
+      {
+        ndbout_c("stopping: curr: %d cnt: %d", curr, cnt);
+        abort();
+      }
+      
+      NdbSleep_MilliSleep(100);
+      curr= ctx->getProperty(NMR_SR_THREADS_STOPPED);
+    }
+
+    CHECK(ctx->isTestStopped() == false);
+    CHECK(startAll() == 0);
+    CHECK(waitClusterStarted(timeout) == 0);
+    
+    cnt = ctx->getProperty(NMR_SR_VALIDATE_THREADS);
+    if (cnt)
+    {
+      ndbout << " -- Validating starts " << endl;
+      ctx->setProperty(NMR_SR_VALIDATE_THREADS_DONE, (Uint32)0);
+      ctx->setProperty(NMR_SR, NdbMixRestarter::SR_VALIDATING);
+      curr = ctx->getProperty(NMR_SR_VALIDATE_THREADS_DONE);
+      while (curr != cnt && !ctx->isTestStopped())
+      {
+        if (curr > cnt)
+        {
+          ndbout_c("validating: curr: %d cnt: %d", curr, cnt);
+          abort();
+        }
+
+        NdbSleep_MilliSleep(100);
+        curr = ctx->getProperty(NMR_SR_VALIDATE_THREADS_DONE);
+      }
+      ndbout << " -- Validating complete " << endl;
+    }
+    CHECK(ctx->isTestStopped() == false);    
+    ctx->setProperty(NMR_SR, NdbMixRestarter::SR_RUNNING);
+
+  } while(0);
+
+  return result;
+}
+
+static
+ndb_mgm_node_state*
+select_node_to_stop(Vector<ndb_mgm_node_state>& nodes)
+{
+  Uint32 i, j;
+  Vector<ndb_mgm_node_state*> alive_nodes;
+  for(i = 0; i<nodes.size(); i++)
+  {
+    ndb_mgm_node_state* node = &nodes[i];
+    if (node->node_status == NDB_MGM_NODE_STATUS_STARTED)
+      alive_nodes.push_back(node);
+  }
+
+  Vector<ndb_mgm_node_state*> victims;
+  // Remove those with one in node group
+  for(i = 0; i<alive_nodes.size(); i++)
+  {
+    int group = alive_nodes[i]->node_group;
+    for(j = 0; j<alive_nodes.size(); j++) 
+    {
+      if (i != j && alive_nodes[j]->node_group == group)
+      {
+	victims.push_back(alive_nodes[i]);
+	break;
+      }
+    }
+  }
+
+  if (victims.size())
+  {
+    int victim = rand() % victims.size();
+    return victims[victim];
+  }
+  else
+  {
+    return 0;
+  }
+}
+
+static
+ndb_mgm_node_state*
+select_node_to_start(Vector<ndb_mgm_node_state>& nodes)
+{
+  Uint32 i;
+  Vector<ndb_mgm_node_state*> victims;
+  for(i = 0; i<nodes.size(); i++)
+  {
+    ndb_mgm_node_state* node = &nodes[i];
+    if (node->node_status == NDB_MGM_NODE_STATUS_NOT_STARTED)
+      victims.push_back(node);
+  }
+
+  if (victims.size())
+  {
+    int victim = rand() % victims.size();
+    return victims[victim];
+  }
+  else
+  {
+    return 0;
+  }
+}
+
+void
+NdbMixRestarter::setRestartTypeMask(Uint32 mask)
+{
+  m_mask = mask;
+}
+
+int
+NdbMixRestarter::runUntilStopped(NDBT_Context* ctx, 
+                                 NDBT_Step* step, 
+                                 Uint32 freq)
+{
+  if (init(ctx, step))
+    return NDBT_FAILED;
+
+  while (!ctx->isTestStopped())
+  {
+    if (dostep(ctx, step))
+      return NDBT_FAILED;
+    NdbSleep_SecSleep(freq);
+  }
+  
+  if (!finish(ctx, step))
+    return NDBT_FAILED;
+  
+  return NDBT_OK;
+}
+
+int
+NdbMixRestarter::runPeriod(NDBT_Context* ctx, 
+                           NDBT_Step* step, 
+                           Uint32 period, Uint32 freq)
+{
+  if (init(ctx, step))
+    return NDBT_FAILED;
+
+  Uint32 stop = time(0) + period;
+  while (!ctx->isTestStopped() && (time(0) < stop))
+  {
+    if (dostep(ctx, step))
+    {
+      return NDBT_FAILED;
+    }
+    NdbSleep_SecSleep(freq);
+  }
+  
+  if (finish(ctx, step))
+  {
+    return NDBT_FAILED;
+  }
+
+  ctx->stopTest();
+  return NDBT_OK;
+}
+
+int
+NdbMixRestarter::init(NDBT_Context* ctx, NDBT_Step* step)
+{
+  waitClusterStarted();
+  m_nodes = ndbNodes;
+  return 0;
+}
+
+int 
+NdbMixRestarter::dostep(NDBT_Context* ctx, NDBT_Step* step)
+{
+  ndb_mgm_node_state* node = 0;
+  int action;
+loop:
+  while(((action = (1 << (rand() % RTM_COUNT))) & m_mask) == 0);
+  switch(action){
+  case RTM_RestartCluster:
+    if (restart_cluster(ctx, step))
+      return NDBT_FAILED;
+    for (Uint32 i = 0; i<m_nodes.size(); i++)
+      m_nodes[i].node_status = NDB_MGM_NODE_STATUS_STARTED;
+    break;
+  case RTM_RestartNode:
+  case RTM_RestartNodeInitial:
+  case RTM_StopNode:
+  case RTM_StopNodeInitial:
+  {
+    if ((node = select_node_to_stop(m_nodes)) == 0)
+      goto loop;
+    
+    if (action == RTM_RestartNode || action == RTM_RestartNodeInitial)
+      ndbout << "Restarting " << node->node_id << endl;
+    else
+      ndbout << "Stopping " << node->node_id << endl;
+    
+    bool initial = 
+      action == RTM_RestartNodeInitial || action == RTM_StopNodeInitial;
+    
+    if (restartOneDbNode(node->node_id, initial, true, true))
+      return NDBT_FAILED;
+      
+    if (waitNodesNoStart(&node->node_id, 1))
+      return NDBT_FAILED;
+    
+    node->node_status = NDB_MGM_NODE_STATUS_NOT_STARTED;
+    
+    if (action == RTM_StopNode || action == RTM_StopNodeInitial)
+      break;
+    else
+      goto start;
+  }
+  case RTM_StartNode:
+    if ((node = select_node_to_start(m_nodes)) == 0)
+      goto loop;
+start:
+    ndbout << "Starting " << node->node_id << endl;
+    if (startNodes(&node->node_id, 1))
+      return NDBT_FAILED;
+    if (waitNodesStarted(&node->node_id, 1))
+      return NDBT_FAILED;
+    
+    node->node_status = NDB_MGM_NODE_STATUS_STARTED;      
+    break;
+  }
+  return NDBT_OK;
+}
+
+int 
+NdbMixRestarter::finish(NDBT_Context* ctx, NDBT_Step* step)
+{
+  Vector<int> not_started;
+  {
+    ndb_mgm_node_state* node = 0;
+    while((node = select_node_to_start(m_nodes)))
+    {
+      not_started.push_back(node->node_id);
+      node->node_status = NDB_MGM_NODE_STATUS_STARTED;
+    }
+  }
+  
+  if (not_started.size())
+  {
+    ndbout << "Starting stopped nodes " << endl;
+    if (startNodes(not_started.getBase(), not_started.size()))
+      return NDBT_FAILED;
+    if (waitClusterStarted())
+      return NDBT_FAILED;
+  }
+  return NDBT_OK;
+}
+
+template class Vector<ndb_mgm_node_state*>;