Update testMaster:

- Fix an inversion in killSecondaryMaster making it kill only primary nodes.
- Add DELAY_SAFETY_MARGIN to make explicit which part of a sleep delay is just here for safety, WRT the "mandatory" part of the delay.
- A disconnected secondary master node must be removed from NM, so its state becomes None.
- Add tests comparing the behaviour of a 2-masters and 3-masters cluster.


git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@1158 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/tests/functional/testMaster.py

@@ -18,6 +18,8 @@ from neo import protocol
 
 neo = NEOCluster([], port_base=20000, master_node_count=3)
 
+DELAY_SAFETY_MARGIN = 5
+
 class MasterTests(unittest.TestCase):
 
     def setUp(self):
@@ -48,10 +50,13 @@ class MasterTests(unittest.TestCase):
         return self._killMaster(primary=True)
 
     def killSecondaryMaster(self, all=False):
-        return self._killMaster(primary=True, all=all)
+        return self._killMaster(primary=False, all=all)
+
+    def getMasterNodeList(self):
+        return self.neoctl.getNodeList(protocol.MASTER_NODE_TYPE)
 
     def getMasterNodeState(self, uuid):
-        node_list = self.neoctl.getNodeList(protocol.MASTER_NODE_TYPE)
+        node_list = self.getMasterNodeList()
         for node_type, address, node_uuid, state in node_list:
             if node_uuid == uuid:
                 break
@@ -65,23 +70,102 @@ class MasterTests(unittest.TestCase):
         self.assertEqual(len(killed_uuid_list), 1)
         uuid = killed_uuid_list[0]
         # Leave some time for the primary master to notice the disconnection.
-        time.sleep(1)
+        time.sleep(DELAY_SAFETY_MARGIN)
         # Check node state has changed.
-        self.assertEqual(self.getMasterNodeState(uuid), protocol.TEMPORARILY_DOWN_STATE)
+        self.assertEqual(self.getMasterNodeState(uuid), None)
 
-    def testStoppingPrimaryMaster(self):
+    def testStoppingPrimaryMasterWithTwoSecondaries(self):
         killed_uuid_list = self.killPrimaryMaster()
         # Test sanity check.
         self.assertEqual(len(killed_uuid_list), 1)
         uuid = killed_uuid_list[0]
-        # Leave some time for the other master to elect a new primary.
-        time.sleep(1)
+        # Leave some time for the disconnection to be noticed by the admin
+        # node.
+        time.sleep(DELAY_SAFETY_MARGIN)
+        # Check the state of the primary we just killed
+        self.assertEqual(self.getMasterNodeState(uuid),
+                         protocol.TEMPORARILY_DOWN_STATE)
+        # Leave some time for the other masters to elect a new primary.
+        time.sleep(10)
         # Check that a primary master arised.
         # (raises if admin node is not connected to a primary master)
         new_uuid = self.neoctl.getPrimaryMaster()
         # Check that the uuid really changed.
         self.assertNotEqual(new_uuid, uuid)
 
+    def testStoppingPrimaryMasterWithOneSecondary(self):
+        # Kill one secondary master.
+        killed_uuid_list = self.killSecondaryMaster()
+        # Test sanity checks.
+        self.assertEqual(len(killed_uuid_list), 1)
+        time.sleep(DELAY_SAFETY_MARGIN)
+        self.assertEqual(self.getMasterNodeState(killed_uuid_list[0]), None)
+        self.assertEqual(len(self.getMasterNodeList()), 2)
+
+        killed_uuid_list = self.killPrimaryMaster()
+        # Test sanity check.
+        self.assertEqual(len(killed_uuid_list), 1)
+        uuid = killed_uuid_list[0]
+        # Leave some time for the disconnection to be noticed by the admin
+        # node.
+        time.sleep(DELAY_SAFETY_MARGIN)
+        # Check the state of the primary we just killed
+        self.assertEqual(self.getMasterNodeState(uuid),
+                         protocol.TEMPORARILY_DOWN_STATE)
+        # Leave some time for the other masters to elect a new primary.
+        time.sleep(10)
+        # Check that a primary master arised.
+        # (raises if admin node is not connected to a primary master)
+        new_uuid = self.neoctl.getPrimaryMaster()
+        # Check that the uuid really changed.
+        self.assertNotEqual(new_uuid, uuid)
+
+    def testMasterSequentialStart(self):
+        # Stop the cluster (so we can start processes manually)
+        neo.stop()
+        master_list = neo.getMasterProcessList()
+        # Test sanity check.
+        self.assertEqual(len(master_list), 3)
+        # Start the admin node.
+        neo.getAdminProcessList()[0].start()
+
+        # Start the first master.
+        first_master = master_list[0]
+        first_master.start()
+        # Leave some time for the master to elect itself without any other
+        # master.
+        time.sleep(60 + DELAY_SAFETY_MARGIN)
+        # Check that the master node we started elected itself.
+        self.assertEqual(self.neoctl.getPrimaryMaster(),
+                         first_master.getUUID())
+        # Check that no other master node is known as running.
+        self.assertEqual(len(self.neoctl.getNodeList(
+            protocol.MASTER_NODE_TYPE)), 1)
+
+        # Start a second master.
+        second_master = master_list[1]
+        second_master.start()
+        # Leave the second master some time to connect to the primary master.
+        time.sleep(DELAY_SAFETY_MARGIN)
+        # Check that the second master is running under his known UUID.
+        self.assertEqual(self.getMasterNodeState(second_master.getUUID()),
+                         protocol.RUNNING_STATE)
+        # Check that the primary master didn't change.
+        self.assertEqual(self.neoctl.getPrimaryMaster(),
+                         first_master.getUUID())
+
+        # Start a third master.
+        third_master = master_list[2]
+        third_master.start()
+        # Leave the third master some time to connect to the primary master.
+        time.sleep(DELAY_SAFETY_MARGIN)
+        # Check that the third master is running under his known UUID.
+        self.assertEqual(self.getMasterNodeState(third_master.getUUID()),
+                         protocol.RUNNING_STATE)
+        # Check that the primary master didn't change.
+        self.assertEqual(self.neoctl.getPrimaryMaster(),
+                         first_master.getUUID())
+
 def test_suite():
     return unittest.makeSuite(MasterTests)