promise/plugin: add check_network_transit promise

slapos/promise/plugin/check_network_transit.py

+import json
+import os
+import psutil
+import time
+
+from psutil._common import bytes2human
+from .util import get_data_interval_json_log
+from .util import JSONRunPromise
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+
+@implementer(interface.IPromise)
+class RunPromise(JSONRunPromise):
+
+  def __init__(self, config):
+
+    super(RunPromise, self).__init__(config)
+
+    self.setPeriodicity(minute=1)
+    self.last_transit_file = self.getConfig('last-transit-file', 'last_transit')
+
+  def sense(self):
+
+    promise_success = True
+    
+    # Get reference values
+    min_threshold_recv = float(self.getConfig('min-threshold-recv', 1e2)) # ≈100 bytes
+    min_threshold_sent = float(self.getConfig('min-threshold-sent', 1e2)) # ≈100 bytes
+    transit_period_sec = int(self.getConfig('transit-period-sec', 0)) # For test
+    if transit_period_sec:
+      transit_period = transit_period_sec
+    else:
+      transit_period = 60*int(self.getConfig('transit-period-minutes', 5)) # 5 min
+    
+    # Get current network statistics, see https://psutil.readthedocs.io/en/latest/#network
+    network_data = psutil.net_io_counters(nowrap=True)
+
+    # Log recv and sent bytes
+    data = json.dumps({'bytes_recv': network_data.bytes_recv, 
+    'bytes_sent': network_data.bytes_sent})
+    self.json_logger.info("Network data", extra={'data': data})
+
+    # Get last timestamp (i.e. last modification) of log file
+    try:
+      t = os.path.getmtime(self.last_transit_file)
+    except OSError:
+      t = 0
+    # Get total bytes recv/sent since transit_period
+    if (time.time() - t) > transit_period:
+      open(self.last_transit_file, 'w').close()
+      temp_list = get_data_interval_json_log(self.log_file, transit_period)
+      if temp_list:
+        if len(temp_list) == 1: # If no previous data in log
+          pass
+        else: 
+          total_recv = temp_list[0]['bytes_recv'] - temp_list[-1]['bytes_recv']
+          total_sent = temp_list[0]['bytes_sent'] - temp_list[-1]['bytes_sent']
+          if total_recv <= min_threshold_recv:
+            self.logger.error("Network congested, received bytes over the last %s seconds "\
+              "reached minimum threshold: %7s (threshold is %7s)" 
+              % (transit_period, bytes2human(total_recv), bytes2human(min_threshold_recv)))
+            promise_success = False
+          if total_sent <= min_threshold_sent:
+            self.logger.error("Network congested, sent bytes over the last %s seconds "\
+              "reached minimum threshold: %7s (threshold is %7s)" 
+              % (transit_period, bytes2human(total_sent), bytes2human(min_threshold_sent)))
+            promise_success = False
+      else:
+        self.logger.error("Couldn't read network data from log")
+        promise_success = False
+
+    if promise_success:
+      self.logger.info("Network transit OK")
+
+  def test(self):
+    """
+    Called after sense() if the instance is still converging.
+    Returns success or failure based on sense results.
+
+    In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+    Called after sense() if the instance has finished converging.
+    Returns success or failure based on sense results.
+    Failure signals the instance has diverged.
+
+    In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

slapos/test/promise/plugin/test_check_network_transit.py

+# -*- coding: utf-8 -*-
+##############################################################################
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+
+import mock
+import os
+import time
+
+from collections import namedtuple
+from slapos.grid.promise import PromiseError
+from slapos.promise.plugin.check_network_transit import RunPromise
+from . import TestPromisePluginMixin
+
+
+class TestCheckNetworkTransit(TestPromisePluginMixin):
+
+  promise_name = "monitor-network-transit.py"
+
+  def setUp(self):
+    super(TestCheckNetworkTransit, self).setUp()
+    self.network_data = namedtuple('network_data', ['bytes_recv', 'bytes_sent'])
+
+  def writePromise(self, **kw):
+    super(TestCheckNetworkTransit, self).writePromise(self.promise_name,
+      "from %s import %s\nextra_config_dict = %r\n"
+      % (RunPromise.__module__, RunPromise.__name__, kw))
+
+  def runPromise(self, summary=None, failed=False):
+    self.configureLauncher(enable_anomaly=True, force=True)
+    with mock.patch('psutil.net_io_counters', return_value=summary):
+      if failed:
+        self.assertRaises(PromiseError, self.launcher.run)
+      else:
+        self.launcher.run()
+    result = self.getPromiseResult(self.promise_name)['result']
+    self.assertEqual(result['failed'], failed)
+    return result['message']
+
+  def test_network_transit_ok(self):
+    message = "Network transit OK"
+    mock_stats = {'bytes_recv':1e3, 'bytes_sent':1e3}
+    self.writePromise(**{
+        'last_transit_file':'last_transit_file',
+        'min-threshold-recv': 500, # ≈500B
+        'min-threshold-sent': 500, # ≈500B
+        'transit-period-sec': 1,
+    })
+    self.runPromise(self.network_data(**{'bytes_recv':300, 'bytes_sent':300}))
+    time.sleep(1)
+    self.assertEqual(message, self.runPromise(self.network_data(**mock_stats)))
+
+  def test_network_upload_nok(self):
+    message = "Network congested, sent bytes over the last 1 seconds"\
+      " reached minimum threshold:  100.0B (threshold is  500.0B)"
+    mock_stats = {'bytes_recv':1e3, 'bytes_sent':400}
+    self.writePromise(**{
+        'last_transit_file':'last_transit_file',
+        'min-threshold-recv': 500, # ≈500B
+        'min-threshold-sent': 500, # ≈500B
+        'transit-period-sec': 1,
+    })
+    self.runPromise(self.network_data(**{'bytes_recv':300, 'bytes_sent':300}))
+    time.sleep(1)
+    self.assertEqual(message, self.runPromise(self.network_data(**mock_stats)))
+
+  def test_network_download_nok(self):
+    message = "Network congested, received bytes over the last 1 seconds"\
+      " reached minimum threshold:  100.0B (threshold is  500.0B)"
+    mock_stats = {'bytes_recv':400, 'bytes_sent':1e3}
+    self.writePromise(**{
+        'last_transit_file':'last_transit_file',
+        'min-threshold-recv': 500, # ≈500B
+        'min-threshold-sent': 500, # ≈500B
+        'transit-period-sec': 1,
+    })
+    self.runPromise(self.network_data(**{'bytes_recv':300, 'bytes_sent':300}))
+    time.sleep(1)
+    self.assertEqual(message, self.runPromise(self.network_data(**mock_stats)))
+
+  def test_network_transit_nok(self):
+    message = "Network congested, received bytes over the last 1 seconds"\
+      " reached minimum threshold:  100.0B (threshold is  500.0B)\n"\
+      "Network congested, sent bytes over the last 1 seconds"\
+      " reached minimum threshold:  100.0B (threshold is  500.0B)"
+    mock_stats = {'bytes_recv':400, 'bytes_sent':400}
+    self.writePromise(**{
+        'last_transit_file':'last_transit_file',
+        'min-threshold-recv': 500, # ≈500B
+        'min-threshold-sent': 500, # ≈500B
+        'transit-period-sec': 1,
+    })
+    self.runPromise(self.network_data(**{'bytes_recv':300, 'bytes_sent':300}))
+    time.sleep(1)
+    self.assertEqual(message, self.runPromise(self.network_data(**mock_stats)))
+
+if __name__ == '__main__':
+  unittest.main()