monitor: Implement new features and moniorings

  * Copy Re6st logs if exist.
  * Include smartmontools, lm-sensors, rrdtools
  * Implement smartmontools logging
  * Implement network benchmark monitoring

component/lmsensors/buildout.cfg

+
+[buildout]
+
+extends = 
+  ../bison/buildout.cfg
+  ../rrdtools/buildout.cfg
+  ../flex/buildout.cfg
+
+parts =
+  lmsensors
+
+[lmsensors]
+recipe = slapos.recipe.cmmi
+md5sum = da506dedceb41822e64865f6ba34828a
+url = http://dl.lm-sensors.org/lm-sensors/releases/lm_sensors-3.3.5.tar.bz2
+configure-command = true
+make-options =
+  PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+  ETCDIR=${buildout:parts-directory}/${:_buildout_section_name_}/etc
+
+environment = 
+  PATH=${bison:location}/bin:${flex:location}/bin:%(PATH)s
+

component/rrdtools/buildout.cfg

+
+[buildout]
+
+parts =
+  rrdtools
+
+[rrdtools]
+recipe = slapos.recipe.cmmi
+md5sum = dbe59386db97fd2f2216729facd74ca8
+url = http://oss.oetiker.ch/rrdtool/pub/rrdtool-1.4.8.tar.gz

component/smartmontools/buildout.cfg

+
+[buildout]
+
+parts =
+  smartmontools
+
+[smartmontools]
+recipe = slapos.recipe.cmmi
+md5sum = 2ea0c62206e110192a97b59291b17f54 
+url = http://freefr.dl.sourceforge.net/project/smartmontools/smartmontools/6.3/smartmontools-6.3.tar.gz 
+
+configure-option = 
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

software/monitor/instance.cfg → software/monitor/instance.cfg.in

@@ -19,6 +19,8 @@ parts =
   monitor-promise
   monitor-instance-log-access
   cron-rsync-logs
+  cron-smart-monitor-logs
+  cron-network-bench
   
 extends = ${monitor-template:output}
 
@@ -34,6 +36,20 @@ name = rsync-logs
 frequency = * * * * *
 command = $${rsync-logs:rendered}
 
+[cron-smart-monitor-logs]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = smartctl-mon
+frequency = 11 * * * *
+command = $${smartmon-logs:rendered}
+
+[cron-network-bench]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = network-bench-test
+frequency = */3 * * * *
+command = ${buildout:bin-directory}/networkbench $${network-bench-configuration:rendered} $${monitor-directory:monitor-log}
+
 [rsync-logs]
 recipe = slapos.recipe.template:jinja2
 template = ${rsync-logs-script:output}
@@ -43,6 +59,27 @@ context =
   key buildout_directory buildout:directory
   raw rsync_binary ${rsync:location}/bin/rsync
   key server_log_directory monitor-directory:server-log 
+  raw re6st_log_directory /var/log/re6stnet
+  key system_log_directory monitor-directory:system-log 
+
+[smartmon-logs]
+recipe = slapos.recipe.template:jinja2
+template = ${smartmon-logs-script:output}
+rendered = $${monitor-directory:bin}/smartmon-logs.py
+mode = 0744
+context =
+  key buildout_directory buildout:directory
+  raw smartctl_binary ${smartmontools:location}/sbin/smartctl
+  key monitor_log_directory monitor-directory:monitor-log 
+  raw python_executable ${buildout:executable}
+
+[network-bench-configuration]
+recipe = slapos.recipe.template:jinja2
+template = ${network-bench-cfg:output}
+rendered = $${monitor-directory:etc}/network_bench.cfg
+mode = 0744
+context =
+  
 
 [pwgen]
 recipe = slapos.cookbook:generate.password
@@ -54,11 +91,12 @@ recipe = slapos.cookbook:generate.password
 user = admin
 bytes = 16
 
-
 [monitor-directory]
 server-log = $${:private-directory}/server-log
+monitor-log = $${:private-directory}/monitor-log
 cache = $${:var}/cache
 mod-ssl = $${:cache}/httpd_mod_ssl
+system-log = $${:private-directory}/system-log 
 
 [slap-parameter]
 private-hash = $${pwgen:passwd}$${pwgen32:passwd}

software/monitor/network_bench.cfg

+[network_bench]
+
+url =
+    http://www.tiolive.com/robots.txt
+    http://www.erp5.com/robots.txt
+    http://www.erp5.cn/robots.txt
+    http://www.nexedi.cn/robots.txt
+    http://www.osoe-project.cn/robots.txt
+    http://www.nexedi.cn/robots.txt
+    http://www.osoe-project.org/robots.txt
+    http://www.osoe-project.cn/robots.txt
+    http://www.google.com/robots.txt
+    http://ipv6.google.com/robots.txt
+
+    https://[2001:67c:1254:e:a7::2e20]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:69::115c]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:8a::2621]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:e:c7::89ae]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:e:93::2e31]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:1f::2723]:9685/monitor-public/rssfeed.html
+    https://[2001:41d0:1:a78b::3c2a]:9685/monitor-public/rssfeed.html
+    http://[2001:67c:1254:4::1]/index.html
+
+dns =
+    erp5.com
+    www.erp5.com
+    erp5.cn
+    www.erp5.cn
+    nexedi.cn
+    www.nexedi.com
+    www.nexedi.cn
+    www.osoe-project.org
+    www.osoe-project.cn
+    free.fr
+    google.com
+
+ping =
+    erp5.com
+    erp5.cn
+    google.com
+    free.fr
+    frontend1.nexedi.com
+    frontend2.nexedi.com
+    frontend3.nexedi.cn
+    42.192.4.15
+    195.208.185.22

software/monitor/rsync-logs.sh.in

@@ -2,6 +2,13 @@
 
 RSYNC_BIN={{ rsync_binary }}
 SLAPGRID_LOGS={{ buildout_directory }}/../var/data-log/
+RE6STLOGS={{ re6st_log_directory }}
+
+SYSTEMLOGS={{ system_log_directory }}
 SERVERLOGS={{ server_log_directory }}
 
 $RSYNC_BIN -a --delete $SLAPGRID_LOGS $SERVERLOGS
+
+if [ -d $RE6STLOGS ]; then
+  $RSYNC_BIN -a --delete $RE6STLOGS $YSTEMLOGS
+fi

software/monitor/smartmon.py.in

+#!{{ python_executable }}
+
+SMARCTBIN={{ smartctl_binary }}
+LOGFOLDER={{ monitor_log_directory }}
+
+import subprocess
+import logging
+import os
+
+logger = logging.getLogger("SMARTMON")
+logger.setLevel(logging.DEBUG)
+
+ch = logging.FileHandler(os.path.join(LOGFOLDER, "smartmontools.log"))
+ch.setLevel(logging.DEBUG)
+
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+def _call(cmd):
+  p = subprocess.Popen(
+     cmd,
+     stdout=subprocess.PIPE,
+     stdin=subprocess.PIPE,
+     shell=True)
+
+  return p.communicate()
+
+def get_disk_list():
+  output, err = _call("%s --scan" % SMARCTBIN)
+  disk_list = []
+  for line in output.split("\n"):
+    if line != '':
+      disk_path = line.split(" ")[0]
+      if disk_path.startswith("/dev/"):
+        disk_list.append(disk_path)
+
+  return disk_list
+
+logger.debug("Starting new Data collection.")
+
+disk_list = get_disk_list()
+
+logger.debug("Identified disks: %s" % disk_list)
+
+for disk in disk_list:
+  logger.info("Starting test for %s" % disk)
+  output, err = _call("%s --all %s" % (SMARCTBIN, disk))
+  for l in output.split("\n"):
+    logger.info("%s - %s" % (disk, l))
+  if err is not None:
+    logger.warning(err)
+
+logger.debug("Stopped to Collect Data.")

software/monitor/software.cfg

@@ -4,18 +4,23 @@ extends =
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
   ../../component/rsync/buildout.cfg
+  ../../component/smartmontools/buildout.cfg
+  ../../component/lmsensors/buildout.cfg
 
 parts = 
   slapos-cookbook
+  slapos-toolbox
   template
   eggs
   rsync
+  smartmontools
+  lmsensors
   
 [template]
 recipe = slapos.recipe.template 
-url = ${:_profile_base_location_}/instance.cfg
+url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-md5sum = 6ad84a0061df0c00a9f41d8b302adc45
+md5sum = 184a27424b48514cfe9374e881e1cc0c 
 mode = 0644
 
 # stupify index for now
@@ -30,10 +35,34 @@ md5sum = 1f30b17f5ea11fd033984d17c38bcfd3
 [rsync-logs-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/rsync-logs.sh.in
-md5sum = 33900b1f30f9d1de19814d6d3bacb03b
+md5sum = f2766c6eecd0b90a5dc2bd9047d17319 
 output = ${buildout:directory}/template-rsync-logs.sh.in
 mode = 0644
 
+[smartmon-logs-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/smartmon.py.in
+md5sum = a3ea2604abf53067af1b952b278fba94 
+output = ${buildout:directory}/template-smartmon-logs.py.in
+mode = 0644
+
+[network-bench-cfg]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/network_bench.cfg
+md5sum  = 44f8db5fca5e16ae87f570623a3ad0b4 
+output = ${buildout:directory}/template-network-bench-cfg.in
+mode = 0644
+
+[slapos-toolbox]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.toolbox
+
+scripts =
+  networkbench
+  onetimedownload
+
 [eggs]
 recipe = z3c.recipe.scripts
 eggs = 
@@ -49,3 +78,26 @@ cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.5
 z3c.recipe.scripts = 1.0.1
+
+apache-libcloud = 0.16.0
+ecdsa = 0.11
+gitdb = 0.6.0
+pycrypto = 2.6.1
+slapos.toolbox = 0.45.1
+smmap = 0.8.3
+
+# Required by:
+# slapos.toolbox==0.45.1
+GitPython = 0.3.2.1
+
+# Required by:
+# slapos.toolbox==0.45.1
+atomize = 0.2.0
+
+# Required by:
+# slapos.toolbox==0.45.1
+feedparser = 5.1.3
+
+# Required by:
+# slapos.toolbox==0.45.1
+paramiko = 1.15.1