squash!

remove unused parts, fix CameCase format

add buildout.hash.cfg for update-hash

remove unused my_server.py

remove global variables, change calling way of env varibale

use subprocess instead of os.system

rename tests, add comments

fix setUp and tearDown for tests

diable debug

get url from comp partition

software/erp5testnode/testsuite/fluentTest/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-profile]
+filename = instance.cfg.in
+md5sum = 45c4adaae557bed5c25c24f4c4506b3d
+
+[template-fluentd-config]
+filename = templates/fluentd-conf.cnf.in
+md5sum = 88235b3b15cccc615acb6a9f9029684a
+
+[template-caddy-service]
+filename = templates/template-caddy-service.sh.in
+md5sum = 3e09969c479931ca4df6546abbd1a117
+
+[template-caddyfile]
+filename = templates/Caddyfile.in
+md5sum = 8644e98038ab76a8e5a499e22f92660a
+
+[template-caddy-fluentd]
+filename = instance-caddy-fluentd.cfg.in
+md5sum = a022edfb2b08e8376b7f4140f8d87917

software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in

@@ -2,11 +2,10 @@
 parts =
   slapos-test-runner
   directory
-  fluentdConfig
+  fluentd-config
   fluentd-service
   caddy-service
   publish-connection-information
-  runTest-instance
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -28,20 +27,13 @@ home = $${:etc}/home
 ssl = $${:etc}/ssl
 working-dir = $${buildout:directory}/tmp/
 
-[runTest-instance]
-recipe = slapos.recipe.template
-url = ${template-test:output}
-output = $${directory:bin}/test.py
-buildout-directory = $${buildout:directory}
-mode = 0700
-
 #################################
 # fluentd service
 #################################
 
-[fluentdConfig]
+[fluentd-config]
 recipe = slapos.recipe.template:jinja2
-template = ${template-fluentdConfig:output}
+template = ${template-fluentd-config:output}
 rendered = $${directory:etc}/fluentd-conf.cnf
 mode = 0600
 context =
@@ -87,7 +79,7 @@ context =
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-secure_access = http://$${caddy-configuration:local_ip}:4443}
+url = http://$${caddy-configuration:local_ip}:4443
 
 [download-source]
 recipe = slapos.recipe.build:gitclone

software/erp5testnode/testsuite/fluentTest/software.cfg

@@ -4,6 +4,7 @@ extends =
   ../../../../stack/slapos.cfg
   ../../../../software/caddy/software.cfg
   ../../../../software/fluentd/software.cfg
+  buildout.hash.cfg
 
 index = https://pypi.python.org/simple/
 
@@ -14,7 +15,7 @@ parts =
   fluentd
   fluentd-plugin-dev-repository
   instance-profile
-  template-fluentdConfig
+  template-fluentd-config
 
 [setup-develop-egg]
 recipe = zc.recipe.egg:develop
@@ -51,6 +52,7 @@ branch = master
 
 [slapos-repository]
 <= git-clone-repository
+#repository = https://lab.nexedi.com/nexedi/slapos.git
 repository = https://lab.nexedi.com/Sokhoyan/slapos.git
 branch = FluentTest_withFixUtil
 
@@ -69,22 +71,13 @@ erp5.util =
 
 [instance-profile]
 recipe = slapos.recipe.template
-md5sum = 45c4adaae557bed5c25c24f4c4506b3d
-url = ${:_profile_base_location_}/instance.cfg.in
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/instance.cfg
 mode = 0644
 
-[template-test]
+[template-fluentd-config]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/tests/test.py
-#md5sum = ff66d13f73982e8257eb5535cdb541c7
-output = ${buildout:directory}/test.py
-mode = 0644
-
-[template-fluentdConfig]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/templates/fluentd-conf.cnf.in
-md5sum = f37d4d69895680abf40b847633876373
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/fluentd-conf.cnf.in
 mode = 0644
 
@@ -104,22 +97,19 @@ gems =
 
 [template-caddy-service]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/templates/template-caddy-service.sh.in
-md5sum = 3e09969c479931ca4df6546abbd1a117
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template-caddy-service.sh.in
 mode = 0644
 
 [template-caddyfile]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/templates/Caddyfile.in
-md5sum = 8644e98038ab76a8e5a499e22f92660a
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/Caddyfile.in
 mode = 0644
 
 [template-caddy-fluentd]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-caddy-fluentd.cfg.in
-md5sum = ccfc6d0a4dfe0217cce5d369eea5cbd1
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/instance-caddy-fluentd.cfg.in
 mode = 0644
 

software/erp5testnode/testsuite/fluentTest/templates/fluentd-conf.cnf.in

@@ -3,13 +3,6 @@
   port 5438
 </source>
 
-#<source>
-#  @type dummy
-#  tag dummy
-#  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
-#</source>
-
-
 # HTTP input
 # http://localhost:8888/<tag>?json=<json>
 #<source>

software/erp5testnode/testsuite/fluentTest/tests/my_server.py

-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-import SocketServer
-import time
-import threading
-import os
-
-class TestServerHandler(BaseHTTPRequestHandler):
-
-    def _set_headers(self):
-        self.send_response(200)
-        self.send_header('Content-type', 'text/html')
-        self.end_headers()
-
-    def do_GET(self):
-        self._set_headers()
-        self.wfile.write("<html><body><h1>hi!</h1></body></html>")
-
-
-    def do_HEAD(self):
-        self._set_headers()
-
-    def do_POST(self):
-        content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
-        post_data = self.rfile.read(content_length) # <--- Gets the data itself
-
-        self._set_headers()
-        self.wfile.write(post_data)
-
-        global posted_data
-        posted_data = post_data
-        print("post data from do_POST")
-        print(posted_data)
-     #   global all_data
-    #    all_data.append(post_data.split(" ")[1])
-        
-    #    global request_tag
-    #    request_tag = find_tag(self.requestline,"=", " ")
-
-
-def run():
-
-  port=9443
-  #server_address = ('0.0.0.0', port)
-  server_address = ("10.0.46.242", port)
-  server = HTTPServer(server_address, TestServerHandler)
-  thread = threading.Thread(target=server.serve_forever)
-  thread.start()
-  print("start server")
-  
-  time.sleep(60)
-  server.shutdown()
-  server.server_close()
-  print("stop server")
-
-
-if __name__ == "__main__":
-    run()        
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/tests/test.py

-#!${buildout:directory}/bin/${eggs:interpreter}
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-
 import unittest
 import requests
 from StringIO import StringIO
@@ -14,13 +10,10 @@ import os
 import time
 import utils
 import threading
+import subprocess
 
 test_msg = "dummyInputSimpleIngest"
-url = "http://" + os.environ.get('LOCAL_IPV4') + ":4443"
 caddy_pidfile = os.environ.get('CADDY_DIR')
-posted_data = None
-all_data = []
-request_tag = ""
 
 if os.environ.get('DEBUG'):
   import logging
@@ -35,6 +28,10 @@ class FluentdPluginTestCase(utils.SlapOSInstanceTestCase):
 
 class TestServerHandler(BaseHTTPRequestHandler):
   
+    posted_data = None
+    all_data = []
+    request_tag = ""
+
     def _set_headers(self):
         self.send_response(200)
         self.send_header('Content-type', 'text/html')
@@ -54,25 +51,21 @@ class TestServerHandler(BaseHTTPRequestHandler):
         self._set_headers()
         self.wfile.write(post_data)
 
-        global posted_data
-        posted_data = post_data
+        TestServerHandler.posted_data = post_data.split(" ")[1]
 
-        global all_data
-        all_data.append(post_data.split(" ")[1])
+        TestServerHandler.all_data.append(post_data.split(" ")[1])
         
-        global request_tag
-        request_tag = find_tag(self.requestline,"=", " ")
+        TestServerHandler.request_tag = find_tag(self.requestline,"=", " ")
 
 class TestIngestion(FluentdPluginTestCase):
     
     @classmethod
     def startServer(cls):
       port=9443
-      server_address = (os.environ.get('LOCAL_IPV4'), port)
+      server_address = (os.environ['LOCAL_IPV4'], port)
       cls.server = HTTPServer(server_address, TestServerHandler)
       cls.thread = threading.Thread(target=cls.server.serve_forever)
       cls.thread.start()
-      time.sleep(10)
       print("server start")
       
     @classmethod
@@ -80,98 +73,118 @@ class TestIngestion(FluentdPluginTestCase):
       cls.server.shutdown()
       cls.server.server_close()
       print("serever shutdown")
-      time.sleep(10)
-
-    def test_1_get(self):
     
+    def setUp(self):
       self.startServer()
-      time.sleep(10)
 
+    def tearDown(self):
+      self.stopServer()
+      
+    def test_get_request_responds_with_200(self):
+      '''
+        simple get request to be sure that server is up
+      '''
       print("############## TEST 1 ##############")
+      url  = self.computer_partition.getConnectionParameterDict()['url']
       resp = requests.get(url)
       self.assertEqual(resp.status_code, 200)
       print (resp.status_code)
 
-    def test_2_ingest(self):
+    def test_simple_ingest(self):
       
-      time.sleep(10)
       print("############## TEST 2 ##############")
-      start_fluentd_cat(test_msg, "tag_test_2")
+      start_fluentd_cat(self, test_msg, "tag_test_2")
       time.sleep(10)
+      self.assertEqual(test_msg, TestServerHandler.posted_data)
       
-      if posted_data:
-        self.assertEqual(test_msg, posted_data.split(" ")[1])
-      else:
-        self.assertEqual(test_msg, posted_data)
-      time.sleep(10)
-      
-    def test_3_keepAlive_on(self):
+    def test_keepAlive_on(self):
       print("############## TEST 3 ##############")
       s = requests.session()
       print("check connection type ")
       print(s.headers['Connection'])
       self.assertEqual('keep-alive', s.headers['Connection'])
 
-    def test_4_delay_15_mins(self):
+    def test_ingest_with_15mins_delay(self):
+      '''
+        sleep 15mins to test that connections doesn't break after long delay
+        and data is ingested correctly after the delay.
+      '''
       print("############## TEST 4 ##############")
-      # sleep 15mins to test that connections doesn't break after long delay
-      # and data is ingested correctly after the delay.
+      
       time.sleep(900)
-      start_fluentd_cat("dummyInputDelay", "tag_test_4")
+      start_fluentd_cat(self, "dummyInputDelay", "tag_test_4")
       time.sleep(15)
-      self.assertEqual("dummyInputDelay", posted_data.split(" ")[1])
-
-    def test_5_caddy_restart(self):
+      self.assertEqual("dummyInputDelay", TestServerHandler.posted_data)
+
+    def test_ingest_while_server_breakage(self):
+      '''
+         stop and then start caddy again to check that
+         fluentd plugin keeps message in a local buffer
+         and correctly sends them when caddy is back online
+      '''
       print("############## TEST 5 ##############")
 
       with open(caddy_pidfile) as f:
         caddy_pid = f.readline()
 
-      time.sleep(10)
-      start_fluentd_cat("dummyInputCaddyRestart1", "tag_test_5_1")
+      start_fluentd_cat(self, "dummyInputCaddyRestart1", "tag_test_5_1")
       time.sleep(10)
 
       kill_caddy(caddy_pid)
       time.sleep(10)
 
-      start_fluentd_cat("dummyInputCaddyRestart2 ", "tag_test_5_2")
+      start_fluentd_cat(self, "dummyInputCaddyRestart2 ", "tag_test_5_2")
       time.sleep(10)
-      start_fluentd_cat("dummyInputCaddyRestart3 ", "tag_test_5_3")
+      start_fluentd_cat(self, "dummyInputCaddyRestart3 ", "tag_test_5_3")
       time.sleep(10)
-      start_fluentd_cat("dummyInputCaddyRestart4 ", "tag_test_5_4")
+      start_fluentd_cat(self, "dummyInputCaddyRestart4 ", "tag_test_5_4")
       time.sleep(130)
 
       start_caddy(caddy_pid)
       time.sleep(15)
 
-      self.assertTrue("dummyInputCaddyRestart1" in all_data)
-      self.assertTrue("dummyInputCaddyRestart2" in all_data)
-      self.assertTrue("dummyInputCaddyRestart3" in all_data)
-      self.assertTrue("dummyInputCaddyRestart4" in all_data)
+      self.assertTrue("dummyInputCaddyRestart1" in TestServerHandler.all_data)
+      self.assertTrue("dummyInputCaddyRestart2" in TestServerHandler.all_data)
+      self.assertTrue("dummyInputCaddyRestart3" in TestServerHandler.all_data)
+      self.assertTrue("dummyInputCaddyRestart4" in TestServerHandler.all_data)
 
-    def test_6_check_diff_tags(self):
+    def test_ingest_with_diff_tags(self):
+      '''
+        ingest data with different tags
+      '''
       print("############## TEST 6 ##############")
       
-      start_fluentd_cat("dummyInputTags_6_1", "tag_Test_6_1")
+      start_fluentd_cat(self, "dummyInputTags_6_1", "tag_Test_6_1")
       time.sleep(10)
-      self.assertEqual("tag_Test_6_1", request_tag)
-      
-      start_fluentd_cat("dummyInputTags_6_2", "tag_Test_6_2")
-      time.sleep(2)
-      self.assertEqual("tag_Test_6_2", request_tag)
+      self.assertEqual("tag_Test_6_1", TestServerHandler.request_tag)
       
-      start_fluentd_cat("dummyInputTags_6_3", "tag_Test_6_3")
-      time.sleep(2)
-      self.assertEqual("tag_Test_6_3", request_tag)
-      self.stopServer()
+      start_fluentd_cat(self, "dummyInputTags_6_2", "tag_Test_6_2")
       time.sleep(10)
+      self.assertEqual("tag_Test_6_2", TestServerHandler.request_tag)
       
-def start_fluentd_cat(test_msg, tag):
+      start_fluentd_cat(self, "dummyInputTags_6_3", "tag_Test_6_3")
+      time.sleep(10)
+      self.assertEqual("tag_Test_6_3", TestServerHandler.request_tag)
       
+def start_fluentd_cat(self, test_msg, tag):
+  """Feeds `test_msg` with `tag` to fluentd.
+  """ 
   fluent_service = os.environ.get('FLUENT_SERVICE')
-    os.environ["GEM_PATH"] = fluent_service + "/lib/ruby/gems/1.8/"
-    fluentd_cat_exec_comand = fluent_service + '/bin/fluent-cat --none ' + tag + " -p 5438 "
-    os.system("echo + " + test_msg + " | " + fluentd_cat_exec_comand)
+  proc = subprocess.Popen(
+    [fluent_service + '/bin/fluent-cat',
+    '--none',
+     tag,
+    '-p',
+    '5438',
+    ],
+    stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    env={"GEM_PATH": fluent_service + "/lib/ruby/gems/1.8/" })
+  stdout, stderr = proc.communicate("+ " + test_msg)
+  self.assertEqual(0, proc.wait())
+  self.assertEqual('', stdout)
+  self.assertEqual('', stderr)
 
 def kill_caddy(caddy_pid):