agent: Refactor to follow up new implementation using ERP5 Task Distributor

slapos/recipe/agent/__init__.py

-##############################################################################
-#
-# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-#############################################################################
-
-import zc.buildout
-from slapos.recipe.librecipe import GenericBaseRecipe
-import sys
-
-class Recipe(GenericBaseRecipe):
-  def install(self):
-    path_list = []
-
-    configuration_path = self.options["config"]
-    header = """[DEFAULT]
-master_url = %s
-key = %s
-
-cert = %s
-
-max_install_duration = %s
-max_uninstall_duration = %s
-max_request_duration = %s
-max_destroy_duration = %s
-""" % (self.options["master-url"],
-      "\n  ".join(self.options["key"].split("\n")),
-       "\n  ".join(self.options["cert"].split("\n")),
-       self.options["default_max_install_duration"],
-       self.options["default_max_uninstall_duration"],
-       self.options["default_max_request_duration"],
-       self.options["default_max_destroy_duration"])
-
-    with open(configuration_path, "w") as configuration:
-      configuration.write(header + self.options["configuration"])
-
-    path_list.append(self.createPythonScript(
-                       self.options['wrapper'],
-                       'slapos.recipe.librecipe.execute.execute',
-                         [self.options["agent_binary"], '--pidfile=%s' % self.options["pidfile"],
-                          "--log=%s" % self.options["log"], configuration_path]))
-
-    path_list.append(configuration_path)
-
-    return path_list

slapos/recipe/agent/catdatefile.py

-import os
-import sys
-import time
-def catdatefile(args):
-  directory = args[0]
-  try:
-    suffix = args[1]
-  except IndexError:
-    suffix = '.log'
-  f = open(os.path.join(directory,
-    time.strftime('%Y-%m-%d.%H:%M.%s') + suffix), 'aw')
-  for line in sys.stdin.read():
-    f.write(line)
-  f.close()

software/agent/README.testing_agent.txt

-testing agent
-=============
-
-Testing agent simulates a normal user interacting with vifib master. It requests software release installation or software instance instiaction randomly from time to time. SlapOS clients then run these commands remotely. It is the testing agent determining whether an error occurs.
-
-Usage
-=====
-Request a testing agent instance from vifib with following parameters:
-<?xml version='1.0' encoding='utf-8'?>
-<instance>
-<parameter id="configuration">[DEFAULT]
-# ConfigParser's magic section.
-computer_list = ["COMP-607"]
-master_url = https://slap.vifib.com/
-# Note that certificates are now literally in the configuration, meaning
-# you may decide to specify different ones for each test. Likewise for
-# master_url.
-key = -----BEGIN PRIVATE KEY-----
-  MII[...]
-  [...]
-  -----END PRIVATE KEY-----
-cert = -----BEGIN CERTIFICATE-----
-  MII[...]
-  [...]
-  -----END CERTIFICATE-----
-
-[agent]
-# This section is special: it contains configuration.
-# Does not make use of values coming from [DEFAULT] (well, it
-# necessarily contains them, but they are not used).
-node_title = ...
-test_title = ...
-project_title = ...
-task_count = 2 # Number of tests to run concurrently
-report_url = # report_url, find details in erp5 for details
-
-# All other sections are individual tests, whatever they are named.
-[test-apache]
-# Software release URL
-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-# Optional request_kw parameters: if not provided, will only test SR
-# build. Depending on your slap version, it may be required to provide
-# a "software_type" parameter, even if you want the default type.
-request_kw = {
-    "filter_kw": {"computer_guid": "..."},
-    "partition_parameter_kw": {
-      "domain": "example.com"
-    }
-  }
-# All are in seconds.
-max_install_duration = 3000
-max_uninstall_duration = 360
-max_request_duration = 700
-max_destroy_duration = 360
-
-[impossible-apache]
-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-max_install_duration = 1
-max_uninstall_duration = 1
-
-[impossible-apache-2]
-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-max_install_duration = 660
-max_uninstall_duration = 1
-</parameter>
-</instance>
-

software/agent/agent.cfg.in

+[agent]
+timeout = ${slap-parameter:timeout} 
+node_title = ${slap-parameter:node_title} 
+project_title = ${slap-parameter:project_title} 
+task_count = ${slap-parameter:task_count} 
+report_url = ${slap-parameter:report_url} 
+
+working_directory = $${directory:testnode}

software/agent/instance-agent.cfg

@@ -6,28 +6,37 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-
 [directory]
 recipe = slapos.cookbook:mkdirectory
 etc = $${buildout:directory}/etc
-run = $${:etc}/run
+service = $${buildout:directory}/etc/service
+run = $${buildout:directory}/etc/run
 agentlog = $${buildout:directory}/var/log/agent
 srv = $${buildout:directory}/srv
 bin = $${buildout:directory}/bin
-
+testnode = $${buildout:directory}/srv/testnode
 
 [instance]
-recipe = slapos.cookbook:agent
-agent_binary = ${buildout:directory}/bin/agent
-pidfile = $${directory:srv}/agent.pid
-log = $${directory:agentlog}/agent.log
-wrapper = $${directory:run}/agent
-config =  $${directory:etc}/agent.cfg
-master-url = $${slap-parameter:master-url}
-key = $${slap-parameter:userkey}
-cert = $${slap-parameter:usercertificate}
-configuration = $${slap-parameter:configuration}
-default_max_install_duration =  $${slap-parameter:default_max_install_duration}
-default_max_uninstall_duration = $${slap-parameter:default_max_uninstall_duration}
-default_max_request_duration = $${slap-parameter:default_max_request_duration}
-default_max_destroy_duration = $${slap-parameter:default_max_destroy_duration}
+recipe = slapos.cookbook:wrapper
+command-line =
+  ${buildout:bin-directory}/agent
+  --pidfile=$${buildout:directory}/srv/agent.pid
+  --log=$${buildout:directory}/var/log/agent/agent.log
+  $${agent-cfg:output}
+
+wrapper-path = $${directory:service}/agent
+output = $${:wrapper-path}
+
+[agent-cfg]
+recipe = slapos.recipe.template
+url = ${agent.cfg.in:output} 
+output = $${directory:etc}/agent.cfg
+
+[slap-parameter]
+timeout = 3600 
+
+node_title = 
+test_title = 
+project_title = 
+task_count = 1 
+report_url = 

software/agent/software.cfg

@@ -4,41 +4,13 @@ extends =
   ../../component/git/buildout.cfg
   ../../stack/slapos.cfg
 
-develop =
-  ${:parts-directory}/slapos.cookbook-repository
-  ${:parts-directory}/slapos.toolbox-repository
-
 parts =
+  agent.cfg.in
   template
   template-agent
-  slapos.cookbook-repository
-  slapos.toolbox-repository
-  check-recipe
   slapos-cookbook
   script
 
-# Local development
-[slapos.cookbook-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.git
-branch = agent
-git-executable = ${git:location}/bin/git
-
-[slapos.toolbox-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
-branch = agent3
-git-executable = ${git:location}/bin/git
-
-
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command =
-  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link;
-  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
@@ -47,10 +19,16 @@ md5sum = bcd3b3cb8a305c83bb048d5ac1c583fe
 mode = 0644
 
 [template-agent]
-recipe = slapos.recipe.template
+recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/instance-agent.cfg
 output = ${buildout:directory}/template-agent.cfg
-md5sum = 7c5c43eb98d5a11961d72fce97a8e67b
+md5sum = dbdd2bf2062c842937bd125b0ac494da
+mode = 0644
+
+[agent.cfg.in]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/agent.cfg.in
+md5sum = e6b92e07e3709e4ab6c59e2e3d3642ab
 mode = 0644
 
 [script]
@@ -60,3 +38,39 @@ eggs =
   slapos.core
   slapos.toolbox[agent]
   erp5.util
+
+
+[versions]
+apache-libcloud = 0.18.0
+ecdsa = 0.13
+erp5.util = 0.4.43
+gitdb = 0.6.4
+pycrypto = 2.6.1
+slapos.recipe.download = 1.0
+slapos.recipe.template = 2.8
+slapos.toolbox = 0.52
+smmap = 0.9.0
+
+# Required by:
+# slapos.toolbox==0.52
+GitPython = 1.0.1
+
+# Required by:
+# slapos.toolbox==0.52
+atomize = 0.2.0
+
+# Required by:
+# slapos.toolbox==0.52
+feedparser = 5.2.1
+
+# Required by:
+# slapos.toolbox==0.52
+lockfile = 0.10.2
+
+# Required by:
+# slapos.toolbox==0.52
+paramiko = 1.15.2
+
+# Required by:
+# slapos.toolbox==0.52
+rpdb = 0.1.5