factorize the code of boinc recipe

slapos/recipe/boinc/__init__.py

@@ -103,7 +103,7 @@ class Recipe(GenericBaseRecipe):
 
     #Define environment variable here
     python = os.path.join(self.home, 'bin/python')
-    python_path = self.boinc_egg + ":" + os.environ['PYTHONPATH']
+    python_path = self.boinc_egg
     if not os.path.exists(python):
       os.symlink(self.pythonbin, python)
     for f in os.listdir(self.developegg):
@@ -112,8 +112,8 @@ class Recipe(GenericBaseRecipe):
         python_path += ":" + dir
     bin_dir = os.path.join(self.home, 'bin')
     environment = dict(
-        PATH=self.svn + ':' + bin_dir + ':' + self.perl + ':' + os.environ['PATH'],
-        PYTHONPATH=python_path,
+        PATH=os.pathsep.join([self.svn, bin_dir, self.perl, os.environ['PATH']]),
+        PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
     )
 
     #Generate wrapper for php
@@ -134,7 +134,7 @@ class Recipe(GenericBaseRecipe):
       dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
           mysql_user=self.username, mysql_password=self.password,
           database=self.database,
-          file_status=file_status, python_path=python_path
+          file_status=file_status, environment=environment
       )
     )
 
@@ -160,6 +160,18 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(install_wrapper)
 
+    #generate sh script for project configuration
+    bash = os.path.join(self.home, 'bin', 'project_config.sh')
+    sh_script = self.createFile(bash,
+        self.substituteTemplate(self.getTemplateFilename('project_config.in'),
+        dict(dash=self.options['dash'].strip(),
+              uldl_pid=self.options['apache-pid'].strip(),
+              user=slapuser, fullname=self.fullname,
+              copyright=self.copyright, installroot=self.installroot))
+    )
+    path_list.append(sh_script)
+    os.chmod(bash , 0700)
+
     #After make_project run configure_script to perform and restart apache php services
     service_status = os.path.join(self.home, '.start_service')
     parameter = dict(
@@ -171,10 +183,8 @@ class Recipe(GenericBaseRecipe):
         xadd=os.path.join(self.installroot, 'bin/xadd'),
         environment=environment,
         service_status=service_status,
-        project=niceprojectname,
-        fullname=self.fullname,
-        copyright=self.copyright,
-        drop_install=drop_install
+        drop_install=drop_install,
+        sedconfig=bash
     )
     start_service = self.createPythonScript(
       os.path.join(self.wrapperdir, 'config_project'),
@@ -191,8 +201,8 @@ class Recipe(GenericBaseRecipe):
         installroot=self.installroot, drop_install=drop_install,
         mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
         mysql_user=self.username, mysql_password=self.password,
-        database=self.database, PATH=environment['PATH'],
-        python_path=python_path, start_boinc=start_boinc)
+        database=self.database, environment=environment,
+        start_boinc=start_boinc)
     start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
         'start_boinc'),
         '%s.configure.restart_boinc' % __name__,
@@ -211,14 +221,14 @@ class App(GenericBaseRecipe):
 
   def downloadFiles(self):
     """This is used to download app files if necessary and update options values"""
-    for key in ('template-result', 'template-wu', 'input-file', 'binary'):
-      option = self.options[key].strip()
-      if option and (option.startswith('http') or option.startswith('ftp')):
+    for key in ('input-file', 'template-result', 'template-wu', 'binary'):
+      param = self.options[key].strip()
+      if param and (param.startswith('http') or param.startswith('ftp')):
         #download the specified file
-        cache = os.path.join(self.options['home'].strip(), 'temp')
+        cache = os.path.join(self.options['home'].strip(), 'tmp')
         downloader = zc.buildout.download.Download(self.buildout['buildout'],
                         hash_name=True, cache=cache)
-        path, _ = downloader(option, md5sum=None)
+        path, _ = downloader(param, md5sum=None)
         mode = 0600
         if key == 'binary':
           mode = 0700
@@ -243,8 +253,9 @@ class App(GenericBaseRecipe):
         print "Invalid argement values...operation cancelled"
         return False
     #write application to install
-    toInstall = open(os.path.join(self.options['home'].strip(),
-                            '.install_' + self.appname + self.version), 'w')
+    request_file = os.path.join(self.options['home'].strip(),
+                        '.install_' + self.appname + self.version)
+    toInstall = open(request_file, 'w')
     toInstall.write('install or update')
     toInstall.close()
     return True
@@ -258,9 +269,8 @@ class App(GenericBaseRecipe):
     path_list = []
     package = self.options['boinc'].strip()
     #Define environment variable here
-    boinc_egg = os.path.join(package, 'lib/python2.7/site-packages')
     developegg = self.options['develop-egg'].strip()
-    python_path = boinc_egg + ":" + os.environ['PYTHONPATH']
+    python_path = os.path.join(package, 'lib/python2.7/site-packages')
     home = self.options['home'].strip()
     user = pwd.getpwuid(os.stat(home).st_uid)[0]
     perl = self.options['perl-binary'].strip()
@@ -271,17 +281,15 @@ class App(GenericBaseRecipe):
         python_path += ":" + dir
     bin_dir = os.path.join(home, 'bin')
     environment = dict(
-        PATH=svn + ':' + bin_dir + ':' + perl + ':' + os.environ['PATH'],
-        PYTHONPATH=python_path,
+        PATH=os.pathsep.join([svn, bin_dir, perl, os.environ['PATH']]),
+        PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
     )
 
     #generate project.xml and config.xml script updater
     bash = os.path.join(home, 'bin', 'update_config.sh')
     sh_script = self.createFile(bash,
         self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
-        dict(dash=self.options['dash'].strip(),
-              uldl_pid=self.options['apache-pid'].strip(),
-              user=user))
+        dict(dash=self.options['dash'].strip()))
     )
     path_list.append(sh_script)
     os.chmod(bash , 0700)
@@ -311,7 +319,7 @@ class App(GenericBaseRecipe):
             bash=bash, home_dir=home,
     )
     deploy_app = self.createPythonScript(
-      os.path.join(wrapperdir, self.appname),
+      os.path.join(wrapperdir, 'boinc_app'),
       '%s.configure.deployApp' % __name__, parameter
     )
     path_list.append(deploy_app)

slapos/recipe/boinc/configure.py

@@ -31,9 +31,10 @@ import subprocess
 import time
 import shutil
 import re
+import filecmp
 
 def checkMysql(args):
-  sys.path += args['python_path'].split(':')
+  sys.path += args['environment']['PYTHONPATH'].split(':')
   import MySQLdb
   #Sleep until mysql server becomes available
   while True:
@@ -65,6 +66,61 @@ def checkFile(file, stime):
       break
 
 
+def restart_boinc(args):
+  """Stop (if currently is running state) and start all Boinc service"""
+  if args['drop_install']:
+    checkFile(args['service_status'], 3)
+  else:
+    checkMysql(args)
+  print "Restart Boinc..."
+  env = os.environ
+  env['PATH'] = args['environment']['PATH']
+  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
+  binstart = os.path.join(args['installroot'], 'bin/start')
+  binstop = os.path.join(args['installroot'], 'bin/stop')
+  os.system(binstop)
+  os.system(binstart)
+  writeFile(args['start_boinc'], "started")
+  print "Done."
+
+
+def check_installRequest(args):
+  print "Cheking if needed to install %s..." % args['appname']
+  install_request_file = os.path.join(args['home_dir'],
+                  '.install_' + args['appname'] + args['version'])
+  if not os.path.exists(install_request_file):
+    print "No install or update request for %s version %s..." % (
+                args['appname'], args['version'])
+    return False
+  os.unlink(install_request_file)
+  return True
+
+def copy_file(source, dest):
+  """"Copy file with source to dest with auto replace
+      return True if file has been copied and dest ha been replaced
+  """
+  result = False
+  if source and os.path.exists(source):
+    if os.path.exists(dest):
+      if filecmp.cmp(dest, source):
+        return False
+      os.unlink(dest)
+    result = True
+    shutil.copy(source, dest)
+  return result
+
+
+def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
+  process = subprocess.Popen(launch_args, stdout=stdout,
+              stderr=subprocess.STDOUT, env=env,
+              cwd=cwd)
+  result = process.communicate()[0]
+  if process.returncode is None or process.returncode != 0:
+    print "Failed to execute executable.\nThe error was: %s" % result
+    return False
+  return True
+
+
 def services(args):
   """This function configure a new installed boinc project instance"""
   print "Checking if needed to install or reinstall Boinc-server..."
@@ -81,11 +137,11 @@ def services(args):
   if not startProcess(htpwd_args):
     return
 
-  print "Running xadd script..."
+  print "execute script xadd..."
   env = os.environ
   env['PATH'] = args['environment']['PATH']
   env['PYTHONPATH'] = args['environment']['PYTHONPATH']
-  if not startProcess([args['xadd']], env):
+  if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
     return
   print "Update files and directories permissions..."
   upload = os.path.join(args['installroot'], 'upload')
@@ -103,11 +159,8 @@ def services(args):
   os.system("chmod -R o+r " + inc)
   os.system("chmod o+x " + languages)
   os.system("chmod o+x " + compiled)
-  os.system("sed -i '/remove the die/d' %s" % forum_file)
-  subprocess.Popen(["sed -i 's#REPLACE WITH PROJECT NAME#%s#' %s" % (args['fullname'],
-      project_inc)], shell=True, stdout=subprocess.PIPE).communicate()[0]
-  subprocess.Popen(["sed -i 's#REPLACE WITH COPYRIGHT HOLDER#%s#' %s" % (args['copyright'],
-      project_inc)], shell=True, stdout=subprocess.PIPE).communicate()[0]
+  sed_args = [args['sedconfig']]
+  startProcess(sed_args)
 
   #Execute php create_forum.php...
   print "Boinc Forum: Execute php create_forum.php..."
@@ -118,32 +171,10 @@ def services(args):
   writeFile(args['service_status'], "started")
 
 
-def restart_boinc(args):
-  """Stop (if currently is running state) and start all Boinc service"""
-  if args['drop_install']:
-    checkFile(args['service_status'], 3)
-  else:
-    checkMysql(args)
-  print "Restart Boinc..."
-  os.environ['PATH'] = args['PATH']
-  binstart = os.path.join(args['installroot'], 'bin/start')
-  binstop = os.path.join(args['installroot'], 'bin/stop')
-  os.system(binstop)
-  os.system(binstart)
-  writeFile(args['start_boinc'], "started")
-  print "Done."
-
-
 def deployApp(args):
   """Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
-  print "Cheking if needed to install %s..." % args['appname']
-  install_request_file = os.path.join(args['home_dir'],
-                  '.install_' + args['appname'] + args['version'])
-  if not os.path.exists(install_request_file):
-    print "No install or update request for %s version %s..." % (
-                args['appname'], args['version'])
+  if not check_installRequest(args):
     return
-  os.unlink(install_request_file)
   token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
   newInstall = False
   if os.path.exists(token):
@@ -156,18 +187,23 @@ def deployApp(args):
     newInstall = True
   #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
   checkFile(args['start_boinc'], 3)
+  env = os.environ
+  env['PATH'] = args['environment']['PATH']
+  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
 
   print "setup directories..."
+  numversion = args['version'].replace('.', '')
   args['inputfile'] = os.path.join(args['installroot'], 'download',
-                        args['appname'] + args['version'] + '_input')
+                        args['appname'] + numversion + '_input')
   base_app = os.path.join(args['installroot'], 'apps', args['appname'])
   base_app_version = os.path.join(base_app, args['version'])
   args['templates'] = os.path.join(args['installroot'], 'templates')
   t_result = os.path.join(args['templates'],
-                          args['appname'] + args['version'] + '_result')
+                          args['appname'] + numversion + '_result')
   t_wu = os.path.join(args['templates'],
-                          args['appname'] + args['version'] + '_wu')
+                          args['appname'] + numversion + '_wu')
   binary = os.path.join(args['application'], args['binary_name'])
+  signBin = False
   if not os.path.exists(base_app):
     os.mkdir(base_app)
   if newInstall:
@@ -177,32 +213,24 @@ def deployApp(args):
     os.mkdir(args['application'])
   if not os.path.exists(args['templates']):
     os.mkdir(args['templates'])
-  if args['t_result']:
-    if os.path.exists(t_result):
-      os.unlink(t_result)
-    shutil.copy(args['t_result'], t_result)
-  if args['t_wu']:
-    if os.path.exists(t_wu):
-      os.unlink(t_wu)
-    shutil.copy(args['t_wu'], t_wu)
+  copy_file(args['t_result'], t_result)
+  copy_file(args['t_wu'], t_wu)
+  signBin = copy_file(args['binary'], binary)
   if args['t_input']:
     if os.path.exists(args['inputfile']):
       os.unlink(args['inputfile'])
     os.symlink(args['t_input'], args['inputfile'])
-  if args['binary'] and args['platform']:
-    if os.path.exists(binary):
-      os.unlink(binary)
-    shutil.copy(args['binary'], binary)
 
-  if newInstall:
+  project_xml = os.path.join(args['installroot'], 'project.xml')
+  findapp = re.search("<name>(%s)</name>" % args['appname'],
+                open(project_xml, 'r').read())
+  if not findapp:
     print "Adding '" + args['appname'] + "' to project.xml..."
     print "Adding deamon for application to config.xml..."
-    project_xml = os.path.join(args['installroot'], 'project.xml')
-    config_xml = os.path.join(args['installroot'], 'config.xml')
     sed_args = [args['bash'], args['appname'], args['installroot']]
     startProcess(sed_args)
 
-  if args['binary'] and args['platform']:
+  if signBin:
     print "Sign the application binary..."
     sign = os.path.join(args['installroot'], 'bin/sign_executable')
     privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
@@ -215,13 +243,10 @@ def deployApp(args):
       return
     output.close()
 
-  print "Running xadd script..."
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
+  print "execute script xadd..."
+
   if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
     return
-
   print "Running script bin/update_versions..."
   updt_version = os.path.join(args['installroot'], 'bin/update_versions')
   p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE,
@@ -240,7 +265,6 @@ def deployApp(args):
   print "Restart Boinc..."
   binstart = os.path.join(args['installroot'], 'bin/start')
   binstop = os.path.join(args['installroot'], 'bin/stop')
-  sys.environ = env
   os.system(binstop)
   os.system(binstart)
 
@@ -249,30 +273,21 @@ def deployApp(args):
 
 
 def create_wu(args, env):
-  t_result = "templates/" + args['appname'] + args['version'] + '_result'
-  t_wu = "templates/" + args['appname'] + args['version'] + '_wu'
+  """Create or update number of work unit for an existing boinc application"""
+  numversion = args['version'].replace('.', '')
+  t_result = "templates/" + args['appname'] + numversion + '_result'
+  t_wu = "templates/" + args['appname'] + numversion + '_wu'
   launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
         '--appname', args['appname'], '--wu_name', '',
         '--wu_template', t_wu, '--result_template', t_result,
         '--min_quorum', '1',  '--target_nresults', '1',
-        args['appname'] + args['version'] + '_input']
+        args['appname'] + numversion + '_input']
   for i in range(args['previous_wu'], args['wu_number']):
     print "Creating project wroker %s..." % str(i+1)
-    launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete'
+    launch_args[4] = args['appname'] + str(i+1) + numversion + '_nodelete'
     startProcess(launch_args, env, args['installroot'])
 
 
-def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
-  process = subprocess.Popen(launch_args, stdout=stdout,
-              stderr=subprocess.STDOUT, env=env,
-              cwd=cwd)
-  result = process.communicate()[0]
-  if process.returncode is None or process.returncode != 0:
-    print "Failed to execute executable.\nThe error was: %s" % result
-    return False
-  return True
-
-
 def runCmd(args):
   """Wait for Boinc Client started and run boinc cmd"""
   client_config = os.path.join(args['installdir'], 'client_state.xml')

slapos/recipe/boinc/template/project_config.in

+#!%(dash)s
+
+sed -i "/remove the die/d" %(installroot)s/html/ops/create_forums.php
+sed -i "s#REPLACE WITH PROJECT NAME#%(fullname)s#" %(installroot)s/html/project/project.inc
+sed -i "s#REPLACE WITH COPYRIGHT HOLDER#%(copyright)s#" %(installroot)s/html/project/project.inc
+sed -i.old -e "/<\/config>/i\<uldl_pid>%(uldl_pid)s</uldl_pid>" %(installroot)s/config.xml
+sed -i.old -e "/<\/config>/i\<httpd_user>%(user)s</httpd_user>" %(installroot)s/config.xml

slapos/recipe/boinc/template/sed_update.in

@@ -10,7 +10,5 @@ sed -i.old -e "/<\/boinc>/i\<app>\n<name>$1</name>\n<user_friendly_name>$1</user
 
 sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_bitwise_validator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
 sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_assimilator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
-sed -i.old -e "/<\/config>/i\<uldl_pid>%(uldl_pid)s</uldl_pid>" $2/config.xml
-sed -i.old -e "/<\/config>/i\<httpd_user>%(user)s</httpd_user>" $2/config.xml
 
 fi
\ No newline at end of file

software/boinc/software.cfg

@@ -43,7 +43,7 @@ exec-extension =
 #Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
 platform = x86_64-pc-linux-gnu
 #Work Unit number number of work unit
-wu-number = 4
+wu-number = 2
 
 [template-base]
 recipe = slapos.recipe.download

stack/boinc/buildout.cfg

@@ -41,7 +41,7 @@ eggs =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-boinc.cfg
 output = ${buildout:directory}/template-boinc.cfg
-md5sum = 003bff525faa1e63913fa5f38c18becd
+md5sum = 22de71bad135ab3c760c63e537dcb182
 mode = 0644
 
 #Template for deploying MySQL Database Server
@@ -57,7 +57,7 @@ recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
 mode = 0644
 filename = apache.in
-md5sum = 0b3825a4a0ec82e279609d1f9dc72da4
+md5sum = 030892494ce87357e6e09dcd89187bf4
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 # Local development

stack/boinc/instance-boinc.cfg

@@ -20,7 +20,9 @@ parts =
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
+#temporary solution
+newest = false
+offline = false
 
 
 # Creation of all needed directories
@@ -226,7 +228,7 @@ wu-number = $${slap-parameter:wu-number}
 binary = $${slap-parameter:binary}
 platform = $${slap-parameter:platform}
 extension = $${slap-parameter:extension}
-template-result = $${slap-parameter:template-wu}
+template-result = $${slap-parameter:template-result}
 template-wu = $${slap-parameter:template-wu}
 input-file = $${slap-parameter:input-file}