Update BOINC, allows to install application with slapparameters

slapos/recipe/boinc/__init__.py

@@ -29,6 +29,7 @@ import os
 import subprocess
 import pwd
 import signal
+import zc.buildout
 
 class Recipe(GenericBaseRecipe):
   """Deploy a fully operational boinc architecture."""
@@ -208,11 +209,51 @@ class App(GenericBaseRecipe):
   """This recipe allow to deploy an scientific applications using boinc
   Note that recipe use depend on boinc-server parameter"""
 
+  def downloadFiles(self):
+    """This is used to download app files if necessary and update options values"""
+    for key in ('template-result', 'template-wu', 'input-file', 'binary'):
+      option = self.options[key].strip()
+      if option and (option.startswith('http') or option.startswith('ftp')):
+        #download the specified file
+        cache = os.path.join(self.options['home'].strip(), 'temp')
+        downloader = zc.buildout.download.Download(self.buildout['buildout'],
+                        hash_name=True, cache=cache)
+        path, _ = downloader(option, md5sum=None)
+        mode = 0600
+        if key == 'binary':
+          mode = 0700
+        os.chmod(path, mode)
+        self.options[key] = path
+
+  def checkOptions(self):
+    """Check if parameter send is valid to install or update application"""
+    if not self.options['app-name'].strip() or \
+              not self.options['version'].strip():
+      return False
+    self.appname = self.options['app-name'].strip()
+    self.version = self.options['version'].strip()
+    #for non exist application, check if parameter is complete
+    appdir = os.path.join(self.options['installroot'].strip(), 'apps',
+                    self.options['app-name'].strip(),
+                    self.options['version'].strip())
+    if not os.path.exists(appdir):
+      if not self.options['template-result'].strip() or not self.options['binary'].strip() \
+          or not self.options['input-file'].strip() or not self.options['template-wu'].strip() \
+          or not self.options['wu-number'].strip() or not self.options['platform'].strip():
+        print "Invalid argement values...operation cancelled"
+        return False
+    #write application to install
+    toInstall = open(os.path.join(self.options['home'].strip(),
+                            '.install_' + self.appname + self.version), 'w')
+    toInstall.write('install or update')
+    toInstall.close()
+    return True
 
   def install(self):
-    if self.options['app-name'].strip() == '' or \
-              self.options['version'].strip() == '':
-      #don't deploy empty application...skipped
+    self.appname = ''
+    self.version = ''
+    if not self.checkOptions():
+      #don't deploy empty or invalid application...skipped
       return []
     path_list = []
     package = self.options['boinc'].strip()
@@ -221,6 +262,7 @@ class App(GenericBaseRecipe):
     developegg = self.options['develop-egg'].strip()
     python_path = boinc_egg + ":" + os.environ['PYTHONPATH']
     home = self.options['home'].strip()
+    user = pwd.getpwuid(os.stat(home).st_uid)[0]
     perl = self.options['perl-binary'].strip()
     svn = self.options['svn-binary'].strip()
     for f in os.listdir(developegg):
@@ -239,26 +281,26 @@ class App(GenericBaseRecipe):
         self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
         dict(dash=self.options['dash'].strip(),
               uldl_pid=self.options['apache-pid'].strip(),
-              user=self.options['user']))
+              user=user))
     )
     path_list.append(sh_script)
     os.chmod(bash , 0700)
 
+    #If useful, download necessary files and update options path
+    self.downloadFiles()
     start_boinc = os.path.join(home, '.start_boinc')
     installroot = self.options['installroot'].strip()
-    version = self.options['version'].strip()
     platform = self.options['platform'].strip()
     apps_dir = os.path.join(installroot, 'apps')
-    appname = self.options['app-name'].strip()
-    bin_name = appname +"_"+ version +"_"+ \
+    bin_name = self.appname +"_"+ self.version +"_"+ \
         platform +  self.options['extension'].strip()
-    application = os.path.join(apps_dir, appname, version, platform)
+    application = os.path.join(apps_dir, self.appname, self.version, platform)
     wrapperdir = self.options['wrapper-dir'].strip()
     project = self.options['project'].strip()
 
     parameter = dict(installroot=installroot, project=project,
-            appname=appname, binary_name=bin_name,
-            version=version, platform=platform,
+            appname=self.appname, binary_name=bin_name,
+            version=self.version, platform=platform,
             application=application, environment=environment,
             start_boinc=start_boinc,
             wu_number=int(self.options['wu-number'].strip()),
@@ -266,10 +308,10 @@ class App(GenericBaseRecipe):
             t_wu=self.options['template-wu'].strip(),
             t_input=self.options['input-file'].strip(),
             binary=self.options['binary'].strip(),
-            bash=bash,
+            bash=bash, home_dir=home,
     )
     deploy_app = self.createPythonScript(
-      os.path.join(wrapperdir, appname),
+      os.path.join(wrapperdir, self.appname),
       '%s.configure.deployApp' % __name__, parameter
     )
     path_list.append(deploy_app)

slapos/recipe/boinc/configure.py

@@ -53,6 +53,7 @@ def checkMysql(args):
       print "Could not connect to MySQL database... sleep for 2 secondes"
       time.sleep(2)
 
+
 def checkFile(file, stime):
   """Loop until 'file' is created (exist)"""
   while True:
@@ -63,6 +64,7 @@ def checkFile(file, stime):
     else:
       break
 
+
 def services(args):
   """This function configure a new installed boinc project instance"""
   print "Checking if needed to install or reinstall Boinc-server..."
@@ -115,6 +117,7 @@ def services(args):
 
   writeFile(args['service_status'], "started")
 
+
 def restart_boinc(args):
   """Stop (if currently is running state) and start all Boinc service"""
   if args['drop_install']:
@@ -130,54 +133,68 @@ def restart_boinc(args):
   writeFile(args['start_boinc'], "started")
   print "Done."
 
+
 def deployApp(args):
   """Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
   print "Cheking if needed to install %s..." % args['appname']
+  install_request_file = os.path.join(args['home_dir'],
+                  '.install_' + args['appname'] + args['version'])
+  if not os.path.exists(install_request_file):
+    print "No install or update request for %s version %s..." % (
+                args['appname'], args['version'])
+    return
+  os.unlink(install_request_file)
   token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
-  dropapp = False
+  newInstall = False
   if os.path.exists(token):
     args['previous_wu'] = int(open(token, 'r').read().strip())
-    if args['previous_wu'] >= args['wu_number']:
-      print args['appname'] + " version " + args['version'] + " is already installed in this Boinc instance... skipped"
-      return
-    else:
+    if args['previous_wu'] < args['wu_number']:
       print args['appname'] + " Work units will be updated from %s to %s" % (
                 args['previous_wu'], args['wu_number'])
   else:
     args['previous_wu'] = 0
-    dropapp = True
+    newInstall = True
   #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
   checkFile(args['start_boinc'], 3)
 
   print "setup directories..."
   args['inputfile'] = os.path.join(args['installroot'], 'download',
-                        args['appname'] + '_input')
+                        args['appname'] + args['version'] + '_input')
   base_app = os.path.join(args['installroot'], 'apps', args['appname'])
   base_app_version = os.path.join(base_app, args['version'])
   args['templates'] = os.path.join(args['installroot'], 'templates')
-  t_result = os.path.join(args['templates'], args['appname']+'_result')
-  t_wu = os.path.join(args['templates'], args['appname']+'_wu')
+  t_result = os.path.join(args['templates'],
+                          args['appname'] + args['version'] + '_result')
+  t_wu = os.path.join(args['templates'],
+                          args['appname'] + args['version'] + '_wu')
+  binary = os.path.join(args['application'], args['binary_name'])
   if not os.path.exists(base_app):
     os.mkdir(base_app)
-  if dropapp:
+  if newInstall:
     if os.path.exists(base_app_version):
       shutil.rmtree(base_app_version)
     os.mkdir(base_app_version)
     os.mkdir(args['application'])
   if not os.path.exists(args['templates']):
     os.mkdir(args['templates'])
-    else:
+  if args['t_result']:
     if os.path.exists(t_result):
       os.unlink(t_result)
-      if os.path.exists(t_result):
-        os.unlink(t_wu)
     shutil.copy(args['t_result'], t_result)
+  if args['t_wu']:
+    if os.path.exists(t_wu):
+      os.unlink(t_wu)
     shutil.copy(args['t_wu'], t_wu)
-    if not os.path.exists(args['inputfile']):
+  if args['t_input']:
+    if os.path.exists(args['inputfile']):
+      os.unlink(args['inputfile'])
     os.symlink(args['t_input'], args['inputfile'])
-    shutil.copy(args['binary'], os.path.join(args['application'],
-          args['binary_name']))
+  if args['binary'] and args['platform']:
+    if os.path.exists(binary):
+      os.unlink(binary)
+    shutil.copy(args['binary'], binary)
 
+  if newInstall:
     print "Adding '" + args['appname'] + "' to project.xml..."
     print "Adding deamon for application to config.xml..."
     project_xml = os.path.join(args['installroot'], 'project.xml')
@@ -185,19 +202,18 @@ def deployApp(args):
     sed_args = [args['bash'], args['appname'], args['installroot']]
     startProcess(sed_args)
 
+  if args['binary'] and args['platform']:
     print "Sign the application binary..."
     sign = os.path.join(args['installroot'], 'bin/sign_executable')
     privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
-    output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w')
-    p_sign = subprocess.Popen([sign, os.path.join(args['application'],
-            args['binary_name']), privateKeyFile], stdout=output,
+    output = open(binary + '.sig', 'w')
+    p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
             stderr=subprocess.STDOUT)
     result = p_sign.communicate()[0]
     if p_sign.returncode is None or p_sign.returncode != 0:
       print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
       return
     output.close()
-  #END if drop-app HERE
 
   print "Running xadd script..."
   env = os.environ
@@ -231,19 +247,21 @@ def deployApp(args):
   print "Boinc Application deployment is done... writing end signal file..."
   writeFile(token, str(args['wu_number']))
 
+
 def create_wu(args, env):
-  t_result = "templates/" + args['appname'] + '_result'
-  t_wu = "templates/" + args['appname'] + '_wu'
+  t_result = "templates/" + args['appname'] + args['version'] + '_result'
+  t_wu = "templates/" + args['appname'] + args['version'] + '_wu'
   launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
         '--appname', args['appname'], '--wu_name', '',
         '--wu_template', t_wu, '--result_template', t_result,
         '--min_quorum', '1',  '--target_nresults', '1',
-        args['appname']+'_input']
+        args['appname'] + args['version'] + '_input']
   for i in range(args['previous_wu'], args['wu_number']):
     print "Creating project wroker %s..." % str(i+1)
     launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete'
     startProcess(launch_args, env, args['installroot'])
 
+
 def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
   process = subprocess.Popen(launch_args, stdout=stdout,
               stderr=subprocess.STDOUT, env=env,
@@ -254,13 +272,14 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
     return False
   return True
 
+
 def runCmd(args):
   """Wait for Boinc Client started and run boinc cmd"""
   client_config = os.path.join(args['installdir'], 'client_state.xml')
   checkFile(client_config, 5)
   time.sleep(10)
   #Scan client state xml to find client ipv4 adress
-  host = result = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
+  host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
                 open(client_config, 'r').read()).group(1)
   args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
   print "Run boinccmd with host at %s " % args['base_cmd'][2]
@@ -271,6 +290,7 @@ def runCmd(args):
     #Load or reload cc_config file
     startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
 
+
 def writeFile(file, content):
   f = open(file, 'w')
   f.write(content)

software/boinc/software.cfg

@@ -38,7 +38,7 @@ download-only = true
 filename = upper_case
 #Application configuration
 app-name = upper_case
-version = 1.0
+version = 1.00
 exec-extension = 
 #Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
 platform = x86_64-pc-linux-gnu

stack/boinc/buildout.cfg

@@ -41,7 +41,7 @@ eggs =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-boinc.cfg
 output = ${buildout:directory}/template-boinc.cfg
-md5sum = b85d32e9509e960459c2ba1e47741838
+md5sum = 003bff525faa1e63913fa5f38c18becd
 mode = 0644
 
 #Template for deploying MySQL Database Server
@@ -57,7 +57,7 @@ recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
 mode = 0644
 filename = apache.in
-md5sum = 2d3efa25bf1cf6a673fb2cef13ae0529
+md5sum = 0b3825a4a0ec82e279609d1f9dc72da4
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 # Local development

stack/boinc/instance-boinc.cfg

@@ -218,7 +218,8 @@ mysql-port = $${stunnel:local-port}
 [slap-application]
 <= boinc-server
 recipe = slapos.cookbook:boinc.app
-#appname and version is require to update wu-number
+#appname and version is require to update any existing application
+#otherwise, the recipe would try to install a new one
 app-name = $${slap-parameter:app-name}
 version = $${slap-parameter:version}
 wu-number = $${slap-parameter:wu-number}