From 65ecb29d6355b173aa5c95cc988bc8813bf4dc63 Mon Sep 17 00:00:00 2001 From: Alain Takoudjou <talino@tiolive.com> Date: Thu, 21 Mar 2013 11:21:14 +0100 Subject: [PATCH] Update request Boinc-application to use Json parameter --- slapos/recipe/boinc/__init__.py | 143 +++++++++++++++++++------------ slapos/recipe/boinc/configure.py | 6 +- software/boinc/instance.cfg | 2 +- software/boinc/software.cfg | 33 ++++--- stack/boinc/buildout.cfg | 11 ++- stack/boinc/instance-boinc.cfg | 58 ++++--------- 6 files changed, 135 insertions(+), 118 deletions(-) diff --git a/slapos/recipe/boinc/__init__.py b/slapos/recipe/boinc/__init__.py index 3062eba56..a2b9b874b 100644 --- a/slapos/recipe/boinc/__init__.py +++ b/slapos/recipe/boinc/__init__.py @@ -28,6 +28,7 @@ from slapos.recipe.librecipe import GenericBaseRecipe import os import subprocess import pwd +import json import signal import zc.buildout @@ -223,10 +224,10 @@ class App(GenericBaseRecipe): """This recipe allow to deploy an scientific applications using boinc Note that recipe use depend on boinc-server parameter""" - def downloadFiles(self): + def downloadFiles(self, app): """This is used to download app files if necessary and update options values""" for key in ('input-file', 'template-result', 'template-wu', 'binary'): - param = self.options[key].strip() + param = app[key] if param and (param.startswith('http') or param.startswith('ftp')): #download the specified file cache = os.path.join(self.options['home'].strip(), 'tmp') @@ -237,39 +238,61 @@ class App(GenericBaseRecipe): if key == 'binary': mode = 0700 os.chmod(path, mode) - self.options[key] = path - - def checkOptions(self): - """Check if parameter send is valid to install or update application""" - if not self.options['app-name'].strip() or \ - not self.options['version'].strip(): - return False - self.appname = self.options['app-name'].strip() - self.version = self.options['version'].strip() - #for non exist application, check if parameter is complete - appdir = os.path.join(self.options['installroot'].strip(), 'apps', - self.options['app-name'].strip(), - self.options['version'].strip()) - if not os.path.exists(appdir): - if not self.options['template-result'].strip() or not self.options['binary'].strip() \ - or not self.options['input-file'].strip() or not self.options['template-wu'].strip() \ - or not self.options['wu-number'].strip() or not self.options['platform'].strip(): - print "Invalid argement values...operation cancelled" - return False - #write application to install - request_file = os.path.join(self.options['home'].strip(), - '.install_' + self.appname + self.version) - toInstall = open(request_file, 'w') - toInstall.write('install or update') - toInstall.close() - return True + app[key] = path + + def getAppList(self): + """Load parameters, + check if parameter send is valid to install or update application""" + app_list = json.loads(self.options['boinc-app-list']) + if not app_list: + return None + default_template_result = self.options.get('default-template-result', '').strip() + default_template_wu = self.options.get('default-template-wu', '').strip() + default_extension = self.options.get('default-extension', '').strip() + default_platform = self.options.get('default-platform', '').strip() + for app in app_list: + for version in app_list[app]: + current_app = app_list[app][version] + #Use default value if empty and Use_default is True + #Initialize all values to empty if not define by the user + if current_app['use_default']: + current_app['template-result'] = current_app.get('template-result', + default_template_result).strip() + current_app['template-wu'] = current_app.get('template-wu', + default_template_wu).strip() + current_app['extension'] = current_app.get('extension', + default_extension).strip() + current_app['platform'] = current_app.get('platform', + default_platform).strip() + else: + current_app['template-result'] = current_app.get('template-result', '').strip() + current_app['template-wu'] = current_app.get('template-wu', '').strip() + current_app['extension'] = current_app.get('extension', '').strip() + current_app['platform'] = current_app.get('platform', '').strip() + current_app['input-file'] = current_app.get('input-file', '').strip() + current_app['wu-number'] = current_app.get('wu-number', 1) + #for new application, check if parameter is complete + appdir = os.path.join(self.options['installroot'].strip(), 'apps', + app, version) + if not os.path.exists(appdir): + if not current_app['template-result'] or not current_app['binary'] \ + or not current_app['input-file'] or not current_app['template-wu'] \ + or not current_app['platform']: + print "BOINC-APP: ERROR - Invalid argements values for % ...operation cancelled" % app + app_list[app][version] = None + continue + #write application to install + request_file = os.path.join(self.options['home'].strip(), + '.install_' + app + version) + toInstall = open(request_file, 'w') + toInstall.write('install or update') + toInstall.close() + return app_list def install(self): - self.appname = '' - self.version = '' - if not self.checkOptions(): - #don't deploy empty or invalid application...skipped - return [] + + app_list = self.getAppList() + path_list = [] package = self.options['boinc'].strip() #Define environment variable here @@ -299,39 +322,45 @@ class App(GenericBaseRecipe): os.chmod(bash , 0700) #If useful, download necessary files and update options path - self.downloadFiles() start_boinc = os.path.join(home, '.start_boinc') installroot = self.options['installroot'].strip() - platform = self.options['platform'].strip() apps_dir = os.path.join(installroot, 'apps') - bin_name = self.appname +"_"+ self.version +"_"+ \ - platform + self.options['extension'].strip() - application = os.path.join(apps_dir, self.appname, self.version, platform) wrapperdir = self.options['wrapper-dir'].strip() project = self.options['project'].strip() lockfile = os.path.join(self.options['home'].strip(), 'app_install.lock') - fd = os.open(lockfile, os.O_RDWR|os.O_CREAT) os.close( fd ) - parameter = dict(installroot=installroot, project=project, - appname=self.appname, binary_name=bin_name, - version=self.version, platform=platform, - application=application, environment=environment, - start_boinc=start_boinc, - wu_number=int(self.options['wu-number'].strip()), - t_result=self.options['template-result'].strip(), - t_wu=self.options['template-wu'].strip(), - t_input=self.options['input-file'].strip(), - binary=self.options['binary'].strip(), - bash=bash, home_dir=home, - lockfile=lockfile, - ) - deploy_app = self.createPythonScript( - os.path.join(wrapperdir, 'boinc_%s' % self.appname), - '%s.configure.deployApp' % __name__, parameter - ) - path_list.append(deploy_app) + for appname in app_list: + for version in app_list[appname]: + if not app_list[appname][version]: + continue + self.downloadFiles(app_list[appname][version]) + platform = app_list[appname][version]['platform'] + application = os.path.join(apps_dir, appname, version, platform) + if app_list[appname][version]['binary'] and not platform: + print "BOINC-APP: WARNING - Cannot specify binary without giving platform value" + app_list[appname][version]['binary'] = '' #Binary will not be updated + + parameter = dict(installroot=installroot, + appname=appname, project=project, + version=version, platform=platform, + application=application, environment=environment, + start_boinc=start_boinc, + wu_number=app_list[appname][version]['wu-number'], + t_result=app_list[appname][version]['template-result'], + t_wu=app_list[appname][version]['template-wu'], + t_input=app_list[appname][version]['input-file'], + binary=app_list[appname][version]['binary'], + extension=app_list[appname][version]['extension'], + bash=bash, home_dir=home, + lockfile=lockfile, + ) + deploy_app = self.createPythonScript( + os.path.join(wrapperdir, 'boinc_%s' % appname), + '%s.configure.deployApp' % __name__, parameter + ) + path_list.append(deploy_app) return path_list diff --git a/slapos/recipe/boinc/configure.py b/slapos/recipe/boinc/configure.py index 8901be568..7bf9549b8 100644 --- a/slapos/recipe/boinc/configure.py +++ b/slapos/recipe/boinc/configure.py @@ -173,7 +173,7 @@ def services(args): writeFile(args['service_status'], "started") def deployApp(args): - """Deploy Boinc App with lock""" + """Deploy Boinc App with lock""" print "Asking to enter in execution with lock mode..." with LockFile(args['lockfile'], wait=True): print "acquire the lock file..." @@ -211,7 +211,9 @@ def deployManagement(args): args['appname'] + numversion + '_result') t_wu = os.path.join(args['templates'], args['appname'] + numversion + '_wu') - binary = os.path.join(args['application'], args['binary_name']) + binary_name = args['appname'] +"_"+ args['version'] +"_"+ \ + args['platform'] + args['extension'] + binary = os.path.join(args['application'], binary_name) signBin = False if not os.path.exists(base_app): os.mkdir(base_app) diff --git a/software/boinc/instance.cfg b/software/boinc/instance.cfg index 083c65b0d..f7a2d73a7 100644 --- a/software/boinc/instance.cfg +++ b/software/boinc/instance.cfg @@ -8,5 +8,5 @@ offline = true [switch_softwaretype] recipe = slapos.cookbook:softwaretype -default = ${boinc-instance:output} +default = ${template-boinc:output} mariadb = ${template-mariadb:output} \ No newline at end of file diff --git a/software/boinc/software.cfg b/software/boinc/software.cfg index 74ce61515..0b67b1909 100644 --- a/software/boinc/software.cfg +++ b/software/boinc/software.cfg @@ -5,9 +5,8 @@ develop = ${:parts-directory}/slapos.cookbook-repository parts = - boinc-instance template - boinc-application + application template_wu template_result template_input @@ -21,22 +20,16 @@ extends = ../../stack/boinc/buildout.cfg #Generate All instances templates -[boinc-instance] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/boinc-app.cfg -output = ${buildout:directory}/template-app.cfg -mode = 0644 -md5sum = ab01f101cc6280ef07ea61a22a1432d0 [template] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg output = ${buildout:directory}/template.cfg mode = 0644 -md5sum = d097ddeeee5d89881d097efe6165caf6 +md5sum = 4a286795a6822ee6d53743191d5374a6 #Download Boinc Application Binary and configure project -[boinc-application] +[application] recipe = hexagonit.recipe.download url = ${boinc:location}/libexec/examples/upper_case download-only = true @@ -44,11 +37,6 @@ filename = upper_case #Application configuration app-name = upper_case version = 1.00 -exec-extension = -#Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms -platform = x86_64-pc-linux-gnu -#Work Unit number number of work unit -wu-number = 2 [template-base] recipe = slapos.recipe.download @@ -60,21 +48,30 @@ mode = 0644 url = ${:_profile_base_location_}/templates/template_result filename = template_result location = ${buildout:parts-directory}/${:_buildout_section_name_} -#md5sum = +md5sum = a3f0e9fd559cadcb2f297b952f8face8 [template_wu] <= template-base url = ${:_profile_base_location_}/templates/template_wu filename = template_wu location = ${buildout:parts-directory}/${:_buildout_section_name_} -#md5sum = +md5sum = 66d7ec85ce15e65d2858c11b75fb9574 [template_input] <= template-base url = ${:_profile_base_location_}/input/input_file filename = input_file location = ${buildout:parts-directory}/${:_buildout_section_name_} -#md5sum = +md5sum = 6f8db599de986fab7a21625b7916589c + +[boinc-application] +app-list = {"${application:app-name}":{"${application:version}":{"use_default":true, "wu-number":2, "input-file":"${template_input:location}/${template_input:filename}", "binary":"${application:location}/${application:filename}"}}} + +[boinc-default] +template-result = ${template_result:location}/${template_result:filename} +template-wu = ${template_wu:location}/${template_wu:filename} +platform = x86_64-pc-linux-gnu +extension = # Local development [slapos.cookbook-repository] diff --git a/stack/boinc/buildout.cfg b/stack/boinc/buildout.cfg index fd663eb9a..fdeaf28a1 100644 --- a/stack/boinc/buildout.cfg +++ b/stack/boinc/buildout.cfg @@ -34,7 +34,7 @@ eggs = recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-boinc.cfg output = ${buildout:directory}/template-boinc.cfg -md5sum = 20e9e2276e3cbc6504bf065eabbe13a7 +md5sum = cc95919d5400c4dfcd16646f25ba8c96 mode = 0644 #Template for deploying MySQL Database Server @@ -53,6 +53,15 @@ filename = apache.in md5sum = 030892494ce87357e6e09dcd89187bf4 location = ${buildout:parts-directory}/${:_buildout_section_name_} +[boinc-default] +template-result = +template-wu = +extension = +platform = x86_64-pc-linux-gnu + +[boinc-application] +app-list = {} + [networkcache] # signature certificates of the following uploaders. # Romain Courteaud diff --git a/stack/boinc/instance-boinc.cfg b/stack/boinc/instance-boinc.cfg index cd1dd36ce..3a5af50f7 100644 --- a/stack/boinc/instance-boinc.cfg +++ b/stack/boinc/instance-boinc.cfg @@ -13,7 +13,6 @@ parts = cron-entry-boinc promise slapmonitor - slapreport frontend-promise content-promise publish-connection-informations @@ -184,7 +183,7 @@ bytes = 4 recipe = slapos.cookbook:boinc home = $${buildout:directory} project = $${slap-parameter:project} -#http://[$${apache-php:ip}]:$${apache-php:port}/ +#url-base = http://[$${apache-php:ip}]:$${apache-php:port}/ url-base = $${request-frontend:connection-site_url} fullname = $${slap-parameter:full-name} copyright = $${slap-parameter:copyright-holder} @@ -223,15 +222,11 @@ mysql-port = $${stunnel:local-port} recipe = slapos.cookbook:boinc.app #appname and version is require to update any existing application #otherwise, the recipe would try to install a new one -app-name = $${slap-parameter:app-name} -version = $${slap-parameter:version} -wu-number = $${slap-parameter:wu-number} -binary = $${slap-parameter:binary} -platform = $${slap-parameter:platform} -extension = $${slap-parameter:extension} -template-result = $${slap-parameter:template-result} -template-wu = $${slap-parameter:template-wu} -input-file = $${slap-parameter:input-file} +boinc-app-list = $${slap-parameter:boinc-app-list} +default-template-result = $${slap-parameter:default-template-result} +default-template-wu = $${slap-parameter:default-template-wu} +default-extension = $${slap-parameter:default-extension} +default-platform = $${slap-parameter:default-platform} # Deploy logrotate, cron, configure it [logrotate] @@ -321,19 +316,6 @@ shell-path = ${dash:location}/bin/dash slapmonitor-path = ${buildout:bin-directory}/slapmonitor path = $${basedirectory:scripts}/slapmonitor -[slapreport] -recipe = slapos.cookbook:slapreport -pid-file = $${basedirectory:run}/apache.pid -database-path = $${basedirectory:log}/slapmonitor.db -consumption-log-path = $${basedirectory:log}/instance_consumption.log -logbox-ip = 87.98.152.12 -logbox-port = 5122 -logbox-user = admin -logbox-passwd = passer -shell-path = ${dash:location}/bin/dash -slapreport-path = ${buildout:bin-directory}/slapreport -path = $${basedirectory:scripts}/slapreport - # Publish all instance parameters (url of instance) [publish-connection-informations] @@ -372,20 +354,18 @@ dbname = boinctest project = boinc_test full-name = Boinc Project SAMPLE copyright-holder = REPLACE WITH COPYRIGHT HOLDER -#This parameter is use to update or to deploy BOINC application -binary = -app-name = -version = -platform = -extension = -template-result = -template-wu = -wu-number = -input-file = +#definition of BOINC parameter +#boinc-app-list is a Json data for all application to deploy +# boinc-app-list = {"MY_APP1":{VERSION1:{ +# use_default:true, "binary":"MY_BINARY", +# "platform":"", "extension":"", "template-result":"", +# "template-wu":"", "wu-number":1, "input-file":"INPUT"}, +# "VERSION2":{use_default:false, ...}}} +# "app-name" parameter is now boinc-app-list[key] +boinc-app-list = ${boinc-application:app-list} +default-template-result = ${boinc-default:template-result} +default-template-wu = ${boinc-default:template-wu} +default-extension = ${boinc-default:extension} +default-platform = ${boinc-default:platform} # Default value if no domain is specified domain = -# Default value if no ssh parameter is specified -logbox-ip = -logbox-port = -logbox-user = -logbox-passwd = -- 2.30.9