Commit aca5d6ec authored by Alain Takoudjou's avatar Alain Takoudjou

Now use Stork app to download file for BOINC apps

parent c8fdc1b3
...@@ -35,8 +35,8 @@ depends = ...@@ -35,8 +35,8 @@ depends =
${dash:location} ${dash:location}
${zlib:location} ${zlib:location}
x86 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/28b36a94ad1a405bac689a76b6c353a7/condor-7.9.5-x86_Debian6-unstripped.tar.gz 227059bb9bebc9033665bb246d90a7bb x86 = https://storage.host.cloud.univ-paris13.fr/data/public/00198e.php?dl=1 957a291290d4da089d004f52adc4a490
x86-64 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/7000a12f9a22765b457e2958762871d5/condor-7.9.5-x86_64_Debian6-stripped.tar.gz 30fa4cebabb8ff4971c769f4ef74ba68 x86-64 = https://storage.host.cloud.univ-paris13.fr/data/public/c1a6ff.php?dl=1 41c697e6c88f750b0e5c98baf0d260a5
script = script =
if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
......
...@@ -26,11 +26,8 @@ ...@@ -26,11 +26,8 @@
############################################################################## ##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
import os import os
import subprocess
import pwd import pwd
import json import json
import signal
import zc.buildout
from urlparse import urlparse from urlparse import urlparse
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
...@@ -232,21 +229,19 @@ class App(GenericBaseRecipe): ...@@ -232,21 +229,19 @@ class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc """This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter""" Note that recipe use depend on boinc-server parameter"""
def downloadFiles(self, app): def getDownloadFiles(self, app, files_dict, prefix):
"""This is used to download app files if necessary and update options values""" """This is used to download app files if necessary and update options values"""
for key in ('input-file', 'template-result', 'template-wu', 'binary'): for key in ('input-file', 'template-result', 'template-wu', 'binary'):
param = app[key] src = app[key]
if param and (param.startswith('http') or param.startswith('ftp')): if not src:
#download the specified file continue
cache = os.path.join(self.options['home'].strip(), 'tmp') if src.startswith('/'):
downloader = zc.buildout.download.Download(self.buildout['buildout'], src = "file:" + src
hash_name=True, cache=cache) filename = (prefix + '_' + key).replace('-', '_')
path, _ = downloader(param, md5sum=None) files_dict[filename] = src
mode = 0600 app[key] = os.path.join(self.options['download-folder'].strip(),
if key == 'binary': filename)
mode = 0700
os.chmod(path, mode)
app[key] = path
def getAppList(self): def getAppList(self):
"""Load parameters, """Load parameters,
...@@ -262,7 +257,7 @@ class App(GenericBaseRecipe): ...@@ -262,7 +257,7 @@ class App(GenericBaseRecipe):
for version in app_list[app]: for version in app_list[app]:
current_app = app_list[app][version] current_app = app_list[app][version]
#Use default value if empty and Use_default is True #Use default value if empty and Use_default is True
#Initialize all values to empty if not define by the user #Initialize all values to empty if not defined by the user
if current_app['use_default']: if current_app['use_default']:
current_app['template-result'] = current_app.get('template-result', current_app['template-result'] = current_app.get('template-result',
default_template_result).strip() default_template_result).strip()
...@@ -300,6 +295,7 @@ class App(GenericBaseRecipe): ...@@ -300,6 +295,7 @@ class App(GenericBaseRecipe):
def install(self): def install(self):
app_list = self.getAppList() app_list = self.getAppList()
download_list_dict = dict()
path_list = [] path_list = []
package = self.options['boinc'].strip() package = self.options['boinc'].strip()
...@@ -343,7 +339,9 @@ class App(GenericBaseRecipe): ...@@ -343,7 +339,9 @@ class App(GenericBaseRecipe):
for version in app_list[appname]: for version in app_list[appname]:
if not app_list[appname][version]: if not app_list[appname][version]:
continue continue
self.downloadFiles(app_list[appname][version]) # Get the list of files to download
self.getDownloadFiles(app_list[appname][version],
download_list_dict, appname+'_'+version)
platform = app_list[appname][version]['platform'] platform = app_list[appname][version]['platform']
application = os.path.join(apps_dir, appname, version, platform) application = os.path.join(apps_dir, appname, version, platform)
if app_list[appname][version]['binary'] and not platform: if app_list[appname][version]['binary'] and not platform:
...@@ -370,6 +368,9 @@ class App(GenericBaseRecipe): ...@@ -370,6 +368,9 @@ class App(GenericBaseRecipe):
) )
path_list.append(deploy_app) path_list.append(deploy_app)
with open(self.options['download-json-file'].strip(), 'w') as file_list:
file_list.write(json.dumps(download_list_dict))
return path_list return path_list
update = install update = install
......
...@@ -191,6 +191,11 @@ def services(args): ...@@ -191,6 +191,11 @@ def services(args):
def deployApp(args): def deployApp(args):
"""Deploy Boinc App with lock""" """Deploy Boinc App with lock"""
print "Check if all files have been downloaded..."
for name in ['t_input', 'binary', 't_result', 't_wu']:
if args[name]:
checkFile(args[name], 5)
print "Asking to enter in execution with lock mode..." print "Asking to enter in execution with lock mode..."
with LockFile(args['lockfile'], wait=True): with LockFile(args['lockfile'], wait=True):
print "acquire the lock file..." print "acquire the lock file..."
......
test string Et est admodum mirum videre plebem innumeram mentibus ardore quodam infuso cum dimicationum curulium eventu pendentem. haec similiaque memorabile nihil vel serium agi Romae permittunt. ergo redeundum ad textum.
\ No newline at end of file
Oportunum est, ut arbitror, explanare nunc causam, quae ad exitium praecipitem Aginatium inpulit iam inde a priscis maioribus nobilem, ut locuta est pertinacior fama. nec enim super hoc ulla documentorum rata est fides.
Iam virtutem ex consuetudine vitae sermonisque nostri interpretemur nec eam, ut quidam docti, verborum magnificentia metiamur virosque bonos eos, qui habentur, numeremus, Paulos, Catones, Galos, Scipiones, Philos; his communis vita contenta est; eos autem omittamus, qui omnino nusquam reperiuntur.
Metuentes igitur idem latrones Lycaoniam magna parte campestrem cum se inpares nostris fore congressione stataria documentis frequentibus scirent, tramitibus deviis petivere Pamphyliam diu quidem intactam sed timore populationum et caedium, milite per omnia diffuso propinqua, magnis undique praesidiis conmunitam.
Cum saepe multa, tum memini domi in hemicyclio sedentem, ut solebat, cum et ego essem una et pauci admodum familiares, in eum sermonem illum incidere qui tum forte multis erat in ore. Meministi enim profecto, Attice, et eo magis, quod P. Sulpicio utebare multum, cum is tribunus plebis capitali odio a Q. Pompeio, qui tum erat consul, dissideret, quocum coniunctissime et amantissime vixerat, quanta esset hominum vel admiratio vel querella.
\ No newline at end of file
...@@ -62,7 +62,7 @@ md5sum = 66d7ec85ce15e65d2858c11b75fb9574 ...@@ -62,7 +62,7 @@ md5sum = 66d7ec85ce15e65d2858c11b75fb9574
url = ${:_profile_base_location_}/input/input_file url = ${:_profile_base_location_}/input/input_file
filename = input_file filename = input_file
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
md5sum = 6f8db599de986fab7a21625b7916589c md5sum = ad3ca0f59b475ada29dbf68f2951e2a7
[boinc-application] [boinc-application]
app-list = {"${application:app-name}":{"${application:version}":{"use_default":true, "wu-number":2, "input-file":"${template_input:location}/${template_input:filename}", "binary":"${application:location}/${application:filename}"}}} app-list = {"${application:app-name}":{"${application:version}":{"use_default":true, "wu-number":2, "input-file":"${template_input:location}/${template_input:filename}", "binary":"${application:location}/${application:filename}"}}}
...@@ -73,6 +73,7 @@ template-wu = ${template_wu:location}/${template_wu:filename} ...@@ -73,6 +73,7 @@ template-wu = ${template_wu:location}/${template_wu:filename}
platform = x86_64-pc-linux-gnu platform = x86_64-pc-linux-gnu
extension = extension =
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git repository = http://git.erp5.org/repos/slapos.git
......
...@@ -7,6 +7,7 @@ parts = ...@@ -7,6 +7,7 @@ parts =
template-mariadb template-mariadb
template-boinc template-boinc
template-httpd-conf template-httpd-conf
stork
extends = extends =
../../component/boinc/buildout.cfg ../../component/boinc/buildout.cfg
...@@ -20,6 +21,8 @@ extends = ...@@ -20,6 +21,8 @@ extends =
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/stunnel/buildout.cfg ../../component/stunnel/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/stork/buildout.cfg
../slapos.cfg ../slapos.cfg
[instance-egg] [instance-egg]
...@@ -34,7 +37,7 @@ eggs = ...@@ -34,7 +37,7 @@ eggs =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg output = ${buildout:directory}/template-boinc.cfg
md5sum = 986d45af5734c917b49e489bd8c10f3d md5sum = bde689de1435bcfc1431692e487ae7ba
mode = 0644 mode = 0644
#Template for deploying MySQL Database Server #Template for deploying MySQL Database Server
...@@ -45,6 +48,9 @@ output = ${buildout:directory}/template-mariadb.cfg ...@@ -45,6 +48,9 @@ output = ${buildout:directory}/template-mariadb.cfg
#md5sum = 8142244a3087a23e002a08225579528a #md5sum = 8142244a3087a23e002a08225579528a
mode = 0644 mode = 0644
[mariadb]
keep-compile-dir = false
[template-httpd-conf] [template-httpd-conf]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template/${:filename} url = ${:_profile_base_location_}/template/${:filename}
...@@ -62,6 +68,24 @@ platform = x86_64-pc-linux-gnu ...@@ -62,6 +68,24 @@ platform = x86_64-pc-linux-gnu
[boinc-application] [boinc-application]
app-list = {} app-list = {}
######################################################################
#### STORK INTEGRATION PART ####
[template-stork]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/../../software/stork/instance-stork.cfg
output = ${buildout:directory}/template-stork.cfg
#md5sum = 42985a425369e4f93211cfaad21d8eb8
mode = 0644
[process-svcdeamon]
<= template-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = ${:_profile_base_location_}/../../software/stork/scripts/${:filename}
filename = svcdaemon.py
#####################END STORK###########################################
[networkcache] [networkcache]
# signature certificates of the following uploaders. # signature certificates of the following uploaders.
# Romain Courteaud # Romain Courteaud
......
[buildout] [buildout]
parts = parts =
stork_server
submit-download
boinc-server boinc-server
boinc-app boinc-app
certificate-authority certificate-authority
...@@ -19,10 +21,11 @@ parts = ...@@ -19,10 +21,11 @@ parts =
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
#temporary solution
newest = false newest = false
offline = false offline = true
extends =
${template-stork:output}
# Creation of all needed directories # Creation of all needed directories
[rootdirectory] [rootdirectory]
...@@ -33,6 +36,7 @@ srv = $${buildout:directory}/srv/ ...@@ -33,6 +36,7 @@ srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/ bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/ tmp = $${buildout:directory}/tmp/
project = $${buildout:directory}/projects/ project = $${buildout:directory}/projects/
download-dir = $${buildout:directory}/files/
[basedirectory] [basedirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -129,7 +133,7 @@ pid_file = $${basedirectory:run}/apache.pid ...@@ -129,7 +133,7 @@ pid_file = $${basedirectory:run}/apache.pid
lock_file = $${basedirectory:run}/apache.lock lock_file = $${basedirectory:run}/apache.lock
cgid_sock = $${basedirectory:run}/cgid.sock cgid_sock = $${basedirectory:run}/cgid.sock
ip = $${slap-network-information:global-ipv6} ip = $${slap-network-information:global-ipv6}
port = 8080 port = 8081
error_log = $${directory:httpd-log}/error.log error_log = $${directory:httpd-log}/error.log
access_log = $${directory:httpd-log}/access.log access_log = $${directory:httpd-log}/access.log
php_ini_dir = $${directory:php-ini-dir} php_ini_dir = $${directory:php-ini-dir}
...@@ -173,6 +177,16 @@ environment-section = ...@@ -173,6 +177,16 @@ environment-section =
template = template =
configuration = configuration =
[submit-download]
<= stork
recipe = slapos.cookbook:stork.submit
wrapper-check = $${basedirectory:scripts}/stork_check_job
wrapper-path = $${basedirectory:scripts}/submit_download_files
json_src_url = $${boinc-app:download-json-file}
dest_url = file:$${rootdirectory:download-dir}
src_from_file = true
data_type= transfer
[boinc-passwd] [boinc-passwd]
recipe = slapos.cookbook:generate.password recipe = slapos.cookbook:generate.password
storage-path = $${rootdirectory:etc}/.boincpasswd storage-path = $${rootdirectory:etc}/.boincpasswd
...@@ -222,6 +236,8 @@ mysql-port = $${stunnel:local-port} ...@@ -222,6 +236,8 @@ mysql-port = $${stunnel:local-port}
[boinc-app] [boinc-app]
<= boinc-server <= boinc-server
recipe = slapos.cookbook:boinc.app recipe = slapos.cookbook:boinc.app
download-folder = $${rootdirectory:download-dir}
download-json-file = $${rootdirectory:download-dir}/download.json
#appname and version is require to update any existing application #appname and version is require to update any existing application
#otherwise, the recipe would try to install a new one #otherwise, the recipe would try to install a new one
boinc-app-list = $${slap-parameter:boinc-app-list} boinc-app-list = $${slap-parameter:boinc-app-list}
...@@ -353,7 +369,7 @@ curl_path = ${curl:location}/bin/curl ...@@ -353,7 +369,7 @@ curl_path = ${curl:location}/bin/curl
[slap-parameter] [slap-parameter]
#Default config value for boinc #Default config value for boinc
force-ipv6 = False force-ipv6 = True
project = boinc_test project = boinc_test
full-name = Boinc Project SAMPLE full-name = Boinc Project SAMPLE
copyright-holder = REPLACE WITH COPYRIGHT HOLDER copyright-holder = REPLACE WITH COPYRIGHT HOLDER
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment