Commit aca5d6ec authored by Alain Takoudjou's avatar Alain Takoudjou

Now use Stork app to download file for BOINC apps

parent c8fdc1b3
......@@ -35,8 +35,8 @@ depends =
${dash:location}
${zlib:location}
x86 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/28b36a94ad1a405bac689a76b6c353a7/condor-7.9.5-x86_Debian6-unstripped.tar.gz 227059bb9bebc9033665bb246d90a7bb
x86-64 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/7000a12f9a22765b457e2958762871d5/condor-7.9.5-x86_64_Debian6-stripped.tar.gz 30fa4cebabb8ff4971c769f4ef74ba68
x86 = https://storage.host.cloud.univ-paris13.fr/data/public/00198e.php?dl=1 957a291290d4da089d004f52adc4a490
x86-64 = https://storage.host.cloud.univ-paris13.fr/data/public/c1a6ff.php?dl=1 41c697e6c88f750b0e5c98baf0d260a5
script =
if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
......
......@@ -26,11 +26,8 @@
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import pwd
import json
import signal
import zc.buildout
from urlparse import urlparse
class Recipe(GenericBaseRecipe):
......@@ -232,21 +229,19 @@ class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter"""
def downloadFiles(self, app):
def getDownloadFiles(self, app, files_dict, prefix):
"""This is used to download app files if necessary and update options values"""
for key in ('input-file', 'template-result', 'template-wu', 'binary'):
param = app[key]
if param and (param.startswith('http') or param.startswith('ftp')):
#download the specified file
cache = os.path.join(self.options['home'].strip(), 'tmp')
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(param, md5sum=None)
mode = 0600
if key == 'binary':
mode = 0700
os.chmod(path, mode)
app[key] = path
src = app[key]
if not src:
continue
if src.startswith('/'):
src = "file:" + src
filename = (prefix + '_' + key).replace('-', '_')
files_dict[filename] = src
app[key] = os.path.join(self.options['download-folder'].strip(),
filename)
def getAppList(self):
"""Load parameters,
......@@ -262,7 +257,7 @@ class App(GenericBaseRecipe):
for version in app_list[app]:
current_app = app_list[app][version]
#Use default value if empty and Use_default is True
#Initialize all values to empty if not define by the user
#Initialize all values to empty if not defined by the user
if current_app['use_default']:
current_app['template-result'] = current_app.get('template-result',
default_template_result).strip()
......@@ -300,6 +295,7 @@ class App(GenericBaseRecipe):
def install(self):
app_list = self.getAppList()
download_list_dict = dict()
path_list = []
package = self.options['boinc'].strip()
......@@ -343,7 +339,9 @@ class App(GenericBaseRecipe):
for version in app_list[appname]:
if not app_list[appname][version]:
continue
self.downloadFiles(app_list[appname][version])
# Get the list of files to download
self.getDownloadFiles(app_list[appname][version],
download_list_dict, appname+'_'+version)
platform = app_list[appname][version]['platform']
application = os.path.join(apps_dir, appname, version, platform)
if app_list[appname][version]['binary'] and not platform:
......@@ -370,6 +368,9 @@ class App(GenericBaseRecipe):
)
path_list.append(deploy_app)
with open(self.options['download-json-file'].strip(), 'w') as file_list:
file_list.write(json.dumps(download_list_dict))
return path_list
update = install
......
......@@ -191,6 +191,11 @@ def services(args):
def deployApp(args):
"""Deploy Boinc App with lock"""
print "Check if all files have been downloaded..."
for name in ['t_input', 'binary', 't_result', 't_wu']:
if args[name]:
checkFile(args[name], 5)
print "Asking to enter in execution with lock mode..."
with LockFile(args['lockfile'], wait=True):
print "acquire the lock file..."
......
test string
\ No newline at end of file
Et est admodum mirum videre plebem innumeram mentibus ardore quodam infuso cum dimicationum curulium eventu pendentem. haec similiaque memorabile nihil vel serium agi Romae permittunt. ergo redeundum ad textum.
Oportunum est, ut arbitror, explanare nunc causam, quae ad exitium praecipitem Aginatium inpulit iam inde a priscis maioribus nobilem, ut locuta est pertinacior fama. nec enim super hoc ulla documentorum rata est fides.
Iam virtutem ex consuetudine vitae sermonisque nostri interpretemur nec eam, ut quidam docti, verborum magnificentia metiamur virosque bonos eos, qui habentur, numeremus, Paulos, Catones, Galos, Scipiones, Philos; his communis vita contenta est; eos autem omittamus, qui omnino nusquam reperiuntur.
Metuentes igitur idem latrones Lycaoniam magna parte campestrem cum se inpares nostris fore congressione stataria documentis frequentibus scirent, tramitibus deviis petivere Pamphyliam diu quidem intactam sed timore populationum et caedium, milite per omnia diffuso propinqua, magnis undique praesidiis conmunitam.
Cum saepe multa, tum memini domi in hemicyclio sedentem, ut solebat, cum et ego essem una et pauci admodum familiares, in eum sermonem illum incidere qui tum forte multis erat in ore. Meministi enim profecto, Attice, et eo magis, quod P. Sulpicio utebare multum, cum is tribunus plebis capitali odio a Q. Pompeio, qui tum erat consul, dissideret, quocum coniunctissime et amantissime vixerat, quanta esset hominum vel admiratio vel querella.
\ No newline at end of file
......@@ -62,7 +62,7 @@ md5sum = 66d7ec85ce15e65d2858c11b75fb9574
url = ${:_profile_base_location_}/input/input_file
filename = input_file
location = ${buildout:parts-directory}/${:_buildout_section_name_}
md5sum = 6f8db599de986fab7a21625b7916589c
md5sum = ad3ca0f59b475ada29dbf68f2951e2a7
[boinc-application]
app-list = {"${application:app-name}":{"${application:version}":{"use_default":true, "wu-number":2, "input-file":"${template_input:location}/${template_input:filename}", "binary":"${application:location}/${application:filename}"}}}
......@@ -73,6 +73,7 @@ template-wu = ${template_wu:location}/${template_wu:filename}
platform = x86_64-pc-linux-gnu
extension =
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
......
......@@ -7,6 +7,7 @@ parts =
template-mariadb
template-boinc
template-httpd-conf
stork
extends =
../../component/boinc/buildout.cfg
......@@ -20,6 +21,8 @@ extends =
../../component/lxml-python/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/dcron/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/stork/buildout.cfg
../slapos.cfg
[instance-egg]
......@@ -34,7 +37,7 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg
md5sum = 986d45af5734c917b49e489bd8c10f3d
md5sum = bde689de1435bcfc1431692e487ae7ba
mode = 0644
#Template for deploying MySQL Database Server
......@@ -45,6 +48,9 @@ output = ${buildout:directory}/template-mariadb.cfg
#md5sum = 8142244a3087a23e002a08225579528a
mode = 0644
[mariadb]
keep-compile-dir = false
[template-httpd-conf]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
......@@ -62,6 +68,24 @@ platform = x86_64-pc-linux-gnu
[boinc-application]
app-list = {}
######################################################################
#### STORK INTEGRATION PART ####
[template-stork]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/../../software/stork/instance-stork.cfg
output = ${buildout:directory}/template-stork.cfg
#md5sum = 42985a425369e4f93211cfaad21d8eb8
mode = 0644
[process-svcdeamon]
<= template-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = ${:_profile_base_location_}/../../software/stork/scripts/${:filename}
filename = svcdaemon.py
#####################END STORK###########################################
[networkcache]
# signature certificates of the following uploaders.
# Romain Courteaud
......
[buildout]
parts =
stork_server
submit-download
boinc-server
boinc-app
certificate-authority
......@@ -19,10 +21,11 @@ parts =
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
#temporary solution
newest = false
offline = false
offline = true
extends =
${template-stork:output}
# Creation of all needed directories
[rootdirectory]
......@@ -33,6 +36,7 @@ srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
project = $${buildout:directory}/projects/
download-dir = $${buildout:directory}/files/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
......@@ -129,7 +133,7 @@ pid_file = $${basedirectory:run}/apache.pid
lock_file = $${basedirectory:run}/apache.lock
cgid_sock = $${basedirectory:run}/cgid.sock
ip = $${slap-network-information:global-ipv6}
port = 8080
port = 8081
error_log = $${directory:httpd-log}/error.log
access_log = $${directory:httpd-log}/access.log
php_ini_dir = $${directory:php-ini-dir}
......@@ -173,6 +177,16 @@ environment-section =
template =
configuration =
[submit-download]
<= stork
recipe = slapos.cookbook:stork.submit
wrapper-check = $${basedirectory:scripts}/stork_check_job
wrapper-path = $${basedirectory:scripts}/submit_download_files
json_src_url = $${boinc-app:download-json-file}
dest_url = file:$${rootdirectory:download-dir}
src_from_file = true
data_type= transfer
[boinc-passwd]
recipe = slapos.cookbook:generate.password
storage-path = $${rootdirectory:etc}/.boincpasswd
......@@ -222,6 +236,8 @@ mysql-port = $${stunnel:local-port}
[boinc-app]
<= boinc-server
recipe = slapos.cookbook:boinc.app
download-folder = $${rootdirectory:download-dir}
download-json-file = $${rootdirectory:download-dir}/download.json
#appname and version is require to update any existing application
#otherwise, the recipe would try to install a new one
boinc-app-list = $${slap-parameter:boinc-app-list}
......@@ -353,7 +369,7 @@ curl_path = ${curl:location}/bin/curl
[slap-parameter]
#Default config value for boinc
force-ipv6 = False
force-ipv6 = True
project = boinc_test
full-name = Boinc Project SAMPLE
copyright-holder = REPLACE WITH COPYRIGHT HOLDER
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment