Merge branch 'slaprunner-paas'

slapos/runner/gittools.py

@@ -137,8 +137,8 @@ def gitCommit(project, msg):
       git.add(f)
     #Commit all modified and untracked files
     git.commit('-a', '-m', msg)
-  else:
     code = 1
+  else:
     json = "Nothing to be commited"
   return jsonify(code=code, result=json)
 
@@ -150,6 +150,8 @@ def gitPush(project):
   code = 0
   json = ""
   try:
+    repo = Repo(project)
+    git = repo.git
     #push changes to repo
     current_branch = repo.active_branch.name
     git.push('origin', current_branch)

slapos/runner/runnertest.py

@@ -17,16 +17,16 @@ import hashlib
 import json
 import os
 import shutil
+import sup_process
 import time
 import unittest
 
-from slapos.runner.utils import (getProfilePath, getRcode,
+from slapos.runner.utils import (getProfilePath,
                                  getSession, isInstanceRunning,
                                  isSoftwareRunning, startProxy,
                                  isSoftwareReleaseReady,
                                  runSlapgridUntilSuccess, runInstanceWithLock,
                                  getBuildAndRunParams, saveBuildAndRunParams)
-from slapos.runner.process import killRunningProcess, isRunning
 from slapos.runner import views
 import slapos.slap
 from slapos.htpasswd import  HtpasswdFile
@@ -137,9 +137,9 @@ class SlaprunnerTestCase(unittest.TestCase):
     if os.path.exists(self.app.config['software_link']):
       shutil.rmtree(self.app.config['software_link'])
     #Stop process
-    killRunningProcess('slapproxy', recursive=True)
-    killRunningProcess('slapgrid-cp', recursive=True)
-    killRunningProcess('slapgrid-sr', recursive=True)
+    sup_process.killRunningProcess(self.app.config, 'slapproxy')
+    sup_process.killRunningProcess(self.app.config, 'slapgrid-cp')
+    sup_process.killRunningProcess(self.app.config, 'slapgrid-sr')
 
   def updateConfigParameter(self, parameter, value):
     config_parser = ConfigParser.SafeConfigParser()
@@ -186,10 +186,10 @@ class SlaprunnerTestCase(unittest.TestCase):
 
   def proxyStatus(self, status=True, sleep_time=0):
     """Helper for testslapproxy status"""
-    proxy = isRunning('slapproxy')
+    proxy = sup_process.isRunning(self.app.config, 'slapproxy')
     if proxy != status and sleep_time != 0:
       time.sleep(sleep_time)
-      proxy = isRunning('slapproxy')
+      proxy = sup_process.isRunning(self.app.config, 'slapproxy')
     self.assertEqual(proxy, status)
 
   def setupProjectFolder(self, withSoftware=False):
@@ -232,7 +232,7 @@ class SlaprunnerTestCase(unittest.TestCase):
 
   def stopSlapproxy(self):
     """Kill slapproxy process"""
-    killRunningProcess('slapproxy', recursive=True)
+    pass
 
   def test_configAccount(self):
     """For the first lauch of slaprunner user need do create first account"""
@@ -256,12 +256,8 @@ class SlaprunnerTestCase(unittest.TestCase):
 
   def test_startStopProxy(self):
     """Test slapproxy"""
-    self.stopSlapproxy()
-    self.proxyStatus(False)
     startProxy(self.app.config)
     self.proxyStatus(True)
-    self.stopSlapproxy()
-    self.proxyStatus(False, sleep_time=1)
 
   def test_cloneProject(self):
     """Start scenario 1 for deploying SR: Clone a project from git repository"""
@@ -408,7 +404,6 @@ class SlaprunnerTestCase(unittest.TestCase):
   def test_requestInstance(self):
     """Scenarion 6: request software instance"""
     self.test_updateInstanceParameter()
-    self.proxyStatus(False, sleep_time=1)
     #run Software profile
     response = loadJson(self.app.post('/runSoftwareProfile',
                                       data=dict(),
@@ -459,7 +454,7 @@ class SlaprunnerTestCase(unittest.TestCase):
     self.assertEqual(response, "2")
     # Test that the new call to isSoftwareReleaseReady
     # doesn't overwrite the previous installed one
-    killRunningProcess('slapgrid-sr')
+    sup_process.killRunningProcess(self.app.config, 'slapgrid-sr')
     completed_path = os.path.join(self.app.config['runner_workdir'],
         'softwareLink', 'slaprunner-test', '.completed')
     completed_text = ".completed file: test"
@@ -584,7 +579,7 @@ class SlaprunnerTestCase(unittest.TestCase):
     only by changing the value of slapos.cfg config file. This can happen when
     slapgrid processes the webrunner's partition.
     """
-    config_file = os.path.join(self.app.config['etc_dir'], 'slapos.cfg')
+    config_file = os.path.join(self.app.config['etc_dir'], 'slapos-test.cfg')
     runner_config_old = os.environ['RUNNER_CONFIG']
     os.environ['RUNNER_CONFIG'] = config_file
     open(config_file, 'w').write("[section]\nvar=value")

slapos/runner/static/js/scripts/process.js

@@ -286,6 +286,3 @@ function runProcess(urlfor) {
         });
     }
 }
-
-getRunningState();
-setInterval('getRunningState()', 3000);

slapos/runner/static/js/scripts/repo.js

@@ -171,7 +171,7 @@ $(document).ready(function () {
     });
     $("#commitbutton").click(function () {
         if ($("input#commitmsg").val() === "" ||
-                $("textarea#commitmsg").val() === "Enter message...") {
+                $("input#commitmsg").val() === "Enter message...") {
             $("#error").Popup("Please Enter the commit message", {type: 'alert', duration: 3000});
             return false;
         }
@@ -181,12 +181,12 @@ $(document).ready(function () {
         send = true;
         var project = $("#project").val();
         $("#imgwaitting").fadeIn('normal');
-        //$("#commit").empty();
-        $("#commitbbutton").attr("value", "Wait...");
+        $("#commitmsg").empty();
+        $("#commitbutton").attr("value", "Wait...");
         $.ajax({
             type: "POST",
             url: $SCRIPT_ROOT + '/commitProjectFiles',
-            data: {project: $("input#workdir").val() + "/" + project, msg: $("textarea#commitmsg").val()},
+            data: {project: $("input#workdir").val() + "/" + project, msg: $("input#commitmsg").val()},
             success: function (data) {
                 if (data.code === 1) {
                     if (data.result !== "") {

slapos/runner/sup_process.py

+# pylint: disable-msg=W0311,C0103
+
+import os
+import signal
+import time
+import xmlrpclib
+
+# This mini-library is used to communicate with supervisord process
+# It aims to replace the file "process.py"
+# For the moment, we keep both for compatibility
+
+
+def isRunning(config, process):
+  """
+  Ask supervisor if given process is currently running
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  state = server.supervisor.getProcessInfo(process)['state']
+  return (True if state in (10, 20) else False)
+
+
+def killRunningProcess(config, process, sig=signal.SIGTERM):
+  """
+  Send signal "sig" to given process.
+  Default signal sent is SIGTERM
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  pid = server.supervisor.getProcessInfo(process)['pid']
+  if pid != 0:
+    os.kill(pid, sig)
+
+
+def returnCode(config, process):
+  """
+  Get the returned code of the last run of given process
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  code = server.supervisor.getProcessInfo(process)['exitstatus']
+  return code
+
+
+def runProcess(config, process):
+  """
+  Start a process registered by supervisor
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  server.supervisor.startProcess(process)
+
+
+def runProcesses(config, processes):
+  """
+  Start by supervisor a list of given processes, one by one
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  for proc in processes:
+    server.supervisor.startProcess(proc)
+    waitForProcessEnd(config, proc)
+
+
+def stopProcess(config, process):
+  """
+  Ask supervisor to stop a process
+  """
+  if isRunning(config, process):
+    server = xmlrpclib.Server(config['supervisord_server'])
+    server.supervisor.stopProcess(process)
+
+
+def stopProcesses(config, processes):
+  """
+  Stop a list of processes
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  for proc in processes:
+    server.supervisor.stopProcess(proc)
+
+
+def waitForProcessEnd(config, process):
+  """
+  Block program's execution until given process quits Running state
+  """
+  server = xmlrpclib.Server(config['supervisord_server'])
+  while True:
+    state = server.supervisor.getProcessInfo(process)['state']
+    if state == 20:
+      time.sleep(3)
+    else:
+      return True
+  return False

slapos/runner/templates/layout.html

@@ -44,6 +44,8 @@
     		   $("#error").Popup($("input#fmsg").val(), {type:'info', duration:5000, load:true});
     		}
     		bindRun();
+            getRunningState();
+            setInterval('getRunningState()', 3000);
         $('ul.sf-menu').superfish({
       		delay:       600,
     			speed:       'fast',

slapos/runner/templates/manageRepository.html

@@ -111,12 +111,12 @@
       	<!--<img class="waitting" id="pullimgwaitting" src="{{ url_for('static', filename='images/waiting.gif') }}" alt="" />-->
       	</div>
       	</div>
-      	<div id="push" style="margin-bottom:20px;">
+      	<div id="commit" style="margin-bottom:20px;">
       	<h2>Commit All your changes (On active branch)</h2>
       	<div style="margin-left:15px;">
       	<label for='commitmsg'>Commit message: </label>
       	<input type="text" name="commitmsg" id="commitmsg" size='40' value="Enter message..." />
-      	<input type="submit" name="commit" id ="commit" value="Commit" class="button"/>
+      	<input type="submit" name="commit" id ="commitbutton" value="Commit" class="button"/>
       	<img class="waitting" id="imgwaitting" src="{{ url_for('static', filename='images/waiting.gif') }}" alt="" />
       	</div>
       	</div>

slapos/runner/utils.py

@@ -7,13 +7,14 @@ import datetime
 import json
 import logging
 import md5
-import multiprocessing
 import os
+import sup_process
 import re
 import shutil
 import thread
 import time
 import urllib
+import xmlrpclib
 from xml.dom import minidom
 
 import xml_marshaller
@@ -21,7 +22,7 @@ from flask import jsonify
 
 from slapos.runner.gittools import cloneRepo
 
-from slapos.runner.process import Popen, isRunning, killRunningProcess, isPidFileProcessRunning
+from slapos.runner.process import Popen
 from slapos.htpasswd import HtpasswdFile
 import slapos.slap
 from slapos.grid.utils import md5digest
@@ -236,20 +237,19 @@ def updateInstanceParameter(config, software_type=None):
     config: Slaprunner configuration.
     software_type: reconfigure Software Instance with software type.
   """
+  time.sleep(1)
   if not (updateProxy(config) and requestInstance(config, software_type)):
     return False
 
 
 def startProxy(config):
   """Start Slapproxy server"""
-  if isRunning('slapproxy'):
+  if sup_process.isRunning(config, 'slapproxy'):
     return
-
-  log = os.path.join(config['log_dir'], 'slapproxy.log')
-  Popen([config['slapos'], 'proxy', 'start', '--logfile', log,
-         '--cfg', config['configuration_file_path']],
-        name='slapproxy',
-        stdout=None)
+  try:
+    sup_process.runProcess(config, "slapproxy")
+  except xmlrpclib.Fault:
+    pass
   time.sleep(4)
 
 
@@ -265,15 +265,11 @@ def removeProxyDb(config):
     os.unlink(config['database_uri'])
 
 
-def isSoftwareRunning(config=None):
+def isSoftwareRunning(config):
   """
     Return True if slapos is still running and false if slapos if not
   """
-  # XXX-Cedric Hardcoded pidfile
-  if config:
-    slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-sr.pid')
-    return isPidFileProcessRunning(slapgrid_pid)
-  return isRunning('slapgrid-sr')
+  return sup_process.isRunning(config, 'slapgrid-sr')
 
 
 def slapgridResultToFile(config, step, returncode, datetime):
@@ -299,17 +295,14 @@ def waitProcess(config, process, step):
   slapgridResultToFile(config, step, process.returncode, date)
 
 
-def runSoftwareWithLock(config, lock=True):
+def runSoftwareWithLock(config, lock=False):
   """
     Use Slapgrid to compile current Software Release and wait until
     compilation is done
   """
-  if isSoftwareRunning():
-    return False
+  if sup_process.isRunning(config, 'slapgrid-sr'):
+    return 1
 
-  slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-sr.pid')
-  if isPidFileProcessRunning(slapgrid_pid):
-    return False
   if not os.path.exists(config['software_root']):
     os.mkdir(config['software_root'])
   stopProxy(config)
@@ -318,21 +311,16 @@ def runSoftwareWithLock(config, lock=True):
   if os.path.exists(config['software_log']):
     os.remove(config['software_log'])
   if not updateProxy(config):
-    return False
-  slapgrid = Popen([config['slapos'], 'node', 'software', '--all',
-                   '--cfg', config['slapos_cfg'], '--pidfile', slapgrid_pid,
-                   '--verbose', '--logfile', config['software_log']],
-                    name='slapgrid-sr', stdout=None)
-  if lock:
-    slapgrid.wait()
-    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    slapgridResultToFile(config, "software", slapgrid.returncode, date)
+    return 1
+  try:
+    sup_process.runProcess(config, "slapgrid-sr")
+    if lock:
+      sup_process.waitForProcessEnd(config, "slapgrid-sr")
     #Saves the current compile software for re-use
     config_SR_folder(config)
-    return ( True if slapgrid.returncode == 0 else False )
-  else:
-    thread.start_new_thread(waitProcess, (config, slapgrid, "software"))
-    return False
+    return sup_process.returnCode(config, "slapgrid-sr")
+  except xmlrpclib.Fault:
+    return 1
 
 
 def config_SR_folder(config):
@@ -396,44 +384,34 @@ def loadSoftwareRList(config):
   return list
 
 
-def isInstanceRunning(config=None):
+def isInstanceRunning(config):
   """
     Return True if slapos is still running and False otherwise
   """
-  # XXX-Cedric Hardcoded pidfile
-  if config:
-    slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-cp.pid')
-    return isPidFileProcessRunning(slapgrid_pid)
-  return isRunning('slapgrid-cp')
+  return sup_process.isRunning(config, 'slapgrid-cp')
 
 
-def runInstanceWithLock(config, lock=True):
+def runInstanceWithLock(config, lock=False):
   """
     Use Slapgrid to deploy current Software Release and wait until
     deployment is done.
   """
-  if isInstanceRunning():
-    return False
+  if sup_process.isRunning(config, 'slapgrid-cp'):
+    return 1
 
-  slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-cp.pid')
   startProxy(config)
   # XXX Hackish and unreliable
   if os.path.exists(config['instance_log']):
     os.remove(config['instance_log'])
   if not (updateProxy(config) and requestInstance(config)):
-    return False
-  slapgrid = Popen([config['slapos'], 'node', 'instance', '--all',
-                   '--cfg', config['slapos_cfg'], '--pidfile', slapgrid_pid,
-                   '--verbose', '--logfile', config['instance_log']],
-                   name='slapgrid-cp', stdout=None)
-  if lock:
-    slapgrid.wait()
-    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    slapgridResultToFile(config, "instance", slapgrid.returncode, date)
-    return ( True if slapgrid.returncode == 0 else False )
-  else:
-    thread.start_new_thread(waitProcess, (config, slapgrid, "instance"))
-    return False
+    return 1
+  try:
+    sup_process.runProcess(config, "slapgrid-cp")
+    if lock:
+      sup_process.waitForProcessEnd(config, "slapgrid-cp")
+    return sup_process.returnCode(config, "slapgrid-cp")
+  except xmlrpclib.Fault:
+    return 1
 
 
 def getProfilePath(projectDir, profile):
@@ -475,8 +453,11 @@ def getSlapStatus(config):
 
 def svcStopAll(config):
   """Stop all Instance processes on this computer"""
-  return Popen([config['supervisor'], config['configuration_file_path'],
-                'shutdown']).communicate()[0]
+  try:
+    return Popen([config['supervisor'], config['configuration_file_path'],
+                  'shutdown']).communicate()[0]
+  except:
+    pass
 
 
 def removeInstanceRoot(config):
@@ -566,10 +547,8 @@ def configNewSR(config, projectpath):
   """
   folder = realpath(config, projectpath)
   if folder:
-    if isInstanceRunning():
-      killRunningProcess('slapgrid-cp')
-    if isSoftwareRunning():
-      killRunningProcess('slapgrid-sr')
+    sup_process.stopProcess(config, 'slapgrid-cp')
+    sup_process.stopProcess(config, 'slapgrid-sr')
     stopProxy(config)
     removeProxyDb(config)
     startProxy(config)
@@ -661,7 +640,7 @@ def removeSoftwareByName(config, md5, folderName):
     config: slaprunner configuration
     foldername: the link name given to the software release
     md5: the md5 filename given by slapgrid to SR folder"""
-  if isSoftwareRunning() or isInstanceRunning():
+  if isSoftwareRunning(config) or isInstanceRunning(config):
     raise Exception("Software installation or instantiation in progress, cannot remove")
   path = os.path.join(config['software_root'], md5)
   linkpath = os.path.join(config['software_link'], folderName)
@@ -899,8 +878,9 @@ def runSlapgridUntilSuccess(config, step):
   # XXX-Nico runSoftwareWithLock can return 0 or False (0==False)
   while counter > 0:
     counter -= 1
-    slapgrid = runSlapgridWithLock(config)
-    if slapgrid:
+    slapgrid = runSlapgridWithLock(config, lock=True)
+    # slapgrid == 0 because EXIT_SUCCESS == 0
+    if slapgrid == 0:
       break
     times_left = int(open(counter_file).read()) - 1
     if times_left > 0 :
@@ -911,7 +891,7 @@ def runSlapgridUntilSuccess(config, step):
   max_tries -= counter
   # run instance only if we are deploying the software release,
   # if it is defined so, and sr is correctly deployed
-  if step == "software" and params['run_instance'] and slapgrid:
+  if step == "software" and params['run_instance'] and slapgrid == 0:
     return (max_tries, runSlapgridUntilSuccess(config, "instance"))
   else:
     return max_tries

slapos/runner/views.py

@@ -7,13 +7,13 @@ import json
 import os
 import shutil
 import subprocess
+import sup_process
 import thread
 import urllib
 
 from flask import (Flask, request, redirect, url_for, render_template,
                    g, flash, jsonify, session, abort, send_file)
 
-from slapos.runner.process import killRunningProcess
 from slapos.runner.utils import (checkSoftwareFolder, configNewSR,
                                  createNewUser, getBuildAndRunParams,
                                  getProfilePath, getSlapgridResult,
@@ -25,8 +25,7 @@ from slapos.runner.utils import (checkSoftwareFolder, configNewSR,
                                  loadSoftwareRList, md5sum, newSoftware,
                                  readFileFrom, readParameters, realpath,
                                  removeInstanceRoot, removeProxyDb,
-                                 removeSoftwareByName, runInstanceWithLock,
-                                 runSoftwareWithLock, runSlapgridUntilSuccess,
+                                 removeSoftwareByName, runSlapgridUntilSuccess,
                                  saveSession, saveBuildAndRunParams,
                                  setMiniShellHistory,
                                  svcStartStopProcess, svcStopAll, tail,
@@ -403,6 +402,14 @@ def getProjectDiff():
                   result="Error: No such file or directory. PERMISSION DENIED!")
 
 
+def commitProjectFiles():
+  path = realpath(app.config, request.form['project'])
+  if path:
+    return gitCommit(path, request.form['msg'])
+  else:
+    return jsonify(code=0, result="Can not read folder: Permission Denied")
+
+
 def pushProjectFiles():
   path = realpath(app.config, request.form['project'])
   if path:
@@ -470,7 +477,7 @@ def slapgridResult():
 def stopSlapgrid():
   counter_file = os.path.join(app.config['runner_workdir'], '.turn-left')
   open(counter_file, 'w+').write(str(0))
-  result = killRunningProcess(request.form['type'])
+  result = sup_process.killRunningProcess(app.config, request.form['type'])
   return jsonify(result=result)
 
 
@@ -787,6 +794,8 @@ app.add_url_rule("/slapgridResult", 'slapgridResult',
 app.add_url_rule("/getmd5sum", 'getmd5sum', getmd5sum, methods=['POST'])
 app.add_url_rule("/checkFileType", 'checkFileType', checkFileType,
                  methods=['POST'])
+app.add_url_rule("/commitProjectFiles", 'commitProjectFiles', commitProjectFiles,
+                 methods=['POST'])
 app.add_url_rule("/pullProjectFiles", 'pullProjectFiles', pullProjectFiles,
                  methods=['POST'])
 app.add_url_rule("/pushProjectFiles", 'pushProjectFiles', pushProjectFiles,