utils.py 19 KB
Newer Older
Łukasz Nowak's avatar
Łukasz Nowak committed
1 2 3 4 5
import slapos.slap
import time
import subprocess
import os
from xml_marshaller import xml_marshaller
6
from xml.dom import minidom
7 8 9 10 11
import re
import urllib
from flask import jsonify
import shutil
import string
12
import hashlib
13
import signal
Łukasz Nowak's avatar
Łukasz Nowak committed
14 15


16

Łukasz Nowak's avatar
Łukasz Nowak committed
17 18 19 20 21 22 23 24 25 26 27
class Popen(subprocess.Popen):
  def __init__(self, *args, **kwargs):
    kwargs['stdin'] = subprocess.PIPE
    kwargs['stderr'] = subprocess.STDOUT
    kwargs.setdefault('stdout', subprocess.PIPE)
    kwargs.setdefault('close_fds', True)
    subprocess.Popen.__init__(self, *args, **kwargs)
    self.stdin.flush()
    self.stdin.close()
    self.stdin = None

28 29 30 31 32 33 34 35 36 37 38 39
html_escape_table = {
  "&": "&",
  '"': """,
  "'": "'",
  ">": ">",
  "<": "&lt;",
}
 
def html_escape(text):
  """Produce entities within text."""
  return "".join(html_escape_table.get(c,c) for c in text)

Łukasz Nowak's avatar
Łukasz Nowak committed
40 41 42 43 44

def updateProxy(config):
  if not os.path.exists(config['instance_root']):
    os.mkdir(config['instance_root'])
  slap = slapos.slap.slap()
45 46 47 48 49 50 51 52 53
  #Get current software release profile
  try:
    software_folder = open(os.path.join(config['runner_workdir'],
                                        ".project")).read()
    profile = realpath(config, os.path.join(software_folder,
                                          config['software_profile']))
  except:
    return False

Łukasz Nowak's avatar
Łukasz Nowak committed
54
  slap.initializeConnection(config['master_url'])
55
  slap.registerSupply().supply(profile, computer_guid=config['computer_id'])
Łukasz Nowak's avatar
Łukasz Nowak committed
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
  computer = slap.registerComputer(config['computer_id'])
  prefix = 'slappart'
  slap_config = {
 'address': config['ipv4_address'],
 'instance_root': config['instance_root'],
 'netmask': '255.255.255.255',
 'partition_list': [
                    ],
 'reference': config['computer_id'],
 'software_root': config['software_root']}
  for i in xrange(0, int(config['partition_amount'])):
    partition_reference = '%s%s' % (prefix, i)
    partition_path = os.path.join(config['instance_root'], partition_reference)
    if not os.path.exists(partition_path):
      os.mkdir(partition_path)
    os.chmod(partition_path, 0750)
    slap_config['partition_list'].append({'address_list': [{'addr': config['ipv4_address'],
                                       'netmask': '255.255.255.255'},
                                      {'addr': config['ipv6_address'],
                                       'netmask': 'ffff:ffff:ffff::'},
                      ],
                     'path': partition_path,
                     'reference': partition_reference,
                     'tap': {'name': partition_reference},
                     })
81 82 83 84 85 86 87
  #get instance parameter
  param_path = os.path.join(config['runner_workdir'], ".parameter.xml")
  xml_result = readParameters(param_path)
  if type(xml_result) != type('') and xml_result.has_key('instance'):
    partition_parameter_kw = xml_result['instance']
  else:
    partition_parameter_kw = None
Łukasz Nowak's avatar
Łukasz Nowak committed
88
  computer.updateConfiguration(xml_marshaller.dumps(slap_config))
89
  sr_request = slap.registerOpenOrder().request(profile, partition_reference=partition_reference,
90 91
                partition_parameter_kw=partition_parameter_kw, software_type=None,
                filter_kw=None, state=None, shared=False)
92 93
  #open(param_path, 'w').write(xml_marshaller.dumps(sr_request.
  #                      getInstanceParameterDict()))
94
  return True
Łukasz Nowak's avatar
Łukasz Nowak committed
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164

def readPid(file):
  if os.path.exists(file):
    data = open(file).read().strip()
    try:
      return int(data)
    except Exception:
      return 0
  return 0


def writePid(file, pid):
  open(file, 'w').write(str(pid))


def startProxy(config):
  proxy_pid = os.path.join(config['runner_workdir'], 'proxy.pid')
  pid = readPid(proxy_pid)
  running = False
  if pid:
    try:
      os.kill(pid, 0)
    except Exception:
      pass
    else:
      running = True
  if not running:
    proxy = Popen([config['slapproxy'], config['configuration_file_path']])
    proxy_pid = os.path.join(config['runner_workdir'], 'proxy.pid')
    writePid(proxy_pid, proxy.pid)
    time.sleep(5)


def stopProxy(config):
  pid = readPid(os.path.join(config['runner_workdir'], 'proxy.pid'))
  if pid:
    try:
      os.kill(pid)
    except:
      pass


def removeProxyDb(config):
  if os.path.exists(config['database_uri']):
    os.unlink(config['database_uri'])

def isSoftwareRunning(config):
  slapgrid_pid = os.path.join(config['runner_workdir'], 'slapgrid-sr.pid')
  pid = readPid(slapgrid_pid)
  if pid:
    try:
      os.kill(pid, 0)
    except Exception:
      running = False
    else:
      running = True
  else:
    running = False
  return running


def runSoftwareWithLock(config):
  slapgrid_pid = os.path.join(config['runner_workdir'], 'slapgrid-sr.pid')
  if not isSoftwareRunning(config):
    if not os.path.exists(config['software_root']):
      os.mkdir(config['software_root'])
    stopProxy(config)
    removeProxyDb(config)
    startProxy(config)
    logfile = open(config['software_log'], 'w')
165 166
    if not updateProxy(config):
      return False
Łukasz Nowak's avatar
Łukasz Nowak committed
167 168 169
    slapgrid = Popen([config['slapgrid_sr'], '-vc', config['configuration_file_path']], stdout=logfile)
    writePid(slapgrid_pid, slapgrid.pid)
    slapgrid.wait()
170
    #Saves the current compile software for re-use
171
    #This uses the new folder create by slapgrid (if not exits yet)
172
    data = loadSoftwareData(config['runner_workdir'])
173
    md5 = ""
174 175 176
    for path in os.listdir(config['software_root']):
      exist = False
      for val in data:
177
	if val['md5'] == path:
178
	  exist = True
179
      conf = os.path.join(config['runner_workdir'], ".project")
180
      if not exist: #save this compile software folder
181 182 183 184 185 186 187
	if os.path.exists(conf):
	  data.append({"title":getProjectTitle(config), "md5":path,
	              "path": open(os.path.join(config['runner_workdir'],
	                                        ".project"), 'r').read()})
	  writeSoftwareData(config['runner_workdir'], data)
	else:
	  shutil.rmtree(os.path.join(config['software_root'], path))
188
	break
Łukasz Nowak's avatar
Łukasz Nowak committed
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
    return True
  return False


def isInstanceRunning(config):
  slapgrid_pid = os.path.join(config['runner_workdir'], 'slapgrid-cp.pid')
  pid = readPid(slapgrid_pid)
  if pid:
    try:
      os.kill(pid, 0)
    except Exception:
      running = False
    else:
      running = True
  else:
    running = False
  return running

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
def killRunningSlapgrid(config, ptype):
  slapgrid_pid = os.path.join(config['runner_workdir'], ptype)
  pid = readPid(slapgrid_pid)
  if pid:
      recursifKill([pid])
  else:
    return False

def recursifKill(pids):
  """Try to kill a list of proccess by the given pid list"""
  if pids == []:
    return
  else:
    for pid in pids:
      ppids = pidppid(pid)
      try:
223
        os.kill(pid, signal.SIGKILL) #kill current process
224
      except Exception:
225
	      pass
226 227 228 229 230 231 232 233
      recursifKill(ppids) #kill all children of this process

def pidppid(pid):
  """get the list of the children pids of a process `pid`"""
  proc = Popen('ps -o pid,ppid ax | grep "%d"' % pid, shell=True,
               stdout=subprocess.PIPE)
  ppid  = [x.split() for x in proc.communicate()[0].split("\n") if x]
  return list(int(p) for p, pp in ppid if int(pp) == pid)
Łukasz Nowak's avatar
Łukasz Nowak committed
234 235 236 237 238 239

def runInstanceWithLock(config):
  slapgrid_pid = os.path.join(config['runner_workdir'], 'slapgrid-cp.pid')
  if not isInstanceRunning(config):
    startProxy(config)
    logfile = open(config['instance_log'], 'w')
240 241
    if not updateProxy(config):
      return False
Łukasz Nowak's avatar
Łukasz Nowak committed
242 243 244 245 246 247
    slapgrid = Popen([config['slapgrid_cp'], '-vc', config['configuration_file_path']], stdout=logfile)
    writePid(slapgrid_pid, slapgrid.pid)
    slapgrid.wait()
    return True
  return False

248 249
def getProfilePath(peojectDir, profile):
  if not os.path.exists(os.path.join(peojectDir, ".project")):
250
    return False
251 252
  projectFolder = open(os.path.join(peojectDir, ".project")).read()
  return os.path.join(projectFolder, profile)
Łukasz Nowak's avatar
Łukasz Nowak committed
253 254 255 256 257 258 259 260 261

def getSlapStatus(config):
  slap = slapos.slap.slap()
  slap.initializeConnection(config['master_url'])
  partition_list = []
  computer = slap.registerComputer(config['computer_id'])
  try:
    for partition in computer.getComputerPartitionList():
      # Note: Internal use of API, as there is no reflexion interface in SLAP
262 263
      partition_list.append((partition.getId(), partition._connection_dict.copy(),
                              partition._parameter_dict.copy()))
Łukasz Nowak's avatar
Łukasz Nowak committed
264 265
  except Exception:
    pass
266 267 268 269 270
  if partition_list:
    for i in xrange(0, int(config['partition_amount'])):
      slappart_id = '%s%s' % ("slappart", i)
      if not [x[0] for x in partition_list if slappart_id==x[0]]:
        partition_list.append((slappart_id, []))
Łukasz Nowak's avatar
Łukasz Nowak committed
271 272
  return partition_list

273 274 275 276 277 278
def runBuildoutAnnotate(config):
  slapgrid_pid = os.path.join(config['runner_workdir'], 'slapgrid-sr.pid')
  if not isSoftwareRunning(config):
	bin_buildout = os.path.join(config['software_root'], "bin/buildout")
	if os.path.exists(bin_buildout):
	  logfile = open(config['annotate_log'], 'w')
279
	  buildout = Popen([bin_buildout, '-vc', config['configuration_file_path'],
280 281 282 283
	                    "annotate"], stdout=logfile)
	  buildout.wait()
	  return True
  return False
Łukasz Nowak's avatar
Łukasz Nowak committed
284 285

def svcStopAll(config):
286
  return Popen([config['supervisor'], config['configuration_file_path'],
287
                'shutdown']).communicate()[0]
Łukasz Nowak's avatar
Łukasz Nowak committed
288

289 290 291 292 293 294 295 296 297 298 299
def removeInstanceRoot(config):
  if os.path.exists(config['instance_root']):
    svcStopAll(config)
    for root, dirs, files in os.walk(config['instance_root']):
      for fname in dirs:
	fullPath = os.path.join(root, fname)
	if not os.access(fullPath, os.W_OK) :
	  # Some directories may be read-only, preventing to remove files in it
	  os.chmod(fullPath, 0744)
    shutil.rmtree(config['instance_root'])

Łukasz Nowak's avatar
Łukasz Nowak committed
300
def getSvcStatus(config):
301
  result = Popen([config['supervisor'], config['configuration_file_path'],
302 303 304 305
                  'status']).communicate()[0]
  regex = "(^unix:.+\.socket)|(^error:).*$"
  supervisord = []
  for item in result.split('\n'):
306 307 308 309 310
    if item.strip() != "":
      if re.search(regex, item, re.IGNORECASE) == None:
        supervisord.append(re.split('[\s,]+', item))
      else:
        return [] #ignore because it is an error message
311 312 313
  return supervisord

def getSvcTailProcess(config, process):
314
  return Popen([config['supervisor'], config['configuration_file_path'],
315 316 317
                "tail", process]).communicate()[0]

def svcStartStopProcess(config, process, action):
318
  cmd = {"RESTART":"restart", "STOPPED":"start", "RUNNING":"stop", "EXITED":"start", "STOP":"stop"}
319
  return Popen([config['supervisor'], config['configuration_file_path'],
320 321
                cmd[action], process]).communicate()[0]

322
def getFolderContent(config, folder):
323 324
  r=['<ul class="jqueryFileTree" style="display: none;">']
  try:
325
    folder = str(folder)
326 327
    r=['<ul class="jqueryFileTree" style="display: none;">']
    d=urllib.unquote(folder)
328 329 330 331 332 333
    realdir = realpath(config, d)
    if not realdir:
      r.append('Could not load directory: Permission denied')
      ldir = []
    else:
      ldir = sorted(os.listdir(realdir), key=str.lower)
334
    for f in ldir:
335
      if f.startswith('.'): #do not displays this file/folder
336
	continue
337
      ff=os.path.join(d,f)
338 339
      if os.path.isdir(os.path.join(realdir,f)):
	r.append('<li class="directory collapsed"><a href="#%s" rel="%s/">%s</a></li>' % (ff, ff,f))
340 341
      else:
	e=os.path.splitext(f)[1][1:] # get .ext and remove dot
342
	r.append('<li class="file ext_%s"><a href="#%s" rel="%s">%s</a></li>' % (e, ff,ff,f))
343 344 345 346 347 348
    r.append('</ul>')
  except Exception,e:
    r.append('Could not load directory: %s' % str(e))
  r.append('</ul>')
  return jsonify(result=''.join(r))

349
def getFolder(config, folder):
350 351
  r=['<ul class="jqueryFileTree" style="display: none;">']
  try:
352 353
    folder = str(folder)
    r=['<ul class="jqueryFileTree" style="display: none;">']
354
    d=urllib.unquote(folder)
355 356 357 358 359 360
    realdir = realpath(config, d)
    if not realdir:
      r.append('Could not load directory: Permission denied')
      ldir = []
    else:
      ldir = sorted(os.listdir(realdir), key=str.lower)
361
    for f in ldir:
362 363 364
      if f.startswith('.'): #do not display this file/folder
	continue
      ff=os.path.join(d,f)
365 366
      if os.path.isdir(os.path.join(realdir,f)):
	r.append('<li class="directory collapsed"><a href="#%s" rel="%s/">%s</a></li>' % (ff, ff, f))
367 368 369 370 371 372 373 374
    r.append('</ul>')
  except Exception,e:
    r.append('Could not load directory: %s' % str(e))
  r.append('</ul>')
  return jsonify(result=''.join(r))

def getProjectList(folder):
  project = []
375 376
  project_list = sorted(os.listdir(folder), key=str.lower)
  for elt in project_list:
377 378 379
    project.append(elt)
  return project

380 381 382 383 384 385 386
def configNewSR(config, projectpath):
  folder = realpath(config, projectpath)
  if folder:
    if isInstanceRunning(config):
      killRunningSlapgrid(config, "slapgrid-cp.pid")
    if isSoftwareRunning(config):
      killRunningSlapgrid(config, "slapgrid-sr.pid")
387 388 389 390
    stopProxy(config)
    removeProxyDb(config)
    startProxy(config)
    removeInstanceRoot(config)
391
    open(os.path.join(config['runner_workdir'], ".project"), 'w').write(projectpath)
392 393 394 395
    return True
  else:
    return False

396
def newSoftware(folder, config, session):
397 398 399 400
  json = ""
  code = 0
  runner_dir = config['runner_workdir']
  try:
401 402 403
    folderPath = realpath(config, folder, check_exist=False)
    if folderPath and not os.path.exists(folderPath):
      os.mkdir(folderPath)
404 405 406 407 408 409 410 411 412 413 414
      #load software.cfg and instance.cfg from http://git.erp5.org
      software = "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/lamp-template/software.cfg"
      instance = "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/lamp-template/instance.cfg"
      softwareContent = ""
      instanceContent = ""
      try:
	softwareContent = urllib.urlopen(software).read()
	instanceContent = urllib.urlopen(instance).read()
      except:
	#Software.cfg and instance.cfg content will be empty
	pass
415 416 417
      open(os.path.join(folderPath, config['software_profile']), 'w').write(softwareContent)
      open(os.path.join(folderPath, config['instance_profile']), 'w').write(instanceContent)
      open(os.path.join(runner_dir, ".project"), 'w').write(folder + "/")
418 419 420
      session['title'] = getProjectTitle(config)
      code = 1
    else:
421 422
      json = "Bad folder or Directory '" + folder + \
        "' already exist, please enter a new name for your software"
423 424
  except Exception, e:
    json = "Can not create your software, please try again! : " + str(e)
425 426
    if os.path.exists(folderPath):
      shutil.rmtree(folderPath)
427 428 429
  return jsonify(code=code, result=json)

def checkSoftwareFolder(path, config):
430 431
  realdir = realpath(config, path)
  if realdir and os.path.exists(os.path.join(realdir, config['software_profile'])):
432 433 434 435 436 437
    return jsonify(result=path)
  return jsonify(result="")

def getProjectTitle(config):
  conf = os.path.join(config['runner_workdir'], ".project")
  if os.path.exists(conf):
438 439 440
    project = open(conf, "r").read().split("/")
    software = project[len(project) - 2]
    return software + " (" + string.join(project[:(len(project) - 2)], '/') + ")"
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
  return "No Profile"

def loadSoftwareData(runner_dir):
  import pickle
  file_path = os.path.join(runner_dir, '.softdata')
  if not os.path.exists(file_path):
    return []
  pkl_file = open(file_path, 'rb')
  data = pickle.load(pkl_file)
  pkl_file.close()
  return data

def writeSoftwareData(runner_dir, data):
  import pickle
  file_path = os.path.join(runner_dir, '.softdata')
  pkl_file = open(file_path, 'wb')
  # Pickle dictionary using protocol 0.
  pickle.dump(data, pkl_file)
  pkl_file.close()
460

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
def removeSoftwareByName(config, folderName):
  if isSoftwareRunning(config) or isInstanceRunning(config):
    return jsonify(code=0, result="Software installation or instantiation in progress, cannot remove")
  path = os.path.join(config['software_root'], folderName)
  if not os.path.exists(path):
    return jsonify(code=0, result="Can not remove software: No such file or directory")
  svcStopAll(config)
  shutil.rmtree(path)
  #update compiled software list
  data = loadSoftwareData(config['runner_workdir'])
  i = 0
  for p in data:
    if p['md5'] == folderName:
      del data[i]
      writeSoftwareData(config['runner_workdir'], data)
      break
    i = i+1
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
  return jsonify(code=1, result=data)

def tail(f, lines=20):
  """
  Returns the last `lines` lines of file `f`. It is an implementation of tail -f n.
  """
  BUFSIZ = 1024
  f.seek(0, 2)
  bytes = f.tell()
  size = lines + 1
  block = -1
  data = []
  while size > 0 and bytes > 0:
      if bytes - BUFSIZ > 0:
	  # Seek back one whole BUFSIZ
	  f.seek(block * BUFSIZ, 2)
	  # read BUFFER
	  data.insert(0, f.read(BUFSIZ))
      else:
	  # file too small, start from begining
	  f.seek(0,0)
	  # only read what was not read
	  data.insert(0, f.read(bytes))
      linesFound = data[0].count('\n')
      size -= linesFound
      bytes -= BUFSIZ
      block -= 1
505 506
  return string.join(''.join(data).splitlines()[-lines:], '\n')

507 508 509 510 511 512 513 514 515 516 517
def readFileFrom(f, lastPosition):
  """
  Returns the last lines of file `f`, from position lastPosition.
  and the last position
  """
  BUFSIZ = 1024
  f.seek(0, 2)
  bytes = f.tell()
  block = -1
  data = ""
  length = bytes
518 519
  if lastPosition <= 0 and length > 30000:
    lastPosition = length-30000
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
  size = bytes - lastPosition
  while bytes > lastPosition:
    if abs(block*BUFSIZ) <= size:
      # Seek back one whole BUFSIZ
      f.seek(block * BUFSIZ, 2)
      data = f.read(BUFSIZ) + data
    else:
      margin = abs(block*BUFSIZ) - size
      if length < BUFSIZ:
	f.seek(0,0)
      else:
	seek = block * BUFSIZ + margin
	f.seek(seek, 2)
      data = f.read(BUFSIZ - margin) + data
    bytes -= BUFSIZ
    block -= 1
  f.close()
  return {"content":data, "position":length}

539 540 541 542 543 544 545 546
def isText(file):
  """Return True if the mimetype of file is Text"""
  if not os.path.exists(file):
    return False
  text_range = ''.join(map(chr, [7,8,9,10,12,13,27] + range(0x20, 0x100)))
  is_binary_string = lambda bytes: bool(bytes.translate(None, text_range))
  try:
    return not is_binary_string(open(file).read(1024))
547 548
  except:
    return False
549

550 551 552 553 554 555 556 557 558 559 560 561 562
def md5sum(file):
  """Compute md5sum of `file` and return hexdigest value"""
  if os.path.isdir(file):
    return False
  try:
    fh = open(file, 'rb')
    m = hashlib.md5()
    while True:
      data = fh.read(8192)
      if not data:
	break
      m.update(data)
    return m.hexdigest()
563
  except:
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
    return False

def realpath(config, path, check_exist=True):
  """Get realpath of path or return False if user is not allowed to access to
  this file"""
  split_path = path.split('/')
  key = split_path[0]
  allow_list = {'software_root':config['software_root'], 'instance_root':
                config['instance_root'], 'workspace': config['workspace']}
  if allow_list.has_key(key):
    del split_path[0]
    path = os.path.join(allow_list[key], string.join(split_path, '/'))
    if check_exist:
      if os.path.exists(path):
	return path
      else:
	return False
    else:
      return path
583
  return False
584 585 586 587 588 589 590

def readParameters(path):
  if os.path.exists(path):
    try:
      xmldoc = minidom.parse(path)
      object = {}
      for elt in xmldoc.childNodes:
591 592 593 594
        sub_object = {}
        for subnode in elt.childNodes:
          if subnode.nodeType != subnode.TEXT_NODE:
            sub_object[str(subnode.getAttribute('id'))] = str(subnode.
595
	                                                 childNodes[0].data)
596
	    object[str(elt.tagName)] = sub_object
597 598 599 600 601
      return object
    except Exception, e:
      return str(e)
  else:
    return "No such file or directory: " + path