slaprunner.py 9.99 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

from .resiliencytestsuite import ResiliencyTestSuite

31
import base64
32
import cookielib
33
import json
34
from lxml import etree
35
import random
36
import ssl
37 38 39 40 41 42 43 44 45 46 47 48
import string
import time
import urllib2

class NotHttpOkException(Exception):
  pass

class SlaprunnerTestSuite(ResiliencyTestSuite):
  """
  Run Slaprunner Resiliency Test.
  It is highly suggested to read ResiliencyTestSuite code.
  """
49
  def __init__(self, *args, **kwargs):
50 51
    # Setup urllib2 with cookie support
    cookie_jar = cookielib.CookieJar()
52 53
    ssl_context = ssl._create_unverified_context()

54
    self._opener_director = urllib2.build_opener(
55 56
        urllib2.HTTPCookieProcessor(cookie_jar),
        urllib2.HTTPSHandler(context=ssl_context)
57
    )
58

59
    ResiliencyTestSuite.__init__(self, *args, **kwargs)
60

61 62 63 64 65 66 67 68 69 70 71 72 73
  def _getPartitionParameterDict(self):
    """
    Helper.
    Return the partition parameter dict of the main root ("resilient") instance.
    """
    # XXX Hardcoded parameters, should be obtained dynamically
    return self.partition.request(
      software_release=self.software,
      software_type='resilient',
      partition_reference=self.root_instance_name,
      partition_parameter_kw={
        'resiliency-backup-periodicity': '*/6 * * * *',
        'auto-deploy-instance': 'false',
74 75 76
        'auto-deploy': 'true',
        # XXX HACK!
        "slapos-reference": 'slaprunner-erp5-resiliency'
77 78 79
        }
    ).getConnectionParameterDict()
    self.deleteTimestamp()
80

81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
  def _connectToSlaprunner(self, resource, data=None):
    """
    Utility.
    Connect through HTTP to the slaprunner instance.
    Require self.slaprunner_backend_url to be set.
    """
    url = "%s/%s" % (self.slaprunner_backend_url, resource)
    if data:
      result = self._opener_director.open(url, data=data)
    else:
      result = self._opener_director.open(url)

    if result.getcode() is not 200:
      raise NotHttpOkException(result.getcode())
    return result.read()

  def _login(self):
    self.logger.debug('Logging in...')
99 100
    b64string = base64.encodestring('%s:%s' % (self.slaprunner_user, self.slaprunner_password))[:-1]
    self._opener_director.addheaders = [('Authorization', 'Basic %s'%b64string)]
101 102 103

  def _retrieveInstanceLogFile(self):
    """
104 105
    Store the logfile (=data) of the instance, check it is not empty nor it is
    html.
106
    """
107
    time.sleep(30)
108
    data = self._connectToSlaprunner(
109 110
        resource='getFileContent',
        data="file=instance_root/slappart0/var/log/log.log"
111
    )
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
    try:
        data = json.loads(data)['result']
        self.logger.info('Retrieved data are:\n%s' % data)
    except (ValueError, KeyError):
        if data.find('<') is not -1:
          raise IOError(
              'Could not retrieve logfile content: retrieved content is html.'
          )
        if data.find('Could not load') is not -1:
          raise IOError(
              'Could not retrieve logfile content: server could not load the file.'
          )
        if data.find('Hello') is -1:
          raise IOError(
              'Could not retrieve logfile content: retrieve content does not match "Hello".'
          )
128 129
    return data

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150

  def _waitForSoftwareBuild(self, limit=5000):
    """
    Wait until SR is built or limit reach 0
    """
    def getSRStatus():
      """
      Return current status (-1 in case of connection problem)
      """
      try:
        return self._connectToSlaprunner(resource='isSRReady')
      except (NotHttpOkException, urllib2.HTTPError) as error:
        # The nginx frontend might timeout before software release is finished.
        self.logger.warning('Problem occured when contacting the server: %s' % error)
        return -1

    status = getSRStatus()
    while limit > 0 and status != '1':
      status = getSRStatus()
      limit -= 1
      self.logger.info('Software release is still building. Sleeping...')
151
      time.sleep(60)
152

153

154
  def _buildSoftwareRelease(self):
155 156 157 158 159 160
    self.logger.info('Building the Software Release...')
    try:
      self._connectToSlaprunner(resource='runSoftwareProfile')
    except (NotHttpOkException, urllib2.HTTPError):
      # The nginx frontend might timeout before software release is finished.
      pass
161
    self._waitForSoftwareBuild()
162

163
  def _deployInstance(self):
164 165 166 167 168 169
    self.logger.info('Deploying instance...')
    try:
      self._connectToSlaprunner(resource='runInstanceProfile')
    except (NotHttpOkException, urllib2.HTTPError):
      # The nginx frontend might timeout before someftware release is finished.
      pass
170
    while True:
171
      time.sleep(15)
172 173 174 175
      result = json.loads(self._connectToSlaprunner(resource='slapgridResult', data='position=0&log='))
      if result['instance']['state'] is False:
        break
      self.logger.info('Buildout is still running. Sleeping...')
176 177
    self.logger.info('Instance has been deployed.')

178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
  def _gitClone(self):
    self.logger.debug('Doing git clone of git.erp5.org/repos/slapos.git...')
    try:
      self._connectToSlaprunner(
          resource='cloneRepository',
          data='repo=http://git.erp5.org/repos/slapos.git&name=workspace/slapos&email=slapos@slapos.org&user=slapos'
      )
    except (NotHttpOkException, urllib2.HTTPError):
      # cloning can be very long.
      # XXX: quite dirty way to check.
      while self._connectToSlaprunner('getProjectStatus', data='project=workspace/slapos').find('On branch master') is -1:
        self.logger.info('git-cloning ongoing, sleeping...')

  def _openSoftwareRelease(self, software_name):
    self.logger.debug('Opening %s software release...' % software_name)
    self._connectToSlaprunner(
        resource='setCurrentProject',
        data='path=workspace/slapos/software/%s/' % software_name
    )

198 199
  def _getRcode(self):
    #XXX-Nicolas: hardcoded url. Best way right now to automate the tests...
200
    monitoring_password = "passwordtochange"
201
    monitor_url = self.monitor_url + "?script=zero-knowledge%2Fsettings.cgi"
202
    result = self._opener_director.open(monitor_url,
203
                                       "password=" + monitoring_password + ";password_2=" + monitoring_password)
204 205 206 207 208

    if result.getcode() is not 200:
      raise NotHttpOkException(result.getcode())

    page = result.read().strip()
209
    html = etree.HTML(page)
210

211 212
    input = html.xpath("//input[@name='recovery-code']")
    return input[0].get('value')
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234

  def generateData(self):
    self.slaprunner_password = ''.join(
        random.SystemRandom().sample(string.ascii_lowercase, 8)
    )
    self.slaprunner_user = 'slapos'
    self.logger.info('Generated slaprunner user is: %s' % self.slaprunner_user)
    self.logger.info('Generated slaprunner password is: %s' % self.slaprunner_password)

  def pushDataOnMainInstance(self):
    """
    Create a dummy Software Release,
    Build it,
    Wait for build to be successful,
    Deploy instance,
    Wait for instance to be started.
    Store the main IP of the slaprunner for future use.
    """
    self.logger.debug('Getting the backend URL and recovery code...')
    parameter_dict = self._getPartitionParameterDict()
    self.slaprunner_backend_url = parameter_dict['backend_url']
    self.logger.info('backend_url is %s.' % self.slaprunner_backend_url)
235
    self.monitor_url = parameter_dict['monitor_backend_url']
236
    slaprunner_recovery_code = self._getRcode()
237

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
    self.logger.debug('Creating the slaprunner account...')
    self._connectToSlaprunner(
        resource='configAccount',
        data='name=slapos&username=%s&email=slapos@slapos.org&password=%s&rcode=%s' % (
            self.slaprunner_user,
            self.slaprunner_password,
            slaprunner_recovery_code
        )
    )

    self._login()

    self._gitClone()
    # XXX should be taken from parameter.
    self._openSoftwareRelease('helloworld')

    self._buildSoftwareRelease()
255
    time.sleep(15)
256 257 258
    self._deployInstance()

    self.data = self._retrieveInstanceLogFile()
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274

  def checkDataOnCloneInstance(self):
    """
    Check that:
      * backend_url is different
      * Software Release profile is the same,
      * Software Release is built and is the same, (?)
      * Instance is deployed and is the same.
    """
    # XXX: does the promise wait for the software to be built and the instance to be ready?
    old_slaprunner_backend_url = self.slaprunner_backend_url
    self.slaprunner_backend_url = self._returnNewInstanceParameter(
        parameter_key='backend_url',
        old_parameter_value=old_slaprunner_backend_url
    )
    self._login()
275
    self._waitForSoftwareBuild()
276
    time.sleep(15)
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
    new_data = self._retrieveInstanceLogFile()

    if new_data == self.data:
      self.logger.info('Data are the same: success.')
      return True
    else:
      self.logger.info('Data are different: failure.')


def runTestSuite(*args, **kwargs):
  """
  Run Slaprunner Resiliency Test.
  """
  return SlaprunnerTestSuite(*args, **kwargs).runTestSuite()