runTestSuite.in 8.18 KB
Newer Older
1 2 3 4 5 6 7 8 9
#!${buildout:directory}/bin/${eggs:interpreter}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
"""
  Script to run jIO/renderJS test suite using Nexedi's test node framework.
"""
import argparse, os, re, shutil, subprocess, sys, traceback
from erp5.util import taskdistribution
from time import gmtime, strftime
10
from lxml import etree
11 12 13 14 15 16 17 18 19 20 21 22 23
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Put a 'firefox' executable in the path
# otherwise, WebDriver refuses to start
os.environ['PATH'] = '$${directory:bin}' + os.pathsep + os.environ['PATH']

FIREFOX_EXECUTABLE = '$${firefox-instance:executable}'
BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'

def main():
24 25 26 27 28 29 30 31 32 33
  parser = argparse.ArgumentParser(description='Run a test suite.')
  parser.add_argument('--test_suite', help='The test suite name')
  parser.add_argument('--test_suite_title', help='The test suite title')
  parser.add_argument('--test_node_title', help='The test node title')
  parser.add_argument('--project_title', help='The project title')
  parser.add_argument('--revision', help='The revision to test',
                      default='dummy_revision')
  parser.add_argument('--node_quantity', help='ignored', type=int)
  parser.add_argument('--master_url',
                      help='The Url of Master controling many suites')
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
  parser.add_argument('--frontend_url',
                      help='The url of frontend of the test suite')
  parser.add_argument('--target',
                      help='Target OS to run tests on',
                      type=str)
  parser.add_argument('--target_version',
                      help='Target OS version to use',
                      type=str,)
  parser.add_argument('--target_browser',
                      help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
                      type=str,)
  parser.add_argument('--target_device',
                      help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
                      type=str,)
  parser.add_argument('--appium_server_auth',
                      help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
                      type=str)
51

52
  args = parser.parse_args()
53

54
  import json
Ivan Tyagov's avatar
Ivan Tyagov committed
55
  parsed_parameters = json.loads('$${instance-parameter:configuration._}')
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70


  if not getattr(args, 'target', None):
    args.target = parsed_parameters.get('target', 'firefox')
  if not getattr(args, 'test_suite', None):
    args.test_suite = parsed_parameters.get('test-suite')
  if not getattr(args, 'target_version', None):
    args.target_version = parsed_parameters.get('target-version')
  if not getattr(args, 'appium_server_auth', None):
    args.appium_server_auth = parsed_parameters.get('appium-server-auth')
  if not getattr(args, 'target_browser', None):
    args.target_browser = parsed_parameters.get('target-browser')
  if not getattr(args, 'target_device', None):
    args.target_device = parsed_parameters.get('target-device')

Ivan Tyagov's avatar
Ivan Tyagov committed
71
  is_browser_running = False
72
  try:
73 74 75 76 77 78 79 80 81 82 83 84 85
    test_suite_title = args.test_suite_title or args.test_suite
    test_suite = args.test_suite
    revision = args.revision

    test_line_dict = {}

    if ('jio' in test_suite):
      url = BASE_URL + 'jio/test/tests.html'
    else:
      url = BASE_URL + 'renderjs/test/'

    date = strftime("%Y/%m/%d %H:%M:%S", gmtime())

86
  
87 88 89
    ##########################
    # Run all tests
    ##########################
90 91 92
    
    is_appium = False
    
93 94 95
    if args.target == 'firefox':
      firefox_binary = webdriver.firefox.firefox_binary.FirefoxBinary(firefox_path=FIREFOX_EXECUTABLE)
      browser = webdriver.Firefox(firefox_binary=firefox_binary)
96 97 98 99
    elif args.target in ['iOS', 'Android']:
      # parameters for mobile emulators have different names then parameters for
      # desktop OSes
      is_appium = True
100 101 102 103 104 105 106
      capabilities = {
        'platformName': args.target,
        'platformVersion': args.target_version,
        'deviceName': args.target_device,
        'browserName': args.target_browser

      }
107 108 109 110 111 112 113 114 115 116
    elif 'Windows' in args.target or 'OS X' in args.target:
      # parameters for mobile emulators have different names then parameters for
      # desktop OSes
      is_appium = True
      capabilities = {
        'browserName': args.target_browser,
        'platform': args.target,
        'version': args.target_version
      }
    if is_appium:
117 118 119 120 121
      if not args.appium_server_auth:
        raise RuntimeError('--appium_server_auth is required.')
      appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
      browser = webdriver.Remote(appium_url, capabilities)

122 123 124 125
      # adjust make path to testnode's frontend
      full_path = '$${runTestSuite-instance:buildout-directory}/software_release/parts/%s' % parsed_parameters['test-url']
      full_path = full_path.split('srv')[-1]
      url = "%s%s" % (args.frontend_url, full_path)
126

127
    is_browser_running = True
128 129
    agent = browser.execute_script("return navigator.userAgent")
    print agent
130
    print url
131 132 133 134 135 136

    browser.get(url)
    WebDriverWait(browser, 60).until(EC.presence_of_element_located((
      By.XPATH, '//p[@id="qunit-testresult" and contains(text(), "completed")]')
    ))

137 138 139 140
    html_parser = etree.HTMLParser(recover=True)
    body = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)

    print ' '.join(body.xpath('//*[@id="qunit-testresult"]//text()'))
141

142
    for elt in body.xpath('.//ol[@id="qunit-tests"]/li'):
143
      test_name = '%s: %s' % (
144 145
        elt.xpath('.//span[@class="module-name"]')[0].text,
        elt.xpath('.//span[@class="test-name"]')[0].text
146
      )
147
      print elt.get('class'), ''.join(elt.xpath('.//strong')[0].itertext())
148 149
      # print elt.find_element_by_tag_name('ol').get_attribute('innerHTML')

150 151
      failure = int(elt.xpath('.//b[@class="failed"]')[0].text)
      success = int(elt.xpath('.//b[@class="passed"]')[0].text)
152 153 154 155 156
      test_line_dict[test_name] = {
        'test_count': success + failure,
        'error_count': 0,
        'failure_count': failure,
        'skip_count': 0,
157 158
        'duration': int(elt.xpath('.//span[@class="runtime"]')[0].text.split()[0]),
        'command': elt.xpath('.//a[text()="Rerun"]')[0].get('href'),
159 160
        'stdout': agent,
        'stderr': '',
161
        'html_test_result': etree.tostring(elt.xpath('.//ol')[0])
162 163
      }

164 165 166 167 168 169 170
    # do quit browser asap as we have results. this is required in case of timeout of
    # remote appium service which will close test session of no command received within
    # usually 90s and thus fail this script. And it costs processing time as well
    # to keep test session needlessly opened.
    browser.quit()
    is_browser_running = False

171 172 173 174 175 176
    tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url)
    test_result = tool.createTestResult(revision = revision,
                                        test_name_list = test_line_dict.keys(),
                                        node_title = args.test_node_title,
                                        test_title = test_suite_title,
                                        project_title = args.project_title)
177
    if test_result is None or not hasattr(args, 'master_url'):
178 179 180 181 182 183 184 185 186 187 188 189
      return
    # report test results
    while 1:
      test_result_line = test_result.start()
      if not test_result_line:
        print 'No test result anymore.'
        break

      print 'Submitting: "%s"' % test_result_line.name
      # report status back to Nexedi ERP5
      test_result_line.stop(**test_line_dict[test_result_line.name])

190
  except:
191 192
    # Catch any exception here, to warn user instead of being silent,
    # by generating fake error result
193
    print traceback.format_exc()
194 195 196 197 198 199
    result = dict(status_code=-1,
                  command=url,
                  stderr=traceback.format_exc(),
                  stdout='')
    # XXX: inform test node master of error
    raise EnvironmentError(result)
200 201
  
  finally:
202
    if is_browser_running:
Ivan Tyagov's avatar
Ivan Tyagov committed
203 204
      # if by any chance browser is still running due to
      # traceback raised make sure we cleanup
205
      browser.quit()
206 207

if __name__ == "__main__":
208
    main()