software/tactic_generator: Add tactic_generator SR

Provide :
* a Python with selenium egg installed
* a Python script to run tactics
* a Fluentd instance
* instance parameters to send generated logs to a Wendelin instance through Fluentd

software/tactic_generator/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance-selenium]
+filename = instance.cfg
+md5sum = 80c32e1a77cfab9e6df77fd4c145deeb

software/tactic_generator/flight_script.js

+var EPSILON = {0},
+  DODGE_DISTANCE = {1};
+
+function distance2D(a, b) {{
+  var R = 6371e3, // meters
+    la1 = a.latitude * Math.PI / 180, // lat, lon in radians
+    la2 = b.latitude * Math.PI / 180,
+    lo1 = a.longitude * Math.PI / 180,
+    lo2 = b.longitude * Math.PI / 180,
+    haversine_phi = Math.pow(Math.sin((la2 - la1) / 2), 2),
+    sin_lon = Math.sin((lo2 - lo1) / 2),
+    h = haversine_phi + Math.cos(la1) * Math.cos(la2) * sin_lon * sin_lon;
+  return 2 * R * Math.asin(Math.sqrt(h));
+}}
+
+function distance(a, b) {{
+  return Math.sqrt(
+    Math.pow(a.altitude - b.altitude, 2) + Math.pow(distance2D(a, b), 2)
+  );
+}}
+
+me.onStart = function (timestamp) {{
+  me.direction_set = false;
+  me.dodging = false;
+  me.ongoing_detection = false;
+  me.takeOff();
+}};
+
+me.onGetMsg = function (msg) {{
+  if (msg && msg.flag_positions) {{
+    me.flag_positions = msg.flag_positions;
+    me.next_checkpoint = me.id % me.flag_positions.length;
+  }}
+}};
+
+me.onUpdate = function (timestamp) {{
+  if (!me.isReadyToFly()) {{
+    return;
+  }}
+  if (!me.flag_positions) return;
+  if (me.dodging) {{
+    me.current_position = me.getCurrentPosition();
+    var dist = distance(
+      me.current_position,
+      me.dodging.position
+    );
+    if (dist >= DODGE_DISTANCE) {{
+      me.dodging = false;
+    }}
+    return;
+  }}
+  if (!me.direction_set) {{
+    if (me.next_checkpoint < me.flag_positions.length) {{
+      me.setTargetCoordinates(
+        me.flag_positions[me.next_checkpoint].position.latitude,
+        me.flag_positions[me.next_checkpoint].position.longitude,
+        me.flag_positions[me.next_checkpoint].position.altitude + me.id,
+        {2}
+      );
+    }}
+    me.direction_set = true;
+    return;
+  }}
+  if (me.next_checkpoint < me.flag_positions.length) {{
+    if (!me.ongoing_detection) {{
+      me.getDroneViewInfo();
+      me.ongoing_detection = true;
+    }}
+  }}
+  if (me.next_checkpoint < me.flag_positions.length) {{
+    me.current_position = me.getCurrentPosition();
+    me.distance = distance(
+      me.current_position,
+      me.flag_positions[me.next_checkpoint].position
+    );
+    if (me.distance <= EPSILON) {{
+      me.next_checkpoint += 1;
+      me.direction_set = false;
+    }}
+    return;
+  }}
+}};
+
+me.onDroneViewInfo = function (drone_view) {{
+  me.ongoing_detection = false;
+  if (drone_view && drone_view.obstacles && drone_view.obstacles.length) {{
+    me.dodging = drone_view.obstacles[0];
+    me.direction_set = false;
+    var random = Math.random() < 0.5, dodge_point = {{}};
+    Object.assign(dodge_point, me.flag_positions[me.next_checkpoint].position);
+    if (random) {{
+      dodge_point.latitude = dodge_point.latitude * -1;
+    }} else {{
+      dodge_point.longitude = dodge_point.longitude * -1;
+    }}
+    me.setTargetCoordinates(dodge_point.latitude, dodge_point.longitude, me.getCurrentPosition().altitude, 16);
+    return;
+  }}
+}};
\ No newline at end of file

software/tactic_generator/generate_tactic.py

+"""Run the web application through remote selenium browsers."""
+
+import argparse
+import itertools
+import logging
+import os
+import traceback
+from datetime import datetime
+from multiprocessing import Queue
+from threading import Thread
+from urllib3 import PoolManager
+from urllib3.exceptions import ProtocolError
+
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.remote.remote_connection import RemoteConnection
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+SELENIUM_INSTANCE_DICT= {
+#  'https://selenium:PASSWORD@[IPV6]:9443/wd/hub': 1,
+}
+
+TIMEOUT = 50
+GRANULARITY = 5
+GAME_INPUT_DICT = {
+  'number_of_drones': (1, 10),
+  'simulation_speed': (100000, 100000),
+}
+SCRIPT_INPUT_DICT = {
+  'epsilon': (1, 30),
+  'dogde': (1, 200),
+  'speed': (1, 16)
+}
+
+parser = argparse.ArgumentParser(
+  prog='run_web_application',
+  description='use selenium to run a web application and collect the logged results',
+)
+parser.add_argument(
+  '-d', '--directory', type=str, default='.', help='location to store results')
+parser.add_argument('-s', '--script', type=str, help='flight script location')
+parser.add_argument(
+  '-u', '--url', type=str, default='https://flock.app.officejs.com/', help='simulator url')
+args = parser.parse_args()
+os.makedirs(args.directory, exist_ok=True)
+
+def createDriver(capabilities, selenium_url):
+  # XXX we are using a self signed certificate, but selenium 3.141.0 does
+  # not expose API to ignore certificate verification
+  executor = RemoteConnection(selenium_url, keep_alive=True)
+  executor._conn = PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
+  return webdriver.Remote(
+    command_executor=executor,
+    desired_capabilities=capabilities,
+    keep_alive=True,
+  )
+
+def chromeCapabilities():
+  options = webdriver.ChromeOptions()
+  options.headless = True
+  capabilities = options.to_capabilities()
+  capabilities['version'] = '120.0.6099.109'
+  return capabilities
+
+def downloadLogs(driver, combination, result_dir):
+  """Collect results from application run"""
+  user_score = driver.find_element(By.ID, 'result_score').get_attribute('textContent').split(' ')[-1]
+  simulation_duration = driver.find_element(By.ID, 'simulation_duration').get_attribute('textContent').split(' ')[-2]
+  crash_score = 0
+  for i in range(combination[0]):
+    id_s = "log_result_" + str(i)
+    result_log = driver.find_element(
+      By.XPATH, '//textarea[@id="log_result_%s"]' % i)
+    if not 'touched' in result_log.get_attribute('value').split('\n')[-1]:
+      crash_score += 1
+
+  with open(
+    os.path.join(
+      result_dir,
+      'simulation_log_%s.log' % '_'.join((
+        '%s_%s' % (k, v) for k, v
+        in zip(list(GAME_INPUT_DICT.keys()) + list(SCRIPT_INPUT_DICT.keys()), combination)
+      ),
+    )
+  ), 'w') as f:
+    f.write('score;duration;crash\n%s;%s;%s\n' % (user_score, simulation_duration, crash_score))
+
+def write_into_CodeMirror(driver, text):
+  WebDriverWait(driver, TIMEOUT).until(
+    EC.presence_of_element_located((By.XPATH, '//iframe'))
+  )
+  frame = driver.find_element(By.XPATH, '//iframe')
+  driver.switch_to.frame(frame)
+  driver.execute_script(
+    "arguments[0].CodeMirror.setValue(arguments[1])",
+    driver.find_element(By.XPATH, '//body//div'),
+    text,
+  )
+  driver.switch_to.default_content()
+
+def selenium_task(driver, app_url, combination, data_queue, error_logger):
+  try:
+    try:
+      driver.set_window_size(1000, 4080)
+      driver.get(app_url)
+
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.XPATH, '//canvas'))
+      )
+
+      driver.find_element(By.XPATH, '//button[text()="Parameters"]').click()
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.ID, 'simulation_speed'))
+      )
+
+      for i, key in enumerate(GAME_INPUT_DICT.keys()):
+        driver.execute_script(
+          'arguments[0].value = ' + str(combination[i]),
+          driver.find_element(By.ID, key),
+        )
+
+      if args.script is not None:
+        driver.find_element(By.XPATH, '//button[text()="Drone Script"]').click()
+        with open(args.script, 'r') as script:
+          write_into_CodeMirror(driver, script.read().format(*(v for i, v in enumerate(combination) if i >= len(GAME_INPUT_DICT.keys()))))
+
+      driver.find_element(By.XPATH, '//button[text()="Run"]').click()
+      WebDriverWait(driver, TIMEOUT).until(
+        EC.presence_of_element_located((By.XPATH, '//div//a[contains(text(), "Download Simulation LOG")]'))
+      )
+      downloadLogs(driver, combination, args.directory)
+      if not data_queue.empty():
+        driver.refresh()
+    except (TimeoutException, WebDriverException):
+      driver.refresh()
+      raise
+    except ProtocolError:
+      driver = createDriver(chromeOptions(), driver.command_executor._url)
+      raise
+  except Exception as e:
+    error_logger.error(driver.command_executor._url)
+    error_logger.error(e)
+    error_logger.error(traceback.format_exc())
+    data_queue.put(combination)
+
+def selenium_queue_listener(data_queue, worker_queue, selenium_worker_list, app_url, combination_nb, error_logger):
+  while not data_queue.empty():
+    current_data = data_queue.get()
+    worker_id = worker_queue.get()
+    selenium_task(selenium_worker_list[worker_id], app_url, current_data, data_queue, error_logger)
+    worker_queue.put(worker_id)
+    update_progress((combination_nb - data_queue.qsize()) / combination_nb)
+  return
+
+def values_in_range(start, end, n):
+  if n == 1:
+    return [start]
+  d = (end - start) / (n - 1)
+  return [start + i*d for i in range(n)]
+
+def update_progress(progress):
+  bar_length = 20
+  if isinstance(progress, int):
+    progress = float(progress)
+  if not isinstance(progress, float):
+    progress = 0
+  if progress < 0:
+    progress = 0
+  if progress >= 1:
+    progress = 1
+
+  block = int(round(bar_length * progress))
+  text = "Progress: [{0}] {1:.1f}%".format( "#" * block + "-" * (bar_length - block), progress * 100)
+  print(text, end='\r')
+
+##### MAIN #####
+worker_queue = Queue()
+selenium_worker_list = []
+if not SELENIUM_INSTANCE_DICT:
+  raise ValueError('SELENIUM_INSTANCE_DICT is empty')
+print('Creating web drivers ' + datetime.now().strftime('%d/%m/%Y %H:%M:%S'))
+for i, url in enumerate(
+  (url for url, slot_quantity in SELENIUM_INSTANCE_DICT.items() for _ in range(slot_quantity))
+):
+  selenium_worker_list.append(createDriver(chromeOptions(), url))
+  worker_queue.put(i)
+
+drone_input_value_tuple = (
+  values_in_range(start, end, GRANULARITY) for start, end
+  in list(GAME_INPUT_DICT.values()) + list(SCRIPT_INPUT_DICT.values())
+)
+combination_list = [combination for combination in itertools.product(*drone_input_value_tuple)]
+combination_nb = len(combination_list)
+print('Total combinations: ' + str(combination_nb))
+selenium_data_queue = Queue()
+for d in combination_list:
+  selenium_data_queue.put(d)
+
+creation_time = datetime.now()
+print('Starting selenium background processes ' + creation_time.strftime('%d/%m/%Y %H:%M:%S'))
+error_logger = logging.getLogger(__name__)
+logging.basicConfig(filename='error.log')
+selenium_processes = [
+  Thread(target=selenium_queue_listener,
+         args=(selenium_data_queue, worker_queue, selenium_worker_list, args.url, combination_nb, error_logger))
+  for _ in range(len(selenium_worker_list))
+]
+update_progress(0)
+for p in selenium_processes:
+  p.daemon = True
+  p.start()
+for p in selenium_processes:
+  p.join()
+elapsed_time = datetime.now() - creation_time
+print('\nTotal time for runs: %s seconds.' % elapsed_time.seconds)
+
+for b in selenium_worker_list:
+  try:
+    b.quit()
+  except WebDriverException as e:
+    error_logger.error(e)
+    error_logger.error(traceback.format_exc())

software/tactic_generator/instance-input-schema.json

+{
+  "$schema": "https://json-schema.org/draft-06/schema#",
+  "type": "object",
+  "description": "Parameters to instantiate Selenium client",
+  "additionalProperties": false,
+  "properties": {
+    "flight-log-dir": {
+      "title": "Fligh log directory",
+      "description": "Directory where Fluentd will find the flight logs to send to Wendelin.",
+      "type": "string",
+      "default": "BUILDOUT_DIRECTORY/etc/flight_log_dir"
+    },
+    "simulation-log-dir": {
+      "title": "Simulation log directory",
+      "description": "Directory where Fluentd will find the simulation logs to send to Wendelin.",
+      "type": "string",
+      "default": "BUILDOUT_DIRECTORY/etc/flight_log_dir/simulation_log_dir"
+    },
+    "wendelin-password": {
+      "title": "Wendelin passwor",
+      "description": "Password for Wendelin instance",
+      "type": "string",
+      "default": ""
+    },
+    "wendelin-url": {
+      "title": "Wendelin url",
+      "description": "Url of the wendelin instance to send logs to",
+      "type": "string",
+      "default": ""
+    },
+    "wendelin-user": {
+      "title": "Wendelin user",
+      "description": "Username for Wendelin instance",
+      "type": "string",
+      "default": ""
+    }
+  }
+}

software/tactic_generator/instance-output-schema.json

+{
+  "$schema": "https://json-schema.org/draft-06/schema#",
+  "description": "Values returned by Selenium client instantiation",
+  "additionalProperties": false,
+  "properties": {
+    "python-with-eggs": {
+      "description": "Path to a Python interpreter with selenium egg installed",
+      "type": "array"
+    }
+  },
+  "type": "object"
+}

software/tactic_generator/instance.cfg

+[buildout]
+extends =
+  ${instance-fluentd:output}
+parts +=
+  publish-connection-information
+
+[dynamic-template-fluentd]
+context =
+  key fluentd_agent_conf selenium-client-conf:output
+  key port_list fluentd-conf:port-list
+  section directory directory
+
+[selenium-client-conf]
+recipe = slapos.recipe.template:jinja2
+output = $${buildout:directory}/selenium-conf.cfg
+context =
+  key slapparameter_dict slap-configuration:configuration
+  key simulation_log_dir directory:simulation-log-dir
+  key flight_log_dir directory:flight-log-dir
+  key tmp_dir directory:tmp
+inline =
+  {% set wendelin_url = slapparameter_dict.get('wendelin-url', '') -%}
+  {% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
+  {% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
+  {% set simulation_log_dir = slapparameter_dict.get('simulation-log-dir', simulation_log_dir) -%}
+  {% set flight_log_dir = slapparameter_dict.get('flight-log-dir', flight_log_dir) -%}
+  <source>
+    @type tail
+    read_from_head true
+    path {{ simulation_log_dir }}/*.log
+    pos_file {{ tmp_dir }}/drone_simulation.pos
+    tag drone_simulation.sample-drone-raw-data
+    format none
+    path_key filepath
+  </source>
+
+  <match drone_simulation.sample-drone-raw-data>
+    @type wendelin
+    streamtool_uri {{ wendelin_url }}/erp5/portal_ingestion_policies/default
+    user {{ wendelin_user }}
+    password {{ wendelin_password }}
+    <buffer>
+      @type memory
+      flush_mode interval
+      flush_interval 1s
+    </buffer>
+  </match>
+
+  <source>
+    @type tail
+    read_from_head true
+    path {{ flight_log_dir }}/*.log
+    pos_file {{ tmp_dir }}/drone_real.pos
+    tag drone_real.bouncy_flight_mavsdk_raw_data
+    format none
+    path_key filepath
+  </source>
+
+  <match drone_real.bouncy_flight_mavsdk_raw_data>
+    @type wendelin
+    streamtool_uri {{ wendelin_url }}/erp5/portal_ingestion_policies/default
+    user {{ wendelin_user }}
+    password {{ wendelin_password }}
+    <buffer>
+      @type memory
+      flush_mode interval
+      flush_interval 1s
+    </buffer>
+  </match>
+
+[directory]
+tmp = $${:home}/tmp
+simulation-log-dir = $${:etc}/simulation_log_dir
+flight-log-dir = $${:etc}/flight_log_dir
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+python-with-eggs = ${python-with-eggs:executable}

software/tactic_generator/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../component/selenium/buildout.cfg
+  ../fluentd/software.cfg
+
+parts +=
+  python-with-eggs
+
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+scripts = ${:interpreter}
+executable = ${buildout:bin-directory}/${:interpreter}
+eggs = ${selenium:egg}
+
+[instance-profile]
+url = ${:_profile_base_location_}/${instance-selenium:filename}
+md5sum = ${instance-selenium:md5sum}
+
+[instance-fluentd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/../fluentd/instance.cfg
+output = ${buildout:directory}/template-fluentd-profile.cfg

software/tactic_generator/software.cfg.json

+{
+  "name": "Selenium client",
+  "description": "Selenium client",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Selenium client",
+      "request": "instance-input-schema.json",
+      "response": "instance-output-schema.json",
+      "index": 0
+    }
+  }
+}